elk: add elasticsearch6, logstash6, kibana6 and the beats at v6.1.0

This change is backwards compatible since the ELK tools at version 5.x
remain unchanged.

The test suite now both tests ELK-5 and ELK-6.

7 files changed, 208 insertions, 1 deletions

diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix
new file mode 100644
index 00000000000..64aeb38439e
--- /dev/null
+++ b/pkgs/development/tools/misc/kibana/6.x.nix
@@ -0,0 +1,40 @@
+{ stdenv, makeWrapper, fetchurl, elk6Version, nodejs, coreutils, which }:
+
+with stdenv.lib;
+let
+  inherit (builtins) elemAt;
+  info = splitString "-" stdenv.system;
+  arch = elemAt info 0;
+  plat = elemAt info 1;
+  shas = {
+    "x86_64-linux"  = "08lkjj9h4ij25b53bgdz825j2ccymlllijbhv9kw1q1liv2irr34";
+    "x86_64-darwin" = "1iqzj01s9walj5arfdlw0dgbmrv6mjp64mch11rx5aybcafv4z9h";
+  };
+in stdenv.mkDerivation rec {
+  name = "kibana-${version}";
+  version = elk6Version;
+
+  src = fetchurl {
+    url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz";
+    sha256 = shas."${stdenv.system}" or (throw "Unknown architecture");
+  };
+
+  buildInputs = [ makeWrapper ];
+
+  installPhase = ''
+    mkdir -p $out/libexec/kibana $out/bin
+    mv * $out/libexec/kibana/
+    rm -r $out/libexec/kibana/node
+    makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
+      --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
+    sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
+  '';
+
+  meta = {
+    description = "Visualize logs and time-stamped data";
+    homepage = http://www.elasticsearch.org/overview/kibana;
+    license = licenses.asl20;
+    maintainers = with maintainers; [ offline rickynils basvandijk ];
+    platforms = with platforms; unix;
+  };
+}
diff --git a/pkgs/misc/logging/beats/default.nix b/pkgs/misc/logging/beats/5.x.nix
index ed2a2eadb65..ed2a2eadb65 100644
--- a/pkgs/misc/logging/beats/default.nix
+++ b/pkgs/misc/logging/beats/5.x.nix
diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix
new file mode 100644
index 00000000000..d9fa98c1497
--- /dev/null
+++ b/pkgs/misc/logging/beats/6.x.nix
@@ -0,0 +1,42 @@
+{ stdenv, fetchFromGitHub, elk6Version, buildGoPackage, libpcap }:
+
+let beat = package : extraArgs : buildGoPackage (rec {
+      name = "${package}-${version}";
+      version = elk6Version;
+
+      src = fetchFromGitHub {
+        owner = "elastic";
+        repo = "beats";
+        rev = "v${version}";
+        sha256 = "0pp4in66byggcfmvf8yx0m1vra98cs77m7mbr45sdla4hinvaqar";
+      };
+
+      goPackagePath = "github.com/elastic/beats";
+
+      subPackages = [ package ];
+
+      meta = with stdenv.lib; {
+        homepage = https://www.elastic.co/products/beats;
+        license = licenses.asl20;
+        maintainers = with maintainers; [ fadenb basvandijk ];
+        platforms = platforms.linux;
+      };
+    } // extraArgs);
+in {
+  filebeat   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
+  heartbeat  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
+  metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
+  packetbeat = beat "packetbeat" {
+    buildInputs = [ libpcap ];
+    meta.description = "Network packet analyzer that ships data to Elasticsearch";
+    meta.longDescription = ''
+      Packetbeat is an open source network packet analyzer that ships the
+      data to Elasticsearch.
+
+      Think of it like a distributed real-time Wireshark with a lot more
+      analytics features. The Packetbeat shippers sniff the traffic between
+      your application processes, parse on the fly protocols like HTTP, MySQL,
+      PostgreSQL, Redis or Thrift and correlate the messages into transactions.
+    '';
+  };
+}
diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix
new file mode 100644
index 00000000000..3b7ff42d4e3
--- /dev/null
+++ b/pkgs/servers/search/elasticsearch/6.x.nix
@@ -0,0 +1,45 @@
+{ stdenv, fetchurl, elk6Version, makeWrapper, jre_headless, utillinux, getopt }:
+
+with stdenv.lib;
+
+stdenv.mkDerivation rec {
+  version = elk6Version;
+  name = "elasticsearch-${version}";
+
+  src = fetchurl {
+    url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz";
+    sha256 = "1mq8lnpv5y82a7d8vxn5np6hrg2pys22v85l5l9jynk3k0kgwyf8";
+  };
+
+  patches = [ ./es-home-6.x.patch ];
+
+  postPatch = ''
+    sed -i "s|ES_CLASSPATH=\"\$ES_HOME/lib/\*\"|ES_CLASSPATH=\"$out/lib/*\"|" ./bin/elasticsearch-env
+  '';
+
+  buildInputs = [ makeWrapper jre_headless ] ++
+    (if (!stdenv.isDarwin) then [utillinux] else [getopt]);
+
+  installPhase = ''
+    mkdir -p $out
+    cp -R bin config lib modules plugins $out
+
+    chmod -x $out/bin/*.*
+
+    wrapProgram $out/bin/elasticsearch \
+      ${if (!stdenv.isDarwin)
+        then ''--prefix PATH : "${utillinux}/bin/"''
+        else ''--prefix PATH : "${getopt}/bin"''} \
+      --set JAVA_HOME "${jre_headless}" \
+      --set ES_JVM_OPTIONS "$out/config/jvm.options"
+
+    wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
+  '';
+
+  meta = {
+    description = "Open Source, Distributed, RESTful Search Engine";
+    license = licenses.asl20;
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ apeschar basvandijk ];
+  };
+}
diff --git a/pkgs/servers/search/elasticsearch/es-home-6.x.patch b/pkgs/servers/search/elasticsearch/es-home-6.x.patch
new file mode 100644
index 00000000000..aba8d396a69
--- /dev/null
+++ b/pkgs/servers/search/elasticsearch/es-home-6.x.patch
@@ -0,0 +1,26 @@
+diff -Naur a/bin/elasticsearch-env b/bin/elasticsearch-env
+--- a/bin/elasticsearch-env	2017-12-12 13:31:51.000000000 +0100
++++ b/bin/elasticsearch-env	2017-12-18 19:51:12.282809695 +0100
+@@ -19,18 +19,10 @@
+   fi
+ done
+ 
+-# determine Elasticsearch home; to do this, we strip from the path until we find
+-# bin, and then strip bin (there is an assumption here that there is no nested
+-# directory under bin also named bin)
+-ES_HOME=`dirname "$SCRIPT"`
+-
+-# now make ES_HOME absolute
+-ES_HOME=`cd "$ES_HOME"; pwd`
+-
+-while [ "`basename "$ES_HOME"`" != "bin" ]; do
+-  ES_HOME=`dirname "$ES_HOME"`
+-done
+-ES_HOME=`dirname "$ES_HOME"`
++if [ -z "$ES_HOME" ]; then
++    echo "You must set the ES_HOME var" >&2
++    exit 1
++fi
+ 
+ # now set the classpath
+ ES_CLASSPATH="$ES_HOME/lib/*"
diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix
new file mode 100644
index 00000000000..3a05bedbdff
--- /dev/null
+++ b/pkgs/tools/misc/logstash/6.x.nix
@@ -0,0 +1,39 @@
+{ stdenv, fetchurl, elk6Version, makeWrapper, jre  }:
+
+stdenv.mkDerivation rec {
+  version = elk6Version;
+  name = "logstash-${version}";
+
+  src = fetchurl {
+    url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
+    sha256 = "1s2w8d2siryg2wy8i9lwqbp4mjf1sv80lf3sllxwa2vqwsv6l64p";
+  };
+
+  dontBuild         = true;
+  dontPatchELF      = true;
+  dontStrip         = true;
+  dontPatchShebangs = true;
+
+  buildInputs = [
+    makeWrapper jre
+  ];
+
+  installPhase = ''
+    mkdir -p $out
+    cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out
+
+    wrapProgram $out/bin/logstash \
+       --set JAVA_HOME "${jre}"
+
+    wrapProgram $out/bin/logstash-plugin \
+       --set JAVA_HOME "${jre}"
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
+    homepage    = https://www.elastic.co/products/logstash;
+    license     = licenses.asl20;
+    platforms   = platforms.unix;
+    maintainers = with maintainers; [ wjlroe offline basvandijk ];
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 4735f583b01..ac7a2fc397c 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -741,12 +741,23 @@ with pkgs;
 
   bchunk = callPackage ../tools/cd-dvd/bchunk { };
 
-  inherit (callPackages ../misc/logging/beats { })
+  inherit (callPackages ../misc/logging/beats/5.x.nix { })
     filebeat
     heartbeat
     metricbeat
     packetbeat;
 
+  inherit (let beats6 = callPackages ../misc/logging/beats/6.x.nix { }; in {
+    filebeat6   = beats6.filebeat;
+    heartbeat6  = beats6.heartbeat;
+    metricbeat6 = beats6.metricbeat;
+    packetbeat6 = beats6.packetbeat;
+  })
+    filebeat6
+    heartbeat6
+    metricbeat6
+    packetbeat6;
+
   bfr = callPackage ../tools/misc/bfr { };
 
   bibtool = callPackage ../tools/misc/bibtool { };
@@ -1949,10 +1960,12 @@ with pkgs;
 
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
   elk5Version = "5.6.5";
+  elk6Version = "6.1.0";
 
   elasticsearch = callPackage ../servers/search/elasticsearch { };
   elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { };
   elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
+  elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { };
 
   elasticsearchPlugins = recurseIntoAttrs (
     callPackage ../servers/search/elasticsearch/plugins.nix { }
@@ -2956,6 +2969,7 @@ with pkgs;
 
   kibana  = callPackage ../development/tools/misc/kibana { };
   kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
+  kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { };
 
   kismet = callPackage ../applications/networking/sniffers/kismet { };
 
@@ -3022,6 +3036,7 @@ with pkgs;
 
   logstash  = callPackage ../tools/misc/logstash { };
   logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
+  logstash6 = callPackage ../tools/misc/logstash/6.x.nix { };
 
   logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { };