elk7: 7.11.1 -> 7.16.1, 6.8.3 -> 6.8.21 + add filebeat module and tests (#150879)

* elk7: 7.11.1 -> 7.16.1

* nixosTests.elk: Improve reliability and compatibility with ELK 7.x

- Use comparisons in jq instead of grepping
- Match for `.hits.total.value` if version >= 7, otherwise it always
  passes
- Make curl fail if requests fails

* nixos/filebeat: Add initial module and test

Filebeat is an open source file harvester, mostly used to fetch logs
files and feed them into logstash.

This module can be used instead of journalbeat if used with
`filebeat7` and configured with the `journald` input.

* python3Packages.parsedmarc.tests: Fix breakage

- Don't use the deprecated elasticsearch7-oss package
- Improve jq query robustness and add tracing

* rl-2205: Note the addition of the filebeat service

* elk6: 6.8.3 -> 6.8.21

The latest version includes a fix for CVE-2021-44228.

* nixos/journalbeat: Add a loose dependency on elasticsearch

Avoid unnecssary back-off when elasticsearch is running on the same
host.

3 files changed, 27 insertions, 27 deletions

diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix
index 4b92592f65d..56f0779c833 100644
--- a/pkgs/servers/search/elasticsearch/6.x.nix
+++ b/pkgs/servers/search/elasticsearch/6.x.nix
@@ -19,8 +19,8 @@ stdenv.mkDerivation (rec {
     url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}.tar.gz";
     sha256 =
       if enableUnfree
-      then "09dy3iyzk460vra6na6vk7d3mzpbv4cl0pl7kjmybxy947j7hh42"
-      else "0s04xz3j4psyhawvy503sp2nl5s0gswmpd9wfvwnavgcrr23wk39";
+      then "1hkcgqsrnnx3zjpgar4424mxfaxrx0zbrp7n7n0dlbhphshwnkmd"
+      else "1pglg60aigy31xmpfchnxcc04nd18zwc3av4m0kyp00yk5mnlyqm";
   };
 
   patches = [ ./es-home-6.x.patch ];
diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix
index c254b733837..592cc947a42 100644
--- a/pkgs/servers/search/elasticsearch/7.x.nix
+++ b/pkgs/servers/search/elasticsearch/7.x.nix
@@ -18,9 +18,9 @@ let
   plat = elemAt info 1;
   shas =
     {
-      x86_64-linux  = "1ld7656b37l67vi4pyv0il865b168niqnbd4hzbvdnwrm35prp10";
-      x86_64-darwin = "11b180y11xw5q01l7aw6lyn15lp9ks8xmakjg1j7gp3z6c90hpn3";
-      aarch64-linux = "0s4ph79x17f90jk31wjwk259dk9dmhnmnkxdcn77m191wvf6m3wy";
+      x86_64-linux  = "1s16l95wc589cr69pfbgmkn9rkvxn6sd6jlbiqpm6p6iyxiaxd6c";
+      x86_64-darwin = "05h7pvq4pb816wgcymnfklp3w6sv54x6138v2infw5219dnk8pfs";
+      aarch64-linux = "0q4xnjzhlx1b2lkikca88qh9glfxaifsm419k2bxxlrfrx31zlkq";
     };
 in
 stdenv.mkDerivation rec {
diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix
index f71d7b9cc76..03bb24d9a39 100644
--- a/pkgs/servers/search/elasticsearch/plugins.nix
+++ b/pkgs/servers/search/elasticsearch/plugins.nix
@@ -38,8 +38,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "0mi6fmnjbqypa4n1w34dvlmyq793pz4wf1r5srcs7i84kkiddysy"
-        else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv"
+        if version == "7.16.1" then "1sz858m9963xqr5kzjlwnq7k0a146rn60v6xijyfbp8y3brg618p"
+        else if version == "6.8.21" then "06b1pavyggzfp4wwdql0q9nm3r7i9px9cagp4yh4nhxhnk4w5fiq"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -55,8 +55,8 @@ in
     src = fetchurl {
       url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip";
       sha256 =
-        if version == "7.11.1" then "0r2k2ndgqiqh27lch8dbay1m09f00h5kjcan87chcvyf623l40a3"
-        else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53"
+        if version == "7.16.1" then "0yjy9yhw77lmalivxnmv2rq8fk93ddxszkk73lgmpffladx2ikir"
+        else if version == "6.8.21" then "0m80cn7vkcvk95v4pdmi6vk5ww7p01k0hj2iqb9g870vs6x2qjzv"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -72,8 +72,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "10ln81zyf04qi9wv10mck8iz0xwfvwp4ni0hl1gkgvh44lf1n855"
-        else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37"
+        if version == "7.16.1" then "1w5ndgffqzj5ijglmykifrk1jsgh7qwn8m7sbpiv0r7n3aayhz1x"
+        else if version == "6.8.21" then "07w8s4a5gvr9lzjzf629y8rx3kvs6zd1vl07ksw1paghp42yb354"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -89,8 +89,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "09grfvqjmm2rznc48z84awh54afh81qa16amfqw3amsb8dr6czm6"
-        else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im"
+        if version == "7.16.1" then "16mv7b9nl96bcygabvjqidxp2sjk340va19mrmliblpq3mxa2sii"
+        else if version == "6.8.21" then "1kdpbrasxwr3dn21zjrklp1s389rwa51fairygdwl8px9liwwfa5"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -106,8 +106,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "0imkf3w2fmspb78vkf9k6kqx1crm4f82qgnbk1qa7gbsa2j47hbs"
-        else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll"
+        if version == "7.16.1" then "0bf8f8cybsp6s2ai3j04yay9kbhsafpgxivxjvzn2iy9qgc84ls4"
+        else if version == "6.8.21" then "0v31yyhjcdlqnjw1f9kihh7z3c6d31whc57hqqd1dn579n4s9rlz"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -123,8 +123,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.11.1" then "0ahyb1plgwvq22id2kcx9g076ybb3kvybwakgcvsdjjdyi4cwgjs"
-        else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x"
+        if version == "7.16.1" then "0sfa0ql3hh8jmha230dyhr51bvsvwmazyycf36ngpmxsysm8ccml"
+        else if version == "6.8.21" then "0sfh1az30q4f34zxig2fz8wn9gk53fmmxyg5pbi1svn9761p5awq"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -140,8 +140,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.11.1" then "0i98b905k1zwm3y9pfhr40v2fm5qdsp3icygibhxf7drffygk4l7"
-        else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk"
+        if version == "7.16.1" then "1b95hjr4qhiavm7r7k19bwk5c64r00f1g5s0ydnb6gzym9hdb5s1"
+        else if version == "6.8.21" then "00lwj00rfdk6850gk1n86chiz2w6afpqn7jn588jdbwv41qh5mrv"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -157,24 +157,24 @@ in
     pluginName = "search-guard";
     version =
       # https://docs.search-guard.com/latest/search-guard-versions
-      if esVersion == "7.11.1" then "${esVersion}-50.0.0"
-      else if esVersion == "6.8.3" then "${esVersion}-25.5"
+      if esVersion == "7.16.1" then "${esVersion}-52.5.0"
+      else if esVersion == "6.8.21" then "${esVersion}-25.6"
       else throw "unsupported version ${esVersion} for plugin ${pluginName}";
     src =
-      if esVersion == "7.11.1" then
+      if esVersion == "7.16.1" then
         fetchurl {
           url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip";
-          sha256 = "1lippygiy0xcxxlakylhvj3bj2i681k6jcfjsprkfk7hlaqsqxkm";
+          sha256 = "1m3nj35qyrkkh3mhmn66nippavima8h8qpaxddalhjsvf70lhnjb";
         }
-      else if esVersion == "6.8.3" then
+      else if esVersion == "6.8.21" then
         fetchurl {
-          url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip";
-          sha256 = "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb";
+          url = "https://maven.search-guard.com/search-guard-release/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip";
+          sha256 = "19nj513wigwd0mzq747zax4fzvv5vi24f7j0636rydd9iv9cyhg2";
         }
       else throw "unsupported version ${version} for plugin ${pluginName}";
     meta = with lib; {
       homepage = "https://search-guard.com";
-      description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. ";
+      description = "Elasticsearch plugin that offers encryption, authentication, and authorisation.";
       license = licenses.asl20;
     };
   };