elk7: 7.11.1 -> 7.16.1, 6.8.3 -> 6.8.21 + add filebeat module and tests (#150879)

* elk7: 7.11.1 -> 7.16.1

* nixosTests.elk: Improve reliability and compatibility with ELK 7.x

- Use comparisons in jq instead of grepping
- Match for `.hits.total.value` if version >= 7, otherwise it always
  passes
- Make curl fail if requests fails

* nixos/filebeat: Add initial module and test

Filebeat is an open source file harvester, mostly used to fetch logs
files and feed them into logstash.

This module can be used instead of journalbeat if used with
`filebeat7` and configured with the `journald` input.

* python3Packages.parsedmarc.tests: Fix breakage

- Don't use the deprecated elasticsearch7-oss package
- Improve jq query robustness and add tracing

* rl-2205: Note the addition of the filebeat service

* elk6: 6.8.3 -> 6.8.21

The latest version includes a fix for CVE-2021-44228.

* nixos/journalbeat: Add a loose dependency on elasticsearch

Avoid unnecssary back-off when elasticsearch is running on the same
host.

18 files changed, 414 insertions, 94 deletions

diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
index d02f951a03c..2dd27649c52 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
@@ -19,7 +19,7 @@
   </section>
   <section xml:id="sec-release-22.05-new-services">
     <title>New Services</title>
-    <itemizedlist spacing="compact">
+    <itemizedlist>
       <listitem>
         <para>
           <link xlink:href="https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw">aesmd</link>,
@@ -28,6 +28,14 @@
           <link linkend="opt-services.aesmd.enable">services.aesmd</link>.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html">filebeat</link>,
+          a lightweight shipper for forwarding and centralizing log
+          data. Available as
+          <link linkend="opt-services.filebeat.enable">services.filebeat</link>.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
   <section xml:id="sec-release-22.05-incompatibilities">
diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md
index 11e5462b331..595785e732a 100644
--- a/nixos/doc/manual/release-notes/rl-2205.section.md
+++ b/nixos/doc/manual/release-notes/rl-2205.section.md
@@ -10,6 +10,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - [aesmd](https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw), the Intel SGX Architectural Enclave Service Manager. Available as [services.aesmd](#opt-services.aesmd.enable).
 
+- [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html), a lightweight shipper for forwarding and centralizing log data. Available as [services.filebeat](#opt-services.filebeat.enable).
+
 ## Backward Incompatibilities {#sec-release-22.05-incompatibilities}
 
 - `pkgs.ghc` now refers to `pkgs.targetPackages.haskellPackages.ghc`.
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 1f826220a0f..cb2dd530de1 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -448,6 +448,7 @@
   ./services/hardware/xow.nix
   ./services/logging/SystemdJournal2Gelf.nix
   ./services/logging/awstats.nix
+  ./services/logging/filebeat.nix
   ./services/logging/fluentd.nix
   ./services/logging/graylog.nix
   ./services/logging/heartbeat.nix
diff --git a/nixos/modules/services/logging/filebeat.nix b/nixos/modules/services/logging/filebeat.nix
new file mode 100644
index 00000000000..223a993c505
--- /dev/null
+++ b/nixos/modules/services/logging/filebeat.nix
@@ -0,0 +1,253 @@
+{ config, lib, utils, pkgs, ... }:
+
+let
+  inherit (lib)
+    attrValues
+    literalExpression
+    mkEnableOption
+    mkIf
+    mkOption
+    types;
+
+  cfg = config.services.filebeat;
+
+  json = pkgs.formats.json {};
+in
+{
+  options = {
+
+    services.filebeat = {
+
+      enable = mkEnableOption "filebeat";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.filebeat;
+        defaultText = literalExpression "pkgs.filebeat";
+        example = literalExpression "pkgs.filebeat7";
+        description = ''
+          The filebeat package to use.
+        '';
+      };
+
+      inputs = mkOption {
+        description = ''
+          Inputs specify how Filebeat locates and processes input data.
+
+          This is like <literal>services.filebeat.settings.filebeat.inputs</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat input.
+
+          An input type can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.inputs._name_.type"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            type = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The input type.
+
+                Look for the value after <literal>type:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            journald.id = "everything";  # Only for filebeat7
+            log = {
+              enabled = true;
+              paths = [
+                "/var/log/*.log"
+              ];
+            };
+          };
+        '';
+      };
+
+      modules = mkOption {
+        description = ''
+          Filebeat modules provide a quick way to get started
+          processing common log formats. They contain default
+          configurations, Elasticsearch ingest pipeline definitions,
+          and Kibana dashboards to help you implement and deploy a log
+          monitoring solution.
+
+          This is like <literal>services.filebeat.settings.filebeat.modules</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat module.
+
+          A module can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.modules._name_.module"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            module = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The name of the module.
+
+                Look for the value after <literal>module:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            nginx = {
+              access = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/access.log*" ];
+              };
+              error = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/error.log*" ];
+              };
+            };
+          };
+        '';
+      };
+
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = json.type;
+
+          options = {
+
+            output.elasticsearch.hosts = mkOption {
+              type = with types; listOf str;
+              default = [ "127.0.0.1:9200" ];
+              example = [ "myEShost:9200" ];
+              description = ''
+                The list of Elasticsearch nodes to connect to.
+
+                The events are distributed to these nodes in round
+                robin order. If one node becomes unreachable, the
+                event is automatically sent to another node. Each
+                Elasticsearch node can be defined as a URL or
+                IP:PORT. For example:
+                <literal>http://192.15.3.2</literal>,
+                <literal>https://es.found.io:9230</literal> or
+                <literal>192.24.3.2:9300</literal>. If no port is
+                specified, <literal>9200</literal> is used.
+              '';
+            };
+
+            filebeat = {
+              inputs = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Inputs specify how Filebeat locates and processes
+                  input data. Use <xref
+                  linkend="opt-services.filebeat.inputs"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+                '';
+              };
+              modules = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Filebeat modules provide a quick way to get started
+                  processing common log formats. They contain default
+                  configurations, Elasticsearch ingest pipeline
+                  definitions, and Kibana dashboards to help you
+                  implement and deploy a log monitoring solution.
+
+                  Use <xref linkend="opt-services.filebeat.modules"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+                '';
+              };
+            };
+          };
+        };
+        default = {};
+        example = literalExpression ''
+          {
+            settings = {
+              output.elasticsearch = {
+                hosts = [ "myEShost:9200" ];
+                username = "filebeat_internal";
+                password = { _secret = "/var/keys/elasticsearch_password"; };
+              };
+              logging.level = "info";
+            };
+          };
+        '';
+
+        description = ''
+          Configuration for filebeat. See
+          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html"/>
+          for supported values.
+
+          Options containing secret data should be set to an attribute
+          set containing the attribute <literal>_secret</literal> - a
+          string pointing to a file containing the value the option
+          should be set to. See the example to get a better picture of
+          this: in the resulting
+          <filename>filebeat.yml</filename> file, the
+          <literal>output.elasticsearch.password</literal>
+          key will be set to the contents of the
+          <filename>/var/keys/elasticsearch_password</filename> file.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs;
+    services.filebeat.settings.filebeat.modules = attrValues cfg.modules;
+
+    systemd.services.filebeat = {
+      description = "Filebeat log shipper";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "elasticsearch.service" ];
+      after = [ "elasticsearch.service" ];
+      serviceConfig = {
+        ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" ''
+          set -euo pipefail
+
+          umask u=rwx,g=,o=
+
+          ${utils.genJqSecretsReplacementSnippet
+              cfg.settings
+              "/var/lib/filebeat/filebeat.yml"
+           }
+        '';
+        ExecStart = ''
+          ${cfg.package}/bin/filebeat -e \
+            -c "/var/lib/filebeat/filebeat.yml" \
+            --path.data "/var/lib/filebeat"
+        '';
+        Restart = "always";
+        StateDirectory = "filebeat";
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix
index 2d98598c1be..48b02142cb4 100644
--- a/nixos/modules/services/logging/journalbeat.nix
+++ b/nixos/modules/services/logging/journalbeat.nix
@@ -28,7 +28,6 @@ in
         type = types.package;
         default = pkgs.journalbeat;
         defaultText = literalExpression "pkgs.journalbeat";
-        example = literalExpression "pkgs.journalbeat7";
         description = ''
           The journalbeat package to use
         '';
@@ -89,6 +88,8 @@ in
     systemd.services.journalbeat = {
       description = "Journalbeat log shipper";
       wantedBy = [ "multi-user.target" ];
+      wants = [ "elasticsearch.service" ];
+      after = [ "elasticsearch.service" ];
       preStart = ''
         mkdir -p ${cfg.stateDir}/data
         mkdir -p ${cfg.stateDir}/logs
diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix
index ae746d7e1f0..f42be00f23b 100644
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@@ -40,9 +40,8 @@ let
 
             services = {
 
-              journalbeat = let lt6 = builtins.compareVersions
-                                        elk.journalbeat.version "6" < 0; in {
-                enable = true;
+              journalbeat = {
+                enable = elk ? journalbeat;
                 package = elk.journalbeat;
                 extraConfig = pkgs.lib.mkOptionDefault (''
                   logging:
@@ -51,14 +50,29 @@ let
                     metrics.enabled: false
                   output.elasticsearch:
                     hosts: [ "127.0.0.1:9200" ]
-                    ${pkgs.lib.optionalString lt6 "template.enabled: false"}
-                '' + pkgs.lib.optionalString (!lt6) ''
                   journalbeat.inputs:
                   - paths: []
                     seek: cursor
                 '');
               };
 
+              filebeat = {
+                enable = elk ? filebeat;
+                package = elk.filebeat;
+                inputs.journald.id = "everything";
+
+                inputs.log = {
+                  enabled = true;
+                  paths = [
+                    "/var/lib/filebeat/test"
+                  ];
+                };
+
+                settings = {
+                  logging.level = "info";
+                };
+              };
+
               metricbeat = {
                 enable = true;
                 package = elk.metricbeat;
@@ -142,27 +156,43 @@ let
       };
 
     passthru.elkPackages = elk;
-    testScript = ''
+    testScript =
+      let
+        valueObject = lib.optionalString (lib.versionAtLeast elk.elasticsearch.version "7") ".value";
+      in ''
       import json
 
 
-      def total_hits(message):
+      def expect_hits(message):
+          dictionary = {"query": {"match": {"message": message}}}
+          return (
+              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
+              + "-H 'Content-Type: application/json' "
+              + "-d '{}' ".format(json.dumps(dictionary))
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
+          )
+
+
+      def expect_no_hits(message):
           dictionary = {"query": {"match": {"message": message}}}
           return (
-              "curl --silent --show-error '${esUrl}/_search' "
+              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
               + "-H 'Content-Type: application/json' "
               + "-d '{}' ".format(json.dumps(dictionary))
-              + "| jq .hits.total"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} == 0 end'"
           )
 
 
       def has_metricbeat():
           dictionary = {"query": {"match": {"event.dataset": {"query": "system.cpu"}}}}
           return (
-              "curl --silent --show-error '${esUrl}/_search' "
+              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
               + "-H 'Content-Type: application/json' "
               + "-d '{}' ".format(json.dumps(dictionary))
-              + "| jq '.hits.total > 0'"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
 
 
@@ -178,7 +208,8 @@ let
       # TODO: extend this test with multiple elasticsearch nodes
       #       and see if the status turns "green".
       one.wait_until_succeeds(
-          "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"
+          "curl --silent --show-error --fail-with-body '${esUrl}/_cluster/health'"
+          + " | jq -es 'if . == [] then null else .[] | .status != \"red\" end'"
       )
 
       with subtest("Perform some simple logstash tests"):
@@ -189,33 +220,50 @@ let
       with subtest("Kibana is healthy"):
           one.wait_for_unit("kibana.service")
           one.wait_until_succeeds(
-              "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
+              "curl --silent --show-error --fail-with-body 'http://localhost:5601/api/status'"
+              + " | jq -es 'if . == [] then null else .[] | .status.overall.state == \"green\" end'"
           )
 
       with subtest("Metricbeat is running"):
           one.wait_for_unit("metricbeat.service")
 
       with subtest("Metricbeat metrics arrive in elasticsearch"):
-          one.wait_until_succeeds(has_metricbeat() + " | tee /dev/console | grep 'true'")
+          one.wait_until_succeeds(has_metricbeat())
 
       with subtest("Logstash messages arive in elasticsearch"):
-          one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
-          one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
+          one.wait_until_succeeds(expect_hits("flowers"))
+          one.wait_until_succeeds(expect_no_hits("dragons"))
 
+    '' + lib.optionalString (elk ? journalbeat) ''
       with subtest(
           "A message logged to the journal is ingested by elasticsearch via journalbeat"
       ):
           one.wait_for_unit("journalbeat.service")
           one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat")
           one.wait_until_succeeds(
-              total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
+              expect_hits("Supercalifragilisticexpialidocious")
           )
-
+    '' + lib.optionalString (elk ? filebeat) ''
+      with subtest(
+          "A message logged to the journal is ingested by elasticsearch via filebeat"
+      ):
+          one.wait_for_unit("filebeat.service")
+          one.execute("echo 'Superdupercalifragilisticexpialidocious' | systemd-cat")
+          one.wait_until_succeeds(
+              expect_hits("Superdupercalifragilisticexpialidocious")
+          )
+          one.execute(
+              "echo 'SuperdupercalifragilisticexpialidociousIndeed' >> /var/lib/filebeat/test"
+          )
+          one.wait_until_succeeds(
+              expect_hits("SuperdupercalifragilisticexpialidociousIndeed")
+          )
+    '' + ''
       with subtest("Elasticsearch-curator works"):
           one.systemctl("stop logstash")
           one.systemctl("start elasticsearch-curator")
           one.wait_until_succeeds(
-              '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep ^'
+              '! curl --silent --show-error --fail-with-body "${esUrl}/_cat/indices" | grep logstash | grep ^'
           )
     '';
   }) { inherit pkgs system; };
@@ -235,7 +283,7 @@ in {
   #   elasticsearch = pkgs.elasticsearch7-oss;
   #   logstash      = pkgs.logstash7-oss;
   #   kibana        = pkgs.kibana7-oss;
-  #   journalbeat   = pkgs.journalbeat7;
+  #   filebeat      = pkgs.filebeat7;
   #   metricbeat    = pkgs.metricbeat7;
   # };
   unfree = lib.dontRecurseIntoAttrs {
@@ -250,7 +298,7 @@ in {
       elasticsearch = pkgs.elasticsearch7;
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
-      journalbeat   = pkgs.journalbeat7;
+      filebeat      = pkgs.filebeat7;
       metricbeat    = pkgs.metricbeat7;
     };
   };
diff --git a/nixos/tests/parsedmarc/default.nix b/nixos/tests/parsedmarc/default.nix
index d838d3b6a39..50b977723e9 100644
--- a/nixos/tests/parsedmarc/default.nix
+++ b/nixos/tests/parsedmarc/default.nix
@@ -4,6 +4,7 @@
 { pkgs, ... }@args:
 let
   inherit (import ../../lib/testing-python.nix args) makeTest;
+  inherit (pkgs) lib;
 
   dmarcTestReport = builtins.fetchurl {
     name = "dmarc-test-report";
@@ -54,7 +55,7 @@ in
   localMail = makeTest
     {
       name = "parsedmarc-local-mail";
-      meta = with pkgs.lib.maintainers; {
+      meta = with lib.maintainers; {
         maintainers = [ talyz ];
       };
 
@@ -83,7 +84,7 @@ in
             };
           };
 
-          services.elasticsearch.package = pkgs.elasticsearch7-oss;
+          services.elasticsearch.package = pkgs.elasticsearch-oss;
 
           environment.systemPackages = [
             (sendEmail "dmarc@localhost")
@@ -94,6 +95,7 @@ in
       testScript = { nodes }:
         let
           esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
+          valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
         in ''
           parsedmarc.start()
           parsedmarc.wait_for_unit("postfix.service")
@@ -104,11 +106,15 @@ in
           )
 
           parsedmarc.fail(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
           parsedmarc.succeed("send-email")
           parsedmarc.wait_until_succeeds(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
         '';
     };
@@ -121,7 +127,7 @@ in
     in
       makeTest {
         name = "parsedmarc-external-mail";
-        meta = with pkgs.lib.maintainers; {
+        meta = with lib.maintainers; {
           maintainers = [ talyz ];
         };
 
@@ -153,7 +159,7 @@ in
                 };
               };
 
-              services.elasticsearch.package = pkgs.elasticsearch7-oss;
+              services.elasticsearch.package = pkgs.elasticsearch-oss;
 
               environment.systemPackages = [
                 pkgs.jq
@@ -201,6 +207,7 @@ in
         testScript = { nodes }:
           let
             esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
+            valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
           in ''
             mail.start()
             mail.wait_for_unit("postfix.service")
@@ -213,11 +220,15 @@ in
             )
 
             parsedmarc.fail(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+                + " | tee /dev/console"
+                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
             )
             mail.succeed("send-email")
             parsedmarc.wait_until_succeeds(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+                + " | tee /dev/console"
+                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
             )
           '';
       };
diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix
index c728f737543..cd81975ee44 100644
--- a/pkgs/development/tools/misc/kibana/6.x.nix
+++ b/pkgs/development/tools/misc/kibana/6.x.nix
@@ -18,12 +18,12 @@ let
   shas =
     if enableUnfree
     then {
-      x86_64-linux  = "1xwklhqxk5rmdrgy2simwvijzq29kyq5w2w3hy53xh2i1zlnyvq3";
-      x86_64-darwin = "1qpdn28mrpggd55khzqqld6r89l0hb870rigxcw2i8p2yx3jv106";
+      x86_64-linux  = "1a501lavxhckb3l93sbrbqyshicwkk6p89frry4x8p037xcfpy0x";
+      x86_64-darwin = "0zm45af30shhcg3mdhcma6rms1hyrx62rm5jzwnz9kxv4d30skbw";
     }
     else {
-      x86_64-linux  = "1wpnwal2rq5v2bsp5qil9j6dplif7ql5394sy4ia5ghp2fzifxmf";
-      x86_64-darwin = "12z8i0wbw10c097glbpdy350p0h3957433f51qfx2p0ghgkzkhzv";
+      x86_64-linux  = "0wfdipf21apyily7mvlqgyc7m5jpr96zgrryzwa854z3xb2vw8zg";
+      x86_64-darwin = "1nklfx4yz6hsxlljvnvwjy7pncv9mzngl84710xad5jlyras3sdj";
     };
 
 in stdenv.mkDerivation rec {
diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix
index 7a2e0d1d364..455f95a69eb 100644
--- a/pkgs/development/tools/misc/kibana/7.x.nix
+++ b/pkgs/development/tools/misc/kibana/7.x.nix
@@ -4,23 +4,23 @@
 , stdenv
 , makeWrapper
 , fetchurl
-, nodejs-14_x
+, nodejs-16_x
 , coreutils
 , which
 }:
 
 with lib;
 let
-  nodejs = nodejs-14_x;
+  nodejs = nodejs-16_x;
   inherit (builtins) elemAt;
   info = splitString "-" stdenv.hostPlatform.system;
   arch = elemAt info 0;
   plat = elemAt info 1;
   shas =
     {
-      x86_64-linux  = "19p9s4sir982bb1zcldrbphhwfs9i11p0q28vgc421iqg10kjlf1";
-      x86_64-darwin = "0qq557ngwwakifidyrccga4cadj9k9pzhjwy4msmbcgf5pb86qyc";
-      aarch64-linux = "183cp1h8d3n7xfcpcys4hf36palczxa409afyp62kzyzckngy0j8";
+      x86_64-linux  = "0jivwsrq31n0qfznrsjfsn65sg3wpbd990afn2wzjnj4drq7plz6";
+      x86_64-darwin = "02483aqzrccq1x6rwznmcazijdd46yxj9vnbihnvp2xyp3w9as45";
+      aarch64-linux = "0iw155gkkl1hshc80lfj95rssg039ig21wz1l3srmmf2x4f934s9";
     };
 
 in stdenv.mkDerivation rec {
diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix
index 1808197498b..f5e31924791 100644
--- a/pkgs/misc/logging/beats/6.x.nix
+++ b/pkgs/misc/logging/beats/6.x.nix
@@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec {
         owner = "elastic";
         repo = "beats";
         rev = "v${version}";
-        sha256 = "0jkiz5dfdi9zsji04ipcmcj7pml9294v455y7s2c22k24gyzbaw8";
+        sha256 = "1vnw9clsc10cfpjf6vxvc6m507b2q17sgsl079iwqbp4v0286il7";
       };
 
       goPackagePath = "github.com/elastic/beats";
diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix
index b8b82ed4b30..9f5e550d972 100644
--- a/pkgs/misc/logging/beats/7.x.nix
+++ b/pkgs/misc/logging/beats/7.x.nix
@@ -8,10 +8,10 @@ let beat = package: extraArgs: buildGoModule (rec {
     owner = "elastic";
     repo = "beats";
     rev = "v${version}";
-    sha256 = "0gjyzprgj9nskvlkm2bf125b7qn3608llz4kh1fyzsvrw6zb7sm8";
+    sha256 = "sha256-9Jl5Xo1iKdOY9ZE5JXKSL4ee+NdsN3KCY2dDYuxlzPI=";
   };
 
-  vendorSha256 = "04cwf96fh60ld3ndjzzssgirc9ssb53yq71j6ksx36m3y1x7fq9c";
+  vendorSha256 = "sha256-tyxyM7RsTHTVVxc9gagPsSvFRaWGTmobKzyv9RODXBk=";
 
   subPackages = [ package ];
 
@@ -24,7 +24,14 @@ let beat = package: extraArgs: buildGoModule (rec {
 } // extraArgs);
 in
 rec {
-  filebeat7 = beat "filebeat" { meta.description = "Lightweight shipper for logfiles"; };
+  filebeat7 = beat "filebeat" {
+    meta.description = "Lightweight shipper for logfiles";
+    buildInputs = [ systemd ];
+    tags = [ "withjournald" ];
+    postFixup = ''
+      patchelf --set-rpath ${lib.makeLibraryPath [ (lib.getLib systemd) ]} "$out/bin/filebeat"
+    '';
+  };
   heartbeat7 = beat "heartbeat" { meta.description = "Lightweight shipper for uptime monitoring"; };
   metricbeat7 = beat "metricbeat" {
     meta.description = "Lightweight shipper for metrics";
@@ -47,15 +54,4 @@ rec {
       PostgreSQL, Redis or Thrift and correlate the messages into transactions.
     '';
   };
-  journalbeat7 = beat "journalbeat" {
-    meta.description = ''
-      Journalbeat is an open source data collector to read and forward
-      journal entries from Linuxes with systemd.
-    '';
-    buildInputs = [ systemd.dev ];
-    postFixup = let libPath = lib.makeLibraryPath [ (lib.getLib systemd) ]; in
-      ''
-        patchelf --set-rpath ${libPath} "$out/bin/journalbeat"
-      '';
-  };
 }
diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix
index 4b92592f65d..56f0779c833 100644
--- a/pkgs/servers/search/elasticsearch/6.x.nix
+++ b/pkgs/servers/search/elasticsearch/6.x.nix
@@ -19,8 +19,8 @@ stdenv.mkDerivation (rec {
     url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}.tar.gz";
     sha256 =
       if enableUnfree
-      then "09dy3iyzk460vra6na6vk7d3mzpbv4cl0pl7kjmybxy947j7hh42"
-      else "0s04xz3j4psyhawvy503sp2nl5s0gswmpd9wfvwnavgcrr23wk39";
+      then "1hkcgqsrnnx3zjpgar4424mxfaxrx0zbrp7n7n0dlbhphshwnkmd"
+      else "1pglg60aigy31xmpfchnxcc04nd18zwc3av4m0kyp00yk5mnlyqm";
   };
 
   patches = [ ./es-home-6.x.patch ];
diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix
index c254b733837..592cc947a42 100644
--- a/pkgs/servers/search/elasticsearch/7.x.nix
+++ b/pkgs/servers/search/elasticsearch/7.x.nix
@@ -18,9 +18,9 @@ let
   plat = elemAt info 1;
   shas =
     {
-      x86_64-linux  = "1ld7656b37l67vi4pyv0il865b168niqnbd4hzbvdnwrm35prp10";
-      x86_64-darwin = "11b180y11xw5q01l7aw6lyn15lp9ks8xmakjg1j7gp3z6c90hpn3";
-      aarch64-linux = "0s4ph79x17f90jk31wjwk259dk9dmhnmnkxdcn77m191wvf6m3wy";
+      x86_64-linux  = "1s16l95wc589cr69pfbgmkn9rkvxn6sd6jlbiqpm6p6iyxiaxd6c";
+      x86_64-darwin = "05h7pvq4pb816wgcymnfklp3w6sv54x6138v2infw5219dnk8pfs";
+      aarch64-linux = "0q4xnjzhlx1b2lkikca88qh9glfxaifsm419k2bxxlrfrx31zlkq";
     };
 in
 stdenv.mkDerivation rec {
diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix
index f71d7b9cc76..03bb24d9a39 100644
--- a/pkgs/servers/search/elasticsearch/plugins.nix
+++ b/pkgs/servers/search/elasticsearch/plugins.nix
@@ -38,8 +38,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "0mi6fmnjbqypa4n1w34dvlmyq793pz4wf1r5srcs7i84kkiddysy"
-        else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv"
+        if version == "7.16.1" then "1sz858m9963xqr5kzjlwnq7k0a146rn60v6xijyfbp8y3brg618p"
+        else if version == "6.8.21" then "06b1pavyggzfp4wwdql0q9nm3r7i9px9cagp4yh4nhxhnk4w5fiq"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -55,8 +55,8 @@ in
     src = fetchurl {
       url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip";
       sha256 =
-        if version == "7.11.1" then "0r2k2ndgqiqh27lch8dbay1m09f00h5kjcan87chcvyf623l40a3"
-        else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53"
+        if version == "7.16.1" then "0yjy9yhw77lmalivxnmv2rq8fk93ddxszkk73lgmpffladx2ikir"
+        else if version == "6.8.21" then "0m80cn7vkcvk95v4pdmi6vk5ww7p01k0hj2iqb9g870vs6x2qjzv"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -72,8 +72,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "10ln81zyf04qi9wv10mck8iz0xwfvwp4ni0hl1gkgvh44lf1n855"
-        else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37"
+        if version == "7.16.1" then "1w5ndgffqzj5ijglmykifrk1jsgh7qwn8m7sbpiv0r7n3aayhz1x"
+        else if version == "6.8.21" then "07w8s4a5gvr9lzjzf629y8rx3kvs6zd1vl07ksw1paghp42yb354"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -89,8 +89,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "09grfvqjmm2rznc48z84awh54afh81qa16amfqw3amsb8dr6czm6"
-        else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im"
+        if version == "7.16.1" then "16mv7b9nl96bcygabvjqidxp2sjk340va19mrmliblpq3mxa2sii"
+        else if version == "6.8.21" then "1kdpbrasxwr3dn21zjrklp1s389rwa51fairygdwl8px9liwwfa5"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -106,8 +106,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "0imkf3w2fmspb78vkf9k6kqx1crm4f82qgnbk1qa7gbsa2j47hbs"
-        else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll"
+        if version == "7.16.1" then "0bf8f8cybsp6s2ai3j04yay9kbhsafpgxivxjvzn2iy9qgc84ls4"
+        else if version == "6.8.21" then "0v31yyhjcdlqnjw1f9kihh7z3c6d31whc57hqqd1dn579n4s9rlz"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -123,8 +123,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.11.1" then "0ahyb1plgwvq22id2kcx9g076ybb3kvybwakgcvsdjjdyi4cwgjs"
-        else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x"
+        if version == "7.16.1" then "0sfa0ql3hh8jmha230dyhr51bvsvwmazyycf36ngpmxsysm8ccml"
+        else if version == "6.8.21" then "0sfh1az30q4f34zxig2fz8wn9gk53fmmxyg5pbi1svn9761p5awq"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -140,8 +140,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.11.1" then "0i98b905k1zwm3y9pfhr40v2fm5qdsp3icygibhxf7drffygk4l7"
-        else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk"
+        if version == "7.16.1" then "1b95hjr4qhiavm7r7k19bwk5c64r00f1g5s0ydnb6gzym9hdb5s1"
+        else if version == "6.8.21" then "00lwj00rfdk6850gk1n86chiz2w6afpqn7jn588jdbwv41qh5mrv"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -157,24 +157,24 @@ in
     pluginName = "search-guard";
     version =
       # https://docs.search-guard.com/latest/search-guard-versions
-      if esVersion == "7.11.1" then "${esVersion}-50.0.0"
-      else if esVersion == "6.8.3" then "${esVersion}-25.5"
+      if esVersion == "7.16.1" then "${esVersion}-52.5.0"
+      else if esVersion == "6.8.21" then "${esVersion}-25.6"
       else throw "unsupported version ${esVersion} for plugin ${pluginName}";
     src =
-      if esVersion == "7.11.1" then
+      if esVersion == "7.16.1" then
         fetchurl {
           url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip";
-          sha256 = "1lippygiy0xcxxlakylhvj3bj2i681k6jcfjsprkfk7hlaqsqxkm";
+          sha256 = "1m3nj35qyrkkh3mhmn66nippavima8h8qpaxddalhjsvf70lhnjb";
         }
-      else if esVersion == "6.8.3" then
+      else if esVersion == "6.8.21" then
         fetchurl {
-          url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip";
-          sha256 = "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb";
+          url = "https://maven.search-guard.com/search-guard-release/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip";
+          sha256 = "19nj513wigwd0mzq747zax4fzvv5vi24f7j0636rydd9iv9cyhg2";
         }
       else throw "unsupported version ${version} for plugin ${pluginName}";
     meta = with lib; {
       homepage = "https://search-guard.com";
-      description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. ";
+      description = "Elasticsearch plugin that offers encryption, authentication, and authorisation.";
       license = licenses.asl20;
     };
   };
diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix
index b35e5a4aea4..0b3e17818dc 100644
--- a/pkgs/tools/misc/logstash/6.x.nix
+++ b/pkgs/tools/misc/logstash/6.x.nix
@@ -17,8 +17,8 @@ let this = stdenv.mkDerivation rec {
     url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}.tar.gz";
     sha256 =
       if enableUnfree
-      then "00pwi7clgdflzzg15bh3y30gzikvvy7p5fl88fww7xhhy47q8053"
-      else "0spxgqsyh72n0l0xh6rljp0lbqz46xmr02sqz25ybycr4qkxdhgk";
+      then "0hij1byw5b3xmk3vshr9p7gxwbjrywr7ylps05ydc2dmnz8q2a79"
+      else "1fa236pvhj7spys54nqi3k64rwzf6zi6gaccmqg4p4sh92jzsybv";
   };
 
   dontBuild         = true;
diff --git a/pkgs/tools/misc/logstash/7.x.nix b/pkgs/tools/misc/logstash/7.x.nix
index 1e69fbc976d..636c380817c 100644
--- a/pkgs/tools/misc/logstash/7.x.nix
+++ b/pkgs/tools/misc/logstash/7.x.nix
@@ -17,14 +17,14 @@ let
   shas =
     if enableUnfree
     then {
-      x86_64-linux  = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p";
-      x86_64-darwin = "0dqm66c89w1nvmbwqzphlqmf7avrycgv1nwd5b0k1z168fj0c3zm";
-      aarch64-linux = "11hjhyb48mjagmvqyxb780n57kr619h6p4adl2vs1zm97g9gslx8";
+      x86_64-linux  = "1vm53alq9q1qy2jcsjg9z339xrkac5r9qqpdafp53ny4zsv1n7vj";
+      x86_64-darwin = "0hhjyl04h3gd66rdk22272rj419br4v2i59lyrmaj6hmnsqbv968";
+      aarch64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p";
     }
     else {
-      x86_64-linux  = "14b1649avjcalcsi0ffkgznq6d93qdk6m3j0i73mwfqka5d3dvy3";
-      x86_64-darwin = "0ypgdfklr5rxvsnc3czh231pa1z2h70366j1c6q5g64b3xnxpphs";
-      aarch64-linux = "01ainayr8fwwfix7dmxfhhmb23ji65dn4lbjwnj2w0pl0ym9h9w2";
+      x86_64-linux  = "1f3659vcgczm7v03q3fvsmp1ndp6wm3i7r2b2vbl4xq7hf9v7azk";
+      x86_64-darwin = "10zw9qc0lc0x9in0nkxc1aiazhyd69l8sya2ni46ivyyjwf0sqsn";
+      aarch64-linux = "1czhgmky2zf3mqykn5ww4257yfhd36mi4x6dq569ymly83pivf8v";
     };
   this = stdenv.mkDerivation rec {
     version = elk7Version;
diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix
index 222c6ac0474..672fdf21b02 100644
--- a/pkgs/top-level/aliases.nix
+++ b/pkgs/top-level/aliases.nix
@@ -384,6 +384,7 @@ mapAliases ({
   jbuilder = dune_1; # added 2018-09-09
   jikes = throw "jikes was deprecated on 2019-10-07: abandoned by upstream";
   joseki = apache-jena-fuseki; # added 2016-02-28
+  journalbeat7 = throw "journalbeat has been removed upstream. Use filebeat with the journald input instead.";
   jvmci8 = throw "graalvm8 and its tools were deprecated in favor of graalvm8-ce"; # added 2021-10-15
   json_glib = json-glib; # added 2018-02-25
   kafkacat = kcat; # added 2021-10-07
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 8d98f53a331..9c117ca4d71 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -2140,8 +2140,7 @@ with pkgs;
     filebeat7
     heartbeat7
     metricbeat7
-    packetbeat7
-    journalbeat7;
+    packetbeat7;
 
   filebeat = filebeat6;
   heartbeat = heartbeat6;
@@ -5000,8 +4999,8 @@ with pkgs;
 
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
   # When updating make sure to update all plugins or they will break!
-  elk6Version = "6.8.3";
-  elk7Version = "7.11.1";
+  elk6Version = "6.8.21";
+  elk7Version = "7.16.1";
 
   elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix {
     util-linux = util-linuxMinimal;