elk7: 7.11.1 -> 7.16.1, 6.8.3 -> 6.8.21 + add filebeat module and tests (#150879)

* elk7: 7.11.1 -> 7.16.1

* nixosTests.elk: Improve reliability and compatibility with ELK 7.x

- Use comparisons in jq instead of grepping
- Match for `.hits.total.value` if version >= 7, otherwise it always
  passes
- Make curl fail if requests fails

* nixos/filebeat: Add initial module and test

Filebeat is an open source file harvester, mostly used to fetch logs
files and feed them into logstash.

This module can be used instead of journalbeat if used with
`filebeat7` and configured with the `journald` input.

* python3Packages.parsedmarc.tests: Fix breakage

- Don't use the deprecated elasticsearch7-oss package
- Improve jq query robustness and add tracing

* rl-2205: Note the addition of the filebeat service

* elk6: 6.8.3 -> 6.8.21

The latest version includes a fix for CVE-2021-44228.

* nixos/journalbeat: Add a loose dependency on elasticsearch

Avoid unnecssary back-off when elasticsearch is running on the same
host.

2 files changed, 255 insertions, 1 deletions

diff --git a/nixos/modules/services/logging/filebeat.nix b/nixos/modules/services/logging/filebeat.nix
new file mode 100644
index 00000000000..223a993c505
--- /dev/null
+++ b/nixos/modules/services/logging/filebeat.nix
@@ -0,0 +1,253 @@
+{ config, lib, utils, pkgs, ... }:
+
+let
+  inherit (lib)
+    attrValues
+    literalExpression
+    mkEnableOption
+    mkIf
+    mkOption
+    types;
+
+  cfg = config.services.filebeat;
+
+  json = pkgs.formats.json {};
+in
+{
+  options = {
+
+    services.filebeat = {
+
+      enable = mkEnableOption "filebeat";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.filebeat;
+        defaultText = literalExpression "pkgs.filebeat";
+        example = literalExpression "pkgs.filebeat7";
+        description = ''
+          The filebeat package to use.
+        '';
+      };
+
+      inputs = mkOption {
+        description = ''
+          Inputs specify how Filebeat locates and processes input data.
+
+          This is like <literal>services.filebeat.settings.filebeat.inputs</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat input.
+
+          An input type can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.inputs._name_.type"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            type = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The input type.
+
+                Look for the value after <literal>type:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            journald.id = "everything";  # Only for filebeat7
+            log = {
+              enabled = true;
+              paths = [
+                "/var/log/*.log"
+              ];
+            };
+          };
+        '';
+      };
+
+      modules = mkOption {
+        description = ''
+          Filebeat modules provide a quick way to get started
+          processing common log formats. They contain default
+          configurations, Elasticsearch ingest pipeline definitions,
+          and Kibana dashboards to help you implement and deploy a log
+          monitoring solution.
+
+          This is like <literal>services.filebeat.settings.filebeat.modules</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat module.
+
+          A module can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.modules._name_.module"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            module = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The name of the module.
+
+                Look for the value after <literal>module:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            nginx = {
+              access = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/access.log*" ];
+              };
+              error = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/error.log*" ];
+              };
+            };
+          };
+        '';
+      };
+
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = json.type;
+
+          options = {
+
+            output.elasticsearch.hosts = mkOption {
+              type = with types; listOf str;
+              default = [ "127.0.0.1:9200" ];
+              example = [ "myEShost:9200" ];
+              description = ''
+                The list of Elasticsearch nodes to connect to.
+
+                The events are distributed to these nodes in round
+                robin order. If one node becomes unreachable, the
+                event is automatically sent to another node. Each
+                Elasticsearch node can be defined as a URL or
+                IP:PORT. For example:
+                <literal>http://192.15.3.2</literal>,
+                <literal>https://es.found.io:9230</literal> or
+                <literal>192.24.3.2:9300</literal>. If no port is
+                specified, <literal>9200</literal> is used.
+              '';
+            };
+
+            filebeat = {
+              inputs = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Inputs specify how Filebeat locates and processes
+                  input data. Use <xref
+                  linkend="opt-services.filebeat.inputs"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+                '';
+              };
+              modules = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Filebeat modules provide a quick way to get started
+                  processing common log formats. They contain default
+                  configurations, Elasticsearch ingest pipeline
+                  definitions, and Kibana dashboards to help you
+                  implement and deploy a log monitoring solution.
+
+                  Use <xref linkend="opt-services.filebeat.modules"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+                '';
+              };
+            };
+          };
+        };
+        default = {};
+        example = literalExpression ''
+          {
+            settings = {
+              output.elasticsearch = {
+                hosts = [ "myEShost:9200" ];
+                username = "filebeat_internal";
+                password = { _secret = "/var/keys/elasticsearch_password"; };
+              };
+              logging.level = "info";
+            };
+          };
+        '';
+
+        description = ''
+          Configuration for filebeat. See
+          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html"/>
+          for supported values.
+
+          Options containing secret data should be set to an attribute
+          set containing the attribute <literal>_secret</literal> - a
+          string pointing to a file containing the value the option
+          should be set to. See the example to get a better picture of
+          this: in the resulting
+          <filename>filebeat.yml</filename> file, the
+          <literal>output.elasticsearch.password</literal>
+          key will be set to the contents of the
+          <filename>/var/keys/elasticsearch_password</filename> file.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs;
+    services.filebeat.settings.filebeat.modules = attrValues cfg.modules;
+
+    systemd.services.filebeat = {
+      description = "Filebeat log shipper";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "elasticsearch.service" ];
+      after = [ "elasticsearch.service" ];
+      serviceConfig = {
+        ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" ''
+          set -euo pipefail
+
+          umask u=rwx,g=,o=
+
+          ${utils.genJqSecretsReplacementSnippet
+              cfg.settings
+              "/var/lib/filebeat/filebeat.yml"
+           }
+        '';
+        ExecStart = ''
+          ${cfg.package}/bin/filebeat -e \
+            -c "/var/lib/filebeat/filebeat.yml" \
+            --path.data "/var/lib/filebeat"
+        '';
+        Restart = "always";
+        StateDirectory = "filebeat";
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix
index 2d98598c1be..48b02142cb4 100644
--- a/nixos/modules/services/logging/journalbeat.nix
+++ b/nixos/modules/services/logging/journalbeat.nix
@@ -28,7 +28,6 @@ in
         type = types.package;
         default = pkgs.journalbeat;
         defaultText = literalExpression "pkgs.journalbeat";
-        example = literalExpression "pkgs.journalbeat7";
         description = ''
           The journalbeat package to use
         '';
@@ -89,6 +88,8 @@ in
     systemd.services.journalbeat = {
       description = "Journalbeat log shipper";
       wantedBy = [ "multi-user.target" ];
+      wants = [ "elasticsearch.service" ];
+      after = [ "elasticsearch.service" ];
       preStart = ''
         mkdir -p ${cfg.stateDir}/data
         mkdir -p ${cfg.stateDir}/logs