elk7: 7.11.1 -> 7.16.1, 6.8.3 -> 6.8.21 + add filebeat module and tests (#150879)

* elk7: 7.11.1 -> 7.16.1

* nixosTests.elk: Improve reliability and compatibility with ELK 7.x

- Use comparisons in jq instead of grepping
- Match for `.hits.total.value` if version >= 7, otherwise it always
  passes
- Make curl fail if requests fails

* nixos/filebeat: Add initial module and test

Filebeat is an open source file harvester, mostly used to fetch logs
files and feed them into logstash.

This module can be used instead of journalbeat if used with
`filebeat7` and configured with the `journald` input.

* python3Packages.parsedmarc.tests: Fix breakage

- Don't use the deprecated elasticsearch7-oss package
- Improve jq query robustness and add tracing

* rl-2205: Note the addition of the filebeat service

* elk6: 6.8.3 -> 6.8.21

The latest version includes a fix for CVE-2021-44228.

* nixos/journalbeat: Add a loose dependency on elasticsearch

Avoid unnecssary back-off when elasticsearch is running on the same
host.

1 files changed, 19 insertions, 8 deletions

diff --git a/nixos/tests/parsedmarc/default.nix b/nixos/tests/parsedmarc/default.nix
index d838d3b6a39..50b977723e9 100644
--- a/nixos/tests/parsedmarc/default.nix
+++ b/nixos/tests/parsedmarc/default.nix
@@ -4,6 +4,7 @@
 { pkgs, ... }@args:
 let
   inherit (import ../../lib/testing-python.nix args) makeTest;
+  inherit (pkgs) lib;
 
   dmarcTestReport = builtins.fetchurl {
     name = "dmarc-test-report";
@@ -54,7 +55,7 @@ in
   localMail = makeTest
     {
       name = "parsedmarc-local-mail";
-      meta = with pkgs.lib.maintainers; {
+      meta = with lib.maintainers; {
         maintainers = [ talyz ];
       };
 
@@ -83,7 +84,7 @@ in
             };
           };
 
-          services.elasticsearch.package = pkgs.elasticsearch7-oss;
+          services.elasticsearch.package = pkgs.elasticsearch-oss;
 
           environment.systemPackages = [
             (sendEmail "dmarc@localhost")
@@ -94,6 +95,7 @@ in
       testScript = { nodes }:
         let
           esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
+          valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
         in ''
           parsedmarc.start()
           parsedmarc.wait_for_unit("postfix.service")
@@ -104,11 +106,15 @@ in
           )
 
           parsedmarc.fail(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
           parsedmarc.succeed("send-email")
           parsedmarc.wait_until_succeeds(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
         '';
     };
@@ -121,7 +127,7 @@ in
     in
       makeTest {
         name = "parsedmarc-external-mail";
-        meta = with pkgs.lib.maintainers; {
+        meta = with lib.maintainers; {
           maintainers = [ talyz ];
         };
 
@@ -153,7 +159,7 @@ in
                 };
               };
 
-              services.elasticsearch.package = pkgs.elasticsearch7-oss;
+              services.elasticsearch.package = pkgs.elasticsearch-oss;
 
               environment.systemPackages = [
                 pkgs.jq
@@ -201,6 +207,7 @@ in
         testScript = { nodes }:
           let
             esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
+            valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
           in ''
             mail.start()
             mail.wait_for_unit("postfix.service")
@@ -213,11 +220,15 @@ in
             )
 
             parsedmarc.fail(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+                + " | tee /dev/console"
+                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
             )
             mail.succeed("send-email")
             parsedmarc.wait_until_succeeds(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+                + " | tee /dev/console"
+                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
             )
           '';
       };