From 2444eab485ed6631fea02577b0d21c808405fd5e Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Tue, 13 Jun 2017 22:36:08 +0200 Subject: ELK: update kibana and the elastic beats to 5.4 (#26252) * Add kibana5 and logstash5 * Upgrade the elastic beats to 5.4 * Make sure all elastic products use the same version (see elk5Version) * Add a test for the ELK stack --- nixos/modules/services/logging/logstash.nix | 108 +++++++++++++++++++++------- nixos/modules/services/search/kibana.nix | 48 ++++++++++++- nixos/tests/elk.nix | 95 ++++++++++++++++++++++++ nixos/tests/logstash.nix | 41 ----------- 4 files changed, 222 insertions(+), 70 deletions(-) create mode 100644 nixos/tests/elk.nix delete mode 100644 nixos/tests/logstash.nix (limited to 'nixos') diff --git a/nixos/modules/services/logging/logstash.nix b/nixos/modules/services/logging/logstash.nix index c9477b9e3ab..b4abd2cd7e5 100644 --- a/nixos/modules/services/logging/logstash.nix +++ b/nixos/modules/services/logging/logstash.nix @@ -4,17 +4,46 @@ with lib; let cfg = config.services.logstash; + atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4"; pluginPath = lib.concatStringsSep ":" cfg.plugins; havePluginPath = lib.length cfg.plugins > 0; ops = lib.optionalString; - verbosityFlag = { - debug = "--debug"; - info = "--verbose"; - warn = ""; # intentionally empty - error = "--quiet"; - fatal = "--silent"; - }."${cfg.logLevel}"; - + verbosityFlag = + if atLeast54 + then "--log.level " + cfg.logLevel + else { + debug = "--debug"; + info = "--verbose"; + warn = ""; # intentionally empty + error = "--quiet"; + fatal = "--silent"; + }."${cfg.logLevel}"; + + pluginsPath = + if atLeast54 + then "--path.plugins ${pluginPath}" + else "--pluginpath ${pluginPath}"; + + logstashConf = pkgs.writeText "logstash.conf" '' + input { + ${cfg.inputConfig} + } + + filter { + ${cfg.filterConfig} + } + + output { + ${cfg.outputConfig} + } + ''; + + logstashSettingsYml = pkgs.writeText "logstash.yml" cfg.extraSettings; + + logstashSettingsDir = pkgs.runCommand "logstash-settings" {inherit logstashSettingsYml;} '' + mkdir -p $out + ln -s $logstashSettingsYml $out/logstash.yml + ''; in { @@ -45,6 +74,15 @@ in description = "The paths to find other logstash plugins in."; }; + dataDir = mkOption { + type = types.str; + default = "/var/lib/logstash"; + description = '' + A path to directory writable by logstash that it uses to store data. + Plugins will also have access to this path. + ''; + }; + logLevel = mkOption { type = types.enum [ "debug" "info" "warn" "error" "fatal" ]; default = "warn"; @@ -116,6 +154,19 @@ in ''; }; + extraSettings = mkOption { + type = types.lines; + default = ""; + description = "Extra Logstash settings in YAML format."; + example = '' + pipeline: + batch: + size: 125 + delay: 5 + ''; + }; + + }; }; @@ -123,31 +174,34 @@ in ###### implementation config = mkIf cfg.enable { + assertions = [ + { assertion = atLeast54 -> !cfg.enableWeb; + message = '' + The logstash web interface is only available for versions older than 5.4. + So either set services.logstash.enableWeb = false, + or set services.logstash.package to an older logstash. + ''; + } + ]; + systemd.services.logstash = with pkgs; { description = "Logstash Daemon"; wantedBy = [ "multi-user.target" ]; environment = { JAVA_HOME = jre; }; path = [ pkgs.bash ]; serviceConfig = { - ExecStart = - "${cfg.package}/bin/logstash agent " + - "-w ${toString cfg.filterWorkers} " + - ops havePluginPath "--pluginpath ${pluginPath} " + - "${verbosityFlag} " + - "-f ${writeText "logstash.conf" '' - input { - ${cfg.inputConfig} - } - - filter { - ${cfg.filterConfig} - } - - output { - ${cfg.outputConfig} - } - ''} " + - ops cfg.enableWeb "-- web -a ${cfg.listenAddress} -p ${cfg.port}"; + ExecStartPre = ''${pkgs.coreutils}/bin/mkdir -p "${cfg.dataDir}" ; ${pkgs.coreutils}/bin/chmod 700 "${cfg.dataDir}"''; + ExecStart = concatStringsSep " " (filter (s: stringLength s != 0) [ + "${cfg.package}/bin/logstash" + (ops (!atLeast54) "agent") + "-w ${toString cfg.filterWorkers}" + (ops havePluginPath pluginsPath) + "${verbosityFlag}" + "-f ${logstashConf}" + (ops atLeast54 "--path.settings ${logstashSettingsDir}") + (ops atLeast54 "--path.data ${cfg.dataDir}") + (ops cfg.enableWeb "-- web -a ${cfg.listenAddress} -p ${cfg.port}") + ]); }; }; }; diff --git a/nixos/modules/services/search/kibana.nix b/nixos/modules/services/search/kibana.nix index d377a6feeb8..9d7d2d79918 100644 --- a/nixos/modules/services/search/kibana.nix +++ b/nixos/modules/services/search/kibana.nix @@ -5,7 +5,11 @@ with lib; let cfg = config.services.kibana; - cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON ( + atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4"; + + cfgFile = if atLeast54 then cfgFile5 else cfgFile4; + + cfgFile4 = pkgs.writeText "kibana.json" (builtins.toJSON ( (filterAttrsRecursive (n: v: v != null) ({ host = cfg.listenAddress; port = cfg.port; @@ -36,6 +40,27 @@ let ]; } // cfg.extraConf) ))); + + cfgFile5 = pkgs.writeText "kibana.json" (builtins.toJSON ( + (filterAttrsRecursive (n: v: v != null) ({ + server.host = cfg.listenAddress; + server.port = cfg.port; + server.ssl.certificate = cfg.cert; + server.ssl.key = cfg.key; + + kibana.index = cfg.index; + kibana.defaultAppId = cfg.defaultAppId; + + elasticsearch.url = cfg.elasticsearch.url; + elasticsearch.username = cfg.elasticsearch.username; + elasticsearch.password = cfg.elasticsearch.password; + + elasticsearch.ssl.certificate = cfg.elasticsearch.cert; + elasticsearch.ssl.key = cfg.elasticsearch.key; + elasticsearch.ssl.certificateAuthorities = cfg.elasticsearch.certificateAuthorities; + } // cfg.extraConf) + ))); + in { options.services.kibana = { enable = mkEnableOption "enable kibana service"; @@ -96,11 +121,29 @@ in { }; ca = mkOption { - description = "CA file to auth against elasticsearch."; + description = '' + CA file to auth against elasticsearch. + + It's recommended to use the option + when using kibana-5.4 or newer. + ''; default = null; type = types.nullOr types.path; }; + certificateAuthorities = mkOption { + description = '' + CA files to auth against elasticsearch. + + Please use the option when using kibana < 5.4 + because those old versions don't support setting multiple CA's. + + This defaults to the singleton list [ca] when the option is defined. + ''; + default = if isNull cfg.elasticsearch.ca then [] else [ca]; + type = types.listOf types.path; + }; + cert = mkOption { description = "Certificate file to auth against elasticsearch."; default = null; @@ -118,6 +161,7 @@ in { description = "Kibana package to use"; default = pkgs.kibana; defaultText = "pkgs.kibana"; + example = "pkgs.kibana5"; type = types.package; }; diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix new file mode 100644 index 00000000000..65ff1cac070 --- /dev/null +++ b/nixos/tests/elk.nix @@ -0,0 +1,95 @@ +# Test the ELK stack: Elasticsearch, Logstash and Kibana. + +import ./make-test.nix ({ pkgs, ...} : +let + esUrl = "http://localhost:9200"; +in { + name = "ELK"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ eelco chaoflow offline basvandijk ]; + }; + + nodes = { + one = + { config, pkgs, ... }: { + # Not giving the machine at least 2060MB results in elasticsearch failing with the following error: + # + # OpenJDK 64-Bit Server VM warning: + # INFO: os::commit_memory(0x0000000085330000, 2060255232, 0) + # failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (mmap) failed to map 2060255232 bytes for committing reserved memory. + # + # When setting this to 2500 I got "Kernel panic - not syncing: Out of + # memory: compulsory panic_on_oom is enabled" so lets give it even a + # bit more room: + virtualisation.memorySize = 3000; + + # For querying JSON objects returned from elasticsearch and kibana. + environment.systemPackages = [ pkgs.jq ]; + + services = { + logstash = { + enable = true; + package = pkgs.logstash5; + inputConfig = '' + exec { command => "echo -n flowers" interval => 1 type => "test" } + exec { command => "echo -n dragons" interval => 1 type => "test" } + ''; + filterConfig = '' + if [message] =~ /dragons/ { + drop {} + } + ''; + outputConfig = '' + file { + path => "/tmp/logstash.out" + codec => line { format => "%{message}" } + } + elasticsearch { + hosts => [ "${esUrl}" ] + } + ''; + }; + + elasticsearch = { + enable = true; + package = pkgs.elasticsearch5; + }; + + kibana = { + enable = true; + package = pkgs.kibana5; + elasticsearch.url = esUrl; + }; + }; + }; + }; + + testScript = '' + startAll; + + $one->waitForUnit("elasticsearch.service"); + + # Continue as long as the status is not "red". The status is probably + # "yellow" instead of "green" because we are using a single elasticsearch + # node which elasticsearch considers risky. + # + # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green". + $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"); + + # Perform some simple logstash tests. + $one->waitForUnit("logstash.service"); + $one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers"); + $one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons"); + + # See if kibana is healthy. + $one->waitForUnit("kibana.service"); + $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"); + + # See if logstash messages arive in elasticsearch. + $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); + $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); + ''; +}) diff --git a/nixos/tests/logstash.nix b/nixos/tests/logstash.nix deleted file mode 100644 index 01f6a0358b2..00000000000 --- a/nixos/tests/logstash.nix +++ /dev/null @@ -1,41 +0,0 @@ -# This test runs logstash and checks if messages flows and -# elasticsearch is started. - -import ./make-test.nix ({ pkgs, ...} : { - name = "logstash"; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ eelco chaoflow offline ]; - }; - - nodes = { - one = - { config, pkgs, ... }: - { - services = { - logstash = { - enable = true; - inputConfig = '' - exec { command => "echo flowers" interval => 1 type => "test" } - exec { command => "echo dragons" interval => 1 type => "test" } - ''; - filterConfig = '' - if [message] =~ /dragons/ { - drop {} - } - ''; - outputConfig = '' - stdout { codec => rubydebug } - ''; - }; - }; - }; - }; - - testScript = '' - startAll; - - $one->waitForUnit("logstash.service"); - $one->waitUntilSucceeds("journalctl -n 20 _SYSTEMD_UNIT=logstash.service | grep flowers"); - $one->fail("journalctl -n 20 _SYSTEMD_UNIT=logstash.service | grep dragons"); - ''; -}) -- cgit 1.4.1