From 3464c602e83687954b711bb9b30c5d85303f6f51 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 6 Jul 2019 20:42:56 +0200 Subject: nixos/graylog: fix startup Until now the startup failed with an error like this: ``` com.github.joschi.jadconfig.ValidationException: Parent path /var/lib/graylog/server for Node ID file at /var/lib/graylog/server/node-id is not a directory ``` This happens since `graylog.service` ensures that `/var/lib/graylog` exists, however it doesn't take care of the directory for `cfg.nodeIdFile`. --- nixos/modules/services/logging/graylog.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixos/modules/services/logging/graylog.nix b/nixos/modules/services/logging/graylog.nix index ee566825498..49f3187fd31 100644 --- a/nixos/modules/services/logging/graylog.nix +++ b/nixos/modules/services/logging/graylog.nix @@ -150,6 +150,9 @@ in rm -rf /var/lib/graylog/plugins || true mkdir -p /var/lib/graylog/plugins -m 755 + mkdir -p "$(dirname ${cfg.nodeIdFile})" + chown -R ${cfg.user} "$(dirname ${cfg.nodeIdFile})" + for declarativeplugin in `ls ${glPlugins}/bin/`; do ln -sf ${glPlugins}/bin/$declarativeplugin /var/lib/graylog/plugins/$declarativeplugin done -- cgit 1.4.1 From 60ed1f425b3d9a024239e86d6ac8566381576d69 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 9 Jul 2019 01:30:23 +0200 Subject: systemd-journal2gelf: 20170413 -> 20190702 --- pkgs/tools/system/systemd-journal2gelf/default.nix | 16 +++++++--------- pkgs/tools/system/systemd-journal2gelf/deps.nix | 11 ----------- 2 files changed, 7 insertions(+), 20 deletions(-) delete mode 100644 pkgs/tools/system/systemd-journal2gelf/deps.nix diff --git a/pkgs/tools/system/systemd-journal2gelf/default.nix b/pkgs/tools/system/systemd-journal2gelf/default.nix index 7230f55d309..2219d623a03 100644 --- a/pkgs/tools/system/systemd-journal2gelf/default.nix +++ b/pkgs/tools/system/systemd-journal2gelf/default.nix @@ -1,19 +1,17 @@ -{ stdenv, buildGoPackage, fetchFromGitHub }: +{ stdenv, buildGoModule, fetchFromGitHub }: -buildGoPackage rec { - name = "SystemdJournal2Gelf-${version}"; - version = "20170413"; - - goPackagePath = "github.com/parse-nl/SystemdJournal2Gelf"; +buildGoModule rec { + pname = "SystemdJournal2Gelf-unstable"; + version = "20190702"; src = fetchFromGitHub { - rev = "862b1d60d2ba12cd8480304ca95041066cc8bdd0"; + rev = "b1aa5ff31307d11a3c9b4dd08c3cd6230d935ec5"; owner = "parse-nl"; repo = "SystemdJournal2Gelf"; - sha256 = "0xvvc7w2sxkhb33nkq5v626l673d5j2z0yc75wvmqzncwfkkv94v"; + sha256 = "0i2pv817fjm2xazxb01dk2gg1xb4d9b6743gqrbsyghbkm7krx29"; }; - goDeps = ./deps.nix; + modSha256 = "0f66bjij3bkjs09xhhp26arivlqrd66z1j5ziy4lq4krg82krsdp"; meta = with stdenv.lib; { description = "Export entries from systemd's journal and send them to a graylog server using gelf"; diff --git a/pkgs/tools/system/systemd-journal2gelf/deps.nix b/pkgs/tools/system/systemd-journal2gelf/deps.nix deleted file mode 100644 index 2e0988e8105..00000000000 --- a/pkgs/tools/system/systemd-journal2gelf/deps.nix +++ /dev/null @@ -1,11 +0,0 @@ -[ -{ - goPackagePath = "github.com/DECK36/go-gelf"; - fetch = { - type = "git"; - url = "https://github.com/DECK36/go-gelf"; - rev = "4bc6123df0946a1c23fd54e0c1d0ed68b44fd99f"; - sha256 = "071zdwcl8ld05gv88yym1p7xq72igd6jj05n5d7v01hn6rvj48ay"; - }; -} -] -- cgit 1.4.1 From 16d0b8dcbd6a8a6be12a2732c7174529c3828051 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 9 Jul 2019 02:01:43 +0200 Subject: nixos/graylog: add test Basic test which confirms new inputs can be created and that messages can be sent to a UDP-GELF input using `netcat`. This test requires 4GB of RAM to avoid issues due insufficient memory (please refer to `nixos/tests/elk.nix` for a detailed explanation of the issue) for elasticsearch. Also it's ensured that elasticsearch has an open HTTP port for communication when starting `graylog`. This is a workaround to ensure that all services are started in proper order, even in test environments with less power. However this shouldn't be implemented in the `nixos/graylog` module as this might be harmful when using elasticsearch clusters that require e.g. authentication and/or run on different servers. --- nixos/tests/all-tests.nix | 1 + nixos/tests/graylog.nix | 111 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 nixos/tests/graylog.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 359f62751b9..5a017639040 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -99,6 +99,7 @@ in graphene = handleTest ./graphene.nix {}; grafana = handleTest ./grafana.nix {}; graphite = handleTest ./graphite.nix {}; + graylog = handleTest ./graylog.nix {}; hadoop.hdfs = handleTestOn [ "x86_64-linux" ] ./hadoop/hdfs.nix {}; hadoop.yarn = handleTestOn [ "x86_64-linux" ] ./hadoop/yarn.nix {}; handbrake = handleTestOn ["x86_64-linux"] ./handbrake.nix {}; diff --git a/nixos/tests/graylog.nix b/nixos/tests/graylog.nix new file mode 100644 index 00000000000..dc54afd1d26 --- /dev/null +++ b/nixos/tests/graylog.nix @@ -0,0 +1,111 @@ +import ./make-test.nix ({ pkgs, lib, ... }: { + name = "graylog"; + meta.maintainers = with lib.maintainers; [ ma27 ]; + + machine = { pkgs, ... }: { + virtualisation.memorySize = 4096; + virtualisation.diskSize = 4096; + + services.mongodb.enable = true; + services.elasticsearch.enable = true; + services.elasticsearch.package = pkgs.elasticsearch-oss; + services.elasticsearch.extraConf = '' + network.publish_host: 127.0.0.1 + network.bind_host: 127.0.0.1 + ''; + + services.graylog = { + enable = true; + passwordSecret = "YGhZ59wXMrYOojx5xdgEpBpDw2N6FbhM4lTtaJ1KPxxmKrUvSlDbtWArwAWMQ5LKx1ojHEVrQrBMVRdXbRyZLqffoUzHfssc"; + elasticsearchHosts = [ "http://localhost:9200" ]; + + # `echo -n "nixos" | shasum -a 256` + rootPasswordSha2 = "6ed332bcfa615381511d4d5ba44a293bb476f368f7e9e304f0dff50230d1a85b"; + }; + + environment.systemPackages = [ pkgs.jq ]; + + systemd.services.graylog.path = [ pkgs.netcat ]; + systemd.services.graylog.preStart = '' + until nc -z localhost 9200; do + sleep 2 + done + ''; + }; + + testScript = let + payloads.login = pkgs.writeText "login.json" (builtins.toJSON { + host = "127.0.0.1:9000"; + username = "admin"; + password = "nixos"; + }); + + payloads.input = pkgs.writeText "input.json" (builtins.toJSON { + title = "Demo"; + global = false; + type = "org.graylog2.inputs.gelf.udp.GELFUDPInput"; + node = "@node@"; + configuration = { + bind_address = "0.0.0.0"; + decompress_size_limit = 8388608; + number_worker_threads = 1; + override_source = null; + port = 12201; + recv_buffer_size = 262144; + }; + }); + + payloads.gelf_message = pkgs.writeText "gelf.json" (builtins.toJSON { + host = "example.org"; + short_message = "A short message"; + full_message = "A long message"; + version = "1.1"; + level = 5; + facility = "Test"; + }); + in '' + $machine->start; + $machine->waitForUnit("graylog.service"); + $machine->waitForOpenPort(9000); + $machine->succeed("curl -sSfL http://127.0.0.1:9000/"); + + my $session = $machine->succeed("curl -X POST " + . "-sSfL http://127.0.0.1:9000/api/system/sessions " + . "-d \$(cat ${payloads.login}) " + . "-H 'Content-Type: application/json' " + . "-H 'Accept: application/json' " + . "-H 'x-requested-by: cli' " + . "| jq .session_id | xargs echo" + ); + + chomp($session); + + $machine->succeed("curl -X POST " + . "-sSfL http://127.0.0.1:9000/api/system/inputs -u $session:session " + . "-d \$(cat ${payloads.input} | sed -e \"s,\@node\@,\$(cat /var/lib/graylog/server/node-id),\") " + . "-H 'Accept: application/json' " + . "-H 'Content-Type: application/json' " + . "-H 'x-requested-by: cli' " + ); + + $machine->waitUntilSucceeds("test \"\$(curl -sSfL 'http://127.0.0.1:9000/api/cluster/inputstates' " + . "-u $session:session " + . "-H 'Accept: application/json' " + . "-H 'Content-Type: application/json' " + . "-H 'x-requested-by: cli'" + . "| jq 'to_entries[]|.value|.[0]|.state' | xargs echo" + . ")\" = \"RUNNING\"" + ); + + $machine->succeed("echo -n \$(cat ${payloads.gelf_message}) | nc -w10 -u 127.0.0.1 12201"); + + $machine->succeed("test \"\$(curl -X GET " + . "-sSfL 'http://127.0.0.1:9000/api/search/universal/relative?query=*' " + . "-u $session:session " + . "-H 'Accept: application/json' " + . "-H 'Content-Type: application/json' " + . "-H 'x-requested-by: cli'" + . " | jq '.total_results' | xargs echo)\" = \"1\"" + ); + ''; +}) -- cgit 1.4.1