diff options
Diffstat (limited to 'nixos/tests/prometheus-exporters.nix')
| -rw-r--r-- | nixos/tests/prometheus-exporters.nix | 928 |
1 files changed, 757 insertions, 171 deletions
diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix index b912e3425e0..e8bc6339ecf 100644 --- a/nixos/tests/prometheus-exporters.nix +++ b/nixos/tests/prometheus-exporters.nix @@ -1,62 +1,65 @@ { system ? builtins.currentSystem -, config ? {} +, config ? { } , pkgs ? import ../.. { inherit system config; } }: let inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest; inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge - removeSuffix replaceChars singleton splitString; - -/* - * The attrset `exporterTests` contains one attribute - * for each exporter test. Each of these attributes - * is expected to be an attrset containing: - * - * `exporterConfig`: - * this attribute set contains config for the exporter itself - * - * `exporterTest` - * this attribute set contains test instructions - * - * `metricProvider` (optional) - * this attribute contains additional machine config - * - * Example: - * exporterTests.<exporterName> = { - * exporterConfig = { - * enable = true; - * }; - * metricProvider = { - * services.<metricProvider>.enable = true; - * }; - * exporterTest = '' - * wait_for_unit("prometheus-<exporterName>-exporter.service") - * wait_for_open_port("1234") - * succeed("curl -sSf 'localhost:1234/metrics'") - * ''; - * }; - * - * # this would generate the following test config: - * - * nodes.<exporterName> = { - * services.prometheus.<exporterName> = { - * enable = true; - * }; - * services.<metricProvider>.enable = true; - * }; - * - * testScript = '' - * <exporterName>.start() - * <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service") - * <exporterName>.wait_for_open_port("1234") - * <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'") - * <exporterName>.shutdown() - * ''; - */ + removeSuffix replaceChars singleton splitString; + + /* + * The attrset `exporterTests` contains one attribute + * for each exporter test. Each of these attributes + * is expected to be an attrset containing: + * + * `exporterConfig`: + * this attribute set contains config for the exporter itself + * + * `exporterTest` + * this attribute set contains test instructions + * + * `metricProvider` (optional) + * this attribute contains additional machine config + * + * `nodeName` (optional) + * override an incompatible testnode name + * + * Example: + * exporterTests.<exporterName> = { + * exporterConfig = { + * enable = true; + * }; + * metricProvider = { + * services.<metricProvider>.enable = true; + * }; + * exporterTest = '' + * wait_for_unit("prometheus-<exporterName>-exporter.service") + * wait_for_open_port("1234") + * succeed("curl -sSf 'localhost:1234/metrics'") + * ''; + * }; + * + * # this would generate the following test config: + * + * nodes.<exporterName> = { + * services.prometheus.<exporterName> = { + * enable = true; + * }; + * services.<metricProvider>.enable = true; + * }; + * + * testScript = '' + * <exporterName>.start() + * <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service") + * <exporterName>.wait_for_open_port("1234") + * <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'") + * <exporterName>.shutdown() + * ''; + */ exporterTests = { - apcupsd = { + apcupsd = { exporterConfig = { enable = true; }; @@ -68,7 +71,22 @@ let wait_for_open_port(3551) wait_for_unit("prometheus-apcupsd-exporter.service") wait_for_open_port(9162) - succeed("curl -sSf http://localhost:9162/metrics | grep -q 'apcupsd_info'") + succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'") + ''; + }; + + artifactory = { + exporterConfig = { + enable = true; + artiUsername = "artifactory-username"; + artiPassword = "artifactory-password"; + }; + exporterTest = '' + wait_for_unit("prometheus-artifactory-exporter.service") + wait_for_open_port(9531) + succeed( + "curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'" + ) ''; }; @@ -88,11 +106,58 @@ let wait_for_unit("prometheus-bind-exporter.service") wait_for_open_port(9119) succeed( - "curl -sSf http://localhost:9119/metrics | grep -q 'bind_query_recursions_total 0'" + "curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'" + ) + ''; + }; + + bird = { + exporterConfig = { + enable = true; + }; + metricProvider = { + services.bird2.enable = true; + services.bird2.config = '' + router id 127.0.0.1; + + protocol kernel MyObviousTestString { + ipv4 { + import all; + export none; + }; + } + + protocol device { + } + ''; + }; + exporterTest = '' + wait_for_unit("prometheus-bird-exporter.service") + wait_for_open_port(9324) + wait_until_succeeds( + "curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'" ) ''; }; + bitcoin = { + exporterConfig = { + enable = true; + rpcUser = "bitcoinrpc"; + rpcPasswordFile = pkgs.writeText "password" "hunter2"; + }; + metricProvider = { + services.bitcoind.default.enable = true; + services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC = "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7"; + }; + exporterTest = '' + wait_for_unit("prometheus-bitcoin-exporter.service") + wait_for_unit("bitcoind-default.service") + wait_for_open_port(9332) + succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '") + ''; + }; + blackbox = { exporterConfig = { enable = true; @@ -107,7 +172,7 @@ let wait_for_unit("prometheus-blackbox-exporter.service") wait_for_open_port(9115) succeed( - "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep -q 'probe_success 1'" + "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'" ) ''; }; @@ -127,20 +192,21 @@ let "plugin":"testplugin", "time":DATE }] - ''; in '' - wait_for_unit("prometheus-collectd-exporter.service") - wait_for_open_port(9103) - succeed( - 'echo \'${postData}\'> /tmp/data.json' - ) - succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json') - succeed( - "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd" - ) - succeed( - "curl -sSf localhost:9103/metrics | grep -q 'collectd_testplugin_gauge{instance=\"testhost\"} 23'" - ) - ''; + ''; in + '' + wait_for_unit("prometheus-collectd-exporter.service") + wait_for_open_port(9103) + succeed( + 'echo \'${postData}\'> /tmp/data.json' + ) + succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json') + succeed( + "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd" + ) + succeed( + "curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'" + ) + ''; }; dnsmasq = { @@ -154,7 +220,23 @@ let exporterTest = '' wait_for_unit("prometheus-dnsmasq-exporter.service") wait_for_open_port(9153) - succeed("curl -sSf http://localhost:9153/metrics | grep -q 'dnsmasq_leases 0'") + succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'") + ''; + }; + + # Access to WHOIS server is required to properly test this exporter, so + # just perform basic sanity check that the exporter is running and returns + # a failure. + domain = { + exporterConfig = { + enable = true; + }; + exporterTest = '' + wait_for_unit("prometheus-domain-exporter.service") + wait_for_open_port(9222) + succeed( + "curl -sSf 'http://localhost:9222/probe?target=nixos.org' | grep 'domain_probe_success 0'" + ) ''; }; @@ -172,12 +254,13 @@ let wait_for_unit("prometheus-dovecot-exporter.service") wait_for_open_port(9166) succeed( - "curl -sSf http://localhost:9166/metrics | grep -q 'dovecot_up{scope=\"global\"} 1'" + "curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'" ) ''; }; - fritzbox = { # TODO add proper test case + fritzbox = { + # TODO add proper test case exporterConfig = { enable = true; }; @@ -185,19 +268,43 @@ let wait_for_unit("prometheus-fritzbox-exporter.service") wait_for_open_port(9133) succeed( - "curl -sSf http://localhost:9133/metrics | grep -q 'fritzbox_exporter_collect_errors 0'" + "curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'" ) ''; }; + jitsi = { + exporterConfig = { + enable = true; + }; + metricProvider = { + systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ]; + services.jitsi-videobridge = { + enable = true; + apis = [ "colibri" "rest" ]; + }; + }; + exporterTest = '' + wait_for_unit("jitsi-videobridge2.service") + wait_for_open_port(8080) + wait_for_unit("prometheus-jitsi-exporter.service") + wait_for_open_port(9700) + wait_until_succeeds( + 'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"' + ) + succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'") + ''; + }; + json = { exporterConfig = { enable = true; url = "http://localhost"; - configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON [{ - name = "json_test_metric"; - path = "$.test"; - }]); + configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON { + metrics = [ + { name = "json_test_metric"; path = "{ .test }"; } + ]; + }); }; metricProvider = { systemd.services.prometheus-json-exporter.after = [ "nginx.service" ]; @@ -213,7 +320,100 @@ let wait_for_open_port(80) wait_for_unit("prometheus-json-exporter.service") wait_for_open_port(7979) - succeed("curl -sSf localhost:7979/metrics | grep -q 'json_test_metric 1'") + succeed( + "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'" + ) + ''; + }; + + kea = let + controlSocketPath = "/run/kea/dhcp6.sock"; + in + { + exporterConfig = { + enable = true; + controlSocketPaths = [ + controlSocketPath + ]; + }; + metricProvider = { + systemd.services.prometheus-kea-exporter.after = [ "kea-dhcp6-server.service" ]; + + services.kea = { + enable = true; + dhcp6 = { + enable = true; + settings = { + control-socket = { + socket-type = "unix"; + socket-name = controlSocketPath; + }; + }; + }; + }; + }; + + exporterTest = '' + wait_for_unit("kea-dhcp6-server.service") + wait_for_file("${controlSocketPath}") + wait_for_unit("prometheus-kea-exporter.service") + wait_for_open_port(9547) + succeed( + "curl --fail localhost:9547/metrics | grep 'packets_received_total'" + ) + ''; + }; + + knot = { + exporterConfig = { + enable = true; + }; + metricProvider = { + services.knot = { + enable = true; + extraArgs = [ "-v" ]; + extraConfig = '' + server: + listen: 127.0.0.1@53 + + template: + - id: default + global-module: mod-stats + dnssec-signing: off + zonefile-sync: -1 + journal-db: /var/lib/knot/journal + kasp-db: /var/lib/knot/kasp + timer-db: /var/lib/knot/timer + zonefile-load: difference + storage: ${pkgs.buildEnv { + name = "foo"; + paths = [ + (pkgs.writeTextDir "test.zone" '' + @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800 + @ NS ns1 + @ NS ns2 + ns1 A 192.168.0.1 + '') + ]; + }} + + mod-stats: + - id: custom + edns-presence: on + query-type: on + + zone: + - domain: test + file: test.zone + module: mod-stats/custom + ''; + }; + }; + exporterTest = '' + wait_for_unit("knot.service") + wait_for_unit("prometheus-knot-exporter.service") + wait_for_open_port(9433) + succeed("curl -sSf 'localhost:9433' | grep 'knot_server_zone_count 1.0'") ''; }; @@ -228,10 +428,10 @@ let wait_for_unit("prometheus-keylight-exporter.service") wait_for_open_port(9288) succeed( - "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep -q '400'" + "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'" ) succeed( - "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep -q '500'" + "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'" ) ''; }; @@ -241,42 +441,66 @@ let enable = true; lndTlsPath = "/var/lib/lnd/tls.cert"; lndMacaroonDir = "/var/lib/lnd"; + extraFlags = [ "--lnd.network=regtest" ]; }; metricProvider = { - systemd.services.prometheus-lnd-exporter.serviceConfig.DynamicUser = false; - services.bitcoind.enable = true; - services.bitcoind.extraConfig = '' - rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7 - bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 - bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 - ''; + virtualisation.memorySize = 1024; + systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15; + systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ]; + services.bitcoind.regtest = { + enable = true; + extraConfig = '' + rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7 + zmqpubrawblock=tcp://127.0.0.1:28332 + zmqpubrawtx=tcp://127.0.0.1:28333 + ''; + extraCmdlineOptions = [ "-regtest" ]; + }; systemd.services.lnd = { serviceConfig.ExecStart = '' - ${pkgs.lnd}/bin/lnd \ - --datadir=/var/lib/lnd \ - --tlscertpath=/var/lib/lnd/tls.cert \ - --tlskeypath=/var/lib/lnd/tls.key \ - --logdir=/var/log/lnd \ - --bitcoin.active \ - --bitcoin.mainnet \ - --bitcoin.node=bitcoind \ - --bitcoind.rpcuser=bitcoinrpc \ - --bitcoind.rpcpass=hunter2 \ - --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \ - --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \ - --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon + ${pkgs.lnd}/bin/lnd \ + --datadir=/var/lib/lnd \ + --tlscertpath=/var/lib/lnd/tls.cert \ + --tlskeypath=/var/lib/lnd/tls.key \ + --logdir=/var/log/lnd \ + --bitcoin.active \ + --bitcoin.regtest \ + --bitcoin.node=bitcoind \ + --bitcoind.rpcuser=bitcoinrpc \ + --bitcoind.rpcpass=hunter2 \ + --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \ + --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \ + --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon ''; serviceConfig.StateDirectory = "lnd"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; }; + # initialize wallet, creates macaroon needed by exporter + systemd.services.lnd.postStart = '' + ${pkgs.curl}/bin/curl \ + --retry 20 \ + --retry-delay 1 \ + --retry-connrefused \ + --cacert /var/lib/lnd/tls.cert \ + -X GET \ + https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed + ${pkgs.curl}/bin/curl \ + --retry 20 \ + --retry-delay 1 \ + --retry-connrefused \ + --cacert /var/lib/lnd/tls.cert \ + -X POST \ + -d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \ + https://localhost:8080/v1/initwallet + ''; }; exporterTest = '' wait_for_unit("lnd.service") wait_for_open_port(10009) wait_for_unit("prometheus-lnd-exporter.service") wait_for_open_port(9092) - succeed("curl -sSf localhost:9092/metrics | grep -q '^promhttp_metric_handler'") + succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'") ''; }; @@ -286,14 +510,14 @@ let configuration = { monitoringInterval = "2s"; mailCheckTimeout = "10s"; - servers = [ { + servers = [{ name = "testserver"; server = "localhost"; port = 25; from = "mail-exporter@localhost"; to = "mail-exporter@localhost"; detectionDir = "/var/spool/mail/mail-exporter/new"; - } ]; + }]; }; }; metricProvider = { @@ -318,7 +542,7 @@ let wait_for_unit("prometheus-mail-exporter.service") wait_for_open_port(9225) wait_until_succeeds( - "curl -sSf http://localhost:9225/metrics | grep -q 'mail_deliver_success{configname=\"testserver\"} 1'" + "curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'" ) ''; }; @@ -358,7 +582,7 @@ let wait_for_unit("prometheus-mikrotik-exporter.service") wait_for_open_port(9436) succeed( - "curl -sSf http://localhost:9436/metrics | grep -q 'mikrotik_scrape_collector_success{device=\"router\"} 0'" + "curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'" ) ''; }; @@ -383,7 +607,7 @@ let wait_for_unit("prometheus-modemmanager-exporter.service") wait_for_open_port(9539) succeed( - "curl -sSf http://localhost:9539/metrics | grep -q 'modemmanager_info'" + "curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'" ) ''; }; @@ -392,24 +616,27 @@ let exporterConfig = { enable = true; passwordFile = "/var/nextcloud-pwfile"; - url = "http://localhost/negative-space.xml"; + url = "http://localhost"; }; metricProvider = { - systemd.services.nc-pwfile = let - passfile = (pkgs.writeText "pwfile" "snakeoilpw"); - in { - requiredBy = [ "prometheus-nextcloud-exporter.service" ]; - before = [ "prometheus-nextcloud-exporter.service" ]; - serviceConfig.ExecStart = '' - ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile - ''; - }; + systemd.services.nc-pwfile = + let + passfile = (pkgs.writeText "pwfile" "snakeoilpw"); + in + { + requiredBy = [ "prometheus-nextcloud-exporter.service" ]; + before = [ "prometheus-nextcloud-exporter.service" ]; + serviceConfig.ExecStart = '' + ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile + ''; + }; services.nginx = { enable = true; virtualHosts."localhost" = { basicAuth.nextcloud-exporter = "snakeoilpw"; locations."/" = { root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata"; + tryFiles = "/negative-space.xml =404"; }; }; }; @@ -418,7 +645,7 @@ let wait_for_unit("nginx.service") wait_for_unit("prometheus-nextcloud-exporter.service") wait_for_open_port(9205) - succeed("curl -sSf http://localhost:9205/metrics | grep -q 'nextcloud_up 1'") + succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'") ''; }; @@ -437,7 +664,68 @@ let wait_for_unit("nginx.service") wait_for_unit("prometheus-nginx-exporter.service") wait_for_open_port(9113) - succeed("curl -sSf http://localhost:9113/metrics | grep -q 'nginx_up 1'") + succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up 1'") + ''; + }; + + nginxlog = { + exporterConfig = { + enable = true; + group = "nginx"; + settings = { + namespaces = [ + { + name = "filelogger"; + source = { + files = [ "/var/log/nginx/filelogger.access.log" ]; + }; + } + { + name = "syslogger"; + source = { + syslog = { + listen_address = "udp://127.0.0.1:10000"; + format = "rfc3164"; + tags = [ "nginx" ]; + }; + }; + } + ]; + }; + }; + metricProvider = { + services.nginx = { + enable = true; + httpConfig = '' + server { + listen 80; + server_name filelogger.local; + access_log /var/log/nginx/filelogger.access.log; + } + server { + listen 81; + server_name syslogger.local; + access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info; + } + ''; + }; + }; + exporterTest = '' + wait_for_unit("nginx.service") + wait_for_unit("prometheus-nginxlog-exporter.service") + wait_for_open_port(9117) + wait_for_open_port(80) + wait_for_open_port(81) + succeed("curl http://localhost") + execute("sleep 1") + succeed( + "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1" + ) + succeed("curl http://localhost:81") + execute("sleep 1") + succeed( + "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1" + ) ''; }; @@ -449,11 +737,96 @@ let wait_for_unit("prometheus-node-exporter.service") wait_for_open_port(9100) succeed( - "curl -sSf http://localhost:9100/metrics | grep -q 'node_exporter_build_info{.\\+} 1'" + "curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'" ) ''; }; + openldap = { + exporterConfig = { + enable = true; + ldapCredentialFile = "${pkgs.writeText "exporter.yml" '' + ldapUser: "cn=root,dc=example" + ldapPass: "notapassword" + ''}"; + }; + metricProvider = { + services.openldap = { + enable = true; + settings.children = { + "cn=schema".includes = [ + "${pkgs.openldap}/etc/schema/core.ldif" + "${pkgs.openldap}/etc/schema/cosine.ldif" + "${pkgs.openldap}/etc/schema/inetorgperson.ldif" + "${pkgs.openldap}/etc/schema/nis.ldif" + ]; + "olcDatabase={1}mdb" = { + attrs = { + objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ]; + olcDatabase = "{1}mdb"; + olcDbDirectory = "/var/db/openldap"; + olcSuffix = "dc=example"; + olcRootDN = { + # cn=root,dc=example + base64 = "Y249cm9vdCxkYz1leGFtcGxl"; + }; + olcRootPW = { + path = "${pkgs.writeText "rootpw" "notapassword"}"; + }; + }; + }; + "olcDatabase={2}monitor".attrs = { + objectClass = [ "olcDatabaseConfig" ]; + olcDatabase = "{2}monitor"; + olcAccess = [ "to dn.subtree=cn=monitor by users read" ]; + }; + }; + declarativeContents."dc=example" = '' + dn: dc=example + objectClass: domain + dc: example + + dn: ou=users,dc=example + objectClass: organizationalUnit + ou: users + ''; + }; + }; + exporterTest = '' + wait_for_unit("prometheus-openldap-exporter.service") + wait_for_open_port(389) + wait_for_open_port(9330) + wait_until_succeeds( + "curl -sSf http://localhost:9330/metrics | grep 'openldap_scrape{result=\"ok\"} 1'" + ) + ''; + }; + + openvpn = { + exporterConfig = { + enable = true; + group = "openvpn"; + statusPaths = [ "/run/openvpn-test" ]; + }; + metricProvider = { + users.groups.openvpn = { }; + services.openvpn.servers.test = { + config = '' + dev tun + status /run/openvpn-test + status-version 3 + ''; + up = "chmod g+r /run/openvpn-test"; + }; + systemd.services."openvpn-test".serviceConfig.Group = "openvpn"; + }; + exporterTest = '' + wait_for_unit("openvpn-test.service") + wait_for_unit("prometheus-openvpn-exporter.service") + succeed("curl -sSf http://localhost:9176/metrics | grep 'openvpn_up{.*} 1'") + ''; + }; + postfix = { exporterConfig = { enable = true; @@ -463,10 +836,12 @@ let }; exporterTest = '' wait_for_unit("prometheus-postfix-exporter.service") + wait_for_file("/var/lib/postfix/queue/public/showq") wait_for_open_port(9154) succeed( - "curl -sSf http://localhost:9154/metrics | grep -q 'postfix_smtpd_connects_total 0'" + "curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'" ) + succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'") ''; }; @@ -483,20 +858,54 @@ let wait_for_open_port(9187) wait_for_unit("postgresql.service") succeed( - "curl -sSf http://localhost:9187/metrics | grep -q 'pg_exporter_last_scrape_error 0'" + "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'" ) - succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 1'") + succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'") systemctl("stop postgresql.service") succeed( - "curl -sSf http://localhost:9187/metrics | grep -qv 'pg_exporter_last_scrape_error 0'" + "curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'" ) - succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 0'") + succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'") systemctl("start postgresql.service") wait_for_unit("postgresql.service") succeed( - "curl -sSf http://localhost:9187/metrics | grep -q 'pg_exporter_last_scrape_error 0'" + "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'" + ) + succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'") + ''; + }; + + process = { + exporterConfig = { + enable = true; + settings.process_names = [ + # Remove nix store path from process name + { name = "{{.Matches.Wrapped}} {{ .Matches.Args }}"; cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ]; } + ]; + }; + exporterTest = '' + wait_for_unit("prometheus-process-exporter.service") + wait_for_open_port(9256) + wait_until_succeeds( + "curl -sSf localhost:9256/metrics | grep -q '{}'".format( + 'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter ' + ) + ) + ''; + }; + + py-air-control = { + nodeName = "py_air_control"; + exporterConfig = { + enable = true; + deviceHostname = "127.0.0.1"; + }; + exporterTest = '' + wait_for_unit("prometheus-py-air-control-exporter.service") + wait_for_open_port(9896) + succeed( + "curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'" ) - succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 1'") ''; }; @@ -510,7 +919,7 @@ let wait_for_unit("prometheus-redis-exporter.service") wait_for_open_port(6379) wait_for_open_port(9121) - wait_until_succeeds("curl -sSf localhost:9121/metrics | grep -q 'redis_up 1'") + wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'") ''; }; @@ -528,7 +937,79 @@ let wait_for_open_port(11334) wait_for_open_port(7980) wait_until_succeeds( - "curl -sSf localhost:7980/metrics | grep -q 'rspamd_scanned{host=\"rspamd\"} 0'" + "curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep 'rspamd_scanned{host=\"rspamd\"} 0'" + ) + ''; + }; + + rtl_433 = { + exporterConfig = { + enable = true; + }; + metricProvider = { + # Mock rtl_433 binary to return a dummy metric stream. + nixpkgs.overlays = [ + (self: super: { + rtl_433 = self.runCommand "rtl_433" { } '' + mkdir -p "$out/bin" + cat <<EOF > "$out/bin/rtl_433" + #!/bin/sh + while true; do + printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n' + sleep 4 + done + EOF + chmod +x "$out/bin/rtl_433" + ''; + }) + ]; + }; + exporterTest = '' + wait_for_unit("prometheus-rtl_433-exporter.service") + wait_for_open_port(9550) + wait_until_succeeds( + "curl -sSf localhost:9550/metrics | grep '{}'".format( + 'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18' + ) + ) + ''; + }; + + script = { + exporterConfig = { + enable = true; + settings.scripts = [ + { name = "success"; script = "sleep 1"; } + ]; + }; + exporterTest = '' + wait_for_unit("prometheus-script-exporter.service") + wait_for_open_port(9172) + wait_until_succeeds( + "curl -sSf 'localhost:9172/probe?name=success' | grep -q '{}'".format( + 'script_success{script="success"} 1' + ) + ) + ''; + }; + + smokeping = { + exporterConfig = { + enable = true; + hosts = [ "127.0.0.1" ]; + }; + exporterTest = '' + wait_for_unit("prometheus-smokeping-exporter.service") + wait_for_open_port(9374) + wait_until_succeeds( + "curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format( + 'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1"} ' + ) + ) + wait_until_succeeds( + "curl -sSf localhost:9374/metrics | grep '{}'".format( + 'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1"}' + ) ) ''; }; @@ -544,7 +1025,51 @@ let exporterTest = '' wait_for_unit("prometheus-snmp-exporter.service") wait_for_open_port(9116) - succeed("curl -sSf localhost:9116/metrics | grep -q 'snmp_request_errors_total 0'") + succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'") + ''; + }; + + sql = { + exporterConfig = { + configuration.jobs.points = { + interval = "1m"; + connections = [ + "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable" + ]; + queries = { + points = { + labels = [ "name" ]; + help = "Amount of points accumulated per person"; + values = [ "amount" ]; + query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name"; + }; + }; + }; + enable = true; + user = "prometheus-sql-exporter"; + }; + metricProvider = { + services.postgresql = { + enable = true; + initialScript = builtins.toFile "init.sql" '' + CREATE DATABASE data; + \c data; + CREATE TABLE points (amount INT, name TEXT); + INSERT INTO points(amount, name) VALUES (1, 'jack'); + INSERT INTO points(amount, name) VALUES (2, 'jill'); + INSERT INTO points(amount, name) VALUES (3, 'jack'); + + CREATE USER "prometheus-sql-exporter"; + GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter"; + GRANT SELECT ON points TO "prometheus-sql-exporter"; + ''; + }; + systemd.services.prometheus-sql-exporter.after = [ "postgresql.service" ]; + }; + exporterTest = '' + wait_for_unit("prometheus-sql-exporter.service") + wait_for_open_port(9237) + succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2") ''; }; @@ -567,7 +1092,23 @@ let wait_for_open_port(80) wait_for_unit("prometheus-surfboard-exporter.service") wait_for_open_port(9239) - succeed("curl -sSf localhost:9239/metrics | grep -q 'surfboard_up 1'") + succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'") + ''; + }; + + systemd = { + exporterConfig = { + enable = true; + }; + metricProvider = { }; + exporterTest = '' + wait_for_unit("prometheus-systemd-exporter.service") + wait_for_open_port(9558) + succeed( + "curl -sSf localhost:9558/metrics | grep '{}'".format( + 'systemd_unit_state{name="basic.target",state="active",type="target"} 1' + ) + ) ''; }; @@ -579,14 +1120,50 @@ let # Note: this does not connect the test environment to the Tor network. # Client, relay, bridge or exit connectivity are disabled by default. services.tor.enable = true; - services.tor.controlPort = 9051; + services.tor.settings.ControlPort = 9051; }; exporterTest = '' wait_for_unit("tor.service") wait_for_open_port(9051) wait_for_unit("prometheus-tor-exporter.service") wait_for_open_port(9130) - succeed("curl -sSf localhost:9130/metrics | grep -q 'tor_version{.\\+} 1'") + succeed("curl -sSf localhost:9130/metrics | grep 'tor_version{.\\+} 1'") + ''; + }; + + unifi-poller = { + nodeName = "unifi_poller"; + exporterConfig.enable = true; + exporterConfig.controllers = [{ }]; + exporterTest = '' + wait_for_unit("prometheus-unifi-poller-exporter.service") + wait_for_open_port(9130) + succeed( + "curl -sSf localhost:9130/metrics | grep 'unifipoller_build_info{.\\+} 1'" + ) + ''; + }; + + unbound = { + exporterConfig = { + enable = true; + fetchType = "uds"; + controlInterface = "/run/unbound/unbound.ctl"; + }; + metricProvider = { + services.unbound = { + enable = true; + localControlSocketPath = "/run/unbound/unbound.ctl"; + }; + systemd.services.prometheus-unbound-exporter.serviceConfig = { + SupplementaryGroups = [ "unbound" ]; + }; + }; + exporterTest = '' + wait_for_unit("unbound.service") + wait_for_unit("prometheus-unbound-exporter.service") + wait_for_open_port(9167) + succeed("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'") ''; }; @@ -615,55 +1192,64 @@ let wait_for_unit("prometheus-varnish-exporter.service") wait_for_open_port(6081) wait_for_open_port(9131) - succeed("curl -sSf http://localhost:9131/metrics | grep -q 'varnish_up 1'") + succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'") ''; }; - wireguard = let snakeoil = import ./wireguard/snakeoil-keys.nix; in { - exporterConfig.enable = true; - metricProvider = { - networking.wireguard.interfaces.wg0 = { - ips = [ "10.23.42.1/32" "fc00::1/128" ]; - listenPort = 23542; + wireguard = let snakeoil = import ./wireguard/snakeoil-keys.nix; in + { + exporterConfig.enable = true; + metricProvider = { + networking.wireguard.interfaces.wg0 = { + ips = [ "10.23.42.1/32" "fc00::1/128" ]; + listenPort = 23542; - inherit (snakeoil.peer0) privateKey; + inherit (snakeoil.peer0) privateKey; - peers = singleton { - allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ]; + peers = singleton { + allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ]; - inherit (snakeoil.peer1) publicKey; + inherit (snakeoil.peer1) publicKey; + }; }; + systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ]; }; - systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ]; + exporterTest = '' + wait_for_unit("prometheus-wireguard-exporter.service") + wait_for_open_port(9586) + wait_until_succeeds( + "curl -sSf http://localhost:9586/metrics | grep '${snakeoil.peer1.publicKey}'" + ) + ''; }; - exporterTest = '' - wait_for_unit("prometheus-wireguard-exporter.service") - wait_for_open_port(9586) - wait_until_succeeds( - "curl -sSf http://localhost:9586/metrics | grep '${snakeoil.peer1.publicKey}'" - ) - ''; - }; }; in -mapAttrs (exporter: testConfig: (makeTest { - name = "prometheus-${exporter}-exporter"; - - nodes.${exporter} = mkMerge [{ - services.prometheus.exporters.${exporter} = testConfig.exporterConfig; - } testConfig.metricProvider or {}]; - - testScript = '' - ${exporter}.start() - ${concatStringsSep "\n" (map (line: - if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")") - then line - else "${exporter}.${line}" - ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))} - ${exporter}.shutdown() - ''; - - meta = with maintainers; { - maintainers = [ willibutz ]; - }; -})) exporterTests +mapAttrs + (exporter: testConfig: (makeTest ( + let + nodeName = testConfig.nodeName or exporter; + + in + { + name = "prometheus-${exporter}-exporter"; + + nodes.${nodeName} = mkMerge [{ + services.prometheus.exporters.${exporter} = testConfig.exporterConfig; + } testConfig.metricProvider or { }]; + + testScript = '' + ${nodeName}.start() + ${concatStringsSep "\n" (map (line: + if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")") + then line + else "${nodeName}.${line}" + ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))} + ${nodeName}.shutdown() + ''; + + meta = with maintainers; { + maintainers = [ willibutz elseym ]; + }; + } + ))) + exporterTests |