From f12e976aded8113cb466efcdc2abc1932d9beb05 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Wed, 27 Oct 2021 14:18:00 +0200 Subject: module/prometheus: optionally support reloading on config changes The new option `services.prometheus.enableReload` has been introduced which, when enabled, causes the prometheus systemd service to reload when its config file changes. More specifically the following property holds: switching to a configuration (`switch-to-configuration`) that changes the prometheus configuration only finishes successully when prometheus has finished loading the new configuration. `enableReload` is `false` by default in which case the old semantics of restarting the prometheus systemd service are in effect. --- .../services/monitoring/prometheus/default.nix | 97 +++++++++++++++++++++- 1 file changed, 94 insertions(+), 3 deletions(-) (limited to 'nixos/modules/services/monitoring/prometheus/default.nix') diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix index d2b37cf688b..5f7bda1acbc 100644 --- a/nixos/modules/services/monitoring/prometheus/default.nix +++ b/nixos/modules/services/monitoring/prometheus/default.nix @@ -7,6 +7,30 @@ let workingDir = "/var/lib/" + cfg.stateDir; + prometheusYmlOut = "${workingDir}/prometheus-substituted.yaml"; + + writeConfig = pkgs.writeShellScriptBin "write-prometheus-config" '' + PATH="${makeBinPath (with pkgs; [ coreutils envsubst ])}" + touch '${prometheusYmlOut}' + chmod 600 '${prometheusYmlOut}' + envsubst -o '${prometheusYmlOut}' -i '${prometheusYml}' + ''; + + triggerReload = pkgs.writeShellScriptBin "trigger-reload-prometheus" '' + PATH="${makeBinPath (with pkgs; [ systemd ])}" + if systemctl -q is-active prometheus.service; then + systemctl reload prometheus.service + fi + ''; + + reload = pkgs.writeShellScriptBin "reload-prometheus" '' + PATH="${makeBinPath (with pkgs; [ systemd coreutils gnugrep ])}" + cursor=$(journalctl --show-cursor -n0 | grep -oP "cursor: \K.*") + kill -HUP $MAINPID + journalctl -u prometheus.service --after-cursor="$cursor" -f \ + | grep -m 1 "Completed loading of configuration file" > /dev/null + ''; + # a wrapper that verifies that the configuration is valid promtoolCheck = what: name: file: if cfg.checkConfig then @@ -47,7 +71,11 @@ let cmdlineArgs = cfg.extraFlags ++ [ "--storage.tsdb.path=${workingDir}/data/" - "--config.file=/run/prometheus/prometheus-substituted.yaml" + "--config.file=${ + if cfg.enableReload + then prometheusYmlOut + else "/run/prometheus/prometheus-substituted.yaml" + }" "--web.listen-address=${cfg.listenAddress}:${builtins.toString cfg.port}" "--alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}" "--alertmanager.timeout=${toString cfg.alertmanagerTimeout}s" @@ -731,6 +759,25 @@ in { ''; }; + enableReload = mkOption { + default = false; + type = types.bool; + description = '' + Reload prometheus when configuration file changes (instead of restart). + + The following property holds: switching to a configuration + (switch-to-configuration) that changes the prometheus + configuration only finishes successully when prometheus has finished + loading the new configuration. + + Note that prometheus will also get reloaded when the location of the + changes but not when its contents + changes. So when you change it contents make sure to reload prometheus + manually or include the hash of in its + name. + ''; + }; + environmentFile = mkOption { type = types.nullOr types.path; default = null; @@ -928,7 +975,7 @@ in { systemd.services.prometheus = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; - preStart = '' + preStart = mkIf (!cfg.enableReload) '' ${lib.getBin pkgs.envsubst}/bin/envsubst -o "/run/prometheus/prometheus-substituted.yaml" \ -i "${prometheusYml}" ''; @@ -936,9 +983,10 @@ in { ExecStart = "${cfg.package}/bin/prometheus" + optionalString (length cmdlineArgs != 0) (" \\\n " + concatStringsSep " \\\n " cmdlineArgs); + ExecReload = mkIf cfg.enableReload "+${reload}/bin/reload-prometheus"; User = "prometheus"; Restart = "always"; - EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ]; + EnvironmentFile = mkIf (cfg.environmentFile != null && !cfg.enableReload) [ cfg.environmentFile ]; RuntimeDirectory = "prometheus"; RuntimeDirectoryMode = "0700"; WorkingDirectory = workingDir; @@ -946,5 +994,48 @@ in { StateDirectoryMode = "0700"; }; }; + systemd.services.prometheus-config-write = mkIf cfg.enableReload { + wantedBy = [ "prometheus.service" ]; + before = [ "prometheus.service" ]; + serviceConfig = { + Type = "oneshot"; + User = "prometheus"; + StateDirectory = cfg.stateDir; + StateDirectoryMode = "0700"; + EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ]; + ExecStart = "${writeConfig}/bin/write-prometheus-config"; + }; + }; + # prometheus-config-reload will activate after prometheus. However, what we + # don't want is that on startup it immediately reloads prometheus because + # prometheus itself might have just started. + # + # Instead we only want to reload prometheus when the config file has + # changed. So on startup prometheus-config-reload will just output a + # harmless message and then stay active (RemainAfterExit). + # + # Then, when the config file has changed, switch-to-configuration notices + # that this service has changed and needs to be reloaded + # (reloadIfChanged). The reload command then actually writes the new config + # and reloads prometheus. + systemd.services.prometheus-config-reload = mkIf cfg.enableReload { + wantedBy = [ "prometheus.service" ]; + after = [ "prometheus.service" ]; + reloadIfChanged = true; + serviceConfig = { + Type = "oneshot"; + User = "prometheus"; + StateDirectory = cfg.stateDir; + StateDirectoryMode = "0700"; + EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ]; + RemainAfterExit = true; + TimeoutSec = 60; + ExecStart = "${pkgs.logger}/bin/logger 'prometheus-config-reload will only reload prometheus when reloaded itself.'"; + ExecReload = [ + "${writeConfig}/bin/write-prometheus-config" + "+${triggerReload}/bin/trigger-reload-prometheus" + ]; + }; + }; }; } -- cgit 1.4.1