module/prometheus: optionally support reloading on config changes

The new option `services.prometheus.enableReload` has been introduced
which, when enabled, causes the prometheus systemd service to reload
when its config file changes.

More specifically the following property holds: switching to a
configuration (`switch-to-configuration`) that changes the prometheus
configuration only finishes successully when prometheus has finished
loading the new configuration.

`enableReload` is `false` by default in which case the old semantics
of restarting the prometheus systemd service are in effect.

1 files changed, 94 insertions, 3 deletions

diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix
index d2b37cf688b..5f7bda1acbc 100644
--- a/nixos/modules/services/monitoring/prometheus/default.nix
+++ b/nixos/modules/services/monitoring/prometheus/default.nix
@@ -7,6 +7,30 @@ let
 
   workingDir = "/var/lib/" + cfg.stateDir;
 
+  prometheusYmlOut = "${workingDir}/prometheus-substituted.yaml";
+
+  writeConfig = pkgs.writeShellScriptBin "write-prometheus-config" ''
+    PATH="${makeBinPath (with pkgs; [ coreutils envsubst ])}"
+    touch '${prometheusYmlOut}'
+    chmod 600 '${prometheusYmlOut}'
+    envsubst -o '${prometheusYmlOut}' -i '${prometheusYml}'
+  '';
+
+  triggerReload = pkgs.writeShellScriptBin "trigger-reload-prometheus" ''
+    PATH="${makeBinPath (with pkgs; [ systemd ])}"
+    if systemctl -q is-active prometheus.service; then
+      systemctl reload prometheus.service
+    fi
+  '';
+
+  reload = pkgs.writeShellScriptBin "reload-prometheus" ''
+    PATH="${makeBinPath (with pkgs; [ systemd coreutils gnugrep ])}"
+    cursor=$(journalctl --show-cursor -n0 | grep -oP "cursor: \K.*")
+    kill -HUP $MAINPID
+    journalctl -u prometheus.service --after-cursor="$cursor" -f \
+      | grep -m 1 "Completed loading of configuration file" > /dev/null
+  '';
+
   # a wrapper that verifies that the configuration is valid
   promtoolCheck = what: name: file:
     if cfg.checkConfig then
@@ -47,7 +71,11 @@ let
 
   cmdlineArgs = cfg.extraFlags ++ [
     "--storage.tsdb.path=${workingDir}/data/"
-    "--config.file=/run/prometheus/prometheus-substituted.yaml"
+    "--config.file=${
+      if cfg.enableReload
+      then prometheusYmlOut
+      else "/run/prometheus/prometheus-substituted.yaml"
+    }"
     "--web.listen-address=${cfg.listenAddress}:${builtins.toString cfg.port}"
     "--alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
     "--alertmanager.timeout=${toString cfg.alertmanagerTimeout}s"
@@ -731,6 +759,25 @@ in {
       '';
     };
 
+    enableReload = mkOption {
+      default = false;
+      type = types.bool;
+      description = ''
+        Reload prometheus when configuration file changes (instead of restart).
+
+        The following property holds: switching to a configuration
+        (<literal>switch-to-configuration</literal>) that changes the prometheus
+        configuration only finishes successully when prometheus has finished
+        loading the new configuration.
+
+        Note that prometheus will also get reloaded when the location of the
+        <option>environmentFile</option> changes but not when its contents
+        changes. So when you change it contents make sure to reload prometheus
+        manually or include the hash of <option>environmentFile</option> in its
+        name.
+      '';
+    };
+
     environmentFile = mkOption {
       type = types.nullOr types.path;
       default = null;
@@ -928,7 +975,7 @@ in {
     systemd.services.prometheus = {
       wantedBy = [ "multi-user.target" ];
       after    = [ "network.target" ];
-      preStart = ''
+      preStart = mkIf (!cfg.enableReload) ''
          ${lib.getBin pkgs.envsubst}/bin/envsubst -o "/run/prometheus/prometheus-substituted.yaml" \
                                                   -i "${prometheusYml}"
       '';
@@ -936,9 +983,10 @@ in {
         ExecStart = "${cfg.package}/bin/prometheus" +
           optionalString (length cmdlineArgs != 0) (" \\\n  " +
             concatStringsSep " \\\n  " cmdlineArgs);
+        ExecReload = mkIf cfg.enableReload "+${reload}/bin/reload-prometheus";
         User = "prometheus";
         Restart  = "always";
-        EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ];
+        EnvironmentFile = mkIf (cfg.environmentFile != null && !cfg.enableReload) [ cfg.environmentFile ];
         RuntimeDirectory = "prometheus";
         RuntimeDirectoryMode = "0700";
         WorkingDirectory = workingDir;
@@ -946,5 +994,48 @@ in {
         StateDirectoryMode = "0700";
       };
     };
+    systemd.services.prometheus-config-write = mkIf cfg.enableReload {
+      wantedBy = [ "prometheus.service" ];
+      before = [ "prometheus.service" ];
+      serviceConfig = {
+        Type = "oneshot";
+        User = "prometheus";
+        StateDirectory = cfg.stateDir;
+        StateDirectoryMode = "0700";
+        EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ];
+        ExecStart = "${writeConfig}/bin/write-prometheus-config";
+      };
+    };
+    # prometheus-config-reload will activate after prometheus. However, what we
+    # don't want is that on startup it immediately reloads prometheus because
+    # prometheus itself might have just started.
+    #
+    # Instead we only want to reload prometheus when the config file has
+    # changed. So on startup prometheus-config-reload will just output a
+    # harmless message and then stay active (RemainAfterExit).
+    #
+    # Then, when the config file has changed, switch-to-configuration notices
+    # that this service has changed and needs to be reloaded
+    # (reloadIfChanged). The reload command then actually writes the new config
+    # and reloads prometheus.
+    systemd.services.prometheus-config-reload = mkIf cfg.enableReload {
+      wantedBy = [ "prometheus.service" ];
+      after = [ "prometheus.service" ];
+      reloadIfChanged = true;
+      serviceConfig = {
+        Type = "oneshot";
+        User = "prometheus";
+        StateDirectory = cfg.stateDir;
+        StateDirectoryMode = "0700";
+        EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ];
+        RemainAfterExit = true;
+        TimeoutSec = 60;
+        ExecStart = "${pkgs.logger}/bin/logger 'prometheus-config-reload will only reload prometheus when reloaded itself.'";
+        ExecReload = [
+          "${writeConfig}/bin/write-prometheus-config"
+          "+${triggerReload}/bin/trigger-reload-prometheus"
+        ];
+      };
+    };
   };
 }