Merge branch 'master' into staging

39 files changed, 493 insertions, 587 deletions

diff --git a/nixos/modules/services/amqp/rabbitmq.nix b/nixos/modules/services/amqp/rabbitmq.nix
index 7373be2a9b0..302b94de196 100644
--- a/nixos/modules/services/amqp/rabbitmq.nix
+++ b/nixos/modules/services/amqp/rabbitmq.nix
@@ -179,11 +179,11 @@ in {
       } //  optionalAttrs (cfg.config != "") { RABBITMQ_ADVANCED_CONFIG_FILE = advanced_config_file; };
 
       serviceConfig = {
-        PermissionsStartOnly = true; # preStart must be run as root
         ExecStart = "${cfg.package}/sbin/rabbitmq-server";
         ExecStop = "${cfg.package}/sbin/rabbitmqctl shutdown";
         User = "rabbitmq";
         Group = "rabbitmq";
+        LogsDirectory = "rabbitmq";
         WorkingDirectory = cfg.dataDir;
         Type = "notify";
         NotifyAccess = "all";
@@ -197,11 +197,8 @@ in {
       preStart = ''
         ${optionalString (cfg.cookie != "") ''
             echo -n ${cfg.cookie} > ${cfg.dataDir}/.erlang.cookie
-            chown rabbitmq:rabbitmq ${cfg.dataDir}/.erlang.cookie
             chmod 600 ${cfg.dataDir}/.erlang.cookie
         ''}
-        mkdir -p /var/log/rabbitmq
-        chown rabbitmq:rabbitmq /var/log/rabbitmq
       '';
     };
 
diff --git a/nixos/modules/services/audio/liquidsoap.nix b/nixos/modules/services/audio/liquidsoap.nix
index 66f84ef2076..3a047d10a63 100644
--- a/nixos/modules/services/audio/liquidsoap.nix
+++ b/nixos/modules/services/audio/liquidsoap.nix
@@ -14,15 +14,10 @@ let
         description = "${name} liquidsoap stream";
         wantedBy = [ "multi-user.target" ];
         path = [ pkgs.wget ];
-        preStart =
-          ''
-            mkdir -p /var/log/liquidsoap
-            chown liquidsoap -R /var/log/liquidsoap
-          '';
         serviceConfig = {
-          PermissionsStartOnly="true";
           ExecStart = "${pkgs.liquidsoap}/bin/liquidsoap ${stream}";
           User = "liquidsoap";
+          LogsDirectory = "liquidsoap";
         };
       };
     };
diff --git a/nixos/modules/services/audio/mpd.nix b/nixos/modules/services/audio/mpd.nix
index 5bfe2b6a22a..0df8f9688d2 100644
--- a/nixos/modules/services/audio/mpd.nix
+++ b/nixos/modules/services/audio/mpd.nix
@@ -158,18 +158,18 @@ in {
       };
     };
 
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' - ${cfg.user} ${cfg.group} - -"
+      "d '${cfg.playlistDirectory}' - ${cfg.user} ${cfg.group} - -"
+    ];
+
     systemd.services.mpd = {
       after = [ "network.target" "sound.target" ];
       description = "Music Player Daemon";
       wantedBy = optional (!cfg.startWhenNeeded) "multi-user.target";
 
-      preStart = ''
-        mkdir -p "${cfg.dataDir}" && chown -R ${cfg.user}:${cfg.group} "${cfg.dataDir}"
-        mkdir -p "${cfg.playlistDirectory}" && chown -R ${cfg.user}:${cfg.group} "${cfg.playlistDirectory}"
-      '';
       serviceConfig = {
         User = "${cfg.user}";
-        PermissionsStartOnly = true;
         ExecStart = "${pkgs.mpd}/bin/mpd --no-daemon ${mpdConf}";
         Type = "notify";
         LimitRTPRIO = 50;
diff --git a/nixos/modules/services/backup/mysql-backup.nix b/nixos/modules/services/backup/mysql-backup.nix
index f0c273ffebf..ba6e154f6b3 100644
--- a/nixos/modules/services/backup/mysql-backup.nix
+++ b/nixos/modules/services/backup/mysql-backup.nix
@@ -117,14 +117,12 @@ in
         enable = true;
         serviceConfig = {
           User = cfg.user;
-          PermissionsStartOnly = true;
         };
-        preStart = ''
-          mkdir -m 0700 -p ${cfg.location}
-          chown -R ${cfg.user} ${cfg.location}
-        '';
         script = backupScript;
       };
+      tmpfiles.rules = [
+        "d ${cfg.location} 0700 ${cfg.user} - - -"
+      ];
     };
   };
 
diff --git a/nixos/modules/services/backup/postgresql-backup.nix b/nixos/modules/services/backup/postgresql-backup.nix
index 11efa47ec5b..17b410a97f3 100644
--- a/nixos/modules/services/backup/postgresql-backup.nix
+++ b/nixos/modules/services/backup/postgresql-backup.nix
@@ -14,11 +14,6 @@ let
 
       requires = [ "postgresql.service" ];
 
-      preStart = ''
-        mkdir -m 0700 -p ${cfg.location}
-        chown postgres ${cfg.location}
-      '';
-
       script = ''
         umask 0077 # ensure backup is only readable by postgres user
 
@@ -32,7 +27,6 @@ let
 
       serviceConfig = {
         Type = "oneshot";
-        PermissionsStartOnly = "true";
         User = "postgres";
       };
 
@@ -107,6 +101,11 @@ in {
         message = "config.services.postgresqlBackup.backupAll cannot be used together with config.services.postgresqlBackup.databases";
       }];
     }
+    (mkIf cfg.enable {
+      systemd.tmpfiles.rules = [
+        "d '${cfg.location}' 0700 postgres - - -"
+      ];
+    })
     (mkIf (cfg.enable && cfg.backupAll) {
       systemd.services.postgresqlBackup =
         postgresqlBackupService "all" "${config.services.postgresql.package}/bin/pg_dumpall";
diff --git a/nixos/modules/services/databases/clickhouse.nix b/nixos/modules/services/databases/clickhouse.nix
index 21e0cee3415..dbabcae43ee 100644
--- a/nixos/modules/services/databases/clickhouse.nix
+++ b/nixos/modules/services/databases/clickhouse.nix
@@ -1,8 +1,6 @@
 { config, lib, pkgs, ... }:
 let
   cfg = config.services.clickhouse;
-  confDir = "/etc/clickhouse-server";
-  stateDir = "/var/lib/clickhouse";
 in
 with lib;
 {
@@ -43,20 +41,13 @@ with lib;
 
       after = [ "network.target" ];
 
-      preStart = ''
-        mkdir -p ${stateDir}
-        chown clickhouse:clickhouse ${confDir} ${stateDir}
-      '';
-
-      script = ''
-        cd "${confDir}"
-        exec ${pkgs.clickhouse}/bin/clickhouse-server
-      '';
-
       serviceConfig = {
         User = "clickhouse";
         Group = "clickhouse";
-        PermissionsStartOnly = true;
+        ConfigurationDirectory = "clickhouse-server";
+        StateDirectory = "clickhouse";
+        LogsDirectory = "clickhouse";
+        ExecStart = "${pkgs.clickhouse}/bin/clickhouse-server --config-file=${pkgs.clickhouse}/etc/clickhouse-server/config.xml";
       };
     };
 
diff --git a/nixos/modules/services/databases/couchdb.nix b/nixos/modules/services/databases/couchdb.nix
index 84d108d9c74..5ddf8ba4bfb 100644
--- a/nixos/modules/services/databases/couchdb.nix
+++ b/nixos/modules/services/databases/couchdb.nix
@@ -158,27 +158,21 @@ in {
     services.couchdb.configFile = mkDefault
       (if useVersion2 then "/var/lib/couchdb/local.ini" else "/var/lib/couchdb/couchdb.ini");
 
+    systemd.tmpfiles.rules = [
+      "d '${dirOf cfg.uriFile}' - ${cfg.user} ${cfg.group} - -"
+      "d '${dirOf cfg.logFile}' - ${cfg.user} ${cfg.group} - -"
+      "d '${cfg.databaseDir}' -  ${cfg.user} ${cfg.group} - -"
+      "d '${cfg.viewIndexDir}' -  ${cfg.user} ${cfg.group} - -"
+    ];
+
     systemd.services.couchdb = {
       description = "CouchDB Server";
       wantedBy = [ "multi-user.target" ];
 
       preStart =
         ''
-        mkdir -p `dirname ${cfg.uriFile}`;
-        mkdir -p `dirname ${cfg.logFile}`;
-        mkdir -p ${cfg.databaseDir};
-        mkdir -p ${cfg.viewIndexDir};
         touch ${cfg.configFile}
         touch -a ${cfg.logFile}
-
-        if [ "$(id -u)" = 0 ]; then
-          chown ${cfg.user}:${cfg.group} `dirname ${cfg.uriFile}`;
-          (test -f ${cfg.uriFile} && chown ${cfg.user}:${cfg.group} ${cfg.uriFile}) || true
-          chown ${cfg.user}:${cfg.group} ${cfg.databaseDir}
-          chown ${cfg.user}:${cfg.group} ${cfg.viewIndexDir}
-          chown ${cfg.user}:${cfg.group} ${cfg.configFile}
-          chown ${cfg.user}:${cfg.group} ${cfg.logFile}
-        fi
         '';
 
       environment = mkIf useVersion2 {
@@ -191,7 +185,6 @@ in {
       };
 
       serviceConfig = {
-        PermissionsStartOnly = true;
         User = cfg.user;
         Group = cfg.group;
         ExecStart = executable;
diff --git a/nixos/modules/services/databases/influxdb.nix b/nixos/modules/services/databases/influxdb.nix
index 888bf13c3df..6868050c844 100644
--- a/nixos/modules/services/databases/influxdb.nix
+++ b/nixos/modules/services/databases/influxdb.nix
@@ -157,20 +157,19 @@ in
 
   config = mkIf config.services.influxdb.enable {
 
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
+    ];
+
     systemd.services.influxdb = {
       description = "InfluxDB Server";
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
       serviceConfig = {
         ExecStart = ''${cfg.package}/bin/influxd -config "${configFile}"'';
-        User = "${cfg.user}";
-        Group = "${cfg.group}";
-        PermissionsStartOnly = true;
+        User = cfg.user;
+        Group = cfg.group;
       };
-      preStart = ''
-        mkdir -m 0770 -p ${cfg.dataDir}
-        if [ "$(id -u)" = 0 ]; then chown -R ${cfg.user}:${cfg.group} ${cfg.dataDir}; fi
-      '';
       postStart =
         let
           scheme = if configOptions.http.https-enabled then "-k https" else "http";
diff --git a/nixos/modules/services/databases/memcached.nix b/nixos/modules/services/databases/memcached.nix
index 7af452e4dce..052ff1f308e 100644
--- a/nixos/modules/services/databases/memcached.nix
+++ b/nixos/modules/services/databases/memcached.nix
@@ -78,11 +78,6 @@ in
       after = [ "network.target" ];
 
       serviceConfig = {
-        PermissionsStartOnly = true;
-        ExecStartPre = optionals cfg.enableUnixSocket [
-          "${pkgs.coreutils}/bin/install -d -o ${cfg.user} /run/memcached/"
-          "${pkgs.coreutils}/bin/chown -R ${cfg.user} /run/memcached/"
-        ];
         ExecStart =
         let
           networking = if cfg.enableUnixSocket
@@ -91,12 +86,13 @@ in
         in "${memcached}/bin/memcached ${networking} -m ${toString cfg.maxMemory} -c ${toString cfg.maxConnections} ${concatStringsSep " " cfg.extraOptions}";
 
         User = cfg.user;
+        RuntimeDirectory = "memcached";
       };
     };
   };
   imports = [
     (mkRemovedOptionModule ["services" "memcached" "socket"] ''
-      This option was replaced by a fixed unix socket path at /run/memcached/memcached.sock enabled using services.memached.enableUnixSocket.
+      This option was replaced by a fixed unix socket path at /run/memcached/memcached.sock enabled using services.memcached.enableUnixSocket.
     '')
   ];
 
diff --git a/nixos/modules/services/databases/stanchion.nix b/nixos/modules/services/databases/stanchion.nix
index 9fe49f51edd..97e55bc70c4 100644
--- a/nixos/modules/services/databases/stanchion.nix
+++ b/nixos/modules/services/databases/stanchion.nix
@@ -98,7 +98,7 @@ in
         type = types.path;
         default = "/var/log/stanchion";
         description = ''
-          Log directory for Stanchino.
+          Log directory for Stanchion.
         '';
       };
 
@@ -152,6 +152,11 @@ in
 
     users.groups.stanchion.gid = config.ids.gids.stanchion;
 
+    systemd.tmpfiles.rules = [
+      "d '${cfg.logDir}' - stanchion stanchion --"
+      "d '${cfg.dataDir}' 0700 stanchion stanchion --"
+    ];
+
     systemd.services.stanchion = {
       description = "Stanchion Server";
 
@@ -168,25 +173,12 @@ in
       environment.STANCHION_LOG_DIR = "${cfg.logDir}";
       environment.STANCHION_ETC_DIR = "/etc/stanchion";
 
-      preStart = ''
-        if ! test -e ${cfg.logDir}; then
-          mkdir -m 0755 -p ${cfg.logDir}
-          chown -R stanchion:stanchion ${cfg.logDir}
-        fi
-
-        if ! test -e ${cfg.dataDir}; then
-          mkdir -m 0700 -p ${cfg.dataDir}
-          chown -R stanchion:stanchion ${cfg.dataDir}
-        fi
-      '';
-
       serviceConfig = {
         ExecStart = "${cfg.package}/bin/stanchion console";
         ExecStop = "${cfg.package}/bin/stanchion stop";
         StandardInput = "tty";
         User = "stanchion";
         Group = "stanchion";
-        PermissionsStartOnly = true;
         # Give Stanchion a decent amount of time to clean up.
         TimeoutStopSec = 120;
         LimitNOFILE = 65536;
diff --git a/nixos/modules/services/mail/nullmailer.nix b/nixos/modules/services/mail/nullmailer.nix
index 418c02af4b7..9997d287013 100644
--- a/nixos/modules/services/mail/nullmailer.nix
+++ b/nixos/modules/services/mail/nullmailer.nix
@@ -212,6 +212,10 @@ with lib;
       };
     };
 
+    systemd.tmpfiles.rules = [
+      "d /var/spool/nullmailer - ${cfg.user} - - -"
+    ];
+
     systemd.services.nullmailer = {
       description = "nullmailer";
       wantedBy = [ "multi-user.target" ];
@@ -220,13 +224,11 @@ with lib;
       preStart = ''
         mkdir -p /var/spool/nullmailer/{queue,tmp}
         rm -f /var/spool/nullmailer/trigger && mkfifo -m 660 /var/spool/nullmailer/trigger
-        chown ${cfg.user} /var/spool/nullmailer/*
       '';
 
       serviceConfig = {
         User = cfg.user;
         Group = cfg.group;
-        PermissionsStartOnly=true;
         ExecStart = "${pkgs.nullmailer}/bin/nullmailer-send";
         Restart = "always";
       };
diff --git a/nixos/modules/services/mail/rss2email.nix b/nixos/modules/services/mail/rss2email.nix
index 5f3b2877008..a123736005a 100644
--- a/nixos/modules/services/mail/rss2email.nix
+++ b/nixos/modules/services/mail/rss2email.nix
@@ -94,6 +94,10 @@ in {
 
     services.rss2email.config.to = cfg.to;
 
+    systemd.tmpfiles.rules = [
+      "d /var/rss2email 0700 rss2email rss2email - -"
+    ];
+
     systemd.services.rss2email = let
       conf = pkgs.writeText "rss2email.cfg" (lib.generators.toINI {} ({
           DEFAULT = cfg.config;
@@ -105,22 +109,16 @@ in {
     in
     {
       preStart = ''
-        mkdir -p /var/rss2email
-        chmod 700 /var/rss2email
-
         cp ${conf} /var/rss2email/conf.cfg
         if [ ! -f /var/rss2email/db.json ]; then
           echo '{"version":2,"feeds":[]}' > /var/rss2email/db.json
         fi
-
-        chown -R rss2email:rss2email /var/rss2email
       '';
       path = [ pkgs.system-sendmail ];
       serviceConfig = {
         ExecStart =
           "${pkgs.rss2email}/bin/r2e -c /var/rss2email/conf.cfg -d /var/rss2email/db.json run";
         User = "rss2email";
-        PermissionsStartOnly = "true";
       };
     };
 
diff --git a/nixos/modules/services/misc/etcd.nix b/nixos/modules/services/misc/etcd.nix
index 2d1893dae64..e4d5322f9b5 100644
--- a/nixos/modules/services/misc/etcd.nix
+++ b/nixos/modules/services/misc/etcd.nix
@@ -142,6 +142,10 @@ in {
   };
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0700 etcd - - -"
+    ];
+
     systemd.services.etcd = {
       description = "etcd key-value store";
       wantedBy = [ "multi-user.target" ];
@@ -176,14 +180,8 @@ in {
         Type = "notify";
         ExecStart = "${pkgs.etcd.bin}/bin/etcd";
         User = "etcd";
-        PermissionsStartOnly = true;
         LimitNOFILE = 40000;
       };
-
-      preStart = ''
-        mkdir -m 0700 -p ${cfg.dataDir}
-        if [ "$(id -u)" = 0 ]; then chown etcd ${cfg.dataDir}; fi
-      '';
     };
 
     environment.systemPackages = [ pkgs.etcdctl ];
diff --git a/nixos/modules/services/misc/jackett.nix b/nixos/modules/services/misc/jackett.nix
index b18ce2b1f81..a07f20e5c24 100644
--- a/nixos/modules/services/misc/jackett.nix
+++ b/nixos/modules/services/misc/jackett.nix
@@ -38,24 +38,19 @@ in
   };
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0700 ${cfg.user} ${cfg.group} - -"
+    ];
+
     systemd.services.jackett = {
       description = "Jackett";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      preStart = ''
-        test -d ${cfg.dataDir} || {
-          echo "Creating jackett data directory in ${cfg.dataDir}"
-          mkdir -p ${cfg.dataDir}
-        }
-        chown -R ${cfg.user}:${cfg.group} ${cfg.dataDir}
-        chmod 0700 ${cfg.dataDir}
-      '';
 
       serviceConfig = {
         Type = "simple";
         User = cfg.user;
         Group = cfg.group;
-        PermissionsStartOnly = "true";
         ExecStart = "${pkgs.jackett}/bin/Jackett --NoUpdates --DataFolder '${cfg.dataDir}'";
         Restart = "on-failure";
       };
diff --git a/nixos/modules/services/misc/lidarr.nix b/nixos/modules/services/misc/lidarr.nix
index 627f22334fe..f466402abfc 100644
--- a/nixos/modules/services/misc/lidarr.nix
+++ b/nixos/modules/services/misc/lidarr.nix
@@ -17,20 +17,15 @@ in
       description = "Lidarr";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      preStart = ''
-        [ ! -d /var/lib/lidarr ] && mkdir -p /var/lib/lidarr
-        chown -R lidarr:lidarr /var/lib/lidarr
-      '';
 
       serviceConfig = {
         Type = "simple";
         User = "lidarr";
         Group = "lidarr";
-        PermissionsStartOnly = "true";
         ExecStart = "${pkgs.lidarr}/bin/Lidarr";
         Restart = "on-failure";
 
-        StateDirectory = "/var/lib/lidarr/";
+        StateDirectory = "lidarr";
         StateDirectoryMode = "0770";
       };
     };
diff --git a/nixos/modules/services/misc/mesos-master.nix b/nixos/modules/services/misc/mesos-master.nix
index 0523c6549ed..572a9847e46 100644
--- a/nixos/modules/services/misc/mesos-master.nix
+++ b/nixos/modules/services/misc/mesos-master.nix
@@ -95,6 +95,9 @@ in {
 
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.workDir}' 0700 - - - -"
+    ];
     systemd.services.mesos-master = {
       description = "Mesos Master";
       wantedBy = [ "multi-user.target" ];
@@ -114,11 +117,7 @@ in {
             ${toString cfg.extraCmdLineOptions}
         '';
         Restart = "on-failure";
-        PermissionsStartOnly = true;
       };
-      preStart = ''
-        mkdir -m 0700 -p ${cfg.workDir}
-      '';
     };
   };
 
diff --git a/nixos/modules/services/misc/mesos-slave.nix b/nixos/modules/services/misc/mesos-slave.nix
index 468c7f36ecc..170065d0065 100644
--- a/nixos/modules/services/misc/mesos-slave.nix
+++ b/nixos/modules/services/misc/mesos-slave.nix
@@ -184,6 +184,9 @@ in {
   };
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.workDir}' 0701 - - - -"
+    ];
     systemd.services.mesos-slave = {
       description = "Mesos Slave";
       wantedBy = [ "multi-user.target" ];
@@ -210,11 +213,7 @@ in {
             --executor_environment_variables=${lib.escapeShellArg (builtins.toJSON cfg.executorEnvironmentVariables)} \
             ${toString cfg.extraCmdLineOptions}
         '';
-        PermissionsStartOnly = true;
       };
-      preStart = ''
-        mkdir -m 0701 -p ${cfg.workDir}
-      '';
     };
   };
 
diff --git a/nixos/modules/services/misc/radarr.nix b/nixos/modules/services/misc/radarr.nix
index 9ab26d84832..74444e24043 100644
--- a/nixos/modules/services/misc/radarr.nix
+++ b/nixos/modules/services/misc/radarr.nix
@@ -38,24 +38,19 @@ in
   };
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0700 ${cfg.user} ${cfg.group} - -"
+    ];
+
     systemd.services.radarr = {
       description = "Radarr";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      preStart = ''
-        test -d ${cfg.dataDir} || {
-          echo "Creating radarr data directory in ${cfg.dataDir}"
-          mkdir -p ${cfg.dataDir}
-        }
-        chown -R ${cfg.user}:${cfg.group} ${cfg.dataDir}
-        chmod 0700 ${cfg.dataDir}
-      '';
 
       serviceConfig = {
         Type = "simple";
         User = cfg.user;
         Group = cfg.group;
-        PermissionsStartOnly = "true";
         ExecStart = "${pkgs.radarr}/bin/Radarr -nobrowser -data='${cfg.dataDir}'";
         Restart = "on-failure";
       };
diff --git a/nixos/modules/services/misc/sonarr.nix b/nixos/modules/services/misc/sonarr.nix
index a99445a268d..77c7f0582d0 100644
--- a/nixos/modules/services/misc/sonarr.nix
+++ b/nixos/modules/services/misc/sonarr.nix
@@ -39,24 +39,19 @@ in
   };
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0700 ${cfg.user} ${cfg.group} - -"
+    ];
+
     systemd.services.sonarr = {
       description = "Sonarr";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      preStart = ''
-        test -d ${cfg.dataDir} || {
-          echo "Creating sonarr data directory in ${cfg.dataDir}"
-          mkdir -p ${cfg.dataDir}
-        }
-        chown -R ${cfg.user}:${cfg.group} ${cfg.dataDir}
-        chmod 0700 ${cfg.dataDir}
-      '';
 
       serviceConfig = {
         Type = "simple";
         User = cfg.user;
         Group = cfg.group;
-        PermissionsStartOnly = "true";
         ExecStart = "${pkgs.sonarr}/bin/NzbDrone -nobrowser -data='${cfg.dataDir}'";
         Restart = "on-failure";
       };
diff --git a/nixos/modules/services/misc/zookeeper.nix b/nixos/modules/services/misc/zookeeper.nix
index cb7cc97d5a5..50c84e3c6b8 100644
--- a/nixos/modules/services/misc/zookeeper.nix
+++ b/nixos/modules/services/misc/zookeeper.nix
@@ -119,6 +119,10 @@ in {
   config = mkIf cfg.enable {
     environment.systemPackages = [cfg.package];
 
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0700 zookeeper - - -"
+    ];
+
     systemd.services.zookeeper = {
       description = "Zookeeper Daemon";
       wantedBy = [ "multi-user.target" ];
@@ -135,11 +139,8 @@ in {
             ${configDir}/zoo.cfg
         '';
         User = "zookeeper";
-        PermissionsStartOnly = true;
       };
       preStart = ''
-        mkdir -m 0700 -p ${cfg.dataDir}
-        if [ "$(id -u)" = 0 ]; then chown zookeeper ${cfg.dataDir}; fi
         echo "${toString cfg.id}" > ${cfg.dataDir}/myid
       '';
     };
diff --git a/nixos/modules/services/monitoring/collectd.nix b/nixos/modules/services/monitoring/collectd.nix
index 45e3312c0f4..6a4c678eb21 100644
--- a/nixos/modules/services/monitoring/collectd.nix
+++ b/nixos/modules/services/monitoring/collectd.nix
@@ -79,6 +79,10 @@ in {
   };
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' - ${cfg.user} - - -"
+    ];
+
     systemd.services.collectd = {
       description = "Collectd Monitoring Agent";
       after = [ "network.target" ];
@@ -87,16 +91,9 @@ in {
       serviceConfig = {
         ExecStart = "${cfg.package}/sbin/collectd -C ${conf} -f";
         User = cfg.user;
-        PermissionsStartOnly = true;
         Restart = "on-failure";
         RestartSec = 3;
       };
-
-      preStart = ''
-        mkdir -p "${cfg.dataDir}"
-        chmod 755 "${cfg.dataDir}"
-        chown -R ${cfg.user} "${cfg.dataDir}"
-      '';
     };
 
     users.users = optional (cfg.user == "collectd") {
diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix
index e7ac12c07d3..d8384e0d35b 100644
--- a/nixos/modules/services/monitoring/prometheus/default.nix
+++ b/nixos/modules/services/monitoring/prometheus/default.nix
@@ -22,9 +22,6 @@ let
   workingDir  = stateDirBase + stateDir;
   workingDir2 = stateDirBase + cfg2.stateDir;
 
-  # Get a submodule without any embedded metadata:
-  _filter = x: filterAttrs (k: v: k != "_module") x;
-
   # a wrapper that verifies that the configuration is valid
   promtoolCheck = what: name: file: pkgs.runCommand "${name}-${what}-checked"
     { buildInputs = [ cfg.package ]; } ''
@@ -50,11 +47,11 @@ let
 
   # This becomes the main config file for Prometheus 1
   promConfig = {
-    global = cfg.globalConfig;
+    global = filterValidPrometheus cfg.globalConfig;
     rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
       (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg.rules))
     ]);
-    scrape_configs = filterEmpty cfg.scrapeConfigs;
+    scrape_configs = filterValidPrometheus cfg.scrapeConfigs;
   };
 
   generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
@@ -77,11 +74,11 @@ let
 
   # This becomes the main config file for Prometheus 2
   promConfig2 = {
-    global = cfg2.globalConfig;
+    global = filterValidPrometheus cfg2.globalConfig;
     rule_files = map (prom2toolCheck "check rules" "rules") (cfg2.ruleFiles ++ [
       (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg2.rules))
     ]);
-    scrape_configs = filterEmpty cfg2.scrapeConfigs;
+    scrape_configs = filterValidPrometheus cfg2.scrapeConfigs;
     alerting = optionalAttrs (cfg2.alertmanagerURL != []) {
       alertmanagers = [{
         static_configs = [{
@@ -108,7 +105,7 @@ let
   ] ++
   optional (cfg2.webExternalUrl != null) "--web.external-url=${cfg2.webExternalUrl}";
 
-  filterEmpty = filterAttrsListRecursive (_n: v: !(v == null || v == [] || v == {}));
+  filterValidPrometheus = filterAttrsListRecursive (n: v: !(n == "_module" || v == null));
   filterAttrsListRecursive = pred: x:
     if isAttrs x then
       listToAttrs (
@@ -123,41 +120,37 @@ let
       map (filterAttrsListRecursive pred) x
     else x;
 
-  promTypes.globalConfig = types.submodule {
-    options = {
-      scrape_interval = mkOption {
-        type = types.str;
-        default = "1m";
-        description = ''
-          How frequently to scrape targets by default.
-        '';
-      };
+  mkDefOpt = type : defaultStr : description : mkOpt type (description + ''
 
-      scrape_timeout = mkOption {
-        type = types.str;
-        default = "10s";
-        description = ''
-          How long until a scrape request times out.
-        '';
-      };
+    Defaults to <literal>${defaultStr}</literal> in prometheus
+    when set to <literal>null</literal>.
+  '');
 
-      evaluation_interval = mkOption {
-        type = types.str;
-        default = "1m";
-        description = ''
-          How frequently to evaluate rules by default.
-        '';
-      };
+  mkOpt = type : description : mkOption {
+    type = types.nullOr type;
+    default = null;
+    inherit description;
+  };
 
-      external_labels = mkOption {
-        type = types.attrsOf types.str;
-        description = ''
-          The labels to add to any time series or alerts when
-          communicating with external systems (federation, remote
-          storage, Alertmanager).
-        '';
-        default = {};
-      };
+  promTypes.globalConfig = types.submodule {
+    options = {
+      scrape_interval = mkDefOpt types.str "1m" ''
+        How frequently to scrape targets by default.
+      '';
+
+      scrape_timeout = mkDefOpt types.str "10s" ''
+        How long until a scrape request times out.
+      '';
+
+      evaluation_interval = mkDefOpt types.str "1m" ''
+        How frequently to evaluate rules by default.
+      '';
+
+      external_labels = mkOpt (types.attrsOf types.str) ''
+        The labels to add to any time series or alerts when
+        communicating with external systems (federation, remote
+        storage, Alertmanager).
+      '';
     };
   };
 
@@ -169,145 +162,127 @@ let
           The job name assigned to scraped metrics by default.
         '';
       };
-      scrape_interval = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          How frequently to scrape targets from this job. Defaults to the
-          globally configured default.
-        '';
-      };
-      scrape_timeout = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Per-target timeout when scraping this job. Defaults to the
-          globally configured default.
-        '';
-      };
-      metrics_path = mkOption {
-        type = types.str;
-        default = "/metrics";
-        description = ''
-          The HTTP resource path on which to fetch metrics from targets.
-        '';
-      };
-      honor_labels = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Controls how Prometheus handles conflicts between labels
-          that are already present in scraped data and labels that
-          Prometheus would attach server-side ("job" and "instance"
-          labels, manually configured target labels, and labels
-          generated by service discovery implementations).
-
-          If honor_labels is set to "true", label conflicts are
-          resolved by keeping label values from the scraped data and
-          ignoring the conflicting server-side labels.
-
-          If honor_labels is set to "false", label conflicts are
-          resolved by renaming conflicting labels in the scraped data
-          to "exported_&lt;original-label&gt;" (for example
-          "exported_instance", "exported_job") and then attaching
-          server-side labels. This is useful for use cases such as
-          federation, where all labels specified in the target should
-          be preserved.
-        '';
-      };
-      scheme = mkOption {
-        type = types.enum ["http" "https"];
-        default = "http";
-        description = ''
-          The URL scheme with which to fetch metrics from targets.
-        '';
-      };
-      params = mkOption {
-        type = types.attrsOf (types.listOf types.str);
-        default = {};
-        description = ''
-          Optional HTTP URL parameters.
-        '';
-      };
-      basic_auth = mkOption {
-        type = types.nullOr (types.submodule {
-          options = {
-            username = mkOption {
-              type = types.str;
-              description = ''
-                HTTP username
-              '';
-            };
-            password = mkOption {
-              type = types.str;
-              description = ''
-                HTTP password
-              '';
-            };
+      scrape_interval = mkOpt types.str ''
+        How frequently to scrape targets from this job. Defaults to the
+        globally configured default.
+      '';
+
+      scrape_timeout = mkOpt types.str ''
+        Per-target timeout when scraping this job. Defaults to the
+        globally configured default.
+      '';
+
+      metrics_path = mkDefOpt types.str "/metrics" ''
+        The HTTP resource path on which to fetch metrics from targets.
+      '';
+
+      honor_labels = mkDefOpt types.bool "false" ''
+        Controls how Prometheus handles conflicts between labels
+        that are already present in scraped data and labels that
+        Prometheus would attach server-side ("job" and "instance"
+        labels, manually configured target labels, and labels
+        generated by service discovery implementations).
+
+        If honor_labels is set to "true", label conflicts are
+        resolved by keeping label values from the scraped data and
+        ignoring the conflicting server-side labels.
+
+        If honor_labels is set to "false", label conflicts are
+        resolved by renaming conflicting labels in the scraped data
+        to "exported_&lt;original-label&gt;" (for example
+        "exported_instance", "exported_job") and then attaching
+        server-side labels. This is useful for use cases such as
+        federation, where all labels specified in the target should
+        be preserved.
+      '';
+
+      honor_timestamps = mkDefOpt types.bool "true" ''
+        honor_timestamps controls whether Prometheus respects the timestamps present
+        in scraped data.
+
+        If honor_timestamps is set to <literal>true</literal>, the timestamps of the metrics exposed
+        by the target will be used.
+
+        If honor_timestamps is set to <literal>false</literal>, the timestamps of the metrics exposed
+        by the target will be ignored.
+      '';
+
+      scheme = mkDefOpt (types.enum ["http" "https"]) "http" ''
+        The URL scheme with which to fetch metrics from targets.
+      '';
+
+      params = mkOpt (types.attrsOf (types.listOf types.str)) ''
+        Optional HTTP URL parameters.
+      '';
+
+      basic_auth = mkOpt (types.submodule {
+        options = {
+          username = mkOption {
+            type = types.str;
+            description = ''
+              HTTP username
+            '';
           };
-        });
-        default = null;
-        apply = x: mapNullable _filter x;
-        description = ''
-          Optional http login credentials for metrics scraping.
-        '';
-      };
-      tls_config = mkOption {
-        type = types.nullOr promTypes.tls_config;
-        default = null;
-        apply = x: mapNullable _filter x;
-        description = ''
-          Configures the scrape request's TLS settings.
-        '';
-      };
-      dns_sd_configs = mkOption {
-        type = types.listOf promTypes.dns_sd_config;
-        default = [];
-        apply = x: map _filter x;
-        description = ''
-          List of DNS service discovery configurations.
-        '';
-      };
-      consul_sd_configs = mkOption {
-        type = types.listOf promTypes.consul_sd_config;
-        default = [];
-        apply = x: map _filter x;
-        description = ''
-          List of Consul service discovery configurations.
-        '';
-      };
-      file_sd_configs = mkOption {
-        type = types.listOf promTypes.file_sd_config;
-        default = [];
-        apply = x: map _filter x;
-        description = ''
-          List of file service discovery configurations.
-        '';
-      };
-      static_configs = mkOption {
-        type = types.listOf promTypes.static_config;
-        default = [];
-        apply = x: map _filter x;
-        description = ''
-          List of labeled target groups for this job.
-        '';
-      };
-      ec2_sd_configs = mkOption {
-        type = types.listOf promTypes.ec2_sd_config;
-        default = [];
-        apply = x: map _filter x;
-        description = ''
-          List of EC2 service discovery configurations.
-        '';
-      };
-      relabel_configs = mkOption {
-        type = types.listOf promTypes.relabel_config;
-        default = [];
-        apply = x: map _filter x;
-        description = ''
-          List of relabel configurations.
-        '';
-      };
+          password = mkOption {
+            type = types.str;
+            description = ''
+              HTTP password
+            '';
+          };
+        };
+      }) ''
+        Optional http login credentials for metrics scraping.
+      '';
+
+      bearer_token = mkOpt types.str ''
+        Sets the `Authorization` header on every scrape request with
+        the configured bearer token. It is mutually exclusive with
+        <option>bearer_token_file</option>.
+      '';
+
+      bearer_token_file = mkOpt types.str ''
+        Sets the `Authorization` header on every scrape request with
+        the bearer token read from the configured file. It is mutually
+        exclusive with <option>bearer_token</option>.
+      '';
+
+      tls_config = mkOpt promTypes.tls_config ''
+        Configures the scrape request's TLS settings.
+      '';
+
+      proxy_url = mkOpt types.str ''
+        Optional proxy URL.
+      '';
+
+      ec2_sd_configs = mkOpt (types.listOf promTypes.ec2_sd_config) ''
+        List of EC2 service discovery configurations.
+      '';
+
+      dns_sd_configs = mkOpt (types.listOf promTypes.dns_sd_config) ''
+        List of DNS service discovery configurations.
+      '';
+
+      consul_sd_configs = mkOpt (types.listOf promTypes.consul_sd_config) ''
+        List of Consul service discovery configurations.
+      '';
+
+      file_sd_configs = mkOpt (types.listOf promTypes.file_sd_config) ''
+        List of file service discovery configurations.
+      '';
+
+      static_configs = mkOpt (types.listOf promTypes.static_config) ''
+        List of labeled target groups for this job.
+      '';
+
+      relabel_configs = mkOpt (types.listOf promTypes.relabel_config) ''
+        List of relabel configurations.
+      '';
+
+      sample_limit = mkDefOpt types.int "0" ''
+        Per-scrape limit on number of scraped samples that will be accepted.
+        If more than this number of samples are present after metric relabelling
+        the entire scrape will be treated as failed. 0 means no limit.
+      '';
     };
   };
 
@@ -337,66 +312,41 @@ let
           The AWS Region.
         '';
       };
-      endpoint = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Custom endpoint to be used.
-        '';
-      };
-      access_key = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          The AWS API key id. If blank, the environment variable
-          <literal>AWS_ACCESS_KEY_ID</literal> is used.
-        '';
-      };
-      secret_key = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          The AWS API key secret. If blank, the environment variable
-           <literal>AWS_SECRET_ACCESS_KEY</literal> is used.
-        '';
-      };
-      profile = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Named AWS profile used to connect to the API.
-        '';
-      };
-      role_arn = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          AWS Role ARN, an alternative to using AWS API keys.
-        '';
-      };
-      refresh_interval = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Refresh interval to re-read the instance list.
-        '';
-      };
-      port = mkOption {
-        type = types.int;
-        default = 80;
-        description = ''
-          The port to scrape metrics from. If using the public IP
-          address, this must instead be specified in the relabeling
-          rule.
-        '';
-      };
-      filters = mkOption {
-        type = types.nullOr (types.listOf promTypes.filter);
-        default = null;
-        description = ''
-          Filters can be used optionally to filter the instance list by other criteria.
-        '';
-      };
+      endpoint = mkOpt types.str ''
+        Custom endpoint to be used.
+      '';
+
+      access_key = mkOpt types.str ''
+        The AWS API key id. If blank, the environment variable
+        <literal>AWS_ACCESS_KEY_ID</literal> is used.
+      '';
+
+      secret_key = mkOpt types.str ''
+        The AWS API key secret. If blank, the environment variable
+         <literal>AWS_SECRET_ACCESS_KEY</literal> is used.
+      '';
+
+      profile = mkOpt  types.str ''
+        Named AWS profile used to connect to the API.
+      '';
+
+      role_arn = mkOpt types.str ''
+        AWS Role ARN, an alternative to using AWS API keys.
+      '';
+
+      refresh_interval = mkDefOpt types.str "60s" ''
+        Refresh interval to re-read the instance list.
+      '';
+
+      port = mkDefOpt types.int "80" ''
+        The port to scrape metrics from. If using the public IP
+        address, this must instead be specified in the relabeling
+        rule.
+      '';
+
+      filters = mkOpt (types.listOf promTypes.filter) ''
+        Filters can be used optionally to filter the instance list by other criteria.
+      '';
     };
   };
 
@@ -409,6 +359,7 @@ let
           for the available filters.
         '';
       };
+
       value = mkOption {
         type = types.listOf types.str;
         default = [];
@@ -427,56 +378,63 @@ let
           A list of DNS SRV record names to be queried.
         '';
       };
-      refresh_interval = mkOption {
-        type = types.str;
-        default = "30s";
-        description = ''
-          The time after which the provided names are refreshed.
-        '';
-      };
+
+      refresh_interval = mkDefOpt types.str "30s" ''
+        The time after which the provided names are refreshed.
+      '';
     };
   };
 
   promTypes.consul_sd_config = types.submodule {
     options = {
-      server = mkOption {
-        type = types.str;
-        description = "Consul server to query.";
-      };
-      token = mkOption {
-        type = types.nullOr types.str;
-        description = "Consul token";
-      };
-      datacenter = mkOption {
-        type = types.nullOr types.str;
-        description = "Consul datacenter";
-      };
-      scheme = mkOption {
-        type = types.nullOr types.str;
-        description = "Consul scheme";
-      };
-      username = mkOption {
-        type = types.nullOr types.str;
-        description = "Consul username";
-      };
-      password = mkOption {
-        type = types.nullOr types.str;
-        description = "Consul password";
-      };
+      server = mkDefOpt types.str "localhost:8500" ''
+        Consul server to query.
+      '';
 
-      services = mkOption {
-        type = types.listOf types.str;
-        description = ''
-          A list of services for which targets are retrieved.
-        '';
-      };
-      tag_separator = mkOption {
-        type = types.str;
-        default = ",";
-        description = ''
-          The string by which Consul tags are joined into the tag label.
-        '';
-      };
+      token = mkOpt types.str "Consul token";
+
+      datacenter = mkOpt types.str "Consul datacenter";
+
+      scheme = mkDefOpt types.str "http" "Consul scheme";
+
+      username = mkOpt types.str "Consul username";
+
+      password = mkOpt types.str "Consul password";
+
+      tls_config = mkOpt promTypes.tls_config ''
+        Configures the Consul request's TLS settings.
+      '';
+
+      services = mkOpt (types.listOf types.str) ''
+        A list of services for which targets are retrieved.
+      '';
+
+      tags = mkOpt (types.listOf types.str) ''
+        An optional list of tags used to filter nodes for a given
+        service. Services must contain all tags in the list.
+      '';
+
+      node_meta = mkOpt (types.attrsOf types.str) ''
+        Node metadata used to filter nodes for a given service.
+      '';
+
+      tag_separator = mkDefOpt types.str "," ''
+        The string by which Consul tags are joined into the tag label.
+      '';
+
+      allow_stale = mkOpt types.bool ''
+        Allow stale Consul results
+        (see <link xlink:href="https://www.consul.io/api/index.html#consistency-modes"/>).
+
+        Will reduce load on Consul.
+      '';
+
+      refresh_interval = mkDefOpt types.str "30s" ''
+        The time after which the provided names are refreshed.
+
+        On large setup it might be a good idea to increase this value
+        because the catalog will change all the time.
+      '';
     };
   };
 
@@ -488,108 +446,74 @@ let
           Patterns for files from which target groups are extracted. Refer
           to the Prometheus documentation for permitted filename patterns
           and formats.
-
-        '';
-      };
-      refresh_interval = mkOption {
-        type = types.str;
-        default = "30s";
-        description = ''
-          Refresh interval to re-read the files.
         '';
       };
+
+      refresh_interval = mkDefOpt types.str "5m" ''
+        Refresh interval to re-read the files.
+      '';
     };
   };
 
   promTypes.relabel_config = types.submodule {
     options = {
-      source_labels = mkOption {
-        type = with types; nullOr (listOf str);
-        default = null;
-        description = ''
-          The source labels select values from existing labels. Their content
-          is concatenated using the configured separator and matched against
-          the configured regular expression.
-        '';
-      };
-      separator = mkOption {
-        type = types.str;
-        default = ";";
-        description = ''
-          Separator placed between concatenated source label values.
-        '';
-      };
-      target_label = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Label to which the resulting value is written in a replace action.
-          It is mandatory for replace actions.
-        '';
-      };
-      regex = mkOption {
-        type = types.str;
-        default = "(.*)";
-        description = ''
-          Regular expression against which the extracted value is matched.
-        '';
-      };
-      replacement = mkOption {
-        type = types.str;
-        default = "$1";
-        description = ''
-          Replacement value against which a regex replace is performed if the
-          regular expression matches.
-        '';
-      };
-      action = mkOption {
-        type = types.enum ["replace" "keep" "drop"];
-        default = "replace";
-        description = ''
-          Action to perform based on regex matching.
-        '';
-      };
+      source_labels = mkOpt (types.listOf types.str) ''
+        The source labels select values from existing labels. Their content
+        is concatenated using the configured separator and matched against
+        the configured regular expression.
+      '';
+
+      separator = mkDefOpt types.str ";" ''
+        Separator placed between concatenated source label values.
+      '';
+
+      target_label = mkOpt types.str ''
+        Label to which the resulting value is written in a replace action.
+        It is mandatory for replace actions.
+      '';
+
+      regex = mkDefOpt types.str "(.*)" ''
+        Regular expression against which the extracted value is matched.
+      '';
+
+      modulus = mkOpt types.int ''
+        Modulus to take of the hash of the source label values.
+      '';
+
+      replacement = mkDefOpt types.str "$1" ''
+        Replacement value against which a regex replace is performed if the
+        regular expression matches.
+      '';
+
+      action = mkDefOpt (types.enum ["replace" "keep" "drop"]) "replace" ''
+        Action to perform based on regex matching.
+      '';
+
     };
   };
 
   promTypes.tls_config = types.submodule {
     options = {
-      ca_file = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          CA certificate to validate API server certificate with.
-        '';
-      };
-      cert_file = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Certificate file for client cert authentication to the server.
-        '';
-      };
-      key_file = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Key file for client cert authentication to the server.
-        '';
-      };
-      server_name = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          ServerName extension to indicate the name of the server.
-          http://tools.ietf.org/html/rfc4366#section-3.1
-        '';
-      };
-      insecure_skip_verify = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Disable validation of the server certificate.
-        '';
-      };
+      ca_file = mkOpt types.str ''
+        CA certificate to validate API server certificate with.
+      '';
+
+      cert_file = mkOpt types.str ''
+        Certificate file for client cert authentication to the server.
+      '';
+
+      key_file = mkOpt types.str ''
+        Key file for client cert authentication to the server.
+      '';
+
+      server_name = mkOpt types.str ''
+        ServerName extension to indicate the name of the server.
+        http://tools.ietf.org/html/rfc4366#section-3.1
+      '';
+
+      insecure_skip_verify = mkOpt types.bool ''
+        Disable validation of the server certificate.
+      '';
     };
   };
 
@@ -662,7 +586,6 @@ in {
       globalConfig = mkOption {
         type = promTypes.globalConfig;
         default = {};
-        apply = _filter;
         description = ''
           Parameters that are valid in all  configuration contexts. They
           also serve as defaults for other configuration sections
@@ -688,7 +611,6 @@ in {
       scrapeConfigs = mkOption {
         type = types.listOf promTypes.scrape_config;
         default = [];
-        apply = x: map _filter x;
         description = ''
           A list of scrape configurations.
         '';
@@ -786,7 +708,6 @@ in {
       globalConfig = mkOption {
         type = promTypes.globalConfig;
         default = {};
-        apply = _filter;
         description = ''
           Parameters that are valid in all  configuration contexts. They
           also serve as defaults for other configuration sections
@@ -812,7 +733,6 @@ in {
       scrapeConfigs = mkOption {
         type = types.listOf promTypes.scrape_config;
         default = [];
-        apply = x: map _filter x;
         description = ''
           A list of scrape configurations.
         '';
diff --git a/nixos/modules/services/network-filesystems/ipfs.nix b/nixos/modules/services/network-filesystems/ipfs.nix
index d4fa1eccdf3..f19bf9d8139 100644
--- a/nixos/modules/services/network-filesystems/ipfs.nix
+++ b/nixos/modules/services/network-filesystems/ipfs.nix
@@ -226,18 +226,19 @@ in {
       ipfs.gid = config.ids.gids.ipfs;
     };
 
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' - ${cfg.user} ${cfg.group} - -"
+    ] ++ optionals cfg.autoMount [
+      "d '${cfg.ipfsMountDir}' - ${cfg.user} ${cfg.group} - -"
+      "d '${cfg.ipnsMountDir}' - ${cfg.user} ${cfg.group} - -"
+    ];
+
     systemd.services.ipfs-init = recursiveUpdate commonEnv {
       description = "IPFS Initializer";
 
       after = [ "local-fs.target" ];
       before = [ "ipfs.service" "ipfs-offline.service" "ipfs-norouting.service" ];
 
-      preStart = ''
-        install -m 0755 -o ${cfg.user} -g ${cfg.group} -d ${cfg.dataDir}
-      '' + optionalString cfg.autoMount ''
-        install -m 0755 -o ${cfg.user} -g ${cfg.group} -d ${cfg.ipfsMountDir}
-        install -m 0755 -o ${cfg.user} -g ${cfg.group} -d ${cfg.ipnsMountDir}
-      '';
       script = ''
         if [[ ! -f ${cfg.dataDir}/config ]]; then
           ipfs init ${optionalString cfg.emptyRepo "-e"} \
@@ -253,7 +254,6 @@ in {
       serviceConfig = {
         Type = "oneshot";
         RemainAfterExit = true;
-        PermissionsStartOnly = true;
       };
     };
 
diff --git a/nixos/modules/services/networking/mxisd.nix b/nixos/modules/services/networking/mxisd.nix
index 0aa6d0d9ecd..0b9824f29fd 100644
--- a/nixos/modules/services/networking/mxisd.nix
+++ b/nixos/modules/services/networking/mxisd.nix
@@ -116,7 +116,6 @@ in {
         Group = "mxisd";
         ExecStart = "${cfg.package}/bin/mxisd --spring.config.location=${cfg.dataDir}/ --spring.profiles.active=systemd --java.security.egd=file:/dev/./urandom";
         WorkingDirectory = cfg.dataDir;
-        PermissionsStartOnly = true;
         SuccessExitStatus = 143;
         Restart = "on-failure";
       };
diff --git a/nixos/modules/services/networking/smokeping.nix b/nixos/modules/services/networking/smokeping.nix
index 9ba6e48f417..fab3ed5bb39 100644
--- a/nixos/modules/services/networking/smokeping.nix
+++ b/nixos/modules/services/networking/smokeping.nix
@@ -285,12 +285,12 @@ in
       uid = config.ids.uids.smokeping;
       description = "smokeping daemon user";
       home = smokepingHome;
+      createHome = true;
     };
     systemd.services.smokeping = {
       wantedBy = [ "multi-user.target"];
       serviceConfig = {
         User = cfg.user;
-        PermissionsStartOnly = true;
         Restart = "on-failure";
       };
       preStart = ''
@@ -300,7 +300,6 @@ in
         cp ${cgiHome} ${smokepingHome}/smokeping.fcgi
         ${cfg.package}/bin/smokeping --check --config=${configPath}
         ${cfg.package}/bin/smokeping --static --config=${configPath}
-        chown -R ${cfg.user} ${smokepingHome}
       '';
       script = ''${cfg.package}/bin/smokeping --config=${configPath} --nodaemon'';
     };
diff --git a/nixos/modules/services/networking/syncthing.nix b/nixos/modules/services/networking/syncthing.nix
index 702481ec517..114a64dfb17 100644
--- a/nixos/modules/services/networking/syncthing.nix
+++ b/nixos/modules/services/networking/syncthing.nix
@@ -151,7 +151,6 @@ in {
           RestartForceExitStatus="3 4";
           User = cfg.user;
           Group = cfg.group;
-          PermissionsStartOnly = true;
           ExecStart = ''
             ${cfg.package}/bin/syncthing \
               -no-browser \
diff --git a/nixos/modules/services/security/munge.nix b/nixos/modules/services/security/munge.nix
index 504bc66c6d1..1c4f8e20552 100644
--- a/nixos/modules/services/security/munge.nix
+++ b/nixos/modules/services/security/munge.nix
@@ -49,21 +49,16 @@ in
 
       path = [ pkgs.munge pkgs.coreutils ];
 
-      preStart = ''
-        chmod 0400 ${cfg.password}
-        mkdir -p /var/lib/munge -m 0711
-        chown -R munge:munge /var/lib/munge
-        mkdir -p /run/munge -m 0755
-        chown -R munge:munge /run/munge
-      '';
-
       serviceConfig = {
+        ExecStartPre = "+${pkgs.coreutils}/bin/chmod 0400 ${cfg.password}";
         ExecStart = "${pkgs.munge}/bin/munged --syslog --key-file ${cfg.password}";
         PIDFile = "/run/munge/munged.pid";
         ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
-        PermissionsStartOnly = "true";
         User = "munge";
         Group = "munge";
+        StateDirectory = "munge";
+        StateDirectoryMode = "0711";
+        RuntimeDirectory = "munge";
       };
 
     };
diff --git a/nixos/modules/services/security/vault.nix b/nixos/modules/services/security/vault.nix
index 0b28bc89445..8176c168ca9 100644
--- a/nixos/modules/services/security/vault.nix
+++ b/nixos/modules/services/security/vault.nix
@@ -119,6 +119,10 @@ in
     };
     users.groups.vault.gid = config.ids.gids.vault;
 
+    systemd.tmpfiles.rules = optional (cfg.storagePath != null) [
+      "d '${cfg.storagePath}' 0700 vault vault - -"
+    ];
+
     systemd.services.vault = {
       description = "Vault server daemon";
 
@@ -128,14 +132,9 @@ in
 
       restartIfChanged = false; # do not restart on "nixos-rebuild switch". It would seal the storage and disrupt the clients.
 
-      preStart = optionalString (cfg.storagePath != null) ''
-        install -d -m0700 -o vault -g vault "${cfg.storagePath}"
-      '';
-
       serviceConfig = {
         User = "vault";
         Group = "vault";
-        PermissionsStartOnly = true;
         ExecStart = "${cfg.package}/bin/vault server -config ${configFile}";
         PrivateDevices = true;
         PrivateTmp = true;
diff --git a/nixos/modules/services/torrent/peerflix.nix b/nixos/modules/services/torrent/peerflix.nix
index bed6661f84d..a74f6598432 100644
--- a/nixos/modules/services/torrent/peerflix.nix
+++ b/nixos/modules/services/torrent/peerflix.nix
@@ -39,6 +39,10 @@ in {
   ###### implementation
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.stateDir}' - peerflix - - -"
+    ];
+
     systemd.services.peerflix = {
       description = "Peerflix Daemon";
       wantedBy = [ "multi-user.target" ];
@@ -47,13 +51,11 @@ in {
 
       preStart = ''
         mkdir -p "${cfg.stateDir}"/{torrents,.config/peerflix-server}
-        if [ "$(id -u)" = 0 ]; then chown -R peerflix "${cfg.stateDir}"; fi
         ln -fs "${configFile}" "${cfg.stateDir}/.config/peerflix-server/config.json"
       '';
 
       serviceConfig = {
         ExecStart = "${pkgs.nodePackages.peerflix-server}/bin/peerflix-server";
-        PermissionsStartOnly = true;
         User = "peerflix";
       };
     };
diff --git a/nixos/modules/services/web-apps/codimd.nix b/nixos/modules/services/web-apps/codimd.nix
index ee2fc2b9d85..7ae7cd9c52d 100644
--- a/nixos/modules/services/web-apps/codimd.nix
+++ b/nixos/modules/services/web-apps/codimd.nix
@@ -899,10 +899,6 @@ in
       description = "CodiMD Service";
       wantedBy = [ "multi-user.target" ];
       after = [ "networking.target" ];
-      preStart = ''
-        mkdir -p ${cfg.workDir}
-        chown -R codimd: ${cfg.workDir}
-      '';
       serviceConfig = {
         WorkingDirectory = cfg.workDir;
         ExecStart = "${pkgs.codimd}/bin/codimd";
@@ -912,7 +908,6 @@ in
         ];
         Restart = "always";
         User = "codimd";
-        PermissionsStartOnly = true;
         PrivateTmp = true;
       };
     };
diff --git a/nixos/modules/services/web-apps/nexus.nix b/nixos/modules/services/web-apps/nexus.nix
index 050f8757fa5..052dbed6d4f 100644
--- a/nixos/modules/services/web-apps/nexus.nix
+++ b/nixos/modules/services/web-apps/nexus.nix
@@ -83,6 +83,8 @@ in
     users.users."${cfg.user}" = {
       isSystemUser = true;
       group = cfg.group;
+      home = cfg.home;
+      createHome = true;
     };
 
     users.groups."${cfg.group}" = {};
@@ -104,8 +106,6 @@ in
       preStart = ''
         mkdir -p ${cfg.home}/nexus3/etc
 
-        chown -R ${cfg.user}:${cfg.group} ${cfg.home}
-
         if [ ! -f ${cfg.home}/nexus3/etc/nexus.properties ]; then
           echo "# Jetty section" > ${cfg.home}/nexus3/etc/nexus.properties
           echo "application-port=${toString cfg.listenPort}" >> ${cfg.home}/nexus3/etc/nexus.properties
@@ -124,7 +124,6 @@ in
         User = cfg.user;
         Group = cfg.group;
         PrivateTmp = true;
-        PermissionsStartOnly = true;
         LimitNOFILE = 102642;
       };
     };
diff --git a/nixos/modules/services/web-servers/minio.nix b/nixos/modules/services/web-servers/minio.nix
index f78a966989b..cd123000f00 100644
--- a/nixos/modules/services/web-servers/minio.nix
+++ b/nixos/modules/services/web-servers/minio.nix
@@ -72,19 +72,16 @@ in
   };
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.configDir}' - minio minio - -"
+      "d '${cfg.dataDir}' - minio minio - -"
+    ];
+
     systemd.services.minio = {
       description = "Minio Object Storage";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      preStart = ''
-        # Make sure directories exist with correct owner
-        mkdir -p ${cfg.configDir}
-        chown -R minio:minio ${cfg.configDir}
-        mkdir -p ${cfg.dataDir}
-        chown minio:minio ${cfg.dataDir}
-      '';
       serviceConfig = {
-        PermissionsStartOnly = true;
         ExecStart = "${cfg.package}/bin/minio server --json --address ${cfg.listenAddress} --config-dir=${cfg.configDir} ${cfg.dataDir}";
         Type = "simple";
         User = "minio";
diff --git a/nixos/modules/services/web-servers/traefik.nix b/nixos/modules/services/web-servers/traefik.nix
index 4674ed0177e..5bac895d43a 100644
--- a/nixos/modules/services/web-servers/traefik.nix
+++ b/nixos/modules/services/web-servers/traefik.nix
@@ -84,18 +84,16 @@ in {
   };
 
   config = mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0700 traefik traefik - -"
+    ];
+
     systemd.services.traefik = {
       description = "Traefik web server";
       after = [ "network-online.target" ];
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
-        PermissionsStartOnly = true;
         ExecStart = ''${cfg.package.bin}/bin/traefik --configfile=${configFile}'';
-        ExecStartPre = [
-          ''${pkgs.coreutils}/bin/mkdir -p "${cfg.dataDir}"''
-          ''${pkgs.coreutils}/bin/chmod 700 "${cfg.dataDir}"''
-          ''${pkgs.coreutils}/bin/chown -R traefik:traefik "${cfg.dataDir}"''
-        ];
         Type = "simple";
         User = "traefik";
         Group = cfg.group;
diff --git a/nixos/modules/services/x11/desktop-managers/maxx.nix b/nixos/modules/services/x11/desktop-managers/maxx.nix
index d7bd2fc5eb0..6a698658bdd 100644
--- a/nixos/modules/services/x11/desktop-managers/maxx.nix
+++ b/nixos/modules/services/x11/desktop-managers/maxx.nix
@@ -13,6 +13,12 @@ in {
   config = mkIf (xcfg.enable && cfg.enable) {
     environment.systemPackages = [ pkgs.maxx ];
 
+    # there is hardcoded path in binaries
+    system.activationScripts.setup-maxx = ''
+      mkdir -p /opt
+      ln -sfn ${pkgs.maxx}/opt/MaXX /opt
+    '';
+
     services.xserver.desktopManager.session = [
     { name = "MaXX";
       start = ''
diff --git a/nixos/modules/system/boot/stage-1-init.sh b/nixos/modules/system/boot/stage-1-init.sh
index 6dafc6cddde..67cbe720ddc 100644
--- a/nixos/modules/system/boot/stage-1-init.sh
+++ b/nixos/modules/system/boot/stage-1-init.sh
@@ -555,7 +555,7 @@ echo /sbin/modprobe > /proc/sys/kernel/modprobe
 # Start stage 2.  `switch_root' deletes all files in the ramfs on the
 # current root.  Note that $stage2Init might be an absolute symlink,
 # in which case "-e" won't work because we're not in the chroot yet.
-if ! test -e "$targetRoot/$stage2Init" -o ! -L "$targetRoot/$stage2Init"; then
+if [ ! -e "$targetRoot/$stage2Init" ] && [ ! -L "$targetRoot/$stage2Init" ] ; then
     echo "stage 2 init script ($targetRoot/$stage2Init) not found"
     fail
 fi
diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix
index 8702abd3df8..788e3f4a2ab 100644
--- a/nixos/modules/system/boot/stage-1.nix
+++ b/nixos/modules/system/boot/stage-1.nix
@@ -32,7 +32,7 @@ let
   fileSystems = filter utils.fsNeededForBoot config.system.build.fileSystems;
 
   # A utility for enumerating the shared-library dependencies of a program
-  findLibs = pkgs.writeShellScriptBin "find-libs" ''
+  findLibs = pkgs.buildPackages.writeShellScriptBin "find-libs" ''
     set -euo pipefail
 
     declare -A seen
diff --git a/nixos/modules/virtualisation/ec2-amis.nix b/nixos/modules/virtualisation/ec2-amis.nix
index aaea06bb9a6..f640bb21b13 100644
--- a/nixos/modules/virtualisation/ec2-amis.nix
+++ b/nixos/modules/virtualisation/ec2-amis.nix
@@ -274,5 +274,22 @@ let self = {
   "18.09".sa-east-1.hvm-ebs = "ami-0e4a8a47fd6db6112";
   "18.09".ap-south-1.hvm-ebs = "ami-0880a678d3f555313";
 
-  latest = self."18.09";
+  # 19.03.172286.8ea36d73256
+  "19.03".eu-west-1.hvm-ebs = "ami-0fe40176548ff0940";
+  "19.03".eu-west-2.hvm-ebs = "ami-03a40fd3a02fe95ba";
+  "19.03".eu-west-3.hvm-ebs = "ami-0436f9da0f20a638e";
+  "19.03".eu-central-1.hvm-ebs = "ami-0022b8ea9efde5de4";
+  "19.03".us-east-1.hvm-ebs = "ami-0efc58fb70ae9a217";
+  "19.03".us-east-2.hvm-ebs = "ami-0abf711b1b34da1af";
+  "19.03".us-west-1.hvm-ebs = "ami-07d126e8838c40ec5";
+  "19.03".us-west-2.hvm-ebs = "ami-03f8a737546e47fb0";
+  "19.03".ca-central-1.hvm-ebs = "ami-03f9fd0ef2e035ede";
+  "19.03".ap-southeast-1.hvm-ebs = "ami-0cff66114c652c262";
+  "19.03".ap-southeast-2.hvm-ebs = "ami-054c73a7f8d773ea9";
+  "19.03".ap-northeast-1.hvm-ebs = "ami-00db62688900456a4";
+  "19.03".ap-northeast-2.hvm-ebs = "ami-0485cdd1a5fdd2117";
+  "19.03".sa-east-1.hvm-ebs = "ami-0c6a43c6e0ad1f4e2";
+  "19.03".ap-south-1.hvm-ebs = "ami-0303deb1b5890f878";
+
+  latest = self."19.03";
 }; in self
diff --git a/nixos/modules/virtualisation/virtualbox-image.nix b/nixos/modules/virtualisation/virtualbox-image.nix
index 903411799d3..840ac2e291d 100644
--- a/nixos/modules/virtualisation/virtualbox-image.nix
+++ b/nixos/modules/virtualisation/virtualbox-image.nix
@@ -100,6 +100,11 @@ in {
     boot.growPartition = true;
     boot.loader.grub.device = "/dev/sda";
 
+    swap.swapDevices = {
+      device = "/var/swap";
+      size = 2048;
+    };
+
     virtualisation.virtualbox.guest.enable = true;
 
   };
diff --git a/nixos/tests/nginx.nix b/nixos/tests/nginx.nix
index a4d14986a14..d66d99821c1 100644
--- a/nixos/tests/nginx.nix
+++ b/nixos/tests/nginx.nix
@@ -1,18 +1,19 @@
 # verifies:
 #   1. nginx generates config file with shared http context definitions above
 #      generated virtual hosts config.
+#   2. whether the ETag header is properly generated whenever we're serving
+#      files in Nix store paths
 
-import ./make-test.nix ({ pkgs, ...} : {
+import ./make-test.nix ({ pkgs, ... }: {
   name = "nginx";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ mbbx6spp ];
   };
 
-  nodes = {
-    webserver =
-      { ... }:
-      { services.nginx.enable = true;
-        services.nginx.commonHttpConfig = ''
+  nodes = let
+    commonConfig = { pkgs, ... }: {
+      services.nginx.enable = true;
+      services.nginx.commonHttpConfig = ''
         log_format ceeformat '@cee: {"status":"$status",'
           '"request_time":$request_time,'
           '"upstream_response_time":$upstream_response_time,'
@@ -24,20 +25,61 @@ import ./make-test.nix ({ pkgs, ...} : {
           '"request":"$request",'
           '"http_referer":"$http_referer",'
           '"upstream_addr":"$upstream_addr"}';
+      '';
+      services.nginx.virtualHosts."0.my.test" = {
+        extraConfig = ''
+          access_log syslog:server=unix:/dev/log,facility=user,tag=mytag,severity=info ceeformat;
+          location /favicon.ico { allow all; access_log off; log_not_found off; }
         '';
-        services.nginx.virtualHosts."0.my.test" = {
-          extraConfig = ''
-            access_log syslog:server=unix:/dev/log,facility=user,tag=mytag,severity=info ceeformat;
-            location /favicon.ico { allow all; access_log off; log_not_found off; }
-          '';
-        };
       };
+      services.nginx.virtualHosts.localhost = {
+        root = pkgs.runCommand "testdir" {} ''
+          mkdir "$out"
+          echo hello world > "$out/index.html"
+        '';
+      };
+    };
+  in {
+    webserver = commonConfig;
+
+    newwebserver = { pkgs, lib, ... }: {
+      imports = [ commonConfig ];
+      services.nginx.virtualHosts.localhost = {
+        root = lib.mkForce (pkgs.runCommand "testdir2" {} ''
+          mkdir "$out"
+          echo hello world > "$out/index.html"
+        '');
+      };
+    };
   };
 
-  testScript = ''
-    startAll;
+  testScript = { nodes, ... }: let
+    newServerSystem = nodes.newwebserver.config.system.build.toplevel;
+    switch = "${newServerSystem}/bin/switch-to-configuration test";
+  in ''
+    my $url = 'http://localhost/index.html';
+
+    sub checkEtag {
+      my $etag = $webserver->succeed(
+        'curl -v '.$url.' 2>&1 | sed -n -e "s/^< [Ee][Tt][Aa][Gg]: *//p"'
+      );
+      $etag =~ s/\r?\n$//;
+      my $httpCode = $webserver->succeed(
+        'curl -w "%{http_code}" -X HEAD -H \'If-None-Match: '.$etag.'\' '.$url
+      );
+      chomp $httpCode;
+      die "HTTP code is not 304" unless $httpCode == 304;
+      return $etag;
+    }
 
     $webserver->waitForUnit("nginx");
     $webserver->waitForOpenPort("80");
+
+    subtest "check ETag if serving Nix store paths", sub {
+      my $oldEtag = checkEtag;
+      $webserver->succeed('${switch}');
+      my $newEtag = checkEtag;
+      die "Old ETag $oldEtag is the same as $newEtag" if $oldEtag eq $newEtag;
+    };
   '';
 })