Merge master into staging-next

8 files changed, 218 insertions, 8 deletions

diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix
index 0ad0cf1fef5..3f50bb431c5 100644
--- a/nixos/lib/make-disk-image.nix
+++ b/nixos/lib/make-disk-image.nix
@@ -18,9 +18,13 @@
   bootSize ? "256M"
 
 , # The files and directories to be placed in the target file system.
-  # This is a list of attribute sets {source, target} where `source'
-  # is the file system object (regular file or directory) to be
-  # grafted in the file system at path `target'.
+  # This is a list of attribute sets {source, target, mode, user, group} where
+  # `source' is the file system object (regular file or directory) to be
+  # grafted in the file system at path `target', `mode' is a string containing
+  # the permissions that will be set (ex. "755"), `user' and `group' are the
+  # user and group name that will be set as owner of the files.
+  # `mode', `user', and `group' are optional.
+  # When setting one of `user' or `group', the other needs to be set too.
   contents ? []
 
 , # Type of partition table to use; either "legacy", "efi", or "none".
@@ -60,6 +64,11 @@
 assert partitionTableType == "legacy" || partitionTableType == "legacy+gpt" || partitionTableType == "efi" || partitionTableType == "hybrid" || partitionTableType == "none";
 # We use -E offset=X below, which is only supported by e2fsprogs
 assert partitionTableType != "none" -> fsType == "ext4";
+# Either both or none of {user,group} need to be set
+assert lib.all
+         (attrs: ((attrs.user  or null) == null)
+              == ((attrs.group or null) == null))
+         contents;
 
 with lib;
 
@@ -148,6 +157,9 @@ let format' = format; in let
   # !!! should use XML.
   sources = map (x: x.source) contents;
   targets = map (x: x.target) contents;
+  modes   = map (x: x.mode  or "''") contents;
+  users   = map (x: x.user  or "''") contents;
+  groups  = map (x: x.group or "''") contents;
 
   closureInfo = pkgs.closureInfo { rootPaths = [ config.system.build.toplevel channelSources ]; };
 
@@ -174,22 +186,33 @@ let format' = format; in let
     set -f
     sources_=(${concatStringsSep " " sources})
     targets_=(${concatStringsSep " " targets})
+    modes_=(${concatStringsSep " " modes})
     set +f
 
     for ((i = 0; i < ''${#targets_[@]}; i++)); do
       source="''${sources_[$i]}"
       target="''${targets_[$i]}"
+      mode="''${modes_[$i]}"
 
+      if [ -n "$mode" ]; then
+        rsync_chmod_flags="--chmod=$mode"
+      else
+        rsync_chmod_flags=""
+      fi
+      # Unfortunately cptofs only supports modes, not ownership, so we can't use
+      # rsync's --chown option. Instead, we change the ownerships in the
+      # VM script with chown.
+      rsync_flags="-a --no-o --no-g $rsync_chmod_flags"
       if [[ "$source" =~ '*' ]]; then
         # If the source name contains '*', perform globbing.
         mkdir -p $root/$target
         for fn in $source; do
-          rsync -a --no-o --no-g "$fn" $root/$target/
+          rsync $rsync_flags "$fn" $root/$target/
         done
       else
         mkdir -p $root/$(dirname $target)
         if ! [ -e $root/$target ]; then
-          rsync -a --no-o --no-g $source $root/$target
+          rsync $rsync_flags $source $root/$target
         else
           echo "duplicate entry $target -> $source"
           exit 1
@@ -284,6 +307,21 @@ in pkgs.vmTools.runInLinuxVM (
       # The above scripts will generate a random machine-id and we don't want to bake a single ID into all our images
       rm -f $mountPoint/etc/machine-id
 
+      # Set the ownerships of the contents. The modes are set in preVM.
+      # No globbing on targets, so no need to set -f
+      targets_=(${concatStringsSep " " targets})
+      users_=(${concatStringsSep " " users})
+      groups_=(${concatStringsSep " " groups})
+      for ((i = 0; i < ''${#targets_[@]}; i++)); do
+        target="''${targets_[$i]}"
+        user="''${users_[$i]}"
+        group="''${groups_[$i]}"
+        if [ -n "$user$group" ]; then
+          # We have to nixos-enter since we need to use the user and group of the VM
+          nixos-enter --root $mountPoint -- chown -R "$user:$group" "$target"
+        fi
+      done
+
       umount -R /mnt
 
       # Make sure resize2fs works. Note that resize2fs has stricter criteria for resizing than a normal
diff --git a/nixos/modules/services/databases/mongodb.nix b/nixos/modules/services/databases/mongodb.nix
index 4453a182990..4af0b9d44e1 100644
--- a/nixos/modules/services/databases/mongodb.nix
+++ b/nixos/modules/services/databases/mongodb.nix
@@ -176,7 +176,7 @@ in
         postStart = ''
             if test -e "${cfg.dbpath}/.first_startup"; then
               ${optionalString (cfg.initialScript != null) ''
-                ${mongodb}/bin/mongo -u root -p ${cfg.initialRootPassword} admin "${cfg.initialScript}"
+                ${mongodb}/bin/mongo ${optionalString (cfg.enableAuth) "-u root -p ${cfg.initialRootPassword}"} admin "${cfg.initialScript}"
               ''}
               rm -f "${cfg.dbpath}/.first_startup"
             fi
diff --git a/nixos/modules/services/hardware/thermald.nix b/nixos/modules/services/hardware/thermald.nix
index 6be255cebf1..aa936ac09d1 100644
--- a/nixos/modules/services/hardware/thermald.nix
+++ b/nixos/modules/services/hardware/thermald.nix
@@ -23,19 +23,26 @@ in {
         default = null;
         description = "the thermald manual configuration file.";
       };
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.thermald;
+        defaultText = "pkgs.thermald";
+        description = "Which thermald package to use.";
+      };
     };
   };
 
   ###### implementation
   config = mkIf cfg.enable {
-    services.dbus.packages = [ pkgs.thermald ];
+    services.dbus.packages = [ cfg.package ];
 
     systemd.services.thermald = {
       description = "Thermal Daemon Service";
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
         ExecStart = ''
-          ${pkgs.thermald}/sbin/thermald \
+          ${cfg.package}/sbin/thermald \
             --no-daemon \
             ${optionalString cfg.debug "--loglevel=debug"} \
             ${optionalString (cfg.configFile != null) "--config-file ${cfg.configFile}"} \
diff --git a/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixos/modules/services/monitoring/prometheus/exporters.nix
index 1f8c5aeded1..683dc0390c5 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters.nix
@@ -37,6 +37,7 @@ let
     "modemmanager"
     "nextcloud"
     "nginx"
+    "nginxlog"
     "node"
     "openvpn"
     "postfix"
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/nginxlog.nix b/nixos/modules/services/monitoring/prometheus/exporters/nginxlog.nix
new file mode 100644
index 00000000000..8c1f552d58a
--- /dev/null
+++ b/nixos/modules/services/monitoring/prometheus/exporters/nginxlog.nix
@@ -0,0 +1,51 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.nginxlog;
+in {
+  port = 9117;
+  extraOpts = {
+    settings = mkOption {
+      type = types.attrs;
+      default = {};
+      description = ''
+        All settings of nginxlog expressed as an Nix attrset.
+
+        Check the official documentation for the corresponding YAML
+        settings that can all be used here: https://github.com/martin-helmich/prometheus-nginxlog-exporter
+
+        The `listen` object is already generated by `port`, `listenAddress` and `metricsEndpoint` and
+        will be merged with the value of `settings` before writting it as JSON.
+      '';
+    };
+
+    metricsEndpoint = mkOption {
+      type = types.str;
+      default = "/metrics";
+      description = ''
+        Path under which to expose metrics.
+      '';
+    };
+  };
+
+  serviceOpts = let
+    listenConfig = {
+      listen = {
+        port = cfg.port;
+        address = cfg.listenAddress;
+        metrics_endpoint = cfg.metricsEndpoint;
+      };
+    };
+    completeConfig = pkgs.writeText "nginxlog-exporter.yaml" (builtins.toJSON (lib.recursiveUpdate listenConfig cfg.settings));
+  in {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-nginxlog-exporter}/bin/prometheus-nginxlog-exporter -config-file ${completeConfig}
+      '';
+      Restart="always";
+      ProtectSystem="full";
+    };
+  };
+}
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index e2328608703..f878b63e02b 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -280,6 +280,7 @@ in
   openssh = handleTest ./openssh.nix {};
   openstack-image-metadata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).metadata or {};
   openstack-image-userdata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).userdata or {};
+  image-contents = handleTest ./image-contents.nix {};
   orangefs = handleTest ./orangefs.nix {};
   os-prober = handleTestOn ["x86_64-linux"] ./os-prober.nix {};
   osrm-backend = handleTest ./osrm-backend.nix {};
diff --git a/nixos/tests/image-contents.nix b/nixos/tests/image-contents.nix
new file mode 100644
index 00000000000..90908968a7e
--- /dev/null
+++ b/nixos/tests/image-contents.nix
@@ -0,0 +1,51 @@
+# Tests the contents attribute of nixos/lib/make-disk-image.nix
+# including its user, group, and mode attributes.
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
+with pkgs.lib;
+
+with import common/ec2.nix { inherit makeTest pkgs; };
+
+let
+  config = (import ../lib/eval-config.nix {
+    inherit system;
+    modules = [
+      ../modules/testing/test-instrumentation.nix
+      ../modules/profiles/qemu-guest.nix
+      {
+        fileSystems."/".device = "/dev/disk/by-label/nixos";
+        boot.loader.grub.device = "/dev/vda";
+        boot.loader.timeout = 0;
+      }
+    ];
+  }).config;
+  image = (import ../lib/make-disk-image.nix {
+    inherit pkgs config;
+    lib = pkgs.lib;
+    format = "qcow2";
+    contents = [{
+      source = pkgs.writeText "testFile" "contents";
+      target = "/testFile";
+      user = "1234";
+      group = "5678";
+      mode = "755";
+    }];
+  }) + "/nixos.qcow2";
+
+in makeEc2Test {
+  name = "image-contents";
+  inherit image;
+  userData = null;
+  script = ''
+    machine.start()
+    assert "content" in machine.succeed("cat /testFile")
+    fileDetails = machine.succeed("ls -l /testFile")
+    assert "1234" in fileDetails
+    assert "5678" in fileDetails
+    assert "rwxr-xr-x" in fileDetails
+  '';
+}
diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix
index 3eb4341e39c..ffa7f420c09 100644
--- a/nixos/tests/prometheus-exporters.nix
+++ b/nixos/tests/prometheus-exporters.nix
@@ -444,6 +444,67 @@ let
       '';
     };
 
+    nginxlog = {
+      exporterConfig = {
+        enable = true;
+        group = "nginx";
+        settings = {
+          namespaces = [
+            {
+              name = "filelogger";
+              source = {
+                files = [ "/var/log/nginx/filelogger.access.log" ];
+              };
+            }
+            {
+              name = "syslogger";
+              source = {
+                syslog = {
+                  listen_address = "udp://127.0.0.1:10000";
+                  format = "rfc3164";
+                  tags = ["nginx"];
+                };
+              };
+            }
+          ];
+        };
+      };
+      metricProvider = {
+        services.nginx = {
+          enable = true;
+          httpConfig = ''
+            server {
+              listen 80;
+              server_name filelogger.local;
+              access_log /var/log/nginx/filelogger.access.log;
+            }
+            server {
+              listen 81;
+              server_name syslogger.local;
+              access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
+            }
+          '';
+        };
+      };
+      exporterTest = ''
+        wait_for_unit("nginx.service")
+        wait_for_unit("prometheus-nginxlog-exporter.service")
+        wait_for_open_port(9117)
+        wait_for_open_port(80)
+        wait_for_open_port(81)
+        succeed("curl http://localhost")
+        execute("sleep 1")
+        succeed(
+            "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep -q 1"
+        )
+        succeed("curl http://localhost:81")
+        execute("sleep 1")
+        succeed(
+            "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep -q 1"
+        )
+      '';
+    };
+
     node = {
       exporterConfig = {
         enable = true;