Merge branch 'nixpkgs-update' into master

12 files changed, 735 insertions, 69 deletions

diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix
index e1950b91382..f668e69e5df 100644
--- a/nixos/modules/services/continuous-integration/buildbot/master.nix
+++ b/nixos/modules/services/continuous-integration/buildbot/master.nix
@@ -223,6 +223,7 @@ in {
       };
 
       pythonPackages = mkOption {
+        type = types.functionTo (types.listOf types.package);
         default = pythonPackages: with pythonPackages; [ ];
         defaultText = "pythonPackages: with pythonPackages; [ ]";
         description = "Packages to add the to the PYTHONPATH of the buildbot process.";
@@ -282,5 +283,5 @@ in {
     '')
   ];
 
-  meta.maintainers = with lib.maintainers; [ nand0p mic92 ];
+  meta.maintainers = with lib.maintainers; [ mic92 lopsided98 ];
 }
diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix
index 7b8a35f54bf..708b3e1cc18 100644
--- a/nixos/modules/services/continuous-integration/buildbot/worker.nix
+++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix
@@ -191,6 +191,6 @@ in {
     };
   };
 
-  meta.maintainers = with lib.maintainers; [ nand0p ];
+  meta.maintainers = with lib.maintainers; [ ];
 
 }
diff --git a/nixos/modules/services/continuous-integration/buildkite-agents.nix b/nixos/modules/services/continuous-integration/buildkite-agents.nix
index b0045409ae6..b8982d757db 100644
--- a/nixos/modules/services/continuous-integration/buildkite-agents.nix
+++ b/nixos/modules/services/continuous-integration/buildkite-agents.nix
@@ -76,7 +76,7 @@ let
       };
 
       tags = mkOption {
-        type = types.attrsOf types.str;
+        type = types.attrsOf (types.either types.str (types.listOf types.str));
         default = {};
         example = { queue = "default"; docker = "true"; ruby2 ="true"; };
         description = ''
@@ -230,18 +230,21 @@ in
         ##     don't end up in the Nix store.
         preStart = let
           sshDir = "${cfg.dataDir}/.ssh";
-          tagStr = lib.concatStringsSep "," (lib.mapAttrsToList (name: value: "${name}=${value}") cfg.tags);
+          tagStr = name: value:
+            if lib.isList value
+            then lib.concatStringsSep "," (builtins.map (v: "${name}=${v}") value)
+            else "${name}=${value}";
+          tagsStr = lib.concatStringsSep "," (lib.mapAttrsToList tagStr cfg.tags);
         in
           optionalString (cfg.privateSshKeyPath != null) ''
             mkdir -m 0700 -p "${sshDir}"
-            cp -f "${toString cfg.privateSshKeyPath}" "${sshDir}/id_rsa"
-            chmod 600 "${sshDir}"/id_rsa
+            install -m600 "${toString cfg.privateSshKeyPath}" "${sshDir}/id_rsa"
           '' + ''
             cat > "${cfg.dataDir}/buildkite-agent.cfg" <<EOF
             token="$(cat ${toString cfg.tokenPath})"
             name="${cfg.name}"
             shell="${cfg.shell}"
-            tags="${tagStr}"
+            tags="${tagsStr}"
             build-path="${cfg.dataDir}/builds"
             hooks-path="${cfg.hooksPath}"
             ${cfg.extraConfig}
diff --git a/nixos/modules/services/continuous-integration/github-runner.nix b/nixos/modules/services/continuous-integration/github-runner.nix
new file mode 100644
index 00000000000..9627b723f8f
--- /dev/null
+++ b/nixos/modules/services/continuous-integration/github-runner.nix
@@ -0,0 +1,299 @@
+{ config, pkgs, lib, ... }:
+with lib;
+let
+  cfg = config.services.github-runner;
+  svcName = "github-runner";
+  systemdDir = "${svcName}/${cfg.name}";
+  # %t: Runtime directory root (usually /run); see systemd.unit(5)
+  runtimeDir = "%t/${systemdDir}";
+  # %S: State directory root (usually /var/lib); see systemd.unit(5)
+  stateDir = "%S/${systemdDir}";
+  # %L: Log directory root (usually /var/log); see systemd.unit(5)
+  logsDir = "%L/${systemdDir}";
+in
+{
+  options.services.github-runner = {
+    enable = mkOption {
+      default = false;
+      example = true;
+      description = ''
+        Whether to enable GitHub Actions runner.
+
+        Note: GitHub recommends using self-hosted runners with private repositories only. Learn more here:
+        <link xlink:href="https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners"
+        >About self-hosted runners</link>.
+      '';
+      type = lib.types.bool;
+    };
+
+    url = mkOption {
+      type = types.str;
+      description = ''
+        Repository to add the runner to.
+
+        Changing this option triggers a new runner registration.
+      '';
+      example = "https://github.com/nixos/nixpkgs";
+    };
+
+    tokenFile = mkOption {
+      type = types.path;
+      description = ''
+        The full path to a file which contains the runner registration token.
+        The file should contain exactly one line with the token without any newline.
+        The token can be used to re-register a runner of the same name but is time-limited.
+
+        Changing this option or the file's content triggers a new runner registration.
+      '';
+      example = "/run/secrets/github-runner/nixos.token";
+    };
+
+    name = mkOption {
+      # Same pattern as for `networking.hostName`
+      type = types.strMatching "^$|^[[:alnum:]]([[:alnum:]_-]{0,61}[[:alnum:]])?$";
+      description = ''
+        Name of the runner to configure. Defaults to the hostname.
+
+        Changing this option triggers a new runner registration.
+      '';
+      example = "nixos";
+      default = config.networking.hostName;
+    };
+
+    runnerGroup = mkOption {
+      type = types.nullOr types.str;
+      description = ''
+        Name of the runner group to add this runner to (defaults to the default runner group).
+
+        Changing this option triggers a new runner registration.
+      '';
+      default = null;
+    };
+
+    extraLabels = mkOption {
+      type = types.listOf types.str;
+      description = ''
+        Extra labels in addition to the default (<literal>["self-hosted", "Linux", "X64"]</literal>).
+
+        Changing this option triggers a new runner registration.
+      '';
+      example = literalExample ''[ "nixos" ]'';
+      default = [ ];
+    };
+
+    replace = mkOption {
+      type = types.bool;
+      description = ''
+        Replace any existing runner with the same name.
+
+        Without this flag, registering a new runner with the same name fails.
+      '';
+      default = false;
+    };
+
+    extraPackages = mkOption {
+      type = types.listOf types.package;
+      description = ''
+        Extra packages to add to <literal>PATH</literal> of the service to make them available to workflows.
+      '';
+      default = [ ];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    warnings = optionals (isStorePath cfg.tokenFile) [
+      ''
+        `services.github-runner.tokenFile` points to the Nix store and, therefore, is world-readable.
+        Consider using a path outside of the Nix store to keep the token private.
+      ''
+    ];
+
+    systemd.services.${svcName} = {
+      description = "GitHub Actions runner";
+
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [ "network.target" "network-online.target" ];
+
+      environment = {
+        HOME = runtimeDir;
+        RUNNER_ROOT = runtimeDir;
+      };
+
+      path = (with pkgs; [
+        bash
+        coreutils
+        git
+        gnutar
+        gzip
+      ]) ++ [
+        config.nix.package
+      ] ++ cfg.extraPackages;
+
+      serviceConfig = rec {
+        ExecStart = "${pkgs.github-runner}/bin/runsvc.sh";
+
+        # Does the following, sequentially:
+        # - Copy the current and the previous `tokenFile` to the $RUNTIME_DIRECTORY
+        #   and make it accessible to the service user to allow for a content
+        #   comparison.
+        # - If the module configuration or the token has changed, clear the state directory.
+        # - Configure the runner.
+        # - Copy the configured `tokenFile` to the $STATE_DIRECTORY and make it
+        #   inaccessible to the service user.
+        # - Set up the directory structure by creating the necessary symlinks.
+        ExecStartPre =
+          let
+            # Wrapper script which expects the full path of the state, runtime and logs
+            # directory as arguments. Overrides the respective systemd variables to provide
+            # unambiguous directory names. This becomes relevant, for example, if the
+            # caller overrides any of the StateDirectory=, RuntimeDirectory= or LogDirectory=
+            # to contain more than one directory. This causes systemd to set the respective
+            # environment variables with the path of all of the given directories, separated
+            # by a colon.
+            writeScript = name: lines: pkgs.writeShellScript "${svcName}-${name}.sh" ''
+              set -euo pipefail
+
+              STATE_DIRECTORY="$1"
+              RUNTIME_DIRECTORY="$2"
+              LOGS_DIRECTORY="$3"
+
+              ${lines}
+            '';
+            currentConfigPath = "$STATE_DIRECTORY/.nixos-current-config.json";
+            runnerRegistrationConfig = getAttrs [ "name" "tokenFile" "url" "runnerGroup" "extraLabels" ] cfg;
+            newConfigPath = builtins.toFile "${svcName}-config.json" (builtins.toJSON runnerRegistrationConfig);
+            currentConfigTokenFilename = ".current-token";
+            newConfigTokenFilename = ".new-token";
+            runnerCredFiles = [
+              ".credentials"
+              ".credentials_rsaparams"
+              ".runner"
+            ];
+            ownConfigTokens = writeScript "own-config-tokens" ''
+              # Copy current and new token file to runtime dir and make it accessible to the service user
+              cp ${escapeShellArg cfg.tokenFile} "$RUNTIME_DIRECTORY/${newConfigTokenFilename}"
+              chmod 600 "$RUNTIME_DIRECTORY/${newConfigTokenFilename}"
+              chown "$USER" "$RUNTIME_DIRECTORY/${newConfigTokenFilename}"
+
+              if [[ -e "$STATE_DIRECTORY/${currentConfigTokenFilename}" ]]; then
+                cp "$STATE_DIRECTORY/${currentConfigTokenFilename}" "$RUNTIME_DIRECTORY/${currentConfigTokenFilename}"
+                chmod 600 "$RUNTIME_DIRECTORY/${currentConfigTokenFilename}"
+                chown "$USER" "$RUNTIME_DIRECTORY/${currentConfigTokenFilename}"
+              fi
+            '';
+            disownConfigTokens = writeScript "disown-config-tokens" ''
+              # Make the token inaccessible to the runner service user
+              chmod 600 "$STATE_DIRECTORY/${currentConfigTokenFilename}"
+              chown root:root "$STATE_DIRECTORY/${currentConfigTokenFilename}"
+            '';
+            unconfigureRunner = writeScript "unconfigure" ''
+              differs=
+              # Set `differs = 1` if current and new runner config differ or if `currentConfigPath` does not exist
+              ${pkgs.diffutils}/bin/diff -q '${newConfigPath}' "${currentConfigPath}" >/dev/null 2>&1 || differs=1
+              # Also trigger a registration if the token content changed
+              ${pkgs.diffutils}/bin/diff -q \
+                "$RUNTIME_DIRECTORY"/{${currentConfigTokenFilename},${newConfigTokenFilename}} \
+                >/dev/null 2>&1 || differs=1
+
+              if [[ -n "$differs" ]]; then
+                echo "Config has changed, removing old runner state."
+                echo "The old runner will still appear in the GitHub Actions UI." \
+                  "You have to remove it manually."
+                find "$STATE_DIRECTORY/" -mindepth 1 -delete
+              fi
+            '';
+            configureRunner = writeScript "configure" ''
+              empty=$(ls -A "$STATE_DIRECTORY")
+              if [[ -z "$empty" ]]; then
+                echo "Configuring GitHub Actions Runner"
+                token=$(< "$RUNTIME_DIRECTORY"/${newConfigTokenFilename})
+                RUNNER_ROOT="$STATE_DIRECTORY" ${pkgs.github-runner}/bin/config.sh \
+                  --unattended \
+                  --work "$RUNTIME_DIRECTORY" \
+                  --url ${escapeShellArg cfg.url} \
+                  --token "$token" \
+                  --labels ${escapeShellArg (concatStringsSep "," cfg.extraLabels)} \
+                  --name ${escapeShellArg cfg.name} \
+                  ${optionalString cfg.replace "--replace"} \
+                  ${optionalString (cfg.runnerGroup != null) "--runnergroup ${escapeShellArg cfg.runnerGroup}"}
+
+                # Move the automatically created _diag dir to the logs dir
+                mkdir -p  "$STATE_DIRECTORY/_diag"
+                cp    -r  "$STATE_DIRECTORY/_diag/." "$LOGS_DIRECTORY/"
+                rm    -rf "$STATE_DIRECTORY/_diag/"
+
+                # Cleanup token from config
+                rm -f "$RUNTIME_DIRECTORY"/${currentConfigTokenFilename}
+                mv    "$RUNTIME_DIRECTORY"/${newConfigTokenFilename} "$STATE_DIRECTORY/${currentConfigTokenFilename}"
+
+                # Symlink to new config
+                ln -s '${newConfigPath}' "${currentConfigPath}"
+              fi
+            '';
+            setupRuntimeDir = writeScript "setup-runtime-dirs" ''
+              # Link _diag dir
+              ln -s "$LOGS_DIRECTORY" "$RUNTIME_DIRECTORY/_diag"
+
+              # Link the runner credentials to the runtime dir
+              ln -s "$STATE_DIRECTORY"/{${lib.concatStringsSep "," runnerCredFiles}} "$RUNTIME_DIRECTORY/"
+            '';
+          in
+          map (x: "${x} ${escapeShellArgs [ stateDir runtimeDir logsDir ]}") [
+            "+${ownConfigTokens}" # runs as root
+            unconfigureRunner
+            configureRunner
+            "+${disownConfigTokens}" # runs as root
+            setupRuntimeDir
+          ];
+
+        # Contains _diag
+        LogsDirectory = [ systemdDir ];
+        # Default RUNNER_ROOT which contains ephemeral Runner data
+        RuntimeDirectory = [ systemdDir ];
+        # Home of persistent runner data, e.g., credentials
+        StateDirectory = [ systemdDir ];
+        StateDirectoryMode = "0700";
+        WorkingDirectory = runtimeDir;
+
+        # By default, use a dynamically allocated user
+        DynamicUser = true;
+
+        KillMode = "process";
+        KillSignal = "SIGTERM";
+
+        # Hardening (may overlap with DynamicUser=)
+        # The following options are only for optimizing:
+        # systemd-analyze security github-runner
+        AmbientCapabilities = "";
+        CapabilityBoundingSet = "";
+        # ProtectClock= adds DeviceAllow=char-rtc r
+        DeviceAllow = "";
+        LockPersonality = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectSystem = "strict";
+        RemoveIPC = true;
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        UMask = "0066";
+
+        # Needs network access
+        PrivateNetwork = false;
+        # Cannot be true due to Node
+        MemoryDenyWriteExecute = false;
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/continuous-integration/gitlab-runner.nix b/nixos/modules/services/continuous-integration/gitlab-runner.nix
index 431555309cc..2c6d9530a6b 100644
--- a/nixos/modules/services/continuous-integration/gitlab-runner.nix
+++ b/nixos/modules/services/continuous-integration/gitlab-runner.nix
@@ -66,10 +66,10 @@ let
             ++ optional service.debugTraceDisabled
             "--debug-trace-disabled"
             ++ map (e: "--env ${escapeShellArg e}") (mapAttrsToList (name: value: "${name}=${value}") service.environmentVariables)
-            ++ optionals (service.executor == "docker") (
+            ++ optionals (hasPrefix "docker" service.executor) (
               assert (
                 assertMsg (service.dockerImage != null)
-                  "dockerImage option is required for docker executor (${name})");
+                  "dockerImage option is required for ${service.executor} executor (${name})");
               [ "--docker-image ${service.dockerImage}" ]
               ++ optional service.dockerDisableCache
               "--docker-disable-cache"
@@ -541,7 +541,7 @@ in
         jq
         moreutils
         remarshal
-        utillinux
+        util-linux
         cfg.package
       ] ++ cfg.extraPackages;
       reloadIfChanged = true;
diff --git a/nixos/modules/services/continuous-integration/gocd-agent/default.nix b/nixos/modules/services/continuous-integration/gocd-agent/default.nix
index 2e9e1c94857..8cae08bf1fa 100644
--- a/nixos/modules/services/continuous-integration/gocd-agent/default.nix
+++ b/nixos/modules/services/continuous-integration/gocd-agent/default.nix
@@ -90,6 +90,7 @@ in {
       };
 
       startupOptions = mkOption {
+        type = types.listOf types.str;
         default = [
           "-Xms${cfg.initialJavaHeapSize}"
           "-Xmx${cfg.maxJavaHeapMemory}"
@@ -105,6 +106,7 @@ in {
 
       extraOptions = mkOption {
         default = [ ];
+        type = types.listOf types.str;
         example = [
           "-X debug"
           "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5006"
diff --git a/nixos/modules/services/continuous-integration/gocd-server/default.nix b/nixos/modules/services/continuous-integration/gocd-server/default.nix
index 4fa41ac49ed..4c829664a0a 100644
--- a/nixos/modules/services/continuous-integration/gocd-server/default.nix
+++ b/nixos/modules/services/continuous-integration/gocd-server/default.nix
@@ -27,6 +27,7 @@ in {
 
       extraGroups = mkOption {
         default = [ ];
+        type = types.listOf types.str;
         example = [ "wheel" "docker" ];
         description = ''
           List of extra groups that the "gocd-server" user should be a part of.
@@ -92,6 +93,7 @@ in {
       };
 
       startupOptions = mkOption {
+        type = types.listOf types.str;
         default = [
           "-Xms${cfg.initialJavaHeapSize}"
           "-Xmx${cfg.maxJavaHeapMemory}"
@@ -113,6 +115,7 @@ in {
 
       extraOptions = mkOption {
         default = [ ];
+        type = types.listOf types.str;
         example = [
           "-X debug"
           "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"
diff --git a/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix b/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix
new file mode 100644
index 00000000000..70d85a97f3b
--- /dev/null
+++ b/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix
@@ -0,0 +1,210 @@
+/*
+
+This file is for options that NixOS and nix-darwin have in common.
+
+Platform-specific code is in the respective default.nix files.
+
+ */
+
+{ config, lib, options, pkgs, ... }:
+let
+  inherit (lib)
+    filterAttrs
+    literalExample
+    mkIf
+    mkOption
+    mkRemovedOptionModule
+    mkRenamedOptionModule
+    types
+    ;
+
+  cfg =
+    config.services.hercules-ci-agent;
+
+  format = pkgs.formats.toml { };
+
+  settingsModule = { config, ... }: {
+    freeformType = format.type;
+    options = {
+      baseDirectory = mkOption {
+        type = types.path;
+        default = "/var/lib/hercules-ci-agent";
+        description = ''
+          State directory (secrets, work directory, etc) for agent
+        '';
+      };
+      concurrentTasks = mkOption {
+        description = ''
+          Number of tasks to perform simultaneously.
+
+          A task is a single derivation build, an evaluation or an effect run.
+          At minimum, you need 2 concurrent tasks for <literal>x86_64-linux</literal>
+          in your cluster, to allow for import from derivation.
+
+          <literal>concurrentTasks</literal> can be around the CPU core count or lower if memory is
+          the bottleneck.
+
+          The optimal value depends on the resource consumption characteristics of your workload,
+          including memory usage and in-task parallelism. This is typically determined empirically.
+
+          When scaling, it is generally better to have a double-size machine than two machines,
+          because each split of resources causes inefficiencies; particularly with regards
+          to build latency because of extra downloads.
+        '';
+        type = types.either types.ints.positive (types.enum [ "auto" ]);
+        default = "auto";
+      };
+      workDirectory = mkOption {
+        description = ''
+          The directory in which temporary subdirectories are created for task state. This includes sources for Nix evaluation.
+        '';
+        type = types.path;
+        default = config.baseDirectory + "/work";
+        defaultText = literalExample ''baseDirectory + "/work"'';
+      };
+      staticSecretsDirectory = mkOption {
+        description = ''
+          This is the default directory to look for statically configured secrets like <literal>cluster-join-token.key</literal>.
+        '';
+        type = types.path;
+        default = config.baseDirectory + "/secrets";
+        defaultText = literalExample ''baseDirectory + "/secrets"'';
+      };
+      clusterJoinTokenPath = mkOption {
+        description = ''
+          Location of the cluster-join-token.key file.
+        '';
+        type = types.path;
+        default = config.staticSecretsDirectory + "/cluster-join-token.key";
+        defaultText = literalExample ''staticSecretsDirectory + "/cluster-join-token.key"'';
+        # internal: It's a bit too detailed to show by default in the docs,
+        # but useful to define explicitly to allow reuse by other modules.
+        internal = true;
+      };
+      binaryCachesPath = mkOption {
+        description = ''
+          Location of the binary-caches.json file.
+        '';
+        type = types.path;
+        default = config.staticSecretsDirectory + "/binary-caches.json";
+        defaultText = literalExample ''staticSecretsDirectory + "/binary-caches.json"'';
+        # internal: It's a bit too detailed to show by default in the docs,
+        # but useful to define explicitly to allow reuse by other modules.
+        internal = true;
+      };
+    };
+  };
+
+  # TODO (roberth, >=2022) remove
+  checkNix =
+    if !cfg.checkNix
+    then ""
+    else if lib.versionAtLeast config.nix.package.version "2.3.10"
+    then ""
+    else
+      pkgs.stdenv.mkDerivation {
+        name = "hercules-ci-check-system-nix-src";
+        inherit (config.nix.package) src patches;
+        dontConfigure = true;
+        buildPhase = ''
+          echo "Checking in-memory pathInfoCache expiry"
+          if ! grep 'PathInfoCacheValue' src/libstore/store-api.hh >/dev/null; then
+            cat 1>&2 <<EOF
+
+            You are deploying Hercules CI Agent on a system with an incompatible
+            nix-daemon. Please make sure nix.package is set to a Nix version of at
+            least 2.3.10 or a master version more recent than Mar 12, 2020.
+          EOF
+            exit 1
+          fi
+        '';
+        installPhase = "touch $out";
+      };
+
+in
+{
+  imports = [
+    (mkRenamedOptionModule [ "services" "hercules-ci-agent" "extraOptions" ] [ "services" "hercules-ci-agent" "settings" ])
+    (mkRenamedOptionModule [ "services" "hercules-ci-agent" "baseDirectory" ] [ "services" "hercules-ci-agent" "settings" "baseDirectory" ])
+    (mkRenamedOptionModule [ "services" "hercules-ci-agent" "concurrentTasks" ] [ "services" "hercules-ci-agent" "settings" "concurrentTasks" ])
+    (mkRemovedOptionModule [ "services" "hercules-ci-agent" "patchNix" ] "Nix versions packaged in this version of Nixpkgs don't need a patched nix-daemon to work correctly in Hercules CI Agent clusters.")
+  ];
+
+  options.services.hercules-ci-agent = {
+    enable = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Enable to run Hercules CI Agent as a system service.
+
+        <link xlink:href="https://hercules-ci.com">Hercules CI</link> is a
+        continuous integation service that is centered around Nix.
+
+        Support is available at <link xlink:href="mailto:help@hercules-ci.com">help@hercules-ci.com</link>.
+      '';
+    };
+    checkNix = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Whether to make sure that the system's Nix (nix-daemon) is compatible.
+
+        If you set this to false, please keep up with the change log.
+      '';
+    };
+    package = mkOption {
+      description = ''
+        Package containing the bin/hercules-ci-agent executable.
+      '';
+      type = types.package;
+      default = pkgs.hercules-ci-agent;
+      defaultText = literalExample "pkgs.hercules-ci-agent";
+    };
+    settings = mkOption {
+      description = ''
+        These settings are written to the <literal>agent.toml</literal> file.
+
+        Not all settings are listed as options, can be set nonetheless.
+
+        For the exhaustive list of settings, see <link xlink:href="https://docs.hercules-ci.com/hercules-ci/reference/agent-config/"/>.
+      '';
+      type = types.submoduleWith { modules = [ settingsModule ]; };
+    };
+
+    /*
+      Internal and/or computed values.
+
+      These are written as options instead of let binding to allow sharing with
+      default.nix on both NixOS and nix-darwin.
+     */
+    tomlFile = mkOption {
+      type = types.path;
+      internal = true;
+      defaultText = "generated hercules-ci-agent.toml";
+      description = ''
+        The fully assembled config file.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    nix.extraOptions = lib.addContextFrom checkNix ''
+      # A store path that was missing at first may well have finished building,
+      # even shortly after the previous lookup. This *also* applies to the daemon.
+      narinfo-cache-negative-ttl = 0
+    '';
+    services.hercules-ci-agent = {
+      tomlFile =
+        format.generate "hercules-ci-agent.toml" cfg.settings;
+
+      settings.labels = {
+        agent.source =
+          if options.services.hercules-ci-agent.package.highestPrio == (lib.modules.mkOptionDefault { }).priority
+          then "nixpkgs"
+          else lib.mkOptionDefault "override";
+        pkgs.version = pkgs.lib.version;
+        lib.version = lib.version;
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix b/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix
new file mode 100644
index 00000000000..06c174e7d37
--- /dev/null
+++ b/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix
@@ -0,0 +1,101 @@
+/*
+
+This file is for NixOS-specific options and configs.
+
+Code that is shared with nix-darwin goes in common.nix.
+
+ */
+
+{ pkgs, config, lib, ... }:
+let
+  inherit (lib) mkIf mkDefault;
+
+  cfg = config.services.hercules-ci-agent;
+
+  command = "${cfg.package}/bin/hercules-ci-agent --config ${cfg.tomlFile}";
+  testCommand = "${command} --test-configuration";
+
+in
+{
+  imports = [
+    ./common.nix
+    (lib.mkRenamedOptionModule [ "services" "hercules-ci-agent" "user" ] [ "systemd" "services" "hercules-ci-agent" "serviceConfig" "User" ])
+  ];
+
+  config = mkIf cfg.enable {
+    systemd.services.hercules-ci-agent = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      path = [ config.nix.package ];
+      startLimitBurst = 30 * 1000000; # practically infinite
+      serviceConfig = {
+        User = "hercules-ci-agent";
+        ExecStart = command;
+        ExecStartPre = testCommand;
+        Restart = "on-failure";
+        RestartSec = 120;
+      };
+    };
+
+    # Changes in the secrets do not affect the unit in any way that would cause
+    # a restart, which is currently necessary to reload the secrets.
+    systemd.paths.hercules-ci-agent-restart-files = {
+      wantedBy = [ "hercules-ci-agent.service" ];
+      pathConfig = {
+        Unit = "hercules-ci-agent-restarter.service";
+        PathChanged = [ cfg.settings.clusterJoinTokenPath cfg.settings.binaryCachesPath ];
+      };
+    };
+    systemd.services.hercules-ci-agent-restarter = {
+      serviceConfig.Type = "oneshot";
+      script = ''
+        # Wait a bit, with the effect of bundling up file changes into a single
+        # run of this script and hopefully a single restart.
+        sleep 10
+        if systemctl is-active --quiet hercules-ci-agent.service; then
+          if ${testCommand}; then
+            systemctl restart hercules-ci-agent.service
+          else
+            echo 1>&2 "WARNING: Not restarting agent because config is not valid at this time."
+          fi
+        else
+          echo 1>&2 "Not restarting hercules-ci-agent despite config file update, because it is not already active."
+        fi
+      '';
+    };
+
+    # Trusted user allows simplified configuration and better performance
+    # when operating in a cluster.
+    nix.trustedUsers = [ config.systemd.services.hercules-ci-agent.serviceConfig.User ];
+    services.hercules-ci-agent = {
+      settings = {
+        nixUserIsTrusted = true;
+        labels =
+          let
+            mkIfNotNull = x: mkIf (x != null) x;
+          in
+          {
+            nixos.configurationRevision = mkIfNotNull config.system.configurationRevision;
+            nixos.release = config.system.nixos.release;
+            nixos.label = mkIfNotNull config.system.nixos.label;
+            nixos.codeName = config.system.nixos.codeName;
+            nixos.tags = config.system.nixos.tags;
+            nixos.systemName = mkIfNotNull config.system.name;
+          };
+      };
+    };
+
+    users.users.hercules-ci-agent = {
+      home = cfg.settings.baseDirectory;
+      createHome = true;
+      group = "hercules-ci-agent";
+      description = "Hercules CI Agent system user";
+      isSystemUser = true;
+    };
+
+    users.groups.hercules-ci-agent = { };
+  };
+
+  meta.maintainers = [ lib.maintainers.roberth ];
+}
diff --git a/nixos/modules/services/continuous-integration/hydra/default.nix b/nixos/modules/services/continuous-integration/hydra/default.nix
index 502a5898a5d..0103cd723d2 100644
--- a/nixos/modules/services/continuous-integration/hydra/default.nix
+++ b/nixos/modules/services/continuous-integration/hydra/default.nix
@@ -37,8 +37,6 @@ let
 
   haveLocalDB = cfg.dbi == localDB;
 
-  inherit (config.system) stateVersion;
-
   hydra-package =
   let
     makeWrapperArgs = concatStringsSep " " (mapAttrsToList (key: value: "--set \"${key}\" \"${value}\"") hydraEnv);
@@ -91,12 +89,18 @@ in
         example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
         description = ''
           The DBI string for Hydra database connection.
+
+          NOTE: Attempts to set `application_name` will be overridden by
+          `hydra-TYPE` (where TYPE is e.g. `evaluator`, `queue-runner`,
+          etc.) in all hydra services to more easily distinguish where
+          queries are coming from.
         '';
       };
 
       package = mkOption {
         type = types.package;
-        defaultText = "pkgs.hydra";
+        default = pkgs.hydra-unstable;
+        defaultText = "pkgs.hydra-unstable";
         description = "The Hydra package.";
       };
 
@@ -225,34 +229,6 @@ in
 
   config = mkIf cfg.enable {
 
-    warnings = optional (cfg.package.migration or false) ''
-      You're currently deploying an older version of Hydra which is needed to
-      make some required database changes[1]. As soon as this is done, it's recommended
-      to run `hydra-backfill-ids` and set `services.hydra.package` to `pkgs.hydra-unstable`
-      after that.
-
-      [1] https://github.com/NixOS/hydra/pull/711
-    '';
-
-    services.hydra.package = with pkgs;
-      mkDefault (
-        if pkgs ? hydra
-          then throw ''
-            The Hydra package doesn't exist anymore in `nixpkgs`! It probably exists
-            due to an overlay. To upgrade Hydra, you need to take two steps as some
-            bigger changes in the database schema were implemented recently[1]. You first
-            need to deploy `pkgs.hydra-migration`, run `hydra-backfill-ids` on the server
-            and then deploy `pkgs.hydra-unstable`.
-
-            If you want to use `pkgs.hydra` from your overlay, please set `services.hydra.package`
-            explicitly to `pkgs.hydra` and make sure you know what you're doing.
-
-            [1] https://github.com/NixOS/hydra/pull/711
-          ''
-        else if versionOlder stateVersion "20.03" then hydra-migration
-        else hydra-unstable
-      );
-
     users.groups.hydra = {
       gid = config.ids.gids.hydra;
     };
@@ -260,7 +236,7 @@ in
     users.users.hydra =
       { description = "Hydra";
         group = "hydra";
-        createHome = true;
+        # We don't enable `createHome` here because the creation of the home directory is handled by the hydra-init service below.
         home = baseDir;
         useDefaultShell = true;
         uid = config.ids.uids.hydra;
@@ -304,6 +280,8 @@ in
       keep-outputs = true
       keep-derivations = true
 
+
+    '' + optionalString (versionOlder (getVersion config.nix.package.out) "2.4pre") ''
       # The default (`true') slows Nix down a lot since the build farm
       # has so many GC roots.
       gc-check-reachability = false
@@ -313,7 +291,9 @@ in
       { wantedBy = [ "multi-user.target" ];
         requires = optional haveLocalDB "postgresql.service";
         after = optional haveLocalDB "postgresql.service";
-        environment = env;
+        environment = env // {
+          HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init";
+        };
         preStart = ''
           mkdir -p ${baseDir}
           chown hydra.hydra ${baseDir}
@@ -368,7 +348,9 @@ in
       { wantedBy = [ "multi-user.target" ];
         requires = [ "hydra-init.service" ];
         after = [ "hydra-init.service" ];
-        environment = serverEnv;
+        environment = serverEnv // {
+          HYDRA_DBI = "${serverEnv.HYDRA_DBI};application_name=hydra-server";
+        };
         restartTriggers = [ hydraConf ];
         serviceConfig =
           { ExecStart =
@@ -390,6 +372,7 @@ in
         environment = env // {
           PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
           IN_SYSTEMD = "1"; # to get log severity levels
+          HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-queue-runner";
         };
         serviceConfig =
           { ExecStart = "@${hydra-package}/bin/hydra-queue-runner hydra-queue-runner -v";
@@ -409,7 +392,9 @@ in
         after = [ "hydra-init.service" "network.target" ];
         path = with pkgs; [ hydra-package nettools jq ];
         restartTriggers = [ hydraConf ];
-        environment = env;
+        environment = env // {
+          HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
+        };
         serviceConfig =
           { ExecStart = "@${hydra-package}/bin/hydra-evaluator hydra-evaluator";
             User = "hydra";
@@ -421,7 +406,9 @@ in
     systemd.services.hydra-update-gc-roots =
       { requires = [ "hydra-init.service" ];
         after = [ "hydra-init.service" ];
-        environment = env;
+        environment = env // {
+          HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-update-gc-roots";
+        };
         serviceConfig =
           { ExecStart = "@${hydra-package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
             User = "hydra";
@@ -432,7 +419,9 @@ in
     systemd.services.hydra-send-stats =
       { wantedBy = [ "multi-user.target" ];
         after = [ "hydra-init.service" ];
-        environment = env;
+        environment = env // {
+          HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-send-stats";
+        };
         serviceConfig =
           { ExecStart = "@${hydra-package}/bin/hydra-send-stats hydra-send-stats";
             User = "hydra";
@@ -446,6 +435,7 @@ in
         restartTriggers = [ hydraConf ];
         environment = env // {
           PGPASSFILE = "${baseDir}/pgpass-queue-runner";
+          HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify";
         };
         serviceConfig =
           { ExecStart = "@${hydra-package}/bin/hydra-notify hydra-notify";
diff --git a/nixos/modules/services/continuous-integration/jenkins/default.nix b/nixos/modules/services/continuous-integration/jenkins/default.nix
index 1477c471f8a..889688a2685 100644
--- a/nixos/modules/services/continuous-integration/jenkins/default.nix
+++ b/nixos/modules/services/continuous-integration/jenkins/default.nix
@@ -2,6 +2,7 @@
 with lib;
 let
   cfg = config.services.jenkins;
+  jenkinsUrl = "http://${cfg.listenAddress}:${toString cfg.port}${cfg.prefix}";
 in {
   options = {
     services.jenkins = {
@@ -86,8 +87,8 @@ in {
       };
 
       packages = mkOption {
-        default = [ pkgs.stdenv pkgs.git pkgs.jdk config.programs.ssh.package pkgs.nix ];
-        defaultText = "[ pkgs.stdenv pkgs.git pkgs.jdk config.programs.ssh.package pkgs.nix ]";
+        default = [ pkgs.stdenv pkgs.git pkgs.jdk11 config.programs.ssh.package pkgs.nix ];
+        defaultText = "[ pkgs.stdenv pkgs.git pkgs.jdk11 config.programs.ssh.package pkgs.nix ]";
         type = types.listOf types.package;
         description = ''
           Packages to add to PATH for the jenkins process.
@@ -141,14 +142,34 @@ in {
           Additional command line arguments to pass to the Java run time (as opposed to Jenkins).
         '';
       };
+
+      withCLI = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to make the CLI available.
+
+          More info about the CLI available at
+          <link xlink:href="https://www.jenkins.io/doc/book/managing/cli">
+          https://www.jenkins.io/doc/book/managing/cli</link> .
+        '';
+      };
     };
   };
 
   config = mkIf cfg.enable {
-    # server references the dejavu fonts
-    environment.systemPackages = [
-      pkgs.dejavu_fonts
-    ];
+    environment = {
+      # server references the dejavu fonts
+      systemPackages = [
+        pkgs.dejavu_fonts
+      ] ++ optional cfg.withCLI cfg.package;
+
+      variables = {}
+        // optionalAttrs cfg.withCLI {
+          # Make it more convenient to use the `jenkins-cli`.
+          JENKINS_URL = jenkinsUrl;
+        };
+    };
 
     users.groups = optionalAttrs (cfg.group == "jenkins") {
       jenkins.gid = config.ids.gids.jenkins;
@@ -207,7 +228,7 @@ in {
 
       # For reference: https://wiki.jenkins.io/display/JENKINS/JenkinsLinuxStartupScript
       script = ''
-        ${pkgs.jdk}/bin/java ${concatStringsSep " " cfg.extraJavaOptions} -jar ${cfg.package}/webapps/jenkins.war --httpListenAddress=${cfg.listenAddress} \
+        ${pkgs.jdk11}/bin/java ${concatStringsSep " " cfg.extraJavaOptions} -jar ${cfg.package}/webapps/jenkins.war --httpListenAddress=${cfg.listenAddress} \
                                                   --httpPort=${toString cfg.port} \
                                                   --prefix=${cfg.prefix} \
                                                   -Djava.awt.headless=true \
@@ -215,7 +236,7 @@ in {
       '';
 
       postStart = ''
-        until [[ $(${pkgs.curl.bin}/bin/curl -L -s --head -w '\n%{http_code}' http://${cfg.listenAddress}:${toString cfg.port}${cfg.prefix} | tail -n1) =~ ^(200|403)$ ]]; do
+        until [[ $(${pkgs.curl.bin}/bin/curl -L -s --head -w '\n%{http_code}' ${jenkinsUrl} | tail -n1) =~ ^(200|403)$ ]]; do
           sleep 1
         done
       '';
diff --git a/nixos/modules/services/continuous-integration/jenkins/job-builder.nix b/nixos/modules/services/continuous-integration/jenkins/job-builder.nix
index 5d1bfe4ec40..536d394b3fd 100644
--- a/nixos/modules/services/continuous-integration/jenkins/job-builder.nix
+++ b/nixos/modules/services/continuous-integration/jenkins/job-builder.nix
@@ -165,6 +165,42 @@ in {
           '';
         in
           ''
+            joinByString()
+            {
+                local separator="$1"
+                shift
+                local first="$1"
+                shift
+                printf "%s" "$first" "''${@/#/$separator}"
+            }
+
+            # Map a relative directory path in the output from
+            # jenkins-job-builder (jobname) to the layout expected by jenkins:
+            # each directory level gets prepended "jobs/".
+            getJenkinsJobDir()
+            {
+                IFS='/' read -ra input_dirs <<< "$1"
+                printf "jobs/"
+                joinByString "/jobs/" "''${input_dirs[@]}"
+            }
+
+            # The inverse of getJenkinsJobDir (remove the "jobs/" prefixes)
+            getJobname()
+            {
+                IFS='/' read -ra input_dirs <<< "$1"
+                local i=0
+                local nelem=''${#input_dirs[@]}
+                for e in "''${input_dirs[@]}"; do
+                    if [ $((i % 2)) -eq 1 ]; then
+                        printf "$e"
+                        if [ $i -lt $(( nelem - 1 )) ]; then
+                            printf "/"
+                        fi
+                    fi
+                    i=$((i + 1))
+                done
+            }
+
             rm -rf ${jobBuilderOutputDir}
             cur_decl_jobs=/run/jenkins-job-builder/declarative-jobs
             rm -f "$cur_decl_jobs"
@@ -172,27 +208,27 @@ in {
             # Create / update jobs
             mkdir -p ${jobBuilderOutputDir}
             for inputFile in ${yamlJobsFile} ${concatStringsSep " " jsonJobsFiles}; do
-                HOME="${jenkinsCfg.home}" "${pkgs.jenkins-job-builder}/bin/jenkins-jobs" --ignore-cache test -o "${jobBuilderOutputDir}" "$inputFile"
+                HOME="${jenkinsCfg.home}" "${pkgs.jenkins-job-builder}/bin/jenkins-jobs" --ignore-cache test --config-xml -o "${jobBuilderOutputDir}" "$inputFile"
             done
 
-            for file in "${jobBuilderOutputDir}/"*; do
-                test -f "$file" || continue
-                jobname="$(basename $file)"
-                jobdir="${jenkinsCfg.home}/jobs/$jobname"
+            find "${jobBuilderOutputDir}" -type f -name config.xml | while read -r f; do echo "$(dirname "$f")"; done | sort | while read -r dir; do
+                jobname="$(realpath --relative-to="${jobBuilderOutputDir}" "$dir")"
+                jenkinsjobname=$(getJenkinsJobDir "$jobname")
+                jenkinsjobdir="${jenkinsCfg.home}/$jenkinsjobname"
                 echo "Creating / updating job \"$jobname\""
-                mkdir -p "$jobdir"
-                touch "$jobdir/${ownerStamp}"
-                cp "$file" "$jobdir/config.xml"
-                echo "$jobname" >> "$cur_decl_jobs"
+                mkdir -p "$jenkinsjobdir"
+                touch "$jenkinsjobdir/${ownerStamp}"
+                cp "$dir"/config.xml "$jenkinsjobdir/config.xml"
+                echo "$jenkinsjobname" >> "$cur_decl_jobs"
             done
 
             # Remove stale jobs
-            for file in "${jenkinsCfg.home}"/jobs/*/${ownerStamp}; do
-                test -f "$file" || continue
-                jobdir="$(dirname $file)"
-                jobname="$(basename "$jobdir")"
-                grep --quiet --line-regexp "$jobname" "$cur_decl_jobs" 2>/dev/null && continue
+            find "${jenkinsCfg.home}" -type f -name "${ownerStamp}" | while read -r f; do echo "$(dirname "$f")"; done | sort --reverse | while read -r dir; do
+                jenkinsjobname="$(realpath --relative-to="${jenkinsCfg.home}" "$dir")"
+                grep --quiet --line-regexp "$jenkinsjobname" "$cur_decl_jobs" 2>/dev/null && continue
+                jobname=$(getJobname "$jenkinsjobname")
                 echo "Deleting stale job \"$jobname\""
+                jobdir="${jenkinsCfg.home}/$jenkinsjobname"
                 rm -rf "$jobdir"
             done
           '' + (if cfg.accessUser != "" then reloadScript else "");