Merge staging-next into staging

13 files changed, 164 insertions, 82 deletions

diff --git a/nixos/doc/manual/release-notes/rl-2009.xml b/nixos/doc/manual/release-notes/rl-2009.xml
index c65be35bea5..ae55f77b0cc 100644
--- a/nixos/doc/manual/release-notes/rl-2009.xml
+++ b/nixos/doc/manual/release-notes/rl-2009.xml
@@ -446,6 +446,20 @@ systemd.services.nginx.serviceConfig.ReadWritePaths = [ "/var/www" ];
        recommended to only use lower-case characters.
      </para>
    </listitem>
+   <listitem>
+    <para>
+     The GRUB specific option <option>boot.loader.grub.extraInitrd</option>
+     has been replaced with the generic option
+     <option>boot.initrd.secrets</option>. This option creates a secondary
+     initrd from the specified files, rather than using a manually created
+     initrd file.
+
+     Due to an existing bug with <option>boot.loader.grub.extraInitrd</option>,
+     it is not possible to directly boot an older generation that used that
+     option. It is still possible to rollback to that generation if the required
+     initrd file has not been deleted.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 
diff --git a/nixos/modules/config/system-path.nix b/nixos/modules/config/system-path.nix
index 4100ec89701..cba357171d7 100644
--- a/nixos/modules/config/system-path.nix
+++ b/nixos/modules/config/system-path.nix
@@ -8,8 +8,7 @@ with lib;
 let
 
   requiredPackages = map (pkg: setPrio ((pkg.meta.priority or 5) + 3) pkg)
-    [ config.nix.package
-      pkgs.acl
+    [ pkgs.acl
       pkgs.attr
       pkgs.bashInteractive # bash with ncurses support
       pkgs.bzip2
@@ -33,7 +32,6 @@ let
       pkgs.nano
       pkgs.ncurses
       pkgs.netcat
-      pkgs.nix-info
       config.programs.ssh.package
       pkgs.perl
       pkgs.procps
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index e8c4a4c14cf..978d33e7585 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -806,7 +806,6 @@
   ./services/security/torsocks.nix
   ./services/security/usbguard.nix
   ./services/security/vault.nix
-  ./services/system/cgmanager.nix
   ./services/system/cloud-init.nix
   ./services/system/dbus.nix
   ./services/system/earlyoom.nix
diff --git a/nixos/modules/programs/bash/bash.nix b/nixos/modules/programs/bash/bash.nix
index be964ce7f3f..1b3254b54a5 100644
--- a/nixos/modules/programs/bash/bash.nix
+++ b/nixos/modules/programs/bash/bash.nix
@@ -238,9 +238,6 @@ in
       "/share/bash-completion"
     ];
 
-    environment.systemPackages = optional cfg.enableCompletion
-      pkgs.nix-bash-completions;
-
     environment.shells =
       [ "/run/current-system/sw/bin/bash"
         "/run/current-system/sw/bin/sh"
diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix
index 7776c648af8..c91febdb710 100644
--- a/nixos/modules/rename.nix
+++ b/nixos/modules/rename.nix
@@ -24,6 +24,7 @@ with lib;
     (mkRemovedOptionModule [ "environment" "blcr" "enable" ] "The BLCR module has been removed")
     (mkRemovedOptionModule [ "services" "beegfsEnable" ] "The BeeGFS module has been removed")
     (mkRemovedOptionModule [ "services" "beegfs" ] "The BeeGFS module has been removed")
+    (mkRemovedOptionModule ["services" "cgmanager" "enable"] "cgmanager was deprecated by lxc and therefore removed from nixpkgs.")
     (mkRemovedOptionModule [ "services" "osquery" ] "The osquery module has been removed")
     (mkRemovedOptionModule [ "services" "fourStore" ] "The fourStore module has been removed")
     (mkRemovedOptionModule [ "services" "fourStoreEndpoint" ] "The fourStoreEndpoint module has been removed")
diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix
index e3da3092d45..0185f490b0c 100644
--- a/nixos/modules/services/continuous-integration/buildbot/master.nix
+++ b/nixos/modules/services/continuous-integration/buildbot/master.nix
@@ -16,7 +16,7 @@ let
     factory = util.BuildFactory()
     c = BuildmasterConfig = dict(
      workers       = [${concatStringsSep "," cfg.workers}],
-     protocols     = { 'pb': {'port': ${toString cfg.bpPort} } },
+     protocols     = { 'pb': {'port': ${toString cfg.pbPort} } },
      title         = '${escapeStr cfg.title}',
      titleURL      = '${escapeStr cfg.titleUrl}',
      buildbotURL   = '${escapeStr cfg.buildbotUrl}',
@@ -155,10 +155,20 @@ in {
         description = "Specifies the Buildbot directory.";
       };
 
-      bpPort = mkOption {
+      pbPort = mkOption {
         default = 9989;
-        type = types.int;
-        description = "Port where the master will listen to Buildbot Worker.";
+        type = types.either types.str types.int;
+        example = "'tcp:9990:interface=127.0.0.1'";
+        description = ''
+          The buildmaster will listen on a TCP port of your choosing
+          for connections from workers.
+          It can also use this port for connections from remote Change Sources,
+          status clients, and debug tools.
+          This port should be visible to the outside world, and you’ll need to tell
+          your worker admins about your choice.
+          If put in (single) quotes, this can also be used as a connection string,
+          as defined in the <link xlink:href="https://twistedmatrix.com/documents/current/core/howto/endpoints.html">ConnectionStrings guide</link>.
+        '';
       };
 
       listenAddress = mkOption {
@@ -264,5 +274,9 @@ in {
     };
   };
 
+  imports = [
+    (mkRenamedOptionModule [ "services" "buildbot-master" "bpPort" ] [ "services" "buildbot-master" "pbPort" ])
+  ];
+
   meta.maintainers = with lib.maintainers; [ nand0p mic92 ];
 }
diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix
index 7b2bbf89a44..1ada131bd7b 100644
--- a/nixos/modules/services/misc/gitlab.nix
+++ b/nixos/modules/services/misc/gitlab.nix
@@ -43,6 +43,9 @@ let
 
     [gitlab-shell]
     dir = "${cfg.packages.gitlab-shell}"
+    secret_file = "${cfg.statePath}/gitlab_shell_secret"
+    gitlab_url = "http+unix://${pathUrlQuote gitlabSocket}"
+    http_settings = { self_signed_cert = false }
 
     ${concatStringsSep "\n" (attrValues (mapAttrs (k: v: ''
     [[storage]]
diff --git a/nixos/modules/services/misc/nix-daemon.nix b/nixos/modules/services/misc/nix-daemon.nix
index 2577cb78e96..0b3d7f3f03c 100644
--- a/nixos/modules/services/misc/nix-daemon.nix
+++ b/nixos/modules/services/misc/nix-daemon.nix
@@ -442,6 +442,12 @@ in
     nix.binaryCachePublicKeys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ];
     nix.binaryCaches = [ "https://cache.nixos.org/" ];
 
+    environment.systemPackages =
+      [ nix
+        pkgs.nix-info
+      ]
+      ++ optional (config.programs.bash.enableCompletion && !versionAtLeast nixVersion "2.4pre") pkgs.nix-bash-completions;
+
     environment.etc."nix/nix.conf".source = nixConf;
 
     environment.etc."nix/registry.json".text = builtins.toJSON {
diff --git a/nixos/modules/services/system/cgmanager.nix b/nixos/modules/services/system/cgmanager.nix
deleted file mode 100644
index d3d57aa7692..00000000000
--- a/nixos/modules/services/system/cgmanager.nix
+++ /dev/null
@@ -1,26 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-  cfg = config.services.cgmanager;
-in {
-  meta.maintainers = [ maintainers.mic92 ];
-
-  ###### interface
-  options.services.cgmanager.enable = mkEnableOption "cgmanager";
-
-  ###### implementation
-  config = mkIf cfg.enable {
-    systemd.services.cgmanager = {
-      wantedBy = [ "multi-user.target" ];
-      description = "Cgroup management daemon";
-      restartIfChanged = false;
-      serviceConfig = {
-        ExecStart = "${pkgs.cgmanager}/bin/cgmanager -m name=systemd";
-        KillMode = "process";
-        Restart = "on-failure";
-      };
-    };
-  };
-}
diff --git a/nixos/modules/system/boot/loader/grub/grub.nix b/nixos/modules/system/boot/loader/grub/grub.nix
index c775632a4aa..67e8bf6fd65 100644
--- a/nixos/modules/system/boot/loader/grub/grub.nix
+++ b/nixos/modules/system/boot/loader/grub/grub.nix
@@ -60,7 +60,7 @@ let
       inherit (efi) canTouchEfiVariables;
       inherit (cfg)
         version extraConfig extraPerEntryConfig extraEntries forceInstall useOSProber
-        extraEntriesBeforeNixOS extraPrepareConfig extraInitrd configurationLimit copyKernels
+        extraEntriesBeforeNixOS extraPrepareConfig configurationLimit copyKernels
         default fsIdentifier efiSupport efiInstallAsRemovable gfxmodeEfi gfxmodeBios gfxpayloadEfi gfxpayloadBios;
       path = with pkgs; makeBinPath (
         [ coreutils gnused gnugrep findutils diffutils btrfs-progs utillinux mdadm ]
@@ -292,19 +292,6 @@ in
         '';
       };
 
-      extraInitrd = mkOption {
-        type = types.nullOr types.path;
-        default = null;
-        example = "/boot/extra_initramfs.gz";
-        description = ''
-          The path to a second initramfs to be supplied to the kernel.
-          This ramfs will not be copied to the store, so that it can
-          contain secrets such as LUKS keyfiles or ssh keys.
-          This implies that rolling back to a previous configuration
-          won't rollback the state of this file.
-        '';
-      };
-
       useOSProber = mkOption {
         default = false;
         type = types.bool;
@@ -608,6 +595,8 @@ in
         { path = "/boot"; inherit (cfg) devices; inherit (efi) efiSysMountPoint; }
       ];
 
+      boot.loader.supportsInitrdSecrets = true;
+
       system.build.installBootLoader =
         let
           install-grub-pl = pkgs.substituteAll {
@@ -705,6 +694,24 @@ in
       (mkRenamedOptionModule [ "boot" "grubDevice" ] [ "boot" "loader" "grub" "device" ])
       (mkRenamedOptionModule [ "boot" "bootMount" ] [ "boot" "loader" "grub" "bootDevice" ])
       (mkRenamedOptionModule [ "boot" "grubSplashImage" ] [ "boot" "loader" "grub" "splashImage" ])
+      (mkRemovedOptionModule [ "boot" "loader" "grub" "extraInitrd" ] ''
+        This option has been replaced with the bootloader agnostic
+        boot.initrd.secrets option. To migrate to the initrd secrets system,
+        extract the extraInitrd archive into your main filesystem:
+
+          # zcat /boot/extra_initramfs.gz | cpio -idvmD /etc/secrets/initrd
+          /path/to/secret1
+          /path/to/secret2
+
+        then replace boot.loader.grub.extraInitrd with boot.initrd.secrets:
+
+          boot.initrd.secrets = {
+            "/path/to/secret1" = "/etc/secrets/initrd/path/to/secret1";
+            "/path/to/secret2" = "/etc/secrets/initrd/path/to/secret2";
+          };
+
+        See the boot.initrd.secrets option documentation for more information.
+      '')
     ];
 
 }
diff --git a/nixos/modules/system/boot/loader/grub/install-grub.pl b/nixos/modules/system/boot/loader/grub/install-grub.pl
index 8df18cbd901..e469b18abd0 100644
--- a/nixos/modules/system/boot/loader/grub/install-grub.pl
+++ b/nixos/modules/system/boot/loader/grub/install-grub.pl
@@ -49,7 +49,6 @@ my $extraPrepareConfig = get("extraPrepareConfig");
 my $extraPerEntryConfig = get("extraPerEntryConfig");
 my $extraEntries = get("extraEntries");
 my $extraEntriesBeforeNixOS = get("extraEntriesBeforeNixOS") eq "true";
-my $extraInitrd = get("extraInitrd");
 my $splashImage = get("splashImage");
 my $splashMode = get("splashMode");
 my $backgroundColor = get("backgroundColor");
@@ -232,13 +231,6 @@ my $grubStore;
 if ($copyKernels == 0) {
     $grubStore = GrubFs($storePath);
 }
-my $extraInitrdPath;
-if ($extraInitrd) {
-    if (! -f $extraInitrd) {
-        print STDERR "Warning: the specified extraInitrd " . $extraInitrd . " doesn't exist. Your system won't boot without it.\n";
-    }
-    $extraInitrdPath = GrubFs($extraInitrd);
-}
 
 # Generate the header.
 my $conf .= "# Automatically generated.  DO NOT EDIT THIS FILE!\n";
@@ -363,9 +355,30 @@ sub addEntry {
 
     my $kernel = copyToKernelsDir(Cwd::abs_path("$path/kernel"));
     my $initrd = copyToKernelsDir(Cwd::abs_path("$path/initrd"));
-    if ($extraInitrd) {
-        $initrd .= " " .$extraInitrdPath->path;
+
+    # Include second initrd with secrets
+    if (-e -x "$path/append-initrd-secrets") {
+      my $initrdName = basename($initrd);
+      my $initrdSecretsPath = "$bootPath/kernels/$initrdName-secrets";
+
+      mkpath(dirname($initrdSecretsPath), 0, 0755);
+      my $oldUmask = umask;
+      # Make sure initrd is not world readable (won't work if /boot is FAT)
+      umask 0137;
+      my $initrdSecretsPathTemp = File::Temp::mktemp("$initrdSecretsPath.XXXXXXXX");
+      system("$path/append-initrd-secrets", $initrdSecretsPathTemp) == 0 or die "failed to create initrd secrets\n";
+      # Check whether any secrets were actually added
+      if (-e $initrdSecretsPathTemp && ! -z _) {
+        rename $initrdSecretsPathTemp, $initrdSecretsPath or die "failed to move initrd secrets into place\n";
+        $copied{$initrdSecretsPath} = 1;
+        $initrd .= " " . ($grubBoot->path eq "/" ? "" : $grubBoot->path) . "/kernels/$initrdName-secrets";
+      } else {
+        unlink $initrdSecretsPathTemp;
+        rmdir dirname($initrdSecretsPathTemp);
+      }
+      umask $oldUmask;
     }
+
     my $xen = -e "$path/xen.gz" ? copyToKernelsDir(Cwd::abs_path("$path/xen.gz")) : undef;
 
     # FIXME: $confName
@@ -388,9 +401,6 @@ sub addEntry {
         if ($copyKernels == 0) {
             $conf .= $grubStore->search . "\n";
         }
-        if ($extraInitrd) {
-            $conf .= $extraInitrdPath->search . "\n";
-        }
         $conf .= "  $extraPerEntryConfig\n" if $extraPerEntryConfig;
         $conf .= "  multiboot $xen $xenParams\n" if $xen;
         $conf .= "  " . ($xen ? "module" : "linux") . " $kernel $kernelParams\n";
diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix
index dfd158e2d75..9bf3228d1ad 100644
--- a/nixos/modules/system/boot/stage-1.nix
+++ b/nixos/modules/system/boot/stage-1.nix
@@ -517,8 +517,7 @@ in
     };
 
     boot.initrd.secrets = mkOption
-      { internal = true;
-        default = {};
+      { default = {};
         type = types.attrsOf (types.nullOr types.path);
         description =
           ''
diff --git a/nixos/tests/mysql/mysql.nix b/nixos/tests/mysql/mysql.nix
index d236ce94632..50e1c76e9fd 100644
--- a/nixos/tests/mysql/mysql.nix
+++ b/nixos/tests/mysql/mysql.nix
@@ -5,20 +5,34 @@ import ./../make-test-python.nix ({ pkgs, ...} : {
   };
 
   nodes = {
-    mysql =
+    mysql57 =
       { pkgs, ... }:
 
       {
+        users.users.testuser = { };
+        users.users.testuser2 = { };
         services.mysql.enable = true;
         services.mysql.initialDatabases = [
-          { name = "testdb"; schema = ./testdb.sql; }
-          { name = "empty_testdb"; }
+          { name = "testdb3"; schema = ./testdb.sql; }
         ];
         # note that using pkgs.writeText here is generally not a good idea,
         # as it will store the password in world-readable /nix/store ;)
         services.mysql.initialScript = pkgs.writeText "mysql-init.sql" ''
-          CREATE USER 'passworduser'@'localhost' IDENTIFIED BY 'password123';
+          CREATE USER 'testuser3'@'localhost' IDENTIFIED BY 'secure';
+          GRANT ALL PRIVILEGES ON testdb3.* TO 'testuser3'@'localhost';
         '';
+        services.mysql.ensureDatabases = [ "testdb" "testdb2" ];
+        services.mysql.ensureUsers = [{
+          name = "testuser";
+          ensurePermissions = {
+            "testdb.*" = "ALL PRIVILEGES";
+          };
+        } {
+          name = "testuser2";
+          ensurePermissions = {
+            "testdb2.*" = "ALL PRIVILEGES";
+          };
+        }];
         services.mysql.package = pkgs.mysql57;
       };
 
@@ -30,16 +44,30 @@ import ./../make-test-python.nix ({ pkgs, ...} : {
         # Kernel panic - not syncing: Out of memory: compulsory panic_on_oom is enabled
         virtualisation.memorySize = 1024;
 
+        users.users.testuser = { };
+        users.users.testuser2 = { };
         services.mysql.enable = true;
         services.mysql.initialDatabases = [
-          { name = "testdb"; schema = ./testdb.sql; }
-          { name = "empty_testdb"; }
+          { name = "testdb3"; schema = ./testdb.sql; }
         ];
         # note that using pkgs.writeText here is generally not a good idea,
         # as it will store the password in world-readable /nix/store ;)
         services.mysql.initialScript = pkgs.writeText "mysql-init.sql" ''
-          CREATE USER 'passworduser'@'localhost' IDENTIFIED BY 'password123';
+          CREATE USER 'testuser3'@'localhost' IDENTIFIED BY 'secure';
+          GRANT ALL PRIVILEGES ON testdb3.* TO 'testuser3'@'localhost';
         '';
+        services.mysql.ensureDatabases = [ "testdb" "testdb2" ];
+        services.mysql.ensureUsers = [{
+          name = "testuser";
+          ensurePermissions = {
+            "testdb.*" = "ALL PRIVILEGES";
+          };
+        } {
+          name = "testuser2";
+          ensurePermissions = {
+            "testdb2.*" = "ALL PRIVILEGES";
+          };
+        }];
         services.mysql.package = pkgs.mysql80;
       };
 
@@ -81,17 +109,49 @@ import ./../make-test-python.nix ({ pkgs, ...} : {
   testScript = ''
     start_all()
 
-    mysql.wait_for_unit("mysql")
-    mysql.succeed("echo 'use empty_testdb;' | mysql -u root")
-    mysql.succeed("echo 'use testdb; select * from tests;' | mysql -u root -N | grep 4")
-    # ';' acts as no-op, just check whether login succeeds with the user created from the initialScript
-    mysql.succeed("echo ';' | mysql -u passworduser --password=password123")
+    mysql57.wait_for_unit("mysql")
+    mysql57.succeed(
+        "echo 'use testdb; create table tests (test_id INT, PRIMARY KEY (test_id));' | sudo -u testuser mysql -u testuser"
+    )
+    mysql57.succeed(
+        "echo 'use testdb; insert into tests values (41);' | sudo -u testuser mysql -u testuser"
+    )
+    # Ensure testuser2 is not able to insert into testdb as mysql testuser2
+    mysql57.fail(
+        "echo 'use testdb; insert into tests values (22);' | sudo -u testuser2 mysql -u testuser2"
+    )
+    # Ensure testuser2 is not able to authenticate as mysql testuser
+    mysql57.fail(
+        "echo 'use testdb; insert into tests values (22);' | sudo -u testuser2 mysql -u testuser"
+    )
+    mysql57.succeed(
+        "echo 'use testdb; select test_id from tests;' | sudo -u testuser mysql -u testuser -N | grep 41"
+    )
+    mysql57.succeed(
+        "echo 'use testdb3; select * from tests;' | mysql -u testuser3 --password=secure -N | grep 4"
+    )
 
     mysql80.wait_for_unit("mysql")
-    mysql80.succeed("echo 'use empty_testdb;' | mysql -u root")
-    mysql80.succeed("echo 'use testdb; select * from tests;' | mysql -u root -N | grep 4")
-    # ';' acts as no-op, just check whether login succeeds with the user created from the initialScript
-    mysql80.succeed("echo ';' | mysql -u passworduser --password=password123")
+    mysql80.succeed(
+        "echo 'use testdb; create table tests (test_id INT, PRIMARY KEY (test_id));' | sudo -u testuser mysql -u testuser"
+    )
+    mysql80.succeed(
+        "echo 'use testdb; insert into tests values (41);' | sudo -u testuser mysql -u testuser"
+    )
+    # Ensure testuser2 is not able to insert into testdb as mysql testuser2
+    mysql80.fail(
+        "echo 'use testdb; insert into tests values (22);' | sudo -u testuser2 mysql -u testuser2"
+    )
+    # Ensure testuser2 is not able to authenticate as mysql testuser
+    mysql80.fail(
+        "echo 'use testdb; insert into tests values (22);' | sudo -u testuser2 mysql -u testuser"
+    )
+    mysql80.succeed(
+        "echo 'use testdb; select test_id from tests;' | sudo -u testuser mysql -u testuser -N | grep 41"
+    )
+    mysql80.succeed(
+        "echo 'use testdb3; select * from tests;' | mysql -u testuser3 --password=secure -N | grep 4"
+    )
 
     mariadb.wait_for_unit("mysql")
     mariadb.succeed(