diff options
author | Jan Tojnar <jtojnar@gmail.com> | 2020-03-29 00:35:29 +0100 |
---|---|---|
committer | Jan Tojnar <jtojnar@gmail.com> | 2020-03-29 00:35:29 +0100 |
commit | fa4e1bbe07f0128c6ab6628b1601b42a5ca0e6d7 (patch) | |
tree | ee10fc0f2ebc6ad7028408865009fcb1855c0d4a /nixos | |
parent | 72e5b2dc2ccba536159ed93ccc962cfe733f5012 (diff) | |
parent | 3c4ab13243b50d851510dc32c3f7e9605d6155bf (diff) | |
download | nixpkgs-fa4e1bbe07f0128c6ab6628b1601b42a5ca0e6d7.tar nixpkgs-fa4e1bbe07f0128c6ab6628b1601b42a5ca0e6d7.tar.gz nixpkgs-fa4e1bbe07f0128c6ab6628b1601b42a5ca0e6d7.tar.bz2 nixpkgs-fa4e1bbe07f0128c6ab6628b1601b42a5ca0e6d7.tar.lz nixpkgs-fa4e1bbe07f0128c6ab6628b1601b42a5ca0e6d7.tar.xz nixpkgs-fa4e1bbe07f0128c6ab6628b1601b42a5ca0e6d7.tar.zst nixpkgs-fa4e1bbe07f0128c6ab6628b1601b42a5ca0e6d7.zip |
Merge branch 'master' into staging-next
Fix eval of nixos/nginx
Diffstat (limited to 'nixos')
-rw-r--r-- | nixos/doc/manual/release-notes/rl-2003.xml | 60 | ||||
-rw-r--r-- | nixos/modules/services/continuous-integration/hydra/default.nix | 33 | ||||
-rw-r--r-- | nixos/modules/services/networking/connman.nix | 17 | ||||
-rw-r--r-- | nixos/modules/services/web-servers/nginx/default.nix | 2 | ||||
-rw-r--r-- | nixos/tests/all-tests.nix | 6 | ||||
-rw-r--r-- | nixos/tests/hydra/common.nix | 47 | ||||
-rw-r--r-- | nixos/tests/hydra/db-migration.nix | 86 | ||||
-rw-r--r-- | nixos/tests/hydra/default.nix | 137 |
8 files changed, 289 insertions, 99 deletions
diff --git a/nixos/doc/manual/release-notes/rl-2003.xml b/nixos/doc/manual/release-notes/rl-2003.xml index 7674b0a5c0d..76b12af362f 100644 --- a/nixos/doc/manual/release-notes/rl-2003.xml +++ b/nixos/doc/manual/release-notes/rl-2003.xml @@ -697,6 +697,66 @@ auth required pam_succeed_if.so uid >= 1000 quiet </para> </warning> </listitem> + <listitem> + <para> + <package>Hydra</package> has gained a massive performance improvement due to + <link xlink:href="https://github.com/NixOS/hydra/pull/710">some database schema + changes</link> by adding several IDs and better indexing. However, it's necessary + to upgrade Hydra in multiple steps: + <itemizedlist> + <listitem> + <para> + At first, an older version of Hydra needs to be deployed which adds those + (nullable) columns. When having set <link linkend="opt-system.stateVersion">stateVersion + </link> to a value older than <literal>20.03</literal>, this package will be selected + by default from the module when upgrading. Otherwise, the package can be deployed using + the following config: +<programlisting>{ pkgs, ... }: { + <link linkend="opt-services.hydra.package">services.hydra.package</link> = pkgs.hydra-migration; +}</programlisting> + </para> + </listitem> + <listitem> + <para> + Automatically fill the newly added ID columns on the server by running the following + command: +<screen> +<prompt>$ </prompt>hydra-backfill-ids +</screen> + <warning> + <para>Please note that this process can take a while depending on your database-size!</para> + </warning> + </para> + </listitem> + <listitem> + <para> + Deploy a newer version of Hydra to activate the DB optimizations. You can choose from + either <package>hydra-unstable</package> (latest <literal>master</literal> compiled + against <package>nixUnstable</package>) and <package>hydra-flakes</package> (latest + version with flake-support). + <warning> + <para> + If your <link linkend="opt-system.stateVersion">stateVersion</link> is set to + <literal>20.03</literal> or greater, <package>hydra-unstable</package> will be used + automatically! This will break your setup if you didn't run the migration. + </para> + </warning> + Please note that Hydra is currently not available with <package>nixStable</package> + as this doesn't compile anymore. + </para> + </listitem> + </itemizedlist> + <warning> + <para> + <package>pkgs.hydra</package> has been removed to ensure a graceful database-migration + using the dedicated package-attributes. If you still have <package>pkgs.hydra</package> + defined in e.g. an overlay, an assertion error will be thrown. To circumvent this, + you need to set <xref linkend="opt-services.hydra.package" /> to <package>pkgs.hydra</package> + explicitly and make sure you know what you're doing! + </para> + </warning> + </para> + </listitem> </itemizedlist> </section> diff --git a/nixos/modules/services/continuous-integration/hydra/default.nix b/nixos/modules/services/continuous-integration/hydra/default.nix index 8b56207590a..0c335f14f78 100644 --- a/nixos/modules/services/continuous-integration/hydra/default.nix +++ b/nixos/modules/services/continuous-integration/hydra/default.nix @@ -37,6 +37,8 @@ let haveLocalDB = cfg.dbi == localDB; + inherit (config.system) stateVersion; + in { @@ -63,8 +65,7 @@ in }; package = mkOption { - type = types.path; - default = pkgs.hydra; + type = types.package; defaultText = "pkgs.hydra"; description = "The Hydra package."; }; @@ -194,6 +195,34 @@ in config = mkIf cfg.enable { + warnings = optional (cfg.package.migration or false) '' + You're currently deploying an older version of Hydra which is needed to + make some required database changes[1]. As soon as this is done, it's recommended + to run `hydra-backfill-ids` and set `services.hydra.package` to either `pkgs.hydra-unstable` + or `pkgs.hydra-flakes` after that. + + [1] https://github.com/NixOS/hydra/pull/711 + ''; + + services.hydra.package = with pkgs; + mkDefault ( + if pkgs ? hydra + then throw '' + The Hydra package doesn't exist anymore in `nixpkgs`! It probably exists + due to an overlay. To upgrade Hydra, you need to take two steps as some + bigger changes in the database schema were implemented recently[1]. You first + need to deploy `pkgs.hydra-migration`, run `hydra-backfill-ids` on the server + and then deploy either `pkgs.hydra-unstable` or `pkgs.hydra-flakes`. + + If you want to use `pkgs.hydra` from your overlay, please set `services.hydra.package` + explicitly to `pkgs.hydra` and make sure you know what you're doing. + + [1] https://github.com/NixOS/hydra/pull/711 + '' + else if versionOlder stateVersion "20.03" then hydra-migration + else hydra-unstable + ); + users.groups.hydra = { gid = config.ids.gids.hydra; }; diff --git a/nixos/modules/services/networking/connman.nix b/nixos/modules/services/networking/connman.nix index e8eadc4e187..6ccc2dffb26 100644 --- a/nixos/modules/services/networking/connman.nix +++ b/nixos/modules/services/networking/connman.nix @@ -77,6 +77,13 @@ in { ''; }; + package = mkOption { + type = types.path; + description = "The connman package / build flavor"; + default = connman; + example = literalExample "pkgs.connmanFull"; + }; + }; }; @@ -89,11 +96,13 @@ in { assertion = !config.networking.useDHCP; message = "You can not use services.connman with networking.useDHCP"; }{ + # TODO: connman seemingly can be used along network manager and + # connmanFull supports this - so this should be worked out somehow assertion = !config.networking.networkmanager.enable; message = "You can not use services.connman with networking.networkmanager"; }]; - environment.systemPackages = [ connman ]; + environment.systemPackages = [ cfg.package ]; systemd.services.connman = { description = "Connection service"; @@ -105,7 +114,7 @@ in { BusName = "net.connman"; Restart = "on-failure"; ExecStart = toString ([ - "${pkgs.connman}/sbin/connmand" + "${cfg.package}/sbin/connmand" "--config=${configFile}" "--nodaemon" ] ++ optional enableIwd "--wifi=iwd_agent" @@ -122,7 +131,7 @@ in { serviceConfig = { Type = "dbus"; BusName = "net.connman.vpn"; - ExecStart = "${pkgs.connman}/sbin/connman-vpnd -n"; + ExecStart = "${cfg.package}/sbin/connman-vpnd -n"; StandardOutput = "null"; }; }; @@ -132,7 +141,7 @@ in { serviceConfig = { Name = "net.connman.vpn"; before = [ "connman" ]; - ExecStart = "${pkgs.connman}/sbin/connman-vpnd -n"; + ExecStart = "${cfg.package}/sbin/connman-vpnd -n"; User = "root"; SystemdService = "connman-vpn.service"; }; diff --git a/nixos/modules/services/web-servers/nginx/default.nix b/nixos/modules/services/web-servers/nginx/default.nix index fd230a76499..8d49dc66eb1 100644 --- a/nixos/modules/services/web-servers/nginx/default.nix +++ b/nixos/modules/services/web-servers/nginx/default.nix @@ -176,7 +176,7 @@ let ${optionalString (cfg.httpConfig != "") '' http { - ${common.httpConfig} + ${commonHttpConfig} ${cfg.httpConfig} }''} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index b62a3e260aa..a84f2c5d261 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -121,12 +121,16 @@ in handbrake = handleTestOn ["x86_64-linux"] ./handbrake.nix {}; haproxy = handleTest ./haproxy.nix {}; hardened = handleTest ./hardened.nix {}; - hibernate = handleTest ./hibernate.nix {}; + # 9pnet_virtio used to mount /nix partition doesn't support + # hibernation. This test happens to work on x86_64-linux but + # not on other platforms. + hibernate = handleTestOn ["x86_64-linux"] ./hibernate.nix {}; hitch = handleTest ./hitch {}; hocker-fetchdocker = handleTest ./hocker-fetchdocker {}; home-assistant = handleTest ./home-assistant.nix {}; hound = handleTest ./hound.nix {}; hydra = handleTest ./hydra {}; + hydra-db-migration = handleTest ./hydra/db-migration.nix {}; i3wm = handleTest ./i3wm.nix {}; icingaweb2 = handleTest ./icingaweb2.nix {}; iftop = handleTest ./iftop.nix {}; diff --git a/nixos/tests/hydra/common.nix b/nixos/tests/hydra/common.nix new file mode 100644 index 00000000000..f612717dc96 --- /dev/null +++ b/nixos/tests/hydra/common.nix @@ -0,0 +1,47 @@ +{ system, ... }: +{ + baseConfig = { pkgs, ... }: let + trivialJob = pkgs.writeTextDir "trivial.nix" '' + { trivial = builtins.derivation { + name = "trivial"; + system = "${system}"; + builder = "/bin/sh"; + allowSubstitutes = false; + preferLocalBuild = true; + args = ["-c" "echo success > $out; exit 0"]; + }; + } + ''; + + createTrivialProject = pkgs.stdenv.mkDerivation { + name = "create-trivial-project"; + dontUnpack = true; + buildInputs = [ pkgs.makeWrapper ]; + installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh"; + postFixup = '' + wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob} + ''; + }; + in { + virtualisation.memorySize = 2048; + time.timeZone = "UTC"; + environment.systemPackages = [ createTrivialProject pkgs.jq ]; + services.hydra = { + enable = true; + # Hydra needs those settings to start up, so we add something not harmfull. + hydraURL = "example.com"; + notificationSender = "example@example.com"; + extraConfig = '' + email_notification = 1 + ''; + }; + services.postfix.enable = true; + nix = { + buildMachines = [{ + hostName = "localhost"; + systems = [ system ]; + }]; + binaryCaches = []; + }; + }; +} diff --git a/nixos/tests/hydra/db-migration.nix b/nixos/tests/hydra/db-migration.nix new file mode 100644 index 00000000000..aa1c81c9e77 --- /dev/null +++ b/nixos/tests/hydra/db-migration.nix @@ -0,0 +1,86 @@ +{ system ? builtins.currentSystem, ... }: + +let inherit (import ./common.nix { inherit system; }) baseConfig; in + +{ mig = import ../make-test-python.nix ({ pkgs, lib, ... }: { + name = "hydra-db-migration"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ ma27 ]; + }; + + nodes = { + original = { pkgs, lib, ... }: { + imports = [ baseConfig ]; + + # An older version of Hydra before the db change + # for testing purposes. + services.hydra.package = pkgs.hydra-migration.overrideAttrs (old: { + inherit (old) pname; + version = "2020-02-06"; + src = pkgs.fetchFromGitHub { + owner = "NixOS"; + repo = "hydra"; + rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc"; + sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick"; + }; + }); + }; + + migration_phase1 = { pkgs, lib, ... }: { + imports = [ baseConfig ]; + services.hydra.package = pkgs.hydra-migration; + }; + + finished = { pkgs, lib, ... }: { + imports = [ baseConfig ]; + services.hydra.package = pkgs.hydra-unstable; + }; + }; + + testScript = { nodes, ... }: let + next = nodes.migration_phase1.config.system.build.toplevel; + finished = nodes.finished.config.system.build.toplevel; + in '' + original.start() + original.wait_for_unit("multi-user.target") + original.wait_for_unit("postgresql.service") + original.wait_for_unit("hydra-init.service") + original.require_unit_state("hydra-queue-runner.service") + original.require_unit_state("hydra-evaluator.service") + original.require_unit_state("hydra-notify.service") + original.succeed("hydra-create-user admin --role admin --password admin") + original.wait_for_open_port(3000) + original.succeed("create-trivial-project.sh") + original.wait_until_succeeds( + 'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq' + ) + + out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'") + assert "jobset_id" not in out + + original.succeed( + "${next}/bin/switch-to-configuration test >&2" + ) + original.wait_for_unit("hydra-init.service") + + out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'") + assert "jobset_id|integer|||" in out + + original.succeed("hydra-backfill-ids") + + original.succeed( + "${finished}/bin/switch-to-configuration test >&2" + ) + original.wait_for_unit("hydra-init.service") + + out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'") + assert "jobset_id|integer||not null|" in out + + original.wait_until_succeeds( + 'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq' + ) + + original.shutdown() + ''; + }); +} diff --git a/nixos/tests/hydra/default.nix b/nixos/tests/hydra/default.nix index 1c0ed3369b1..5d94eb91bf5 100644 --- a/nixos/tests/hydra/default.nix +++ b/nixos/tests/hydra/default.nix @@ -3,102 +3,57 @@ , pkgs ? import ../../.. { inherit system config; } }: -let - - trivialJob = pkgs.writeTextDir "trivial.nix" '' - { trivial = builtins.derivation { - name = "trivial"; - system = "${system}"; - builder = "/bin/sh"; - allowSubstitutes = false; - preferLocalBuild = true; - args = ["-c" "echo success > $out; exit 0"]; - }; - } - ''; +with import ../../lib/testing-python.nix { inherit system pkgs; }; +with pkgs.lib; - createTrivialProject = pkgs.stdenv.mkDerivation { - name = "create-trivial-project"; - dontUnpack = true; - buildInputs = [ pkgs.makeWrapper ]; - installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh"; - postFixup = '' - wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob} - ''; - }; +let - callTest = f: f { inherit system pkgs; }; + inherit (import ./common.nix { inherit system; }) baseConfig; hydraPkgs = { - inherit (pkgs) nixStable nixUnstable nixFlakes; + inherit (pkgs) hydra-migration hydra-unstable hydra-flakes; }; - tests = pkgs.lib.flip pkgs.lib.mapAttrs hydraPkgs (name: nix: - callTest (import ../make-test-python.nix ({ pkgs, lib, ... }: - { - name = "hydra-with-${name}"; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ pstn lewo ma27 ]; - }; - - machine = { pkgs, ... }: - { - virtualisation.memorySize = 1024; - time.timeZone = "UTC"; - - environment.systemPackages = [ createTrivialProject pkgs.jq ]; - services.hydra = { - enable = true; - - #Hydra needs those settings to start up, so we add something not harmfull. - hydraURL = "example.com"; - notificationSender = "example@example.com"; - - package = pkgs.hydra.override { inherit nix; }; - - extraConfig = '' - email_notification = 1 - ''; - }; - services.postfix.enable = true; - nix = { - buildMachines = [{ - hostName = "localhost"; - systems = [ system ]; - }]; - - binaryCaches = []; - }; - }; - - testScript = '' - # let the system boot up - machine.wait_for_unit("multi-user.target") - # test whether the database is running - machine.wait_for_unit("postgresql.service") - # test whether the actual hydra daemons are running - machine.wait_for_unit("hydra-init.service") - machine.require_unit_state("hydra-queue-runner.service") - machine.require_unit_state("hydra-evaluator.service") - machine.require_unit_state("hydra-notify.service") - - machine.succeed("hydra-create-user admin --role admin --password admin") - - # create a project with a trivial job - machine.wait_for_open_port(3000) - - # make sure the build as been successfully built - machine.succeed("create-trivial-project.sh") - - machine.wait_until_succeeds( - 'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq' - ) - - machine.wait_until_succeeds( - 'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"' - ) - ''; - }))); + makeHydraTest = with pkgs.lib; name: package: makeTest { + name = "hydra-${name}"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ pstn lewo ma27 ]; + }; + + machine = { pkgs, lib, ... }: { + imports = [ baseConfig ]; + services.hydra = { inherit package; }; + }; + + testScript = '' + # let the system boot up + machine.wait_for_unit("multi-user.target") + # test whether the database is running + machine.wait_for_unit("postgresql.service") + # test whether the actual hydra daemons are running + machine.wait_for_unit("hydra-init.service") + machine.require_unit_state("hydra-queue-runner.service") + machine.require_unit_state("hydra-evaluator.service") + machine.require_unit_state("hydra-notify.service") + + machine.succeed("hydra-create-user admin --role admin --password admin") + + # create a project with a trivial job + machine.wait_for_open_port(3000) + + # make sure the build as been successfully built + machine.succeed("create-trivial-project.sh") + + machine.wait_until_succeeds( + 'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq' + ) + + machine.wait_until_succeeds( + 'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"' + ) + ''; + }; in - tests + +mapAttrs makeHydraTest hydraPkgs |