From c772c572cfabba6a1c6b0f5a9a71bb59cd9b9916 Mon Sep 17 00:00:00 2001 From: Winter Date: Tue, 11 Jan 2022 20:43:08 -0500 Subject: nixos/doc: fix mention of reading test logs --- nixos/doc/manual/development/running-nixos-tests.section.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'nixos/doc/manual/development') diff --git a/nixos/doc/manual/development/running-nixos-tests.section.md b/nixos/doc/manual/development/running-nixos-tests.section.md index d6a456f0188..1bec023b613 100644 --- a/nixos/doc/manual/development/running-nixos-tests.section.md +++ b/nixos/doc/manual/development/running-nixos-tests.section.md @@ -24,8 +24,8 @@ After building/downloading all required dependencies, this will perform a build that starts a QEMU/KVM virtual machine containing a NixOS system. The virtual machine mounts the Nix store of the host; this makes VM creation very fast, as no disk image needs to be created. Afterwards, -you can view a pretty-printed log of the test: +you can view a log of the test: ```ShellSession -$ firefox result/log.html +$ nix-store --read-log result ``` -- cgit 1.4.1 From 8d925cc8db5fcc0fe0e091d819d93f8580e62c53 Mon Sep 17 00:00:00 2001 From: Janne Heß Date: Sun, 30 Jan 2022 00:37:55 +0100 Subject: nixos/doc: Document the activation script This may be helpful to new module developers, curious users, and people who just need a reference without having to look at the implementation --- .../development/activation-script.section.md | 72 ++++++++++ nixos/doc/manual/development/development.xml | 1 + .../manual/development/unit-handling.section.md | 57 ++++++++ .../what-happens-during-a-system-switch.chapter.md | 53 ++++++++ .../development/activation-script.section.xml | 150 +++++++++++++++++++++ .../from_md/development/unit-handling.section.xml | 119 ++++++++++++++++ ...what-happens-during-a-system-switch.chapter.xml | 122 +++++++++++++++++ 7 files changed, 574 insertions(+) create mode 100644 nixos/doc/manual/development/activation-script.section.md create mode 100644 nixos/doc/manual/development/unit-handling.section.md create mode 100644 nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md create mode 100644 nixos/doc/manual/from_md/development/activation-script.section.xml create mode 100644 nixos/doc/manual/from_md/development/unit-handling.section.xml create mode 100644 nixos/doc/manual/from_md/development/what-happens-during-a-system-switch.chapter.xml (limited to 'nixos/doc/manual/development') diff --git a/nixos/doc/manual/development/activation-script.section.md b/nixos/doc/manual/development/activation-script.section.md new file mode 100644 index 00000000000..df683662404 --- /dev/null +++ b/nixos/doc/manual/development/activation-script.section.md @@ -0,0 +1,72 @@ +# Activation script {#sec-activation-script} + +The activation script is a bash script called to activate the new +configuration which resides in a NixOS system in `$out/activate`. Since its +contents depend on your system configuration, the contents may differ. +This chapter explains how the script works in general and some common NixOS +snippets. Please be aware that the script is executed on every boot and system +switch, so tasks that can be performed in other places should be performed +there (for example letting a directory of a service be created by systemd using +mechanisms like `StateDirectory`, `CacheDirectory`, ... or if that's not +possible using `preStart` of the service). + +Activation scripts are defined as snippets using +[](#opt-system.activationScripts). They can either be a simple multiline string +or an attribute set that can depend on other snippets. The builder for the +activation script will take these dependencies into account and order the +snippets accordingly. As a simple example: + +```nix +system.activationScripts.my-activation-script = { + deps = [ "etc" ]; + # supportsDryActivation = true; + text = '' + echo "Hallo i bims" + ''; +}; +``` + +This example creates an activation script snippet that is run after the `etc` +snippet. The special variable `supportsDryActivation` can be set so the snippet +is also run when `nixos-rebuild dry-activate` is run. To differentiate between +real and dry activation, the `$NIXOS_ACTION` environment variable can be +read which is set to `dry-activate` when a dry activation is done. + +An activation script can write to special files instructing +`switch-to-configuration` to restart/reload units. The script will take these +requests into account and will incorperate the unit configuration as described +above. This means that the activation script will "fake" a modified unit file +and `switch-to-configuration` will act accordingly. By doing so, configuration +like [systemd.services.\.restartIfChanged](#opt-systemd.services) is +respected. Since the activation script is run **after** services are already +stopped, [systemd.services.\.stopIfChanged](#opt-systemd.services) +cannot be taken into account anymore and the unit is always restarted instead +of being stopped and started afterwards. + +The files that can be written to are `/run/nixos/activation-restart-list` and +`/run/nixos/activation-reload-list` with their respective counterparts for +dry activation being `/run/nixos/dry-activation-restart-list` and +`/run/nixos/dry-activation-reload-list`. Those files can contain +newline-separated lists of unit names where duplicates are being ignored. These +files are not create automatically and activation scripts must take the +possiblility into account that they have to create them first. + +## NixOS snippets {#sec-activation-script-nixos-snippets} + +There are some snippets NixOS enables by default because disabling them would +most likely break you system. This section lists a few of them and what they +do: + +- `binsh` creates `/bin/sh` which points to the runtime shell +- `etc` sets up the contents of `/etc`, this includes systemd units and + excludes `/etc/passwd`, `/etc/group`, and `/etc/shadow` (which are managed by + the `users` snippet) +- `hostname` sets the system's hostname in the kernel (not in `/etc`) +- `modprobe` sets the path to the `modprobe` binary for module auto-loading +- `nix` prepares the nix store and adds a default initial channel +- `specialfs` is responsible for mounting filesystems like `/proc` and `sys` +- `users` creates and removes users and groups by managing `/etc/passwd`, + `/etc/group` and `/etc/shadow`. This also creates home directories +- `usrbinenv` creates `/usr/bin/env` +- `var` creates some directories in `/var` that are not service-specific +- `wrappers` creates setuid wrappers like `ping` and `sudo` diff --git a/nixos/doc/manual/development/development.xml b/nixos/doc/manual/development/development.xml index 0b2ad60a878..21286cdbd2b 100644 --- a/nixos/doc/manual/development/development.xml +++ b/nixos/doc/manual/development/development.xml @@ -12,6 +12,7 @@ + diff --git a/nixos/doc/manual/development/unit-handling.section.md b/nixos/doc/manual/development/unit-handling.section.md new file mode 100644 index 00000000000..d477f2c860f --- /dev/null +++ b/nixos/doc/manual/development/unit-handling.section.md @@ -0,0 +1,57 @@ +# Unit handling {#sec-unit-handling} + +To figure out what units need to be started/stopped/restarted/reloaded, the +script first checks the current state of the system, similar to what `systemctl +list-units` shows. For each of the units, the script goes through the following +checks: + +- Is the unit file still in the new system? If not, **stop** the service unless + it sets `X-StopOnRemoval` in the `[Unit]` section to `false`. + +- Is it a `.target` unit? If so, **start** it unless it sets + `RefuseManualStart` in the `[Unit]` section to `true` or `X-OnlyManualStart` + in the `[Unit]` section to `true`. Also **stop** the unit again unless it + sets `X-StopOnReconfiguration` to `false`. + +- Are the contents of the unit files different? They are compared by parsing + them and comparing their contents. If they are different but only + `X-Reload-Triggers` in the `[Unit]` section is changed, **reload** the unit. + The NixOS module system allows setting these triggers with the option + [systemd.services.\.reloadTriggers](#opt-systemd.services). If the + unit files differ in any way, the following actions are performed: + + - `.path` and `.slice` units are ignored. There is no need to restart them + since changes in their values are applied by systemd when systemd is + reloaded. + + - `.mount` units are **reload**ed. These mostly come from the `/etc/fstab` + parser. + + - `.socket` units are currently ignored. This is to be fixed at a later + point. + + - The rest of the units (mostly `.service` units) are then **reload**ed if + `X-ReloadIfChanged` in the `[Service]` section is set to `true` (exposed + via [systemd.services.\.reloadIfChanged](#opt-systemd.services)). + + - If the reload flag is not set, some more flags decide if the unit is + skipped. These flags are `X-RestartIfChanged` in the `[Service]` section + (exposed via + [systemd.services.\.restartIfChanged](#opt-systemd.services)), + `RefuseManualStop` in the `[Unit]` section, and `X-OnlyManualStart` in the + `[Unit]` section. + + - The rest of the behavior is decided whether the unit has `X-StopIfChanged` + in the `[Service]` section set (exposed via + [systemd.services.\.stopIfChanged](#opt-systemd.services)). This is + set to `true` by default and must be explicitly turned off if not wanted. + If the flag is enabled, the unit is **stop**ped and then **start**ed. If + not, the unit is **restart**ed. The goal of the flag is to make sure that + the new unit never runs in the old environment which is still in place + before the activation script is run. + + - The last thing that is taken into account is whether the unit is a service + and socket-activated. Due to a bug, this is currently only done when + `X-StopIfChanged` is set. If the unit is socket-activated, the socket is + stopped and started, and the service is stopped and to be started by socket + activation. diff --git a/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md b/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md new file mode 100644 index 00000000000..aad82831a3c --- /dev/null +++ b/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md @@ -0,0 +1,53 @@ +# What happens during a system switch? {#sec-switching-systems} + +Running `nixos-rebuild switch` is one of the more common tasks under NixOS. +This chapter explains some of the internals of this command to make it simpler +for new module developers to configure their units correctly and to make it +easier to understand what is happening and why for curious administrators. + +`nixos-rebuild`, like many deployment solutions, calls `switch-to-configuration` +which resides in a NixOS system at `$out/bin/switch-to-configuration`. The +script is called with the action that is to be performed like `switch`, `test`, +`boot`. There is also the `dry-activate` action which does not really perform +the actions but rather prints what it would do if you called it with `test`. +This feature can be used to check what service states would be changed if the +configuration was switched to. + +If the action is `switch` or `boot`, the bootloader is updated first so the +configuration will be the next one to boot. Unless `NIXOS_NO_SYNC` is set to +`1`, `/nix/store` is synced to disk. + +If the action is `switch` or `test`, the currently running system is inspected +and the actions to switch to the new system are calculated. This process takes +two data sources into account: `/etc/fstab` and the current systemd status. +Mounts and swaps are read from `/etc/fstab` and the corresponding actions are +generated. If a new mount is added, for example, the proper `.mount` unit is +marked to be started. The current systemd state is inspected, the difference +between the current system and the desired configuration is calculated and +actions are generated to get to this state. There are a lot of nuances that can +be controlled by the units which are explained here. + +After calculating what should be done, the actions are carried out. The order +of actions is always the same: +- Stop units (`systemctl stop`) +- Run activation script (`$out/activate`) +- See if the activation script requested more units to restart +- Restart systemd if needed (`systemd daemon-reexec`) +- Forget about the failed state of units (`systemctl reset-failed`) +- Reload systemd (`systemctl daemon-reload`) +- Reload systemd user instances (`systemctl --user daemon-reload`) +- Set up tmpfiles (`systemd-tmpfiles --create`) +- Reload units (`systemctl reload`) +- Restart units (`systemctl restart`) +- Start units (`systemctl start`) +- Inspect what changed during these actions and print units that failed and + that were newly started + +Most of these actions are either self-explaining but some of them have to do +with our units or the activation script. For this reason, these topics are +explained in the next sections. + +```{=docbook} + + +``` diff --git a/nixos/doc/manual/from_md/development/activation-script.section.xml b/nixos/doc/manual/from_md/development/activation-script.section.xml new file mode 100644 index 00000000000..0d9e911216e --- /dev/null +++ b/nixos/doc/manual/from_md/development/activation-script.section.xml @@ -0,0 +1,150 @@ +
+ Activation script + + The activation script is a bash script called to activate the new + configuration which resides in a NixOS system in + $out/activate. Since its contents depend on your + system configuration, the contents may differ. This chapter explains + how the script works in general and some common NixOS snippets. + Please be aware that the script is executed on every boot and system + switch, so tasks that can be performed in other places should be + performed there (for example letting a directory of a service be + created by systemd using mechanisms like + StateDirectory, + CacheDirectory, … or if that’s not possible using + preStart of the service). + + + Activation scripts are defined as snippets using + . They can either be + a simple multiline string or an attribute set that can depend on + other snippets. The builder for the activation script will take + these dependencies into account and order the snippets accordingly. + As a simple example: + + +system.activationScripts.my-activation-script = { + deps = [ "etc" ]; + # supportsDryActivation = true; + text = '' + echo "Hallo i bims" + ''; +}; + + + This example creates an activation script snippet that is run after + the etc snippet. The special variable + supportsDryActivation can be set so the snippet + is also run when nixos-rebuild dry-activate is + run. To differentiate between real and dry activation, the + $NIXOS_ACTION environment variable can be read + which is set to dry-activate when a dry + activation is done. + + + An activation script can write to special files instructing + switch-to-configuration to restart/reload units. + The script will take these requests into account and will + incorperate the unit configuration as described above. This means + that the activation script will fake a modified unit + file and switch-to-configuration will act + accordingly. By doing so, configuration like + systemd.services.<name>.restartIfChanged + is respected. Since the activation script is run + after services are already + stopped, + systemd.services.<name>.stopIfChanged + cannot be taken into account anymore and the unit is always + restarted instead of being stopped and started afterwards. + + + The files that can be written to are + /run/nixos/activation-restart-list and + /run/nixos/activation-reload-list with their + respective counterparts for dry activation being + /run/nixos/dry-activation-restart-list and + /run/nixos/dry-activation-reload-list. Those + files can contain newline-separated lists of unit names where + duplicates are being ignored. These files are not create + automatically and activation scripts must take the possiblility into + account that they have to create them first. + +
+ NixOS snippets + + There are some snippets NixOS enables by default because disabling + them would most likely break you system. This section lists a few + of them and what they do: + + + + + binsh creates /bin/sh + which points to the runtime shell + + + + + etc sets up the contents of + /etc, this includes systemd units and + excludes /etc/passwd, + /etc/group, and + /etc/shadow (which are managed by the + users snippet) + + + + + hostname sets the system’s hostname in the + kernel (not in /etc) + + + + + modprobe sets the path to the + modprobe binary for module auto-loading + + + + + nix prepares the nix store and adds a + default initial channel + + + + + specialfs is responsible for mounting + filesystems like /proc and + sys + + + + + users creates and removes users and groups + by managing /etc/passwd, + /etc/group and + /etc/shadow. This also creates home + directories + + + + + usrbinenv creates + /usr/bin/env + + + + + var creates some directories in + /var that are not service-specific + + + + + wrappers creates setuid wrappers like + ping and sudo + + + +
+
diff --git a/nixos/doc/manual/from_md/development/unit-handling.section.xml b/nixos/doc/manual/from_md/development/unit-handling.section.xml new file mode 100644 index 00000000000..a6a654042f6 --- /dev/null +++ b/nixos/doc/manual/from_md/development/unit-handling.section.xml @@ -0,0 +1,119 @@ +
+ Unit handling + + To figure out what units need to be + started/stopped/restarted/reloaded, the script first checks the + current state of the system, similar to what + systemctl list-units shows. For each of the + units, the script goes through the following checks: + + + + + Is the unit file still in the new system? If not, + stop the service unless it + sets X-StopOnRemoval in the + [Unit] section to false. + + + + + Is it a .target unit? If so, + start it unless it sets + RefuseManualStart in the + [Unit] section to true or + X-OnlyManualStart in the + [Unit] section to true. + Also stop the unit again + unless it sets X-StopOnReconfiguration to + false. + + + + + Are the contents of the unit files different? They are compared + by parsing them and comparing their contents. If they are + different but only X-Reload-Triggers in the + [Unit] section is changed, + reload the unit. The NixOS + module system allows setting these triggers with the option + systemd.services.<name>.reloadTriggers. + If the unit files differ in any way, the following actions are + performed: + + + + + .path and .slice units + are ignored. There is no need to restart them since changes + in their values are applied by systemd when systemd is + reloaded. + + + + + .mount units are + reloaded. These mostly + come from the /etc/fstab parser. + + + + + .socket units are currently ignored. This + is to be fixed at a later point. + + + + + The rest of the units (mostly .service + units) are then reloaded + if X-ReloadIfChanged in the + [Service] section is set to + true (exposed via + systemd.services.<name>.reloadIfChanged). + + + + + If the reload flag is not set, some more flags decide if the + unit is skipped. These flags are + X-RestartIfChanged in the + [Service] section (exposed via + systemd.services.<name>.restartIfChanged), + RefuseManualStop in the + [Unit] section, and + X-OnlyManualStart in the + [Unit] section. + + + + + The rest of the behavior is decided whether the unit has + X-StopIfChanged in the + [Service] section set (exposed via + systemd.services.<name>.stopIfChanged). + This is set to true by default and must + be explicitly turned off if not wanted. If the flag is + enabled, the unit is + stopped and then + started. If not, the unit + is restarted. The goal of + the flag is to make sure that the new unit never runs in the + old environment which is still in place before the + activation script is run. + + + + + The last thing that is taken into account is whether the + unit is a service and socket-activated. Due to a bug, this + is currently only done when + X-StopIfChanged is set. If the unit is + socket-activated, the socket is stopped and started, and the + service is stopped and to be started by socket activation. + + + + + +
diff --git a/nixos/doc/manual/from_md/development/what-happens-during-a-system-switch.chapter.xml b/nixos/doc/manual/from_md/development/what-happens-during-a-system-switch.chapter.xml new file mode 100644 index 00000000000..66ba792ddac --- /dev/null +++ b/nixos/doc/manual/from_md/development/what-happens-during-a-system-switch.chapter.xml @@ -0,0 +1,122 @@ + + What happens during a system switch? + + Running nixos-rebuild switch is one of the more + common tasks under NixOS. This chapter explains some of the + internals of this command to make it simpler for new module + developers to configure their units correctly and to make it easier + to understand what is happening and why for curious administrators. + + + nixos-rebuild, like many deployment solutions, + calls switch-to-configuration which resides in a + NixOS system at $out/bin/switch-to-configuration. + The script is called with the action that is to be performed like + switch, test, + boot. There is also the + dry-activate action which does not really perform + the actions but rather prints what it would do if you called it with + test. This feature can be used to check what + service states would be changed if the configuration was switched + to. + + + If the action is switch or + boot, the bootloader is updated first so the + configuration will be the next one to boot. Unless + NIXOS_NO_SYNC is set to 1, + /nix/store is synced to disk. + + + If the action is switch or + test, the currently running system is inspected + and the actions to switch to the new system are calculated. This + process takes two data sources into account: + /etc/fstab and the current systemd status. Mounts + and swaps are read from /etc/fstab and the + corresponding actions are generated. If a new mount is added, for + example, the proper .mount unit is marked to be + started. The current systemd state is inspected, the difference + between the current system and the desired configuration is + calculated and actions are generated to get to this state. There are + a lot of nuances that can be controlled by the units which are + explained here. + + + After calculating what should be done, the actions are carried out. + The order of actions is always the same: + + + + + Stop units (systemctl stop) + + + + + Run activation script ($out/activate) + + + + + See if the activation script requested more units to restart + + + + + Restart systemd if needed + (systemd daemon-reexec) + + + + + Forget about the failed state of units + (systemctl reset-failed) + + + + + Reload systemd (systemctl daemon-reload) + + + + + Reload systemd user instances + (systemctl --user daemon-reload) + + + + + Set up tmpfiles (systemd-tmpfiles --create) + + + + + Reload units (systemctl reload) + + + + + Restart units (systemctl restart) + + + + + Start units (systemctl start) + + + + + Inspect what changed during these actions and print units that + failed and that were newly started + + + + + Most of these actions are either self-explaining but some of them + have to do with our units or the activation script. For this reason, + these topics are explained in the next sections. + + + + -- cgit 1.4.1 From 665344f14839ea286a7aeb329fbf4f44da268ce4 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 2 Aug 2021 21:42:45 +0200 Subject: lib/types: Introduce types.raw for unprocessed values --- lib/tests/modules.sh | 6 +++++ lib/tests/modules/raw.nix | 30 ++++++++++++++++++++++ lib/types.nix | 7 +++++ .../doc/manual/development/option-types.section.md | 11 ++++++++ .../from_md/development/option-types.section.xml | 19 ++++++++++++++ 5 files changed, 73 insertions(+) create mode 100644 lib/tests/modules/raw.nix (limited to 'nixos/doc/manual/development') diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 88d152d3935..a1c592cf4ef 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -293,6 +293,12 @@ checkConfigOutput "{ }" config.submodule.a ./emptyValues.nix checkConfigError 'The option .int.a. is used but not defined' config.int.a ./emptyValues.nix checkConfigError 'The option .nonEmptyList.a. is used but not defined' config.nonEmptyList.a ./emptyValues.nix +## types.raw +checkConfigOutput "{ foo = ; }" config.unprocessedNesting ./raw.nix +checkConfigOutput "10" config.processedToplevel ./raw.nix +checkConfigError "The option .multiple. is defined multiple times" config.multiple ./raw.nix +checkConfigOutput "bar" config.priorities ./raw.nix + cat < + + + types.raw + + + + A type which doesn’t do any checking, merging or nested + evaluation. It accepts a single arbitrary value that is not + recursed into, making it useful for values coming from + outside the module system, such as package sets or arbitrary + data. Options of this type are still evaluated according to + priorities and conditionals, so mkForce, + mkIf and co. still work on the option + value itself, but not for any value nested within it. This + type should only be used when checking, merging and nested + evaluation are not desirable. + + + types.attrs -- cgit 1.4.1 From 6a96ddb67509064c2d445b3fae73d4c4c38c539d Mon Sep 17 00:00:00 2001 From: Minijackson Date: Sat, 10 Apr 2021 16:04:35 +0200 Subject: pkgs-lib: Implement settings format for Elixir --- .../manual/development/settings-options.section.md | 45 +++++ .../development/settings-options.section.xml | 104 +++++++++++ pkgs/pkgs-lib/formats.nix | 207 +++++++++++++++++++++ 3 files changed, 356 insertions(+) (limited to 'nixos/doc/manual/development') diff --git a/nixos/doc/manual/development/settings-options.section.md b/nixos/doc/manual/development/settings-options.section.md index 58a3d8448af..f9bb6ff9cc4 100644 --- a/nixos/doc/manual/development/settings-options.section.md +++ b/nixos/doc/manual/development/settings-options.section.md @@ -66,6 +66,45 @@ have a predefined type and string generator already declared under and returning a set with TOML-specific attributes `type` and `generate` as specified [below](#pkgs-formats-result). +`pkgs.formats.elixirConf { elixir ? pkgs.elixir }` + +: A function taking an attribute set with values + + `elixir` + + : The Elixir package which will be used to format the generated output + + It returns a set with Elixir-Config-specific attributes `type`, `lib`, and + `generate` as specified [below](#pkgs-formats-result). + + The `lib` attribute contains functions to be used in settings, for + generating special Elixir values: + + `mkRaw elixirCode` + + : Outputs the given string as raw Elixir code + + `mkGetEnv { envVariable, fallback ? null }` + + : Makes the configuration fetch an environment variable at runtime + + `mkAtom atom` + + : Outputs the given string as an Elixir atom, instead of the default + Elixir binary string. Note: lowercase atoms still needs to be prefixed + with `:` + + `mkTuple array` + + : Outputs the given array as an Elixir tuple, instead of the default + Elixir list + + `mkMap attrset` + + : Outputs the given attribute set as an Elixir map, instead of the + default Elixir keyword list + + ::: {#pkgs-formats-result} These functions all return an attribute set with these values: ::: @@ -74,6 +113,12 @@ These functions all return an attribute set with these values: : A module system type representing a value of the format +`lib` + +: Utility functions for convenience, or special interactions with the format. + This attribute is optional. It may contain inside a `types` attribute + containing types specific to this format. + `generate` *`filename jsonValue`* : A function that can render a value of the format to a file. Returns diff --git a/nixos/doc/manual/from_md/development/settings-options.section.xml b/nixos/doc/manual/from_md/development/settings-options.section.xml index c9430b77579..746011a2d07 100644 --- a/nixos/doc/manual/from_md/development/settings-options.section.xml +++ b/nixos/doc/manual/from_md/development/settings-options.section.xml @@ -137,6 +137,97 @@ + + + pkgs.formats.elixirConf { elixir ? pkgs.elixir } + + + + A function taking an attribute set with values + + + + + elixir + + + + The Elixir package which will be used to format the + generated output + + + + + + It returns a set with Elixir-Config-specific attributes + type, lib, and + generate as specified + below. + + + The lib attribute contains functions to + be used in settings, for generating special Elixir values: + + + + + mkRaw elixirCode + + + + Outputs the given string as raw Elixir code + + + + + + mkGetEnv { envVariable, fallback ? null } + + + + Makes the configuration fetch an environment variable + at runtime + + + + + + mkAtom atom + + + + Outputs the given string as an Elixir atom, instead of + the default Elixir binary string. Note: lowercase + atoms still needs to be prefixed with + : + + + + + + mkTuple array + + + + Outputs the given array as an Elixir tuple, instead of + the default Elixir list + + + + + + mkMap attrset + + + + Outputs the given attribute set as an Elixir map, + instead of the default Elixir keyword list + + + + + + These functions all return an attribute set with these values: @@ -152,6 +243,19 @@ + + + lib + + + + Utility functions for convenience, or special interactions + with the format. This attribute is optional. It may contain + inside a types attribute containing types + specific to this format. + + + generate diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix index 5e17519d4ce..495a7094f9b 100644 --- a/pkgs/pkgs-lib/formats.nix +++ b/pkgs/pkgs-lib/formats.nix @@ -14,6 +14,15 @@ rec { # The description needs to be overwritten for recursive types type = ...; + # Utility functions for convenience, or special interactions with the + # format (optional) + lib = { + exampleFunction = ... + # Types specific to the format (optional) + types = { ... }; + ... + }; + # generate :: Name -> Value -> Path # A function for generating a file with a value of such a type generate = ...; @@ -147,4 +156,202 @@ rec { ''; }; + + /* For configurations of Elixir project, like config.exs or runtime.exs + + Most Elixir project are configured using the [Config] Elixir DSL + + Since Elixir has more types than Nix, we need a way to map Nix types to + more than 1 Elixir type. To that end, this format provides its own library, + and its own set of types. + + To be more detailed, a Nix attribute set could correspond in Elixir to a + [Keyword list] (the more common type), or it could correspond to a [Map]. + + A Nix string could correspond in Elixir to a [String] (also called + "binary"), an [Atom], or a list of chars (usually discouraged). + + A Nix array could correspond in Elixir to a [List] or a [Tuple]. + + Some more types exists, like records, regexes, but since they are less used, + we can leave the `mkRaw` function as an escape hatch. + + For more information on how to use this format in modules, please refer to + the Elixir section of the Nixos documentation. + + TODO: special Elixir values doesn't show up nicely in the documentation + + [Config]: + [Keyword list]: + [Map]: + [String]: + [Atom]: + [List]: + [Tuple]: + */ + elixirConf = { elixir ? pkgs.elixir }: + with lib; let + toElixir = value: with builtins; + if value == null then "nil" else + if value == true then "true" else + if value == false then "false" else + if isInt value || isFloat value then toString value else + if isString value then string value else + if isAttrs value then attrs value else + if isList value then list value else + abort "formats.elixirConf: should never happen (value = ${value})"; + + escapeElixir = escape [ "\\" "#" "\"" ]; + string = value: "\"${escapeElixir value}\""; + + attrs = set: + if set ? _elixirType then specialType set + else + let + toKeyword = name: value: "${name}: ${toElixir value}"; + keywordList = concatStringsSep ", " (mapAttrsToList toKeyword set); + in + "[" + keywordList + "]"; + + listContent = values: concatStringsSep ", " (map toElixir values); + + list = values: "[" + (listContent values) + "]"; + + specialType = { value, _elixirType }: + if _elixirType == "raw" then value else + if _elixirType == "atom" then value else + if _elixirType == "map" then elixirMap value else + if _elixirType == "tuple" then tuple value else + abort "formats.elixirConf: should never happen (_elixirType = ${_elixirType})"; + + elixirMap = set: + let + toEntry = name: value: "${toElixir name} => ${toElixir value}"; + entries = concatStringsSep ", " (mapAttrsToList toEntry set); + in + "%{${entries}}"; + + tuple = values: "{${listContent values}}"; + + toConf = values: + let + keyConfig = rootKey: key: value: + "config ${rootKey}, ${key}, ${toElixir value}"; + keyConfigs = rootKey: values: mapAttrsToList (keyConfig rootKey) values; + rootConfigs = flatten (mapAttrsToList keyConfigs values); + in + '' + import Config + + ${concatStringsSep "\n" rootConfigs} + ''; + in + { + type = with lib.types; let + valueType = nullOr + (oneOf [ + bool + int + float + str + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "Elixir value"; + }; + in + attrsOf (attrsOf (valueType)); + + lib = + let + mkRaw = value: { + inherit value; + _elixirType = "raw"; + }; + + in + { + inherit mkRaw; + + /* Fetch an environment variable at runtime, with optional fallback + */ + mkGetEnv = { envVariable, fallback ? null }: + mkRaw "System.get_env(${toElixir envVariable}, ${toElixir fallback})"; + + /* Make an Elixir atom. + + Note: lowercase atoms still need to be prefixed by ':' + */ + mkAtom = value: { + inherit value; + _elixirType = "atom"; + }; + + /* Make an Elixir tuple out of a list. + */ + mkTuple = value: { + inherit value; + _elixirType = "tuple"; + }; + + /* Make an Elixir map out of an attribute set. + */ + mkMap = value: { + inherit value; + _elixirType = "map"; + }; + + /* Contains Elixir types. Every type it exports can also be replaced + by raw Elixir code (i.e. every type is `either type rawElixir`). + + It also reexports standard types, wrapping them so that they can + also be raw Elixir. + */ + types = with lib.types; let + isElixirType = type: x: (x._elixirType or "") == type; + + rawElixir = mkOptionType { + name = "rawElixir"; + description = "raw elixir"; + check = isElixirType "raw"; + }; + + elixirOr = other: either other rawElixir; + in + { + inherit rawElixir elixirOr; + + atom = elixirOr (mkOptionType { + name = "elixirAtom"; + description = "elixir atom"; + check = isElixirType "atom"; + }); + + tuple = elixirOr (mkOptionType { + name = "elixirTuple"; + description = "elixir tuple"; + check = isElixirType "tuple"; + }); + + map = elixirOr (mkOptionType { + name = "elixirMap"; + description = "elixir map"; + check = isElixirType "map"; + }); + # Wrap standard types, since anything in the Elixir configuration + # can be raw Elixir + } // lib.mapAttrs (_name: type: elixirOr type) lib.types; + }; + + generate = name: value: pkgs.runCommandNoCC name + { + value = toConf value; + passAsFile = [ "value" ]; + nativeBuildInputs = [ elixir ]; + } '' + cp "$valuePath" "$out" + mix format "$out" + ''; + }; + } -- cgit 1.4.1 From 0c766a100e416611807a184ee35a0edbd11b15a4 Mon Sep 17 00:00:00 2001 From: Janne Heß Date: Wed, 16 Jun 2021 12:27:47 +0200 Subject: lib/options: Throw error for options without a type Makes all options rendered in the manual throw an error if they don't have a type specified. This is a follow-up to #76184 Co-Authored-By: Silvan Mosberger --- lib/options.nix | 2 +- nixos/doc/manual/development/option-declarations.section.md | 7 ++++--- .../manual/from_md/development/option-declarations.section.xml | 8 +++++--- nixos/lib/make-options-doc/mergeJSON.py | 9 ++++++++- 4 files changed, 18 insertions(+), 8 deletions(-) (limited to 'nixos/doc/manual/development') diff --git a/lib/options.nix b/lib/options.nix index 627aac24d2f..9efc1249e58 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -231,7 +231,7 @@ rec { then true else opt.visible or true; readOnly = opt.readOnly or false; - type = opt.type.description or null; + type = opt.type.description or "unspecified"; } // optionalAttrs (opt ? example) { example = scrubOptionValue opt.example; } // optionalAttrs (opt ? default) { default = scrubOptionValue opt.default; } diff --git a/nixos/doc/manual/development/option-declarations.section.md b/nixos/doc/manual/development/option-declarations.section.md index fff06e1ea5b..cb5043b528f 100644 --- a/nixos/doc/manual/development/option-declarations.section.md +++ b/nixos/doc/manual/development/option-declarations.section.md @@ -27,9 +27,10 @@ The function `mkOption` accepts the following arguments. `type` -: The type of the option (see [](#sec-option-types)). It may be - omitted, but that's not advisable since it may lead to errors that - are hard to diagnose. +: The type of the option (see [](#sec-option-types)). This + argument is mandatory for nixpkgs modules. Setting this is highly + recommended for the sake of documentation and type checking. In case it is + not set, a fallback type with unspecified behavior is used. `default` diff --git a/nixos/doc/manual/from_md/development/option-declarations.section.xml b/nixos/doc/manual/from_md/development/option-declarations.section.xml index 0eeffae628e..c7b62192158 100644 --- a/nixos/doc/manual/from_md/development/option-declarations.section.xml +++ b/nixos/doc/manual/from_md/development/option-declarations.section.xml @@ -38,9 +38,11 @@ options = { The type of the option (see - ). It may be omitted, but - that’s not advisable since it may lead to errors that are hard - to diagnose. + ). This argument is + mandatory for nixpkgs modules. Setting this is highly + recommended for the sake of documentation and type checking. + In case it is not set, a fallback type with unspecified + behavior is used. diff --git a/nixos/lib/make-options-doc/mergeJSON.py b/nixos/lib/make-options-doc/mergeJSON.py index 029787a3158..8e2ea322dc8 100644 --- a/nixos/lib/make-options-doc/mergeJSON.py +++ b/nixos/lib/make-options-doc/mergeJSON.py @@ -66,14 +66,21 @@ for (k, v) in overrides.items(): elif ov is not None or cur.get(ok, None) is None: cur[ok] = ov +severity = "error" if warningsAreErrors else "warning" + # check that every option has a description hasWarnings = False for (k, v) in options.items(): if v.value.get('description', None) is None: - severity = "error" if warningsAreErrors else "warning" hasWarnings = True print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr) v.value['description'] = "This option has no description." + if v.value.get('type', "unspecified") == "unspecified": + hasWarnings = True + print( + f"\x1b[1;31m{severity}: option {v.name} has no type. Please specify a valid type, see " + + "https://nixos.org/manual/nixos/stable/index.html#sec-option-types\x1b[0m", file=sys.stderr) + if hasWarnings and warningsAreErrors: print( "\x1b[1;31m" + -- cgit 1.4.1 From 5cbeddfde486ca5524baeaf3da6e8944075cf463 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 8 Dec 2021 19:02:29 +0100 Subject: lib.types: Introduce `types.optionType` This type correctly merges multiple option types together while also annotating them with file information. In a future commit this will be used for `_module.freeformType` --- lib/tests/modules.sh | 7 +++++ lib/tests/modules/optionTypeFile.nix | 28 +++++++++++++++++++ lib/tests/modules/optionTypeMerging.nix | 27 +++++++++++++++++++ lib/types.nix | 31 +++++++++++++++++++++- .../doc/manual/development/option-types.section.md | 7 +++++ .../from_md/development/option-types.section.xml | 14 ++++++++++ 6 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 lib/tests/modules/optionTypeFile.nix create mode 100644 lib/tests/modules/optionTypeMerging.nix (limited to 'nixos/doc/manual/development') diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index a1c592cf4ef..d11f32e5996 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -299,6 +299,13 @@ checkConfigOutput "10" config.processedToplevel ./raw.nix checkConfigError "The option .multiple. is defined multiple times" config.multiple ./raw.nix checkConfigOutput "bar" config.priorities ./raw.nix +# Test that types.optionType merges types correctly +checkConfigOutput '^10$' config.theOption.int ./optionTypeMerging.nix +checkConfigOutput '^"hello"$' config.theOption.str ./optionTypeMerging.nix + +# Test that types.optionType correctly annotates option locations +checkConfigError 'The option .theOption.nested. in .other.nix. is already declared in .optionTypeFile.nix.' config.theOption.nested ./optionTypeFile.nix + cat < + + + types.optionType + + + + The type of an option’s type. Its merging operation ensures + that nested options have the correct file location + annotated, and that if possible, multiple option definitions + are correctly merged together. The main use case is as the + type of the _module.freeformType option. + + + types.attrs -- cgit 1.4.1 From 1def557525157481da42fbd153a00729cce32d87 Mon Sep 17 00:00:00 2001 From: Janne Heß Date: Fri, 25 Feb 2022 14:32:44 +0100 Subject: nixos/switch-to-configuration: Document and test socket-activated services --- .../manual/development/unit-handling.section.md | 15 ++- .../from_md/development/unit-handling.section.xml | 22 ++-- .../system/activation/switch-to-configuration.pl | 9 +- nixos/tests/switch-test.nix | 143 ++++++++++++++++++++- 4 files changed, 169 insertions(+), 20 deletions(-) (limited to 'nixos/doc/manual/development') diff --git a/nixos/doc/manual/development/unit-handling.section.md b/nixos/doc/manual/development/unit-handling.section.md index d477f2c860f..bd4fe9e670f 100644 --- a/nixos/doc/manual/development/unit-handling.section.md +++ b/nixos/doc/manual/development/unit-handling.section.md @@ -41,17 +41,18 @@ checks: `RefuseManualStop` in the `[Unit]` section, and `X-OnlyManualStart` in the `[Unit]` section. - - The rest of the behavior is decided whether the unit has `X-StopIfChanged` - in the `[Service]` section set (exposed via + - Further behavior depends on the unit having `X-StopIfChanged` in the + `[Service]` section set to `true` (exposed via [systemd.services.\.stopIfChanged](#opt-systemd.services)). This is set to `true` by default and must be explicitly turned off if not wanted. If the flag is enabled, the unit is **stop**ped and then **start**ed. If not, the unit is **restart**ed. The goal of the flag is to make sure that the new unit never runs in the old environment which is still in place - before the activation script is run. + before the activation script is run. This behavior is different when the + service is socket-activated, as outlined in the following steps. - The last thing that is taken into account is whether the unit is a service - and socket-activated. Due to a bug, this is currently only done when - `X-StopIfChanged` is set. If the unit is socket-activated, the socket is - stopped and started, and the service is stopped and to be started by socket - activation. + and socket-activated. If `X-StopIfChanged` is **not** set, the service + is **restart**ed with the others. If it is set, both the service and the + socket are **stop**ped and the socket is **start**ed, leaving socket + activation to start the service when it's needed. diff --git a/nixos/doc/manual/from_md/development/unit-handling.section.xml b/nixos/doc/manual/from_md/development/unit-handling.section.xml index a6a654042f6..57c4754c001 100644 --- a/nixos/doc/manual/from_md/development/unit-handling.section.xml +++ b/nixos/doc/manual/from_md/development/unit-handling.section.xml @@ -88,9 +88,10 @@ - The rest of the behavior is decided whether the unit has + Further behavior depends on the unit having X-StopIfChanged in the - [Service] section set (exposed via + [Service] section set to + true (exposed via systemd.services.<name>.stopIfChanged). This is set to true by default and must be explicitly turned off if not wanted. If the flag is @@ -100,17 +101,22 @@ is restarted. The goal of the flag is to make sure that the new unit never runs in the old environment which is still in place before the - activation script is run. + activation script is run. This behavior is different when + the service is socket-activated, as outlined in the + following steps. The last thing that is taken into account is whether the - unit is a service and socket-activated. Due to a bug, this - is currently only done when - X-StopIfChanged is set. If the unit is - socket-activated, the socket is stopped and started, and the - service is stopped and to be started by socket activation. + unit is a service and socket-activated. If + X-StopIfChanged is + not set, the service is + restarted with the + others. If it is set, both the service and the socket are + stopped and the socket is + started, leaving socket + activation to start the service when it’s needed. diff --git a/nixos/modules/system/activation/switch-to-configuration.pl b/nixos/modules/system/activation/switch-to-configuration.pl index a8fe14c58f0..3a5ffe822ed 100644 --- a/nixos/modules/system/activation/switch-to-configuration.pl +++ b/nixos/modules/system/activation/switch-to-configuration.pl @@ -307,6 +307,7 @@ sub handleModifiedUnit { # seem to get applied on daemon-reload. } elsif ($unit =~ /\.mount$/) { # Reload the changed mount unit to force a remount. + # FIXME: only reload when Options= changed, restart otherwise $unitsToReload->{$unit} = 1; recordUnit($reloadListFile, $unit); } elsif ($unit =~ /\.socket$/) { @@ -339,7 +340,7 @@ sub handleModifiedUnit { # If this unit is socket-activated, then stop the # socket unit(s) as well, and restart the # socket(s) instead of the service. - my $socketActivated = 0; + my $socket_activated = 0; if ($unit =~ /\.service$/) { my @sockets = split(/ /, join(" ", @{$unitInfo{Service}{Sockets} // []})); if (scalar @sockets == 0) { @@ -347,13 +348,15 @@ sub handleModifiedUnit { } foreach my $socket (@sockets) { if (defined $activePrev->{$socket}) { + # We can now be sure this is a socket-activate unit + $unitsToStop->{$socket} = 1; # Only restart sockets that actually # exist in new configuration: if (-e "$out/etc/systemd/system/$socket") { $unitsToStart->{$socket} = 1; recordUnit($startListFile, $socket); - $socketActivated = 1; + $socket_activated = 1; } # Remove from units to reload so we don't restart and reload if ($unitsToReload->{$unit}) { @@ -368,7 +371,7 @@ sub handleModifiedUnit { # that this unit needs to be started below. # We write this to a file to ensure that the # service gets restarted if we're interrupted. - if (!$socketActivated) { + if (!$socket_activated) { $unitsToStart->{$unit} = 1; recordUnit($startListFile, $unit); } diff --git a/nixos/tests/switch-test.nix b/nixos/tests/switch-test.nix index b429babce83..090bbe298a4 100644 --- a/nixos/tests/switch-test.nix +++ b/nixos/tests/switch-test.nix @@ -1,6 +1,46 @@ # Test configuration switching. -import ./make-test-python.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : let + + # Simple service that can either be socket-activated or that will + # listen on port 1234 if not socket-activated. + # A connection to the socket causes 'hello' to be written to the client. + socketTest = pkgs.writeScript "socket-test.py" /* python */ '' + #!${pkgs.python3}/bin/python3 + + from socketserver import TCPServer, StreamRequestHandler + import socket + import os + + + class Handler(StreamRequestHandler): + def handle(self): + self.wfile.write("hello".encode("utf-8")) + + + class Server(TCPServer): + def __init__(self, server_address, handler_cls): + listenFds = os.getenv('LISTEN_FDS') + if listenFds is None or int(listenFds) < 1: + print(f'Binding to {server_address}') + TCPServer.__init__( + self, server_address, handler_cls, bind_and_activate=True) + else: + TCPServer.__init__( + self, server_address, handler_cls, bind_and_activate=False) + # Override socket + print(f'Got activated by {os.getenv("LISTEN_FDNAMES")} ' + f'with {listenFds} FDs') + self.socket = socket.fromfd(3, self.address_family, + self.socket_type) + + + if __name__ == "__main__": + server = Server(("localhost", 1234), Handler) + server.serve_forever() + ''; + +in { name = "switch-test"; meta = with pkgs.lib.maintainers; { maintainers = [ gleber das_j ]; @@ -8,6 +48,7 @@ import ./make-test-python.nix ({ pkgs, ...} : { nodes = { machine = { pkgs, lib, ... }: { + environment.systemPackages = [ pkgs.socat ]; # for the socket activation stuff users.mutableUsers = false; specialisation = rec { @@ -231,6 +272,40 @@ import ./make-test-python.nix ({ pkgs, ...} : { systemd.services.reload-triggers-and-restart.serviceConfig.X-Modified = "test"; }; + simple-socket.configuration = { + systemd.services.socket-activated = { + description = "A socket-activated service"; + stopIfChanged = lib.mkDefault false; + serviceConfig = { + ExecStart = socketTest; + ExecReload = "${pkgs.coreutils}/bin/true"; + }; + }; + systemd.sockets.socket-activated = { + wantedBy = [ "sockets.target" ]; + listenStreams = [ "/run/test.sock" ]; + socketConfig.SocketMode = lib.mkDefault "0777"; + }; + }; + + simple-socket-service-modified.configuration = { + imports = [ simple-socket.configuration ]; + systemd.services.socket-activated.serviceConfig.X-Test = "test"; + }; + + simple-socket-stop-if-changed.configuration = { + imports = [ simple-socket.configuration ]; + systemd.services.socket-activated.stopIfChanged = true; + }; + + simple-socket-stop-if-changed-and-reloadtrigger.configuration = { + imports = [ simple-socket.configuration ]; + systemd.services.socket-activated = { + stopIfChanged = true; + reloadTriggers = [ "test" ]; + }; + }; + mount.configuration = { systemd.mounts = [ { @@ -676,7 +751,71 @@ import ./make-test-python.nix ({ pkgs, ...} : { assert_contains(out, "would reload the following units: reload-triggers.service, simple-reload-service.service\n") assert_contains(out, "would restart the following units: reload-triggers-and-restart-by-as.service, reload-triggers-and-restart.service, simple-restart-service.service, simple-service.service\n") assert_lacks(out, "\nwould start the following units:") - assert_lacks(out, "as well:") + + with subtest("socket-activated services"): + # Socket-activated services don't get started, just the socket + machine.fail("[ -S /run/test.sock ]") + out = switch_to_specialisation("${machine}", "simple-socket") + # assert_lacks(out, "stopping the following units:") nobody cares + assert_lacks(out, "NOT restarting the following changed units:") + assert_lacks(out, "reloading the following units:") + assert_lacks(out, "\nrestarting the following units:") + assert_lacks(out, "\nstarting the following units:") + assert_contains(out, "the following new units were started: socket-activated.socket\n") + machine.succeed("[ -S /run/test.sock ]") + + # Changing a non-activated service does nothing + out = switch_to_specialisation("${machine}", "simple-socket-service-modified") + assert_lacks(out, "stopping the following units:") + assert_lacks(out, "NOT restarting the following changed units:") + assert_lacks(out, "reloading the following units:") + assert_lacks(out, "\nrestarting the following units:") + assert_lacks(out, "\nstarting the following units:") + assert_lacks(out, "the following new units were started:") + machine.succeed("[ -S /run/test.sock ]") + # The unit is properly activated when the socket is accessed + if machine.succeed("socat - UNIX-CONNECT:/run/test.sock") != "hello": + raise Exception("Socket was not properly activated") # idk how that would happen tbh + + # Changing an activated service with stopIfChanged=false restarts the service + out = switch_to_specialisation("${machine}", "simple-socket") + assert_lacks(out, "stopping the following units:") + assert_lacks(out, "NOT restarting the following changed units:") + assert_lacks(out, "reloading the following units:") + assert_contains(out, "\nrestarting the following units: socket-activated.service\n") + assert_lacks(out, "\nstarting the following units:") + assert_lacks(out, "the following new units were started:") + machine.succeed("[ -S /run/test.sock ]") + # Socket-activation of the unit still works + if machine.succeed("socat - UNIX-CONNECT:/run/test.sock") != "hello": + raise Exception("Socket was not properly activated after the service was restarted") + + # Changing an activated service with stopIfChanged=true stops the service and + # socket and starts the socket + out = switch_to_specialisation("${machine}", "simple-socket-stop-if-changed") + assert_contains(out, "stopping the following units: socket-activated.service, socket-activated.socket\n") + assert_lacks(out, "NOT restarting the following changed units:") + assert_lacks(out, "reloading the following units:") + assert_lacks(out, "\nrestarting the following units:") + assert_contains(out, "\nstarting the following units: socket-activated.socket\n") + assert_lacks(out, "the following new units were started:") + machine.succeed("[ -S /run/test.sock ]") + # Socket-activation of the unit still works + if machine.succeed("socat - UNIX-CONNECT:/run/test.sock") != "hello": + raise Exception("Socket was not properly activated after the service was restarted") + + # Changing a reload trigger of a socket-activated unit only reloads it + out = switch_to_specialisation("${machine}", "simple-socket-stop-if-changed-and-reloadtrigger") + assert_lacks(out, "stopping the following units:") + assert_lacks(out, "NOT restarting the following changed units:") + assert_contains(out, "reloading the following units: socket-activated.service\n") + assert_lacks(out, "\nrestarting the following units:") + assert_lacks(out, "\nstarting the following units: socket-activated.socket") + assert_lacks(out, "the following new units were started:") + machine.succeed("[ -S /run/test.sock ]") + # Socket-activation of the unit still works + if machine.succeed("socat - UNIX-CONNECT:/run/test.sock") != "hello": + raise Exception("Socket was not properly activated after the service was restarted") with subtest("mounts"): switch_to_specialisation("${machine}", "mount") -- cgit 1.4.1 From f386c42a48397d232869e03f123e2bb5f8bfd3d8 Mon Sep 17 00:00:00 2001 From: Alexandru Scvortov Date: Fri, 4 Mar 2022 20:08:09 +0000 Subject: nixos/doc: improve wording in "Options Types" and "Option Declarations" --- .../development/option-declarations.section.md | 26 +++++++++---------- .../doc/manual/development/option-types.section.md | 6 ++--- .../development/option-declarations.section.xml | 29 +++++++++++----------- .../from_md/development/option-types.section.xml | 8 +++--- 4 files changed, 35 insertions(+), 34 deletions(-) (limited to 'nixos/doc/manual/development') diff --git a/nixos/doc/manual/development/option-declarations.section.md b/nixos/doc/manual/development/option-declarations.section.md index fff06e1ea5b..819fc6d891f 100644 --- a/nixos/doc/manual/development/option-declarations.section.md +++ b/nixos/doc/manual/development/option-declarations.section.md @@ -145,26 +145,26 @@ As an example, we will take the case of display managers. There is a central display manager module for generic display manager options and a module file per display manager backend (sddm, gdm \...). -There are two approach to this module structure: +There are two approaches we could take with this module structure: -- Managing the display managers independently by adding an enable +- Configuring the display managers independently by adding an enable option to every display manager module backend. (NixOS) -- Managing the display managers in the central module by adding an - option to select which display manager backend to use. +- Configuring the display managers in the central module by adding + an option to select which display manager backend to use. Both approaches have problems. Making backends independent can quickly become hard to manage. For -display managers, there can be only one enabled at a time, but the type -system can not enforce this restriction as there is no relation between -each backend `enable` option. As a result, this restriction has to be -done explicitely by adding assertions in each display manager backend -module. +display managers, there can only be one enabled at a time, but the +type system cannot enforce this restriction as there is no relation +between each backend's `enable` option. As a result, this restriction +has to be done explicitly by adding assertions in each display manager +backend module. -On the other hand, managing the display managers backends in the central -module will require to change the central module option every time a new -backend is added or removed. +On the other hand, managing the display manager backends in the +central module will require changing the central module option every +time a new backend is added or removed. By using extensible option types, it is possible to create a placeholder option in the central module @@ -175,7 +175,7 @@ and to extend it in each backend module As a result, `displayManager.enable` option values can be added without changing the main service module file and the type system automatically -enforce that there can only be a single display manager enabled. +enforces that there can only be a single display manager enabled. ::: {#ex-option-declaration-eot-service .example} ::: {.title} diff --git a/nixos/doc/manual/development/option-types.section.md b/nixos/doc/manual/development/option-types.section.md index 071e7751eb6..c34ac0367c4 100644 --- a/nixos/doc/manual/development/option-types.section.md +++ b/nixos/doc/manual/development/option-types.section.md @@ -16,9 +16,9 @@ merging is handled. `types.path` -: A filesystem path, defined as anything that when coerced to a string - starts with a slash. Even if derivations can be considered as path, - the more specific `types.package` should be preferred. +: A filesystem path is anything that starts with a slash when + coerced to a string. Even if derivations can be considered as + paths, the more specific `types.package` should be preferred. `types.package` diff --git a/nixos/doc/manual/from_md/development/option-declarations.section.xml b/nixos/doc/manual/from_md/development/option-declarations.section.xml index 0eeffae628e..554705e2e42 100644 --- a/nixos/doc/manual/from_md/development/option-declarations.section.xml +++ b/nixos/doc/manual/from_md/development/option-declarations.section.xml @@ -215,21 +215,22 @@ lib.mkOption { manager backend (sddm, gdm ...). - There are two approach to this module structure: + There are two approaches we could take with this module + structure: - Managing the display managers independently by adding an - enable option to every display manager module backend. - (NixOS) + Configuring the display managers independently by adding + an enable option to every display manager module + backend. (NixOS) - Managing the display managers in the central module by - adding an option to select which display manager backend - to use. + Configuring the display managers in the central module + by adding an option to select which display manager + backend to use. @@ -238,16 +239,16 @@ lib.mkOption { Making backends independent can quickly become hard to - manage. For display managers, there can be only one enabled - at a time, but the type system can not enforce this - restriction as there is no relation between each backend + manage. For display managers, there can only be one enabled + at a time, but the type system cannot enforce this + restriction as there is no relation between each backend’s enable option. As a result, this - restriction has to be done explicitely by adding assertions + restriction has to be done explicitly by adding assertions in each display manager backend module. - On the other hand, managing the display managers backends in - the central module will require to change the central module + On the other hand, managing the display manager backends in + the central module will require changing the central module option every time a new backend is added or removed. @@ -268,7 +269,7 @@ lib.mkOption { As a result, displayManager.enable option values can be added without changing the main service module - file and the type system automatically enforce that there + file and the type system automatically enforces that there can only be a single display manager enabled. diff --git a/nixos/doc/manual/from_md/development/option-types.section.xml b/nixos/doc/manual/from_md/development/option-types.section.xml index 50a3da0ef5b..e16453df51e 100644 --- a/nixos/doc/manual/from_md/development/option-types.section.xml +++ b/nixos/doc/manual/from_md/development/option-types.section.xml @@ -30,10 +30,10 @@ - A filesystem path, defined as anything that when coerced to - a string starts with a slash. Even if derivations can be - considered as path, the more specific - types.package should be preferred. + A filesystem path is anything that starts with a slash when + coerced to a string. Even if derivations can be considered + as paths, the more specific types.package + should be preferred. -- cgit 1.4.1 From 9c2266c03171dcf492b6accdb0cde0cb28e156b5 Mon Sep 17 00:00:00 2001 From: Naïm Favier Date: Wed, 9 Mar 2022 13:14:22 +0100 Subject: lib.types.package: only call toDerivation when necessary The current logic assumes that everything that isn't a derivation is a store path, but it can also be something that's *coercible* to a store path, like a flake input. Unnecessary uses of `lib.toDerivation` result in errors in pure evaluation mode when `builtins.storePath` is disabled. Also document what a `package` is. --- lib/types.nix | 12 ++++++++++-- nixos/doc/manual/development/option-types.section.md | 3 ++- .../doc/manual/from_md/development/option-types.section.xml | 4 +++- 3 files changed, 15 insertions(+), 4 deletions(-) (limited to 'nixos/doc/manual/development') diff --git a/lib/types.nix b/lib/types.nix index 3fcac9c31b3..bf18866e55e 100644 --- a/lib/types.nix +++ b/lib/types.nix @@ -368,13 +368,21 @@ rec { emptyValue = { value = {}; }; }; - # derivation is a reserved keyword. + # A package is a top-level store path (/nix/store/hash-name). This includes: + # - derivations + # - more generally, attribute sets with an `outPath` or `__toString` attribute + # pointing to a store path, e.g. flake inputs + # - strings with context, e.g. "${pkgs.foo}" or (toString pkgs.foo) + # - hardcoded store path literals (/nix/store/hash-foo) or strings without context + # ("/nix/store/hash-foo"). These get a context added to them using builtins.storePath. package = mkOptionType { name = "package"; check = x: isDerivation x || isStorePath x; merge = loc: defs: let res = mergeOneOption loc defs; - in if isDerivation res then res else toDerivation res; + in if builtins.isPath res || (builtins.isString res && ! builtins.hasContext res) + then toDerivation res + else res; }; shellPackage = package // { diff --git a/nixos/doc/manual/development/option-types.section.md b/nixos/doc/manual/development/option-types.section.md index c34ac0367c4..00f1d85bdb6 100644 --- a/nixos/doc/manual/development/option-types.section.md +++ b/nixos/doc/manual/development/option-types.section.md @@ -22,7 +22,8 @@ merging is handled. `types.package` -: A derivation or a store path. +: A top-level store path. This can be an attribute set pointing + to a store path, like a derivation or a flake input. `types.anything` diff --git a/nixos/doc/manual/from_md/development/option-types.section.xml b/nixos/doc/manual/from_md/development/option-types.section.xml index e16453df51e..44472929270 100644 --- a/nixos/doc/manual/from_md/development/option-types.section.xml +++ b/nixos/doc/manual/from_md/development/option-types.section.xml @@ -43,7 +43,9 @@ - A derivation or a store path. + A top-level store path. This can be an attribute set + pointing to a store path, like a derivation or a flake + input. -- cgit 1.4.1 From c96180c53fcd4f36a7163c3e59a2e6bcd9233f06 Mon Sep 17 00:00:00 2001 From: Janne Heß Date: Sun, 6 Mar 2022 19:22:04 +0100 Subject: nixos/switch-to-configuration: Ignore some unit keys Some unit keys don't need to restart the service to make them effective. Reduce the amount of service restarts by ignoring these keys --- .../manual/development/unit-handling.section.md | 3 +- .../from_md/development/unit-handling.section.xml | 5 +- .../system/activation/switch-to-configuration.pl | 67 +++++++++++++++++----- nixos/tests/switch-test.nix | 14 +++++ 4 files changed, 71 insertions(+), 18 deletions(-) (limited to 'nixos/doc/manual/development') diff --git a/nixos/doc/manual/development/unit-handling.section.md b/nixos/doc/manual/development/unit-handling.section.md index bd4fe9e670f..c51704ad0da 100644 --- a/nixos/doc/manual/development/unit-handling.section.md +++ b/nixos/doc/manual/development/unit-handling.section.md @@ -17,7 +17,8 @@ checks: them and comparing their contents. If they are different but only `X-Reload-Triggers` in the `[Unit]` section is changed, **reload** the unit. The NixOS module system allows setting these triggers with the option - [systemd.services.\.reloadTriggers](#opt-systemd.services). If the + [systemd.services.\.reloadTriggers](#opt-systemd.services). There are + some additional keys in the `[Unit]` section that are ignored as well. If the unit files differ in any way, the following actions are performed: - `.path` and `.slice` units are ignored. There is no need to restart them diff --git a/nixos/doc/manual/from_md/development/unit-handling.section.xml b/nixos/doc/manual/from_md/development/unit-handling.section.xml index 57c4754c001..642cc5cccc7 100644 --- a/nixos/doc/manual/from_md/development/unit-handling.section.xml +++ b/nixos/doc/manual/from_md/development/unit-handling.section.xml @@ -38,8 +38,9 @@ reload the unit. The NixOS module system allows setting these triggers with the option systemd.services.<name>.reloadTriggers. - If the unit files differ in any way, the following actions are - performed: + There are some additional keys in the [Unit] + section that are ignored as well. If the unit files differ in + any way, the following actions are performed: diff --git a/nixos/modules/system/activation/switch-to-configuration.pl b/nixos/modules/system/activation/switch-to-configuration.pl index a1653d451fe..ca45fc9c286 100644 --- a/nixos/modules/system/activation/switch-to-configuration.pl +++ b/nixos/modules/system/activation/switch-to-configuration.pl @@ -226,10 +226,20 @@ sub unrecord_unit { sub compare_units { my ($old_unit, $new_unit) = @_; my $ret = 0; + # Keys to ignore in the [Unit] section + my %unit_section_ignores = map { $_ => 1 } qw( + X-Reload-Triggers + Description Documentation + OnFailure OnSuccess OnFailureJobMode + IgnoreOnIsolate StopWhenUnneeded + RefuseManualStart RefuseManualStop + AllowIsolate CollectMode + SourcePath + ); my $comp_array = sub { my ($a, $b) = @_; - return join("\0", @{$a}) eq join("\0", @{$b}); + return join("\0", @{$a}) eq join "\0", @{$b}; }; # Comparison hash for the sections @@ -238,6 +248,18 @@ sub compare_units { foreach my $section_name (keys %{$old_unit}) { # Missing section in the new unit? if (not exists $section_cmp{$section_name}) { + # If the [Unit] section was removed, make sure that only keys + # were in it that are ignored + if ($section_name eq 'Unit') { + foreach my $ini_key (keys %{$old_unit->{'Unit'}}) { + if (not defined $unit_section_ignores{$ini_key}) { + return 1; + } + } + next; # check the next section + } else { + return 1; + } if ($section_name eq 'Unit' and %{$old_unit->{'Unit'}} == 1 and defined(%{$old_unit->{'Unit'}}{'X-Reload-Triggers'})) { # If a new [Unit] section was removed that only contained X-Reload-Triggers, # do nothing. @@ -255,8 +277,8 @@ sub compare_units { my @old_value = @{$old_unit->{$section_name}{$ini_key}}; # If the key is missing in the new unit, they are different... if (not $new_unit->{$section_name}{$ini_key}) { - # ... unless the key that is now missing was the reload trigger - if ($section_name eq 'Unit' and $ini_key eq 'X-Reload-Triggers') { + # ... unless the key that is now missing is one of the ignored keys + if ($section_name eq 'Unit' and defined $unit_section_ignores{$ini_key}) { next; } return 1; @@ -264,19 +286,30 @@ sub compare_units { my @new_value = @{$new_unit->{$section_name}{$ini_key}}; # If the contents are different, the units are different if (not $comp_array->(\@old_value, \@new_value)) { - # Check if only the reload triggers changed - if ($section_name eq 'Unit' and $ini_key eq 'X-Reload-Triggers') { - $ret = 2; - } else { - return 1; + # Check if only the reload triggers changed or one of the ignored keys + if ($section_name eq 'Unit') { + if ($ini_key eq 'X-Reload-Triggers') { + $ret = 2; + next; + } elsif (defined $unit_section_ignores{$ini_key}) { + next; + } } + return 1; } } # A key was introduced that was missing in the old unit if (%ini_cmp) { - if ($section_name eq 'Unit' and %ini_cmp == 1 and defined($ini_cmp{'X-Reload-Triggers'})) { - # If the newly introduced key was the reload triggers, reload the unit - $ret = 2; + if ($section_name eq 'Unit') { + foreach my $ini_key (keys %ini_cmp) { + if ($ini_key eq 'X-Reload-Triggers') { + $ret = 2; + } elsif (defined $unit_section_ignores{$ini_key}) { + next; + } else { + return 1; + } + } } else { return 1; } @@ -284,10 +317,14 @@ sub compare_units { } # A section was introduced that was missing in the old unit if (%section_cmp) { - if (%section_cmp == 1 and defined($section_cmp{'Unit'}) and %{$new_unit->{'Unit'}} == 1 and defined(%{$new_unit->{'Unit'}}{'X-Reload-Triggers'})) { - # If a new [Unit] section was introduced that only contains X-Reload-Triggers, - # reload instead of restarting - $ret = 2; + if (%section_cmp == 1 and defined $section_cmp{'Unit'}) { + foreach my $ini_key (keys %{$new_unit->{'Unit'}}) { + if (not defined $unit_section_ignores{$ini_key}) { + return 1; + } elsif ($ini_key eq 'X-Reload-Triggers') { + $ret = 2; + } + } } else { return 1; } diff --git a/nixos/tests/switch-test.nix b/nixos/tests/switch-test.nix index 4160e481853..a994fb78160 100644 --- a/nixos/tests/switch-test.nix +++ b/nixos/tests/switch-test.nix @@ -64,6 +64,11 @@ in { }; }; + simpleServiceDifferentDescription.configuration = { + imports = [ simpleService.configuration ]; + systemd.services.test.description = "Test unit"; + }; + simpleServiceModified.configuration = { imports = [ simpleService.configuration ]; systemd.services.test.serviceConfig.X-Test = true; @@ -497,6 +502,15 @@ in { assert_lacks(out, "\nstarting the following units:") assert_lacks(out, "the following new units were started:") + # Only changing the description does nothing + out = switch_to_specialisation("${machine}", "simpleServiceDifferentDescription") + assert_lacks(out, "stopping the following units:") + assert_lacks(out, "NOT restarting the following changed units:") + assert_lacks(out, "reloading the following units:") + assert_lacks(out, "\nrestarting the following units:") + assert_lacks(out, "\nstarting the following units:") + assert_lacks(out, "the following new units were started:") + # Restart the simple service out = switch_to_specialisation("${machine}", "simpleServiceModified") assert_contains(out, "stopping the following units: test.service\n") -- cgit 1.4.1 From bc58430068d0bd0ffd3ef561a92a05f5970d149c Mon Sep 17 00:00:00 2001 From: Janne Heß Date: Sun, 6 Mar 2022 22:43:47 +0100 Subject: nixos/switch-to-configuration: Fix reloading of stopped services --- .../manual/development/unit-handling.section.md | 3 ++ .../from_md/development/unit-handling.section.xml | 5 ++ .../system/activation/switch-to-configuration.pl | 31 ++++++++++++ nixos/tests/switch-test.nix | 59 ++++++++++++++++++++++ 4 files changed, 98 insertions(+) (limited to 'nixos/doc/manual/development') diff --git a/nixos/doc/manual/development/unit-handling.section.md b/nixos/doc/manual/development/unit-handling.section.md index c51704ad0da..a7ccb3dbd04 100644 --- a/nixos/doc/manual/development/unit-handling.section.md +++ b/nixos/doc/manual/development/unit-handling.section.md @@ -34,6 +34,9 @@ checks: - The rest of the units (mostly `.service` units) are then **reload**ed if `X-ReloadIfChanged` in the `[Service]` section is set to `true` (exposed via [systemd.services.\.reloadIfChanged](#opt-systemd.services)). + A little exception is done for units that were deactivated in the meantime, + for example because they require a unit that got stopped before. These + are **start**ed instead of reloaded. - If the reload flag is not set, some more flags decide if the unit is skipped. These flags are `X-RestartIfChanged` in the `[Service]` section diff --git a/nixos/doc/manual/from_md/development/unit-handling.section.xml b/nixos/doc/manual/from_md/development/unit-handling.section.xml index 642cc5cccc7..4c980e1213a 100644 --- a/nixos/doc/manual/from_md/development/unit-handling.section.xml +++ b/nixos/doc/manual/from_md/development/unit-handling.section.xml @@ -72,6 +72,11 @@ [Service] section is set to true (exposed via systemd.services.<name>.reloadIfChanged). + A little exception is done for units that were deactivated + in the meantime, for example because they require a unit + that got stopped before. These are + started instead of + reloaded. diff --git a/nixos/modules/system/activation/switch-to-configuration.pl b/nixos/modules/system/activation/switch-to-configuration.pl index a67a9b05778..d83198bc346 100644 --- a/nixos/modules/system/activation/switch-to-configuration.pl +++ b/nixos/modules/system/activation/switch-to-configuration.pl @@ -104,6 +104,19 @@ sub getActiveUnits { return $res; } +# Returns whether a systemd unit is active +sub unit_is_active { + my ($unit_name) = @_; + + my $mgr = Net::DBus->system->get_service('org.freedesktop.systemd1')->get_object('/org/freedesktop/systemd1'); + my $units = $mgr->ListUnitsByNames([$unit_name]); + if (@{$units} == 0) { + return 0; + } + my $active_state = $units->[0]->[3]; ## no critic (ValuesAndExpressions::ProhibitMagicNumbers) + return $active_state eq 'active' || $active_state eq 'activating'; +} + sub parseFstab { my ($filename) = @_; my ($fss, $swaps); @@ -744,6 +757,24 @@ close $listActiveUsers; print STDERR "setting up tmpfiles\n"; system("@systemd@/bin/systemd-tmpfiles", "--create", "--remove", "--exclude-prefix=/dev") == 0 or $res = 3; +# Before reloading we need to ensure that the units are still active. They may have been +# deactivated because one of their requirements got stopped. If they are inactive +# but should have been reloaded, the user probably expects them to be started. +if (scalar(keys %unitsToReload) > 0) { + for my $unit (keys %unitsToReload) { + if (!unit_is_active($unit)) { + # Figure out if we need to start the unit + my %unit_info = parse_unit("$out/etc/systemd/system/$unit"); + if (!(parseSystemdBool(\%unit_info, 'Unit', 'RefuseManualStart', 0) || parseSystemdBool(\%unit_info, 'Unit', 'X-OnlyManualStart', 0))) { + $unitsToStart{$unit} = 1; + recordUnit($startListFile, $unit); + } + # Don't reload the unit, reloading would fail + delete %unitsToReload{$unit}; + unrecord_unit($reloadListFile, $unit); + } + } +} # Reload units that need it. This includes remounting changed mount # units. if (scalar(keys %unitsToReload) > 0) { diff --git a/nixos/tests/switch-test.nix b/nixos/tests/switch-test.nix index a994fb78160..93eee4babc2 100644 --- a/nixos/tests/switch-test.nix +++ b/nixos/tests/switch-test.nix @@ -208,6 +208,39 @@ in { systemd.services."escaped\\x2ddash".serviceConfig.X-Test = "test"; }; + unitWithRequirement.configuration = { + systemd.services.required-service = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${pkgs.coreutils}/bin/true"; + ExecReload = "${pkgs.coreutils}/bin/true"; + }; + }; + systemd.services.test-service = { + wantedBy = [ "multi-user.target" ]; + requires = [ "required-service.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${pkgs.coreutils}/bin/true"; + ExecReload = "${pkgs.coreutils}/bin/true"; + }; + }; + }; + + unitWithRequirementModified.configuration = { + imports = [ unitWithRequirement.configuration ]; + systemd.services.required-service.serviceConfig.X-Test = "test"; + systemd.services.test-service.reloadTriggers = [ "test" ]; + }; + + unitWithRequirementModifiedNostart.configuration = { + imports = [ unitWithRequirement.configuration ]; + systemd.services.test-service.unitConfig.RefuseManualStart = true; + }; + restart-and-reload-by-activation-script.configuration = { systemd.services = rec { simple-service = { @@ -574,6 +607,32 @@ in { assert_contains(out, "\nstarting the following units: escaped\\x2ddash.service\n") assert_lacks(out, "the following new units were started:") + # Ensure units that require changed units are properly reloaded + out = switch_to_specialisation("${machine}", "unitWithRequirement") + assert_contains(out, "stopping the following units: escaped\\x2ddash.service\n") + assert_lacks(out, "NOT restarting the following changed units:") + assert_lacks(out, "reloading the following units:") + assert_lacks(out, "\nrestarting the following units:") + assert_lacks(out, "\nstarting the following units:") + assert_contains(out, "the following new units were started: required-service.service, test-service.service\n") + + out = switch_to_specialisation("${machine}", "unitWithRequirementModified") + assert_contains(out, "stopping the following units: required-service.service\n") + assert_lacks(out, "NOT restarting the following changed units:") + assert_lacks(out, "reloading the following units:") + assert_lacks(out, "\nrestarting the following units:") + assert_contains(out, "\nstarting the following units: required-service.service, test-service.service\n") + assert_lacks(out, "the following new units were started:") + + # Unless the unit asks to be not restarted + out = switch_to_specialisation("${machine}", "unitWithRequirementModifiedNostart") + assert_contains(out, "stopping the following units: required-service.service\n") + assert_lacks(out, "NOT restarting the following changed units:") + assert_lacks(out, "reloading the following units:") + assert_lacks(out, "\nrestarting the following units:") + assert_contains(out, "\nstarting the following units: required-service.service\n") + assert_lacks(out, "the following new units were started:") + with subtest("failing units"): # Let the simple service fail switch_to_specialisation("${machine}", "simpleServiceModified") -- cgit 1.4.1 From 40a35299fa30421de85a56f084f6c59d05ea883e Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 9 Jan 2022 08:46:55 +0100 Subject: nixos: add functions and documentation for escaping systemd Exec* directives it's really easy to accidentally write the wrong systemd Exec* directive, ones that works most of the time but fails when users include systemd metacharacters in arguments that are interpolated into an Exec* directive. add a few functions analogous to escapeShellArg{,s} and some documentation on how and when to use them. --- .../manual/development/writing-modules.chapter.md | 42 +++++++++++++++++++ .../development/writing-modules.chapter.xml | 49 ++++++++++++++++++++++ nixos/lib/utils.nix | 20 +++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/empty-file | 0 nixos/tests/systemd-escaping.nix | 45 ++++++++++++++++++++ 6 files changed, 157 insertions(+) create mode 100644 nixos/tests/empty-file create mode 100644 nixos/tests/systemd-escaping.nix (limited to 'nixos/doc/manual/development') diff --git a/nixos/doc/manual/development/writing-modules.chapter.md b/nixos/doc/manual/development/writing-modules.chapter.md index 2e3c6b34f1f..0c41cbd3cb7 100644 --- a/nixos/doc/manual/development/writing-modules.chapter.md +++ b/nixos/doc/manual/development/writing-modules.chapter.md @@ -90,6 +90,17 @@ modules: `systemd.services` (the set of all systemd services) and `systemd.timers` (the list of commands to be executed periodically by `systemd`). +Care must be taken when writing systemd services using `Exec*` directives. By +default systemd performs substitution on `%` specifiers in these +directives, expands environment variables from `$FOO` and `${FOO}`, splits +arguments on whitespace, and splits commands on `;`. All of these must be escaped +to avoid unexpected substitution or splitting when interpolating into an `Exec*` +directive, e.g. when using an `extraArgs` option to pass additional arguments to +the service. The functions `utils.escapeSystemdExecArg` and +`utils.escapeSystemdExecArgs` are provided for this, see [Example: Escaping in +Exec directives](#exec-escaping-example) for an example. When using these +functions system environment substitution should *not* be disabled explicitly. + ::: {#locate-example .example} ::: {.title} **Example: NixOS Module for the "locate" Service** @@ -153,6 +164,37 @@ in { ``` ::: +::: {#exec-escaping-example .example} +::: {.title} +**Example: Escaping in Exec directives** +::: +```nix +{ config, lib, pkgs, utils, ... }: + +with lib; + +let + cfg = config.services.echo; + echoAll = pkgs.writeScript "echo-all" '' + #! ${pkgs.runtimeShell} + for s in "$@"; do + printf '%s\n' "$s" + done + ''; + args = [ "a%Nything" "lang=\${LANG}" ";" "/bin/sh -c date" ]; +in { + systemd.services.echo = + { description = "Echo to the journal"; + wantedBy = [ "multi-user.target" ]; + serviceConfig.Type = "oneshot"; + serviceConfig.ExecStart = '' + ${echoAll} ${utils.escapeSystemdExecArgs args} + ''; + }; +} +``` +::: + ```{=docbook} diff --git a/nixos/doc/manual/from_md/development/writing-modules.chapter.xml b/nixos/doc/manual/from_md/development/writing-modules.chapter.xml index e33c24f4f12..367731eda09 100644 --- a/nixos/doc/manual/from_md/development/writing-modules.chapter.xml +++ b/nixos/doc/manual/from_md/development/writing-modules.chapter.xml @@ -122,6 +122,25 @@ services) and systemd.timers (the list of commands to be executed periodically by systemd). + + Care must be taken when writing systemd services using + Exec* directives. By default systemd performs + substitution on %<char> specifiers in these + directives, expands environment variables from + $FOO and ${FOO}, splits + arguments on whitespace, and splits commands on + ;. All of these must be escaped to avoid + unexpected substitution or splitting when interpolating into an + Exec* directive, e.g. when using an + extraArgs option to pass additional arguments to + the service. The functions + utils.escapeSystemdExecArg and + utils.escapeSystemdExecArgs are provided for + this, see Example: Escaping in + Exec directives for an example. When using these functions + system environment substitution should not be + disabled explicitly. + Example: NixOS Module for the @@ -183,6 +202,36 @@ in { }; }; } + + + + Example: Escaping in Exec + directives + + +{ config, lib, pkgs, utils, ... }: + +with lib; + +let + cfg = config.services.echo; + echoAll = pkgs.writeScript "echo-all" '' + #! ${pkgs.runtimeShell} + for s in "$@"; do + printf '%s\n' "$s" + done + ''; + args = [ "a%Nything" "lang=\${LANG}" ";" "/bin/sh -c date" ]; +in { + systemd.services.echo = + { description = "Echo to the journal"; + wantedBy = [ "multi-user.target" ]; + serviceConfig.Type = "oneshot"; + serviceConfig.ExecStart = '' + ${echoAll} ${utils.escapeSystemdExecArgs args} + ''; + }; +} diff --git a/nixos/lib/utils.nix b/nixos/lib/utils.nix index bbebf8ba35a..29135024195 100644 --- a/nixos/lib/utils.nix +++ b/nixos/lib/utils.nix @@ -45,6 +45,26 @@ rec { replaceChars ["/" "-" " "] ["-" "\\x2d" "\\x20"] (removePrefix "/" s); + # Quotes an argument for use in Exec* service lines. + # systemd accepts "-quoted strings with escape sequences, toJSON produces + # a subset of these. + # Additionally we escape % to disallow expansion of % specifiers. Any lone ; + # in the input will be turned it ";" and thus lose its special meaning. + # Every $ is escaped to $$, this makes it unnecessary to disable environment + # substitution for the directive. + escapeSystemdExecArg = arg: + let + s = if builtins.isPath arg then "${arg}" + else if builtins.isString arg then arg + else if builtins.isInt arg || builtins.isFloat arg then toString arg + else throw "escapeSystemdExecArg only allows strings, paths and numbers"; + in + replaceChars [ "%" "$" ] [ "%%" "$$" ] (builtins.toJSON s); + + # Quotes a list of arguments into a single string for use in a Exec* + # line. + escapeSystemdExecArgs = concatMapStringsSep " " escapeSystemdExecArg; + # Returns a system path for a given shell package toShellPath = shell: if types.shellPackage.check shell then diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 9f3e97ceb13..01708fe0679 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -459,6 +459,7 @@ in systemd-boot = handleTest ./systemd-boot.nix {}; systemd-confinement = handleTest ./systemd-confinement.nix {}; systemd-cryptenroll = handleTest ./systemd-cryptenroll.nix {}; + systemd-escaping = handleTest ./systemd-escaping.nix {}; systemd-journal = handleTest ./systemd-journal.nix {}; systemd-networkd = handleTest ./systemd-networkd.nix {}; systemd-networkd-dhcpserver = handleTest ./systemd-networkd-dhcpserver.nix {}; diff --git a/nixos/tests/empty-file b/nixos/tests/empty-file new file mode 100644 index 00000000000..e69de29bb2d diff --git a/nixos/tests/systemd-escaping.nix b/nixos/tests/systemd-escaping.nix new file mode 100644 index 00000000000..7f93eb5e4f7 --- /dev/null +++ b/nixos/tests/systemd-escaping.nix @@ -0,0 +1,45 @@ +import ./make-test-python.nix ({ pkgs, ... }: + +let + echoAll = pkgs.writeScript "echo-all" '' + #! ${pkgs.runtimeShell} + for s in "$@"; do + printf '%s\n' "$s" + done + ''; + # deliberately using a local empty file instead of pkgs.emptyFile to have + # a non-store path in the test + args = [ "a%Nything" "lang=\${LANG}" ";" "/bin/sh -c date" ./empty-file 4.2 23 ]; +in +{ + name = "systemd-escaping"; + + machine = { pkgs, lib, utils, ... }: { + systemd.services.echo = + assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ [] ])).success; + assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ {} ])).success; + assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ null ])).success; + assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ false ])).success; + assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ (_:_) ])).success; + { description = "Echo to the journal"; + serviceConfig.Type = "oneshot"; + serviceConfig.ExecStart = '' + ${echoAll} ${utils.escapeSystemdExecArgs args} + ''; + }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed("systemctl start echo.service") + # skip the first 'Starting ...' line + logs = machine.succeed("journalctl -u echo.service -o cat").splitlines()[1:] + assert "a%Nything" == logs[0] + assert "lang=''${LANG}" == logs[1] + assert ";" == logs[2] + assert "/bin/sh -c date" == logs[3] + assert "/nix/store/ij3gw72f4n5z4dz6nnzl1731p9kmjbwr-empty-file" == logs[4] + assert "4.2" in logs[5] # toString produces extra fractional digits! + assert "23" == logs[6] + ''; +}) -- cgit 1.4.1