nixos/acme: Add defaults and inheritDefaults option

Allows configuring many default settings for certificates,
all of which can still be overridden on a per-cert basis.
Some options have been moved into .defaults from security.acme,
namely email, server, validMinDays and renewInterval. These
changes will not break existing configurations thanks to
mkChangedOptionModule.

With this, it is also now possible to configure DNS-01 with
web servers whose virtualHosts utilise enableACME. The only
requirement is you set `acmeRoot = null` for each vhost.

The test suite has been revamped to cover these additions
and also to generally make it easier to maintain. Test config
for apache and nginx has been fully standardised, and it
is now much easier to add a new web server if it follows
the same configuration patterns as those two. I have also
optimised the use of switch-to-configuration which should
speed up testing.

8 files changed, 448 insertions, 348 deletions

diff --git a/nixos/modules/security/acme.nix b/nixos/modules/security/acme.nix
index 9242a12fd35..c39653d174e 100644
--- a/nixos/modules/security/acme.nix
+++ b/nixos/modules/security/acme.nix
@@ -128,7 +128,7 @@ let
   };
 
   certToConfig = cert: data: let
-    acmeServer = if data.server != null then data.server else cfg.server;
+    acmeServer = data.server;
     useDns = data.dnsProvider != null;
     destPath = "/var/lib/acme/${cert}";
     selfsignedDeps = optionals (cfg.preliminarySelfsigned) [ "acme-selfsigned-${cert}.service" ];
@@ -211,7 +211,7 @@ let
       description = "Renew ACME Certificate for ${cert}";
       wantedBy = [ "timers.target" ];
       timerConfig = {
-        OnCalendar = cfg.renewInterval;
+        OnCalendar = data.renewInterval;
         Unit = "acme-${cert}.service";
         Persistent = "yes";
 
@@ -356,7 +356,7 @@ let
           expiration_s=$[expiration_date - now]
           expiration_days=$[expiration_s / (3600 * 24)]   # rounds down
 
-          [[ $expiration_days -gt ${toString cfg.validMinDays} ]]
+          [[ $expiration_days -gt ${toString data.validMinDays} ]]
         }
 
         ${optionalString (data.webroot != null) ''
@@ -380,11 +380,12 @@ let
           # Even if a cert is not expired, it may be revoked by the CA.
           # Try to renew, and silently fail if the cert is not expired.
           # Avoids #85794 and resolves #129838
-          if ! lego ${renewOpts} --days ${toString cfg.validMinDays}; then
+          if ! lego ${renewOpts} --days ${toString data.validMinDays}; then
             if is_expiration_skippable out/full.pem; then
-              echo 1>&2 "nixos-acme: Ignoring failed renewal because expiration isn't within the coming ${toString cfg.validMinDays} days"
+              echo 1>&2 "nixos-acme: Ignoring failed renewal because expiration isn't within the coming ${toString data.validMinDays} days"
             else
-              exit 3
+              # High number to avoid Systemd reserved codes.
+              exit 11
             fi
           fi
 
@@ -394,8 +395,9 @@ let
           echo Failed to fetch certificates. \
             This may mean your DNS records are set up incorrectly. \
             ${optionalString (cfg.preliminarySelfsigned) "Selfsigned certs are in place and dependant services will still start."}
-          # Exit 2 so that users can potentially amend SuccessExitStatus to ignore this error.
-          exit 2
+          # Exit 10 so that users can potentially amend SuccessExitStatus to ignore this error.
+          # High number to avoid Systemd reserved codes.
+          exit 10
         fi
 
         mv domainhash.txt certificates/
@@ -423,31 +425,33 @@ let
 
   certConfigs = mapAttrs certToConfig cfg.certs;
 
-  certOpts = { name, ... }: {
-    options = {
-      # user option has been removed
-      user = mkOption {
-        visible = false;
-        default = "_mkRemovedOptionModule";
+  # These options can be specified within
+  # security.acme or security.acme.certs.<name>
+  inheritableOpts =
+    { inheritDefaults ? false, defaults ? null }: {
+      validMinDays = mkOption {
+        type = types.int;
+        default = if inheritDefaults then defaults.validMinDays else 30;
+        description = "Minimum remaining validity before renewal in days.";
       };
 
-      # allowKeysForGroup option has been removed
-      allowKeysForGroup = mkOption {
-        visible = false;
-        default = "_mkRemovedOptionModule";
+      renewInterval = mkOption {
+        type = types.str;
+        default = if inheritDefaults then defaults.renewInterval else "daily";
+        description = ''
+          Systemd calendar expression when to check for renewal. See
+          <citerefentry><refentrytitle>systemd.time</refentrytitle>
+          <manvolnum>7</manvolnum></citerefentry>.
+        '';
       };
 
-      # extraDomains was replaced with extraDomainNames
-      extraDomains = mkOption {
-        visible = false;
-        default = "_mkMergedOptionModule";
+      enableDebugLogs = mkEnableOption "debug logging for this certificate" // {
+        default = if inheritDefaults then defaults.enableDebugLogs else true;
       };
 
-      enableDebugLogs = mkEnableOption "debug logging for this certificate" // { default = cfg.enableDebugLogs; };
-
       webroot = mkOption {
         type = types.nullOr types.str;
-        default = null;
+        default = if inheritDefaults then defaults.webroot else null;
         example = "/var/lib/acme/acme-challenge";
         description = ''
           Where the webroot of the HTTP vhost is located.
@@ -458,20 +462,9 @@ let
         '';
       };
 
-      listenHTTP = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        example = ":1360";
-        description = ''
-          Interface and port to listen on to solve HTTP challenges
-          in the form [INTERFACE]:PORT.
-          If you use a port other than 80, you must proxy port 80 to this port.
-        '';
-      };
-
       server = mkOption {
         type = types.nullOr types.str;
-        default = null;
+        default = if inheritDefaults then defaults.server else null;
         description = ''
           ACME Directory Resource URI. Defaults to Let's Encrypt's
           production endpoint,
@@ -479,28 +472,25 @@ let
         '';
       };
 
-      domain = mkOption {
-        type = types.str;
-        default = name;
-        description = "Domain to fetch certificate for (defaults to the entry name).";
-      };
-
       email = mkOption {
-        type = types.nullOr types.str;
-        default = cfg.email;
-        defaultText = literalExpression "config.${opt.email}";
-        description = "Contact email address for the CA to be able to reach you.";
+        type = types.str;
+        default = if inheritDefaults then defaults.email else null;
+        description = ''
+          Email address for account creation and correspondence from the CA.
+          It is recommended to use the same email for all certs to avoid account
+          creation limits.
+        '';
       };
 
       group = mkOption {
         type = types.str;
-        default = "acme";
+        default = if inheritDefaults then defaults.group else "acme";
         description = "Group running the ACME client.";
       };
 
       reloadServices = mkOption {
         type = types.listOf types.str;
-        default = [];
+        default = if inheritDefaults then defaults.reloadServices else [];
         description = ''
           The list of systemd services to call <code>systemctl try-reload-or-restart</code>
           on.
@@ -509,7 +499,7 @@ let
 
       postRun = mkOption {
         type = types.lines;
-        default = "";
+        default = if inheritDefaults then defaults.postRun else "";
         example = "cp full.pem backup.pem";
         description = ''
           Commands to run after new certificates go live. Note that
@@ -519,30 +509,9 @@ let
         '';
       };
 
-      directory = mkOption {
-        type = types.str;
-        readOnly = true;
-        default = "/var/lib/acme/${name}";
-        description = "Directory where certificate and other state is stored.";
-      };
-
-      extraDomainNames = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        example = literalExpression ''
-          [
-            "example.org"
-            "mydomain.org"
-          ]
-        '';
-        description = ''
-          A list of extra domain names, which are included in the one certificate to be issued.
-        '';
-      };
-
       keyType = mkOption {
         type = types.str;
-        default = "ec256";
+        default = if inheritDefaults then defaults.keyType else "ec256";
         description = ''
           Key type to use for private keys.
           For an up to date list of supported values check the --key-type option
@@ -552,7 +521,7 @@ let
 
       dnsProvider = mkOption {
         type = types.nullOr types.str;
-        default = null;
+        default = if inheritDefaults then defaults.dnsProvider else null;
         example = "route53";
         description = ''
           DNS Challenge provider. For a list of supported providers, see the "code"
@@ -562,7 +531,7 @@ let
 
       dnsResolver = mkOption {
         type = types.nullOr types.str;
-        default = null;
+        default = if inheritDefaults then defaults.dnsResolver else null;
         example = "1.1.1.1:53";
         description = ''
           Set the resolver to use for performing recursive DNS queries. Supported:
@@ -573,6 +542,7 @@ let
 
       credentialsFile = mkOption {
         type = types.path;
+        default = if inheritDefaults then defaults.credentialsFile else null;
         description = ''
           Path to an EnvironmentFile for the cert's service containing any required and
           optional environment variables for your selected dnsProvider.
@@ -584,7 +554,7 @@ let
 
       dnsPropagationCheck = mkOption {
         type = types.bool;
-        default = true;
+        default = if inheritDefaults then defaults.dnsPropagationCheck else true;
         description = ''
           Toggles lego DNS propagation check, which is used alongside DNS-01
           challenge to ensure the DNS entries required are available.
@@ -593,7 +563,7 @@ let
 
       ocspMustStaple = mkOption {
         type = types.bool;
-        default = false;
+        default = if inheritDefaults then defaults.ocspMustStaple else false;
         description = ''
           Turns on the OCSP Must-Staple TLS extension.
           Make sure you know what you're doing! See:
@@ -606,7 +576,7 @@ let
 
       extraLegoFlags = mkOption {
         type = types.listOf types.str;
-        default = [];
+        default = if inheritDefaults then defaults.extraLegoFlags else [];
         description = ''
           Additional global flags to pass to all lego commands.
         '';
@@ -614,7 +584,7 @@ let
 
       extraLegoRenewFlags = mkOption {
         type = types.listOf types.str;
-        default = [];
+        default = if inheritDefaults then defaults.extraLegoRenewFlags else [];
         description = ''
           Additional flags to pass to lego renew.
         '';
@@ -622,53 +592,87 @@ let
 
       extraLegoRunFlags = mkOption {
         type = types.listOf types.str;
-        default = [];
+        default = if inheritDefaults then defaults.extraLegoRunFlags else [];
         description = ''
           Additional flags to pass to lego run.
         '';
       };
     };
-  };
-
-in {
 
-  options = {
-    security.acme = {
+  certOpts = { name, ... }: {
+    options = (inheritableOpts { inherit (cfg) defaults; inheritDefaults = cfg.certs."${name}".inheritDefaults; }) // {
+      # user option has been removed
+      user = mkOption {
+        visible = false;
+        default = "_mkRemovedOptionModule";
+      };
 
-      enableDebugLogs = mkEnableOption "debug logging for all certificates by default" // { default = true; };
+      # allowKeysForGroup option has been removed
+      allowKeysForGroup = mkOption {
+        visible = false;
+        default = "_mkRemovedOptionModule";
+      };
 
-      validMinDays = mkOption {
-        type = types.int;
-        default = 30;
-        description = "Minimum remaining validity before renewal in days.";
+      # extraDomains was replaced with extraDomainNames
+      extraDomains = mkOption {
+        visible = false;
+        default = "_mkMergedOptionModule";
       };
 
-      email = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = "Contact email address for the CA to be able to reach you.";
+      directory = mkOption {
+        type = types.str;
+        readOnly = true;
+        default = "/var/lib/acme/${name}";
+        description = "Directory where certificate and other state is stored.";
       };
 
-      renewInterval = mkOption {
+      domain = mkOption {
         type = types.str;
-        default = "daily";
+        default = name;
+        description = "Domain to fetch certificate for (defaults to the entry name).";
+      };
+
+      extraDomainNames = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        example = literalExpression ''
+          [
+            "example.org"
+            "mydomain.org"
+          ]
+        '';
         description = ''
-          Systemd calendar expression when to check for renewal. See
-          <citerefentry><refentrytitle>systemd.time</refentrytitle>
-          <manvolnum>7</manvolnum></citerefentry>.
+          A list of extra domain names, which are included in the one certificate to be issued.
         '';
       };
 
-      server = mkOption {
+      # This setting must be different for each configured certificate, otherwise
+      # two or more renewals may fail to bind to the address. Hence, it is not in
+      # the inheritableOpts.
+      listenHTTP = mkOption {
         type = types.nullOr types.str;
         default = null;
+        example = ":1360";
         description = ''
-          ACME Directory Resource URI. Defaults to Let's Encrypt's
-          production endpoint,
-          <link xlink:href="https://acme-v02.api.letsencrypt.org/directory"/>, if unset.
+          Interface and port to listen on to solve HTTP challenges
+          in the form [INTERFACE]:PORT.
+          If you use a port other than 80, you must proxy port 80 to this port.
         '';
       };
 
+      inheritDefaults = mkOption {
+        default = true;
+        example = true;
+        description = "Whether to inherit values set in `security.acme.defaults` or not.";
+        type = lib.types.bool;
+      };
+    };
+  };
+
+in {
+
+  options = {
+    security.acme = {
       preliminarySelfsigned = mkOption {
         type = types.bool;
         default = true;
@@ -691,6 +695,16 @@ in {
         '';
       };
 
+      defaults = mkOption {
+        type = types.submodule ({ ... }: { options = inheritableOpts {}; });
+        description = ''
+          Default values inheritable by all configured certs. You can
+          use this to define options shared by all your certs. These defaults
+          can also be ignored on a per-cert basis using the
+          `security.acme.certs.''${cert}.inheritDefaults' option.
+        '';
+      };
+
       certs = mkOption {
         default = { };
         type = with types; attrsOf (submodule certOpts);
@@ -724,12 +738,16 @@ in {
 
       To use the let's encrypt staging server, use security.acme.server =
       "https://acme-staging-v02.api.letsencrypt.org/directory".
-    ''
-    )
+    '')
     (mkRemovedOptionModule [ "security" "acme" "directory" ] "ACME Directory is now hardcoded to /var/lib/acme and its permisisons are managed by systemd. See https://github.com/NixOS/nixpkgs/issues/53852 for more info.")
     (mkRemovedOptionModule [ "security" "acme" "preDelay" ] "This option has been removed. If you want to make sure that something executes before certificates are provisioned, add a RequiredBy=acme-\${cert}.service to the service you want to execute before the cert renewal")
     (mkRemovedOptionModule [ "security" "acme" "activationDelay" ] "This option has been removed. If you want to make sure that something executes before certificates are provisioned, add a RequiredBy=acme-\${cert}.service to the service you want to execute before the cert renewal")
-    (mkChangedOptionModule [ "security" "acme" "validMin" ] [ "security" "acme" "validMinDays" ] (config: config.security.acme.validMin / (24 * 3600)))
+    (mkChangedOptionModule [ "security" "acme" "validMin" ] [ "security" "acme" "defaults" "validMinDays" ] (config: config.security.acme.validMin / (24 * 3600)))
+    (mkChangedOptionModule [ "security" "acme" "validMinDays" ] [ "security" "acme" "defaults" "validMinDays" ] (config: config.security.acme.validMinDays))
+    (mkChangedOptionModule [ "security" "acme" "renewInterval" ] [ "security" "acme" "defaults" "renewInterval" ] (config: config.security.acme.renewInterval))
+    (mkChangedOptionModule [ "security" "acme" "email" ] [ "security" "acme" "defaults" "email" ] (config: config.security.acme.email))
+    (mkChangedOptionModule [ "security" "acme" "server" ] [ "security" "acme" "defaults" "server" ] (config: config.security.acme.server))
+    (mkChangedOptionModule [ "security" "acme" "enableDebugLogs" ] [ "security" "acme" "defaults" "enableDebugLogs" ] (config: config.security.acme.enableDebugLogs))
   ];
 
   config = mkMerge [
diff --git a/nixos/modules/services/web-servers/apache-httpd/default.nix b/nixos/modules/services/web-servers/apache-httpd/default.nix
index 992a58875e4..1a49b4ca15c 100644
--- a/nixos/modules/services/web-servers/apache-httpd/default.nix
+++ b/nixos/modules/services/web-servers/apache-httpd/default.nix
@@ -154,7 +154,7 @@ let
       sslServerKey = if useACME then "${sslCertDir}/key.pem" else hostOpts.sslServerKey;
       sslServerChain = if useACME then "${sslCertDir}/chain.pem" else hostOpts.sslServerChain;
 
-      acmeChallenge = optionalString useACME ''
+      acmeChallenge = optionalString (useACME && hostOpts.acmeRoot != null) ''
         Alias /.well-known/acme-challenge/ "${hostOpts.acmeRoot}/.well-known/acme-challenge/"
         <Directory "${hostOpts.acmeRoot}">
             AllowOverride None
@@ -677,9 +677,16 @@ in
     };
 
     security.acme.certs = let
-      acmePairs = map (hostOpts: nameValuePair hostOpts.hostName {
+      acmePairs = map (hostOpts: let
+        hasRoot = hostOpts.acmeRoot != null;
+      in nameValuePair hostOpts.hostName {
         group = mkDefault cfg.group;
-        webroot = hostOpts.acmeRoot;
+        # if acmeRoot is null inherit config.security.acme
+        # Since config.security.acme.certs.<cert>.webroot's own default value
+        # should take precedence set priority higher than mkOptionDefault
+        webroot = mkOverride (if hasRoot then 1000 else 2000) hostOpts.acmeRoot;
+        # Also nudge dnsProvider to null in case it is inherited
+        dnsProvider = mkOverride (if hasRoot then 1000 else 2000) null;
         extraDomainNames = hostOpts.serverAliases;
         # Use the vhost-specific email address if provided, otherwise let
         # security.acme.email or security.acme.certs.<cert>.email be used.
diff --git a/nixos/modules/services/web-servers/apache-httpd/vhost-options.nix b/nixos/modules/services/web-servers/apache-httpd/vhost-options.nix
index 8bb7e91ec9c..c52ab2c596e 100644
--- a/nixos/modules/services/web-servers/apache-httpd/vhost-options.nix
+++ b/nixos/modules/services/web-servers/apache-httpd/vhost-options.nix
@@ -128,9 +128,12 @@ in
     };
 
     acmeRoot = mkOption {
-      type = types.str;
+      type = types.nullOr types.str;
       default = "/var/lib/acme/acme-challenge";
-      description = "Directory for the acme challenge which is PUBLIC, don't put certs or keys in here";
+      description = ''
+        Directory for the acme challenge which is PUBLIC, don't put certs or keys in here.
+        Set to null to inherit from config.security.acme.
+      '';
     };
 
     sslServerCert = mkOption {
diff --git a/nixos/modules/services/web-servers/nginx/default.nix b/nixos/modules/services/web-servers/nginx/default.nix
index ba8e874f2de..05b7870fc3a 100644
--- a/nixos/modules/services/web-servers/nginx/default.nix
+++ b/nixos/modules/services/web-servers/nginx/default.nix
@@ -278,7 +278,7 @@ let
         acmeLocation = optionalString (vhost.enableACME || vhost.useACMEHost != null) ''
           location /.well-known/acme-challenge {
             ${optionalString (vhost.acmeFallbackHost != null) "try_files $uri @acme-fallback;"}
-            root ${vhost.acmeRoot};
+            ${optionalString (vhost.acmeRoot != null) "root ${vhost.acmeRoot};"}
             auth_basic off;
           }
           ${optionalString (vhost.acmeFallbackHost != null) ''
@@ -948,9 +948,16 @@ in
     };
 
     security.acme.certs = let
-      acmePairs = map (vhostConfig: nameValuePair vhostConfig.serverName {
+      acmePairs = map (vhostConfig: let
+        hasRoot = vhostConfig.acmeRoot != null;
+      in nameValuePair vhostConfig.serverName {
         group = mkDefault cfg.group;
-        webroot = vhostConfig.acmeRoot;
+        # if acmeRoot is null inherit config.security.acme
+        # Since config.security.acme.certs.<cert>.webroot's own default value
+        # should take precedence set priority higher than mkOptionDefault
+        webroot = mkOverride (if hasRoot then 1000 else 2000) vhostConfig.acmeRoot;
+        # Also nudge dnsProvider to null in case it is inherited
+        dnsProvider = mkOverride (if hasRoot then 1000 else 2000) null;
         extraDomainNames = vhostConfig.serverAliases;
       # Filter for enableACME-only vhosts. Don't want to create dud certs
       }) (filter (vhostConfig: vhostConfig.useACMEHost == null) acmeEnabledVhosts);
diff --git a/nixos/modules/services/web-servers/nginx/vhost-options.nix b/nixos/modules/services/web-servers/nginx/vhost-options.nix
index 7f49ce9586c..c4e8285dc48 100644
--- a/nixos/modules/services/web-servers/nginx/vhost-options.nix
+++ b/nixos/modules/services/web-servers/nginx/vhost-options.nix
@@ -3,7 +3,7 @@
 # has additional options that affect the web server as a whole, like
 # the user/group to run under.)
 
-{ lib, ... }:
+{ config, lib, ... }:
 
 with lib;
 {
@@ -85,9 +85,12 @@ with lib;
     };
 
     acmeRoot = mkOption {
-      type = types.str;
+      type = types.nullOr types.str;
       default = "/var/lib/acme/acme-challenge";
-      description = "Directory for the acme challenge which is PUBLIC, don't put certs or keys in here";
+      description = ''
+        Directory for the acme challenge which is PUBLIC, don't put certs or keys in here.
+        Set to null to inherit from config.security.acme.
+      '';
     };
 
     acmeFallbackHost = mkOption {
diff --git a/nixos/tests/acme.nix b/nixos/tests/acme.nix
index 80b85502d4e..549fa9e64ee 100644
--- a/nixos/tests/acme.nix
+++ b/nixos/tests/acme.nix
@@ -1,9 +1,9 @@
-let
+import ./make-test-python.nix ({ pkgs, lib, ... }: let
   commonConfig = ./common/acme/client;
 
   dnsServerIP = nodes: nodes.dnsserver.config.networking.primaryIPAddress;
 
-  dnsScript = {pkgs, nodes}: let
+  dnsScript = nodes: let
     dnsAddress = dnsServerIP nodes;
   in pkgs.writeShellScript "dns-hook.sh" ''
     set -euo pipefail
@@ -15,30 +15,137 @@ let
     fi
   '';
 
-  documentRoot = pkgs: pkgs.runCommand "docroot" {} ''
+  dnsConfig = nodes: {
+    dnsProvider = "exec";
+    dnsPropagationCheck = false;
+    credentialsFile = pkgs.writeText "wildcard.env" ''
+      EXEC_PATH=${dnsScript nodes}
+      EXEC_POLLING_INTERVAL=1
+      EXEC_PROPAGATION_TIMEOUT=1
+      EXEC_SEQUENCE_INTERVAL=1
+    '';
+  };
+
+  documentRoot = pkgs.runCommand "docroot" {} ''
     mkdir -p "$out"
     echo hello world > "$out/index.html"
   '';
 
-  vhostBase = pkgs: {
+  vhostBase = {
     forceSSL = true;
-    locations."/".root = documentRoot pkgs;
+    locations."/".root = documentRoot;
+  };
+
+  vhostBaseHttpd = {
+    forceSSL = true;
+    inherit documentRoot;
+  };
+
+  # Base specialisation config for testing general ACME features
+  webserverBasicConfig = {
+    services.nginx.enable = true;
+    services.nginx.virtualHosts."a.example.test" = vhostBase // {
+      enableACME = true;
+    };
+  };
+
+  # Generate specialisations for testing a web server
+  mkServerConfigs = { server, group, vhostBaseData, extraConfig ? {} }: let
+    baseConfig = { nodes, config, specialConfig ? {} }: lib.mkMerge [
+      {
+        security.acme = {
+          defaults = (dnsConfig nodes) // {
+            inherit group;
+          };
+          # One manual wildcard cert
+          certs."example.test" = {
+            domain = "*.example.test";
+          };
+        };
+
+        services."${server}" = {
+          enable = true;
+          virtualHosts = {
+            # Run-of-the-mill vhost using HTTP-01 validation
+            "${server}-http.example.test" = vhostBaseData // {
+              serverAliases = [ "${server}-http-alias.example.test" ];
+              enableACME = true;
+            };
+
+            # Another which inherits the DNS-01 config
+            "${server}-dns.example.test" = vhostBaseData // {
+              serverAliases = [ "${server}-dns-alias.example.test" ];
+              enableACME = true;
+              # Set acmeRoot to null instead of using the default of "/var/lib/acme/acme-challenge"
+              # webroot + dnsProvider are mutually exclusive.
+              acmeRoot = null;
+            };
+
+            # One using the wildcard certificate
+            "${server}-wildcard.example.test" = vhostBaseData // {
+              serverAliases = [ "${server}-wildcard-alias.example.test" ];
+              useACMEHost = "example.test";
+            };
+          };
+        };
+
+        # Used to determine if service reload was triggered
+        systemd.targets."test-renew-${server}" = {
+          wants = [ "acme-${server}-http.example.test.service" ];
+          after = [ "acme-${server}-http.example.test.service" "${server}-config-reload.service" ];
+        };
+      }
+      specialConfig
+      extraConfig
+    ];
+  in {
+    "${server}".configuration = { nodes, config, ... }: baseConfig {
+      inherit nodes config;
+    };
+
+    # Test that server reloads when an alias is removed (and subsequently test removal works in acme)
+    "${server}-remove-alias".configuration = { nodes, config, ... }: baseConfig {
+      inherit nodes config;
+      specialConfig = {
+        # Remove an alias, but create a standalone vhost in its place for testing.
+        # This configuration results in certificate errors as useACMEHost does not imply
+        # append extraDomains, and thus we can validate the SAN is removed.
+        services."${server}" = {
+          virtualHosts."${server}-http.example.test".serverAliases = lib.mkForce [];
+          virtualHosts."${server}-http-alias.example.test" = vhostBaseData // {
+            useACMEHost = "${server}-http.example.test";
+          };
+        };
+      };
+    };
+
+    # Test that the server reloads when only the acme configuration is changed.
+    "${server}-change-acme-conf".configuration = { nodes, config, ... }: baseConfig {
+      inherit nodes config;
+      specialConfig = {
+        security.acme.certs."${server}-http.example.test" = {
+          keyType = "ec384";
+          # Also test that postRun is exec'd as root
+          postRun = "id | grep root";
+        };
+      };
+    };
   };
 
-in import ./make-test-python.nix ({ lib, ... }: {
+in {
   name = "acme";
   meta.maintainers = lib.teams.acme.members;
 
   nodes = {
     # The fake ACME server which will respond to client requests
-    acme = { nodes, lib, ... }: {
+    acme = { nodes, ... }: {
       imports = [ ./common/acme/server ];
       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
     };
 
     # A fake DNS server which can be configured with records as desired
     # Used to test DNS-01 challenge
-    dnsserver = { nodes, pkgs, ... }: {
+    dnsserver = { nodes, ... }: {
       networking.firewall.allowedTCPPorts = [ 8055 53 ];
       networking.firewall.allowedUDPPorts = [ 53 ];
       systemd.services.pebble-challtestsrv = {
@@ -54,7 +161,7 @@ in import ./make-test-python.nix ({ lib, ... }: {
     };
 
     # A web server which will be the node requesting certs
-    webserver = { pkgs, nodes, lib, config, ... }: {
+    webserver = { nodes, config, ... }: {
       imports = [ commonConfig ];
       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
       networking.firewall.allowedTCPPorts = [ 80 443 ];
@@ -63,138 +170,88 @@ in import ./make-test-python.nix ({ lib, ... }: {
       environment.systemPackages = [ pkgs.openssl ];
 
       # Set log level to info so that we can see when the service is reloaded
-      services.nginx.enable = true;
       services.nginx.logError = "stderr info";
 
-      # First tests configure a basic cert and run a bunch of openssl checks
-      services.nginx.virtualHosts."a.example.test" = (vhostBase pkgs) // {
-        enableACME = true;
-      };
-
-      # Used to determine if service reload was triggered
-      systemd.targets.test-renew-nginx = {
-        wants = [ "acme-a.example.test.service" ];
-        after = [ "acme-a.example.test.service" "nginx-config-reload.service" ];
-      };
-
-      # Test that account creation is collated into one service
-      specialisation.account-creation.configuration = { nodes, pkgs, lib, ... }: let
-        email = "newhostmaster@example.test";
-        caDomain = nodes.acme.config.test-support.acme.caDomain;
-        # Exit 99 to make it easier to track if this is the reason a renew failed
-        testScript = ''
-          test -e accounts/${caDomain}/${email}/account.json || exit 99
-        '';
-      in {
-        security.acme.email = lib.mkForce email;
-        systemd.services."b.example.test".preStart = testScript;
-        systemd.services."c.example.test".preStart = testScript;
-
-        services.nginx.virtualHosts."b.example.test" = (vhostBase pkgs) // {
-          enableACME = true;
-        };
-        services.nginx.virtualHosts."c.example.test" = (vhostBase pkgs) // {
-          enableACME = true;
-        };
-      };
-
-      # Cert config changes will not cause the nginx configuration to change.
-      # This tests that the reload service is correctly triggered.
-      # It also tests that postRun is exec'd as root
-      specialisation.cert-change.configuration = { pkgs, ... }: {
-        security.acme.certs."a.example.test".keyType = "ec384";
-        security.acme.certs."a.example.test".postRun = ''
-          set -euo pipefail
-          touch /home/test
-          chown root:root /home/test
-          echo testing > /home/test
-        '';
-      };
-
-      # Now adding an alias to ensure that the certs are updated
-      specialisation.nginx-aliases.configuration = { pkgs, ... }: {
-        services.nginx.virtualHosts."a.example.test" = (vhostBase pkgs) // {
-          serverAliases = [ "b.example.test" ];
-        };
-      };
-
-      # Must be run after nginx-aliases
-      specialisation.remove-extra-domain.configuration = { pkgs, ... } : {
-        # This also validates that useACMEHost doesn't unexpectedly add the domain.
-        services.nginx.virtualHosts."b.example.test" = (vhostBase pkgs) // {
-          useACMEHost = "a.example.test";
-        };
-      };
-
-      # Test OCSP Stapling
-      specialisation.ocsp-stapling.configuration = { pkgs, ... }: {
-        security.acme.certs."a.example.test" = {
-          ocspMustStaple = true;
-        };
-        services.nginx.virtualHosts."a.example.com" = {
-          extraConfig = ''
-            ssl_stapling on;
-            ssl_stapling_verify on;
-          '';
-        };
-      };
-
-      # Test using Apache HTTPD
-      specialisation.httpd-aliases.configuration = { pkgs, config, lib, ... }: {
-        services.nginx.enable = lib.mkForce false;
-        services.httpd.enable = true;
-        services.httpd.adminAddr = config.security.acme.email;
-        services.httpd.virtualHosts."c.example.test" = {
-          serverAliases = [ "d.example.test" ];
-          forceSSL = true;
-          enableACME = true;
-          documentRoot = documentRoot pkgs;
-        };
-
-        # Used to determine if service reload was triggered
-        systemd.targets.test-renew-httpd = {
-          wants = [ "acme-c.example.test.service" ];
-          after = [ "acme-c.example.test.service" "httpd-config-reload.service" ];
-        };
-      };
-
-      # Validation via DNS-01 challenge
-      specialisation.dns-01.configuration = { pkgs, config, nodes, ... }: {
-        security.acme.certs."example.test" = {
-          domain = "*.example.test";
-          group = config.services.nginx.group;
-          dnsProvider = "exec";
-          dnsPropagationCheck = false;
-          credentialsFile = pkgs.writeText "wildcard.env" ''
-            EXEC_PATH=${dnsScript { inherit pkgs nodes; }}
+      specialisation = {
+        # First derivation used to test general ACME features
+        general.configuration = { ... }: let
+          caDomain = nodes.acme.config.test-support.acme.caDomain;
+          email = config.security.acme.defaults.email;
+          # Exit 99 to make it easier to track if this is the reason a renew failed
+          accountCreateTester = ''
+            test -e accounts/${caDomain}/${email}/account.json || exit 99
           '';
-        };
-
-        services.nginx.virtualHosts."dns.example.test" = (vhostBase pkgs) // {
-          useACMEHost = "example.test";
-        };
-      };
-
-      # Validate service relationships by adding a slow start service to nginx' wants.
-      # Reproducer for https://github.com/NixOS/nixpkgs/issues/81842
-      specialisation.slow-startup.configuration = { pkgs, config, nodes, lib, ... }: {
-        systemd.services.my-slow-service = {
-          wantedBy = [ "multi-user.target" "nginx.service" ];
-          before = [ "nginx.service" ];
-          preStart = "sleep 5";
-          script = "${pkgs.python3}/bin/python -m http.server";
-        };
-
-        services.nginx.virtualHosts."slow.example.com" = {
-          forceSSL = true;
-          enableACME = true;
-          locations."/".proxyPass = "http://localhost:8000";
-        };
-      };
+        in lib.mkMerge [
+          webserverBasicConfig
+          {
+            # Used to test that account creation is collated into one service.
+            # These should not run until after acme-finished-a.example.test.target
+            systemd.services."b.example.test".preStart = accountCreateTester;
+            systemd.services."c.example.test".preStart = accountCreateTester;
+
+            services.nginx.virtualHosts."b.example.test" = vhostBase // {
+              enableACME = true;
+            };
+            services.nginx.virtualHosts."c.example.test" = vhostBase // {
+              enableACME = true;
+            };
+          }
+        ];
+
+        # Test OCSP Stapling
+        ocsp-stapling.configuration = { ... }: lib.mkMerge [
+          webserverBasicConfig
+          {
+            security.acme.certs."a.example.test".ocspMustStaple = true;
+            services.nginx.virtualHosts."a.example.com" = {
+              extraConfig = ''
+                ssl_stapling on;
+                ssl_stapling_verify on;
+              '';
+            };
+          }
+        ];
+
+        # Validate service relationships by adding a slow start service to nginx' wants.
+        # Reproducer for https://github.com/NixOS/nixpkgs/issues/81842
+        slow-startup.configuration = { ... }: lib.mkMerge [
+          webserverBasicConfig
+          {
+            systemd.services.my-slow-service = {
+              wantedBy = [ "multi-user.target" "nginx.service" ];
+              before = [ "nginx.service" ];
+              preStart = "sleep 5";
+              script = "${pkgs.python3}/bin/python -m http.server";
+            };
+
+            services.nginx.virtualHosts."slow.example.com" = {
+              forceSSL = true;
+              enableACME = true;
+              locations."/".proxyPass = "http://localhost:8000";
+            };
+          }
+        ];
+
+      # Test compatibility with Nginx
+      } // (mkServerConfigs {
+          server = "nginx";
+          group = "nginx";
+          vhostBaseData = vhostBase;
+        })
+
+      # Test compatibility with Apache HTTPD
+        // (mkServerConfigs {
+          server = "httpd";
+          group = "wwwrun";
+          vhostBaseData = vhostBaseHttpd;
+          extraConfig = {
+            services.httpd.adminAddr = config.security.acme.defaults.email;
+          };
+        });
     };
 
     # The client will be used to curl the webserver to validate configuration
-    client = {nodes, lib, pkgs, ...}: {
+    client = { nodes, ... }: {
       imports = [ commonConfig ];
       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 
@@ -203,7 +260,7 @@ in import ./make-test-python.nix ({ lib, ... }: {
     };
   };
 
-  testScript = {nodes, ...}:
+  testScript = { nodes, ... }:
     let
       caDomain = nodes.acme.config.test-support.acme.caDomain;
       newServerSystem = nodes.webserver.config.system.build.toplevel;
@@ -212,23 +269,26 @@ in import ./make-test-python.nix ({ lib, ... }: {
     # Note, wait_for_unit does not work for oneshot services that do not have RemainAfterExit=true,
     # this is because a oneshot goes from inactive => activating => inactive, and never
     # reaches the active state. Targets do not have this issue.
-
     ''
       import time
 
 
-      has_switched = False
+      def switch_to(node, name):
+          # On first switch, this will create a symlink to the current system so that we can
+          # quickly switch between derivations
+          root_specs = "/tmp/specialisation"
+          node.execute(
+            f"test -e {root_specs}"
+            f" || ln -s $(readlink /run/current-system)/specialisation {root_specs}"
+          )
 
+          switcher_path = f"/run/current-system/specialisation/{name}/bin/switch-to-configuration"
+          rc, _ = node.execute(f"test -e '{switcher_path}'")
+          if rc > 0:
+              switcher_path = f"/tmp/specialisation/{name}/bin/switch-to-configuration"
 
-      def switch_to(node, name):
-          global has_switched
-          if has_switched:
-              node.succeed(
-                  "${switchToNewServer}"
-              )
-          has_switched = True
           node.succeed(
-              f"/run/current-system/specialisation/{name}/bin/switch-to-configuration test"
+              f"{switcher_path} test"
           )
 
 
@@ -318,8 +378,7 @@ in import ./make-test-python.nix ({ lib, ... }: {
               return download_ca_certs(node, retries - 1)
 
 
-      client.start()
-      dnsserver.start()
+      start_all()
 
       dnsserver.wait_for_unit("pebble-challtestsrv.service")
       client.wait_for_unit("default.target")
@@ -328,19 +387,30 @@ in import ./make-test-python.nix ({ lib, ... }: {
           'curl --data \'{"host": "${caDomain}", "addresses": ["${nodes.acme.config.networking.primaryIPAddress}"]}\' http://${dnsServerIP nodes}:8055/add-a'
       )
 
-      acme.start()
-      webserver.start()
-
       acme.wait_for_unit("network-online.target")
       acme.wait_for_unit("pebble.service")
 
       download_ca_certs(client)
 
+      # Perform general tests first
+      switch_to(webserver, "general")
+
       with subtest("Can request certificate with HTTPS-01 challenge"):
           webserver.wait_for_unit("acme-finished-a.example.test.target")
+          check_fullchain(webserver, "a.example.test")
+          check_issuer(webserver, "a.example.test", "pebble")
+          webserver.wait_for_unit("nginx.service")
+          check_connection(client, "a.example.test")
+
+      with subtest("Runs 1 cert for account creation before others"):
+          webserver.wait_for_unit("acme-finished-b.example.test.target")
+          webserver.wait_for_unit("acme-finished-c.example.test.target")
+          check_connection(client, "b.example.test")
+          check_connection(client, "c.example.test")
 
       with subtest("Certificates and accounts have safe + valid permissions"):
-          group = "${nodes.webserver.config.security.acme.certs."a.example.test".group}"
+          # Nginx will set the group appropriately when enableACME is used
+          group = "nginx"
           webserver.succeed(
               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
           )
@@ -354,12 +424,6 @@ in import ./make-test-python.nix ({ lib, ... }: {
               f"test $(find /var/lib/acme/accounts -type f -exec stat -L -c '%a %U %G' {{}} \\; | tee /dev/stderr | grep -v '600 acme {group}' | wc -l) -eq 0"
           )
 
-      with subtest("Certs are accepted by web server"):
-          webserver.succeed("systemctl start nginx.service")
-          check_fullchain(webserver, "a.example.test")
-          check_issuer(webserver, "a.example.test", "pebble")
-          check_connection(client, "a.example.test")
-
       # Selfsigned certs tests happen late so we aren't fighting the system init triggering cert renewal
       with subtest("Can generate valid selfsigned certs"):
           webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
@@ -373,89 +437,80 @@ in import ./make-test-python.nix ({ lib, ... }: {
           # Will succeed if nginx can load the certs
           webserver.succeed("systemctl start nginx-config-reload.service")
 
-      with subtest("Can reload nginx when timer triggers renewal"):
-          webserver.succeed("systemctl start test-renew-nginx.target")
-          check_issuer(webserver, "a.example.test", "pebble")
-          check_connection(client, "a.example.test")
-
-      with subtest("Runs 1 cert for account creation before others"):
-          switch_to(webserver, "account-creation")
-          webserver.wait_for_unit("acme-finished-a.example.test.target")
-          check_connection(client, "a.example.test")
-          webserver.wait_for_unit("acme-finished-b.example.test.target")
-          webserver.wait_for_unit("acme-finished-c.example.test.target")
-          check_connection(client, "b.example.test")
-          check_connection(client, "c.example.test")
-
-      with subtest("Can reload web server when cert configuration changes"):
-          switch_to(webserver, "cert-change")
-          webserver.wait_for_unit("acme-finished-a.example.test.target")
-          check_connection_key_bits(client, "a.example.test", "384")
-          webserver.succeed("grep testing /home/test")
-          # Clean to remove the testing file (and anything else messy we did)
-          webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
-
       with subtest("Correctly implements OCSP stapling"):
           switch_to(webserver, "ocsp-stapling")
           webserver.wait_for_unit("acme-finished-a.example.test.target")
           check_stapling(client, "a.example.test")
 
       with subtest("Can request certificate with HTTPS-01 when nginx startup is delayed"):
+          webserver.execute("systemctl stop nginx")
           switch_to(webserver, "slow-startup")
           webserver.wait_for_unit("acme-finished-slow.example.com.target")
           check_issuer(webserver, "slow.example.com", "pebble")
+          webserver.wait_for_unit("nginx.service")
           check_connection(client, "slow.example.com")
 
-      with subtest("Can request certificate for vhost + aliases (nginx)"):
-          # Check the key hash before and after adding an alias. It should not change.
-          # The previous test reverts the ed384 change
-          webserver.wait_for_unit("acme-finished-a.example.test.target")
-          switch_to(webserver, "nginx-aliases")
-          webserver.wait_for_unit("acme-finished-a.example.test.target")
-          check_issuer(webserver, "a.example.test", "pebble")
-          check_connection(client, "a.example.test")
-          check_connection(client, "b.example.test")
-
-      with subtest("Can remove extra domains from a cert"):
-          switch_to(webserver, "remove-extra-domain")
-          webserver.wait_for_unit("acme-finished-a.example.test.target")
-          webserver.wait_for_unit("nginx.service")
-          check_connection(client, "a.example.test")
-          rc, _ = client.execute(
-              "openssl s_client -CAfile /tmp/ca.crt -connect b.example.test:443"
-              " </dev/null 2>/dev/null | openssl x509 -noout -text"
-              " | grep DNS: | grep b.example.test"
-          )
-          assert rc > 0, "Removed extraDomainName was not removed from the cert"
-
-      with subtest("Can request certificates for vhost + aliases (apache-httpd)"):
-          try:
-              switch_to(webserver, "httpd-aliases")
-              webserver.wait_for_unit("acme-finished-c.example.test.target")
-          except Exception as err:
-              _, output = webserver.execute(
-                  "cat /var/log/httpd/*.log && ls -al /var/lib/acme/acme-challenge"
+      domains = ["http", "dns", "wildcard"]
+      for server, logsrc in [
+          ("nginx", "journalctl -n 30 -u nginx.service"),
+          ("httpd", "tail -n 30 /var/log/httpd/*.log"),
+      ]:
+          wait_for_server = lambda: webserver.wait_for_unit(f"{server}.service")
+          with subtest(f"Works with {server}"):
+              try:
+                  switch_to(webserver, server)
+                  # Skip wildcard domain for this check ([:-1])
+                  for domain in domains[:-1]:
+                      webserver.wait_for_unit(
+                          f"acme-finished-{server}-{domain}.example.test.target"
+                      )
+              except Exception as err:
+                  _, output = webserver.execute(
+                      f"{logsrc} && ls -al /var/lib/acme/acme-challenge"
+                  )
+                  print(output)
+                  raise err
+
+              wait_for_server()
+
+              for domain in domains[:-1]:
+                  check_issuer(webserver, f"{server}-{domain}.example.test", "pebble")
+              for domain in domains:
+                  check_connection(client, f"{server}-{domain}.example.test")
+                  check_connection(client, f"{server}-{domain}-alias.example.test")
+
+          test_domain = f"{server}-{domains[0]}.example.test"
+
+          with subtest(f"Can reload {server} when timer triggers renewal"):
+              # Switch to selfsigned first
+              webserver.succeed(f"systemctl clean acme-{test_domain}.service --what=state")
+              webserver.succeed(f"systemctl start acme-selfsigned-{test_domain}.service")
+              check_issuer(webserver, test_domain, "minica")
+              webserver.succeed(f"systemctl start {server}-config-reload.service")
+              webserver.succeed(f"systemctl start test-renew-{server}.target")
+              check_issuer(webserver, test_domain, "pebble")
+              check_connection(client, test_domain)
+
+          with subtest("Can remove an alias from a domain + cert is updated"):
+              test_alias = f"{server}-{domains[0]}-alias.example.test"
+              switch_to(webserver, f"{server}-remove-alias")
+              webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
+              wait_for_server()
+              check_connection(client, test_domain)
+              rc, _ = client.execute(
+                  f"openssl s_client -CAfile /tmp/ca.crt -connect {test_alias}:443"
+                  " </dev/null 2>/dev/null | openssl x509 -noout -text"
+                  f" | grep DNS: | grep {test_alias}"
               )
-              print(output)
-              raise err
-          check_issuer(webserver, "c.example.test", "pebble")
-          check_connection(client, "c.example.test")
-          check_connection(client, "d.example.test")
-
-      with subtest("Can reload httpd when timer triggers renewal"):
-          # Switch to selfsigned first
-          webserver.succeed("systemctl clean acme-c.example.test.service --what=state")
-          webserver.succeed("systemctl start acme-selfsigned-c.example.test.service")
-          check_issuer(webserver, "c.example.test", "minica")
-          webserver.succeed("systemctl start httpd-config-reload.service")
-          webserver.succeed("systemctl start test-renew-httpd.target")
-          check_issuer(webserver, "c.example.test", "pebble")
-          check_connection(client, "c.example.test")
-
-      with subtest("Can request wildcard certificates using DNS-01 challenge"):
-          switch_to(webserver, "dns-01")
-          webserver.wait_for_unit("acme-finished-example.test.target")
-          check_issuer(webserver, "example.test", "pebble")
-          check_connection(client, "dns.example.test")
+              assert rc > 0, "Removed extraDomainName was not removed from the cert"
+
+          with subtest("security.acme changes reflect on web server"):
+              # Switch back to normal server config first, reset everything.
+              switch_to(webserver, server)
+              wait_for_server()
+              switch_to(webserver, f"{server}-change-acme-conf")
+              webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
+              wait_for_server()
+              check_connection_key_bits(client, test_domain, "384")
     '';
 })
diff --git a/nixos/tests/common/acme/client/default.nix b/nixos/tests/common/acme/client/default.nix
index 1e9885e375c..9dbe345e7a0 100644
--- a/nixos/tests/common/acme/client/default.nix
+++ b/nixos/tests/common/acme/client/default.nix
@@ -5,9 +5,11 @@ let
 
 in {
   security.acme = {
-    server = "https://${caDomain}/dir";
-    email = "hostmaster@example.test";
     acceptTerms = true;
+    defaults = {
+      server = "https://${caDomain}/dir";
+      email = "hostmaster@example.test";
+    };
   };
 
   security.pki.certificateFiles = [ caCert ];
diff --git a/nixos/tests/common/acme/server/default.nix b/nixos/tests/common/acme/server/default.nix
index 1c3bfdf76b7..450d49e6039 100644
--- a/nixos/tests/common/acme/server/default.nix
+++ b/nixos/tests/common/acme/server/default.nix
@@ -120,6 +120,11 @@ in {
         enable = true;
         description = "Pebble ACME server";
         wantedBy = [ "network.target" ];
+        environment = {
+          # We're not testing lego, we're just testing our configuration.
+          # No need to sleep.
+          PEBBLE_VA_NOSLEEP = "1";
+        };
 
         serviceConfig = {
           RuntimeDirectory = "pebble";