1 files changed, 39 insertions, 7 deletions

diff --git a/nixos/modules/services/cluster/k3s/default.nix b/nixos/modules/services/cluster/k3s/default.nix
index 2e8bf20a68f..e5c51441690 100644
--- a/nixos/modules/services/cluster/k3s/default.nix
+++ b/nixos/modules/services/cluster/k3s/default.nix
@@ -35,10 +35,20 @@ in
 
     token = mkOption {
       type = types.str;
-      description = "The k3s token to use when connecting to the server. This option only makes sense for an agent.";
+      description = ''
+        The k3s token to use when connecting to the server. This option only makes sense for an agent.
+        WARNING: This option will expose store your token unencrypted world-readable in the nix store.
+        If this is undesired use the tokenFile option instead.
+      '';
       default = "";
     };
 
+    tokenFile = mkOption {
+      type = types.nullOr types.path;
+      description = "File path containing k3s token to use when connecting to the server. This option only makes sense for an agent.";
+      default = null;
+    };
+
     docker = mkOption {
       type = types.bool;
       default = false;
@@ -47,6 +57,7 @@ in
 
     extraFlags = mkOption {
       description = "Extra flags to pass to the k3s command.";
+      type = types.str;
       default = "";
       example = "--no-deploy traefik --cluster-cidr 10.24.0.0/16";
     };
@@ -56,6 +67,12 @@ in
       default = false;
       description = "Only run the server. This option only makes sense for a server.";
     };
+
+    configPath = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
+    };
   };
 
   # implementation
@@ -63,12 +80,12 @@ in
   config = mkIf cfg.enable {
     assertions = [
       {
-        assertion = cfg.role == "agent" -> cfg.serverAddr != "";
-        message = "serverAddr should be set if role is 'agent'";
+        assertion = cfg.role == "agent" -> (cfg.configPath != null || cfg.serverAddr != "");
+        message = "serverAddr or configPath (with 'server' key) should be set if role is 'agent'";
       }
       {
-        assertion = cfg.role == "agent" -> cfg.token != "";
-        message = "token should be set if role is 'agent'";
+        assertion = cfg.role == "agent" -> cfg.configPath != null || cfg.tokenFile != null || cfg.token != "";
+        message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'";
       }
     ];
 
@@ -76,10 +93,18 @@ in
       enable = mkDefault true;
     };
 
+    # TODO: disable this once k3s supports cgroupsv2, either by docker
+    # supporting it, or their bundled containerd
+    systemd.enableUnifiedCgroupHierarchy = false;
+
+    environment.systemPackages = [ config.services.k3s.package ];
+
     systemd.services.k3s = {
       description = "k3s service";
-      after = mkIf cfg.docker [ "docker.service" ];
+      after = [ "network.service" "firewall.service" ] ++ (optional cfg.docker "docker.service");
+      wants = [ "network.service" "firewall.service" ];
       wantedBy = [ "multi-user.target" ];
+      path = optional config.boot.zfs.enabled config.boot.zfs.package;
       serviceConfig = {
         # See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197
         Type = if cfg.role == "agent" then "exec" else "notify";
@@ -87,12 +112,19 @@ in
         Delegate = "yes";
         Restart = "always";
         RestartSec = "5s";
+        LimitNOFILE = 1048576;
+        LimitNPROC = "infinity";
+        LimitCORE = "infinity";
+        TasksMax = "infinity";
         ExecStart = concatStringsSep " \\\n " (
           [
             "${cfg.package}/bin/k3s ${cfg.role}"
           ] ++ (optional cfg.docker "--docker")
           ++ (optional cfg.disableAgent "--disable-agent")
-          ++ (optional (cfg.role == "agent") "--server ${cfg.serverAddr} --token ${cfg.token}")
+          ++ (optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
+          ++ (optional (cfg.token != "") "--token ${cfg.token}")
+          ++ (optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
+          ++ (optional (cfg.configPath != null) "--config ${cfg.configPath}")
           ++ [ cfg.extraFlags ]
         );
       };