diff options
Diffstat (limited to 'nixos/modules/services/cluster')
12 files changed, 166 insertions, 82 deletions
diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix index bfb73f68371..41ac46e538e 100644 --- a/nixos/modules/services/cluster/hadoop/default.nix +++ b/nixos/modules/services/cluster/hadoop/default.nix @@ -7,6 +7,7 @@ with lib; options.services.hadoop = { coreSite = mkOption { default = {}; + type = types.attrsOf types.anything; example = literalExample '' { "fs.defaultFS" = "hdfs://localhost"; @@ -17,6 +18,7 @@ with lib; hdfsSite = mkOption { default = {}; + type = types.attrsOf types.anything; example = literalExample '' { "dfs.nameservices" = "namenode1"; @@ -27,6 +29,7 @@ with lib; mapredSite = mkOption { default = {}; + type = types.attrsOf types.anything; example = literalExample '' { "mapreduce.map.cpu.vcores" = "1"; @@ -37,6 +40,7 @@ with lib; yarnSite = mkOption { default = {}; + type = types.attrsOf types.anything; example = literalExample '' { "yarn.resourcemanager.ha.id" = "resourcemanager1"; @@ -50,8 +54,7 @@ with lib; default = pkgs.hadoop; defaultText = "pkgs.hadoop"; example = literalExample "pkgs.hadoop"; - description = '' - ''; + description = ""; }; }; diff --git a/nixos/modules/services/cluster/k3s/default.nix b/nixos/modules/services/cluster/k3s/default.nix index 2e8bf20a68f..e5c51441690 100644 --- a/nixos/modules/services/cluster/k3s/default.nix +++ b/nixos/modules/services/cluster/k3s/default.nix @@ -35,10 +35,20 @@ in token = mkOption { type = types.str; - description = "The k3s token to use when connecting to the server. This option only makes sense for an agent."; + description = '' + The k3s token to use when connecting to the server. This option only makes sense for an agent. + WARNING: This option will expose store your token unencrypted world-readable in the nix store. + If this is undesired use the tokenFile option instead. + ''; default = ""; }; + tokenFile = mkOption { + type = types.nullOr types.path; + description = "File path containing k3s token to use when connecting to the server. This option only makes sense for an agent."; + default = null; + }; + docker = mkOption { type = types.bool; default = false; @@ -47,6 +57,7 @@ in extraFlags = mkOption { description = "Extra flags to pass to the k3s command."; + type = types.str; default = ""; example = "--no-deploy traefik --cluster-cidr 10.24.0.0/16"; }; @@ -56,6 +67,12 @@ in default = false; description = "Only run the server. This option only makes sense for a server."; }; + + configPath = mkOption { + type = types.nullOr types.path; + default = null; + description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot)."; + }; }; # implementation @@ -63,12 +80,12 @@ in config = mkIf cfg.enable { assertions = [ { - assertion = cfg.role == "agent" -> cfg.serverAddr != ""; - message = "serverAddr should be set if role is 'agent'"; + assertion = cfg.role == "agent" -> (cfg.configPath != null || cfg.serverAddr != ""); + message = "serverAddr or configPath (with 'server' key) should be set if role is 'agent'"; } { - assertion = cfg.role == "agent" -> cfg.token != ""; - message = "token should be set if role is 'agent'"; + assertion = cfg.role == "agent" -> cfg.configPath != null || cfg.tokenFile != null || cfg.token != ""; + message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"; } ]; @@ -76,10 +93,18 @@ in enable = mkDefault true; }; + # TODO: disable this once k3s supports cgroupsv2, either by docker + # supporting it, or their bundled containerd + systemd.enableUnifiedCgroupHierarchy = false; + + environment.systemPackages = [ config.services.k3s.package ]; + systemd.services.k3s = { description = "k3s service"; - after = mkIf cfg.docker [ "docker.service" ]; + after = [ "network.service" "firewall.service" ] ++ (optional cfg.docker "docker.service"); + wants = [ "network.service" "firewall.service" ]; wantedBy = [ "multi-user.target" ]; + path = optional config.boot.zfs.enabled config.boot.zfs.package; serviceConfig = { # See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197 Type = if cfg.role == "agent" then "exec" else "notify"; @@ -87,12 +112,19 @@ in Delegate = "yes"; Restart = "always"; RestartSec = "5s"; + LimitNOFILE = 1048576; + LimitNPROC = "infinity"; + LimitCORE = "infinity"; + TasksMax = "infinity"; ExecStart = concatStringsSep " \\\n " ( [ "${cfg.package}/bin/k3s ${cfg.role}" ] ++ (optional cfg.docker "--docker") ++ (optional cfg.disableAgent "--disable-agent") - ++ (optional (cfg.role == "agent") "--server ${cfg.serverAddr} --token ${cfg.token}") + ++ (optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}") + ++ (optional (cfg.token != "") "--token ${cfg.token}") + ++ (optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}") + ++ (optional (cfg.configPath != null) "--config ${cfg.configPath}") ++ [ cfg.extraFlags ] ); }; diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix index f55079300b1..1378b5ccfb7 100644 --- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix +++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix @@ -62,7 +62,7 @@ in ''; }; - enable = mkEnableOption "Whether to enable Kubernetes addon manager."; + enable = mkEnableOption "Kubernetes addon manager."; }; ###### implementation diff --git a/nixos/modules/services/cluster/kubernetes/addons/dns.nix b/nixos/modules/services/cluster/kubernetes/addons/dns.nix index f12e866930d..24d86628b21 100644 --- a/nixos/modules/services/cluster/kubernetes/addons/dns.nix +++ b/nixos/modules/services/cluster/kubernetes/addons/dns.nix @@ -3,7 +3,7 @@ with lib; let - version = "1.6.4"; + version = "1.7.1"; cfg = config.services.kubernetes.addons.dns; ports = { dns = 10053; @@ -55,9 +55,9 @@ in { type = types.attrs; default = { imageName = "coredns/coredns"; - imageDigest = "sha256:493ee88e1a92abebac67cbd4b5658b4730e0f33512461442d8d9214ea6734a9b"; + imageDigest = "sha256:4a6e0769130686518325b21b0c1d0688b54e7c79244d48e1b15634e98e40c6ef"; finalImageTag = version; - sha256 = "0fm9zdjavpf5hni8g7fkdd3csjbhd7n7py7llxjc66sbii087028"; + sha256 = "02r440xcdsgi137k5lmmvp0z5w5fmk8g9mysq5pnysq1wl8sj6mw"; }; }; }; @@ -156,7 +156,6 @@ in { health :${toString ports.health} kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa { pods insecure - upstream fallthrough in-addr.arpa ip6.arpa } prometheus :${toString ports.metrics} diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix index 95bdb4c0d14..f1531caa754 100644 --- a/nixos/modules/services/cluster/kubernetes/apiserver.nix +++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix @@ -145,7 +145,7 @@ in extraOpts = mkOption { description = "Kubernetes apiserver extra command line options."; default = ""; - type = str; + type = separatedString " "; }; extraSANs = mkOption { @@ -238,14 +238,40 @@ in type = int; }; + apiAudiences = mkOption { + description = '' + Kubernetes apiserver ServiceAccount issuer. + ''; + default = "api,https://kubernetes.default.svc"; + type = str; + }; + + serviceAccountIssuer = mkOption { + description = '' + Kubernetes apiserver ServiceAccount issuer. + ''; + default = "https://kubernetes.default.svc"; + type = str; + }; + + serviceAccountSigningKeyFile = mkOption { + description = '' + Path to the file that contains the current private key of the service + account token issuer. The issuer will sign issued ID tokens with this + private key. + ''; + type = path; + }; + serviceAccountKeyFile = mkOption { description = '' - Kubernetes apiserver PEM-encoded x509 RSA private or public key file, - used to verify ServiceAccount tokens. By default tls private key file - is used. + File containing PEM-encoded x509 RSA or ECDSA private or public keys, + used to verify ServiceAccount tokens. The specified file can contain + multiple keys, and the flag can be specified multiple times with + different files. If unspecified, --tls-private-key-file is used. + Must be specified when --service-account-signing-key is provided ''; - default = null; - type = nullOr path; + type = path; }; serviceClusterIpRange = mkOption { @@ -357,8 +383,10 @@ in ${optionalString (cfg.runtimeConfig != "") "--runtime-config=${cfg.runtimeConfig}"} \ --secure-port=${toString cfg.securePort} \ - ${optionalString (cfg.serviceAccountKeyFile!=null) - "--service-account-key-file=${cfg.serviceAccountKeyFile}"} \ + --api-audiences=${toString cfg.apiAudiences} \ + --service-account-issuer=${toString cfg.serviceAccountIssuer} \ + --service-account-signing-key-file=${cfg.serviceAccountSigningKeyFile} \ + --service-account-key-file=${cfg.serviceAccountKeyFile} \ --service-cluster-ip-range=${cfg.serviceClusterIpRange} \ --storage-backend=${cfg.storageBackend} \ ${optionalString (cfg.tlsCertFile != null) diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix index a99ef6640e9..0c81fa9ae49 100644 --- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix +++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix @@ -38,7 +38,7 @@ in extraOpts = mkOption { description = "Kubernetes controller manager extra command line options."; default = ""; - type = str; + type = separatedString " "; }; featureGates = mkOption { diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index 3a11a6513a4..33d217ba60e 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -5,6 +5,29 @@ with lib; let cfg = config.services.kubernetes; + defaultContainerdConfigFile = pkgs.writeText "containerd.toml" '' + version = 2 + root = "/var/lib/containerd" + state = "/run/containerd" + oom_score = 0 + + [grpc] + address = "/run/containerd/containerd.sock" + + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "pause:latest" + + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + max_conf_num = 0 + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."io.containerd.runc.v2".options] + SystemdCgroup = true + ''; + mkKubeConfig = name: conf: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON { apiVersion = "v1"; kind = "Config"; @@ -25,8 +48,9 @@ let cluster = "local"; user = name; }; - current-context = "local"; + name = "local"; }]; + current-context = "local"; }); caCert = secret "ca"; @@ -222,14 +246,9 @@ in { }) (mkIf cfg.kubelet.enable { - virtualisation.docker = { + virtualisation.containerd = { enable = mkDefault true; - - # kubernetes needs access to logs - logDriver = mkDefault "json-file"; - - # iptables must be disabled for kubernetes - extraOptions = "--iptables=false --ip-masq=false"; + configFile = mkDefault defaultContainerdConfigFile; }; }) @@ -269,7 +288,6 @@ in { users.users.kubernetes = { uid = config.ids.uids.kubernetes; description = "Kubernetes user"; - extraGroups = [ "docker" ]; group = "kubernetes"; home = cfg.dataDir; createHome = true; diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix index 548ffed1ddb..3f55719027f 100644 --- a/nixos/modules/services/cluster/kubernetes/flannel.nix +++ b/nixos/modules/services/cluster/kubernetes/flannel.nix @@ -8,16 +8,6 @@ let # we want flannel to use kubernetes itself as configuration backend, not direct etcd storageBackend = "kubernetes"; - - # needed for flannel to pass options to docker - mkDockerOpts = pkgs.runCommand "mk-docker-opts" { - buildInputs = [ pkgs.makeWrapper ]; - } '' - mkdir -p $out - - # bashInteractive needed for `compgen` - makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "${pkgs.kubernetes}/bin/mk-docker-opts.sh" - ''; in { ###### interface @@ -43,43 +33,17 @@ in cniVersion = "0.3.1"; delegate = { isDefaultGateway = true; - bridge = "docker0"; + bridge = "mynet"; }; }]; }; - systemd.services.mk-docker-opts = { - description = "Pre-Docker Actions"; - path = with pkgs; [ gawk gnugrep ]; - script = '' - ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker - systemctl restart docker - ''; - serviceConfig.Type = "oneshot"; - }; - - systemd.paths.flannel-subnet-env = { - wantedBy = [ "flannel.service" ]; - pathConfig = { - PathModified = "/run/flannel/subnet.env"; - Unit = "mk-docker-opts.service"; - }; - }; - - systemd.services.docker = { - environment.DOCKER_OPTS = "-b none"; - serviceConfig.EnvironmentFile = "-/run/flannel/docker"; - }; - - # read environment variables generated by mk-docker-opts - virtualisation.docker.extraOptions = "$DOCKER_OPTS"; - networking = { firewall.allowedUDPPorts = [ 8285 # flannel udp 8472 # flannel vxlan ]; - dhcpcd.denyInterfaces = [ "docker*" "flannel*" ]; + dhcpcd.denyInterfaces = [ "mynet*" "flannel*" ]; }; services.kubernetes.pki.certs = { diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix index c3d67552cc8..fcfcc843547 100644 --- a/nixos/modules/services/cluster/kubernetes/kubelet.nix +++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix @@ -23,7 +23,7 @@ let name = "pause"; tag = "latest"; contents = top.package.pause; - config.Cmd = "/bin/pause"; + config.Cmd = ["/bin/pause"]; }; kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig; @@ -125,12 +125,24 @@ in }; }; + containerRuntime = mkOption { + description = "Which container runtime type to use"; + type = enum ["docker" "remote"]; + default = "remote"; + }; + + containerRuntimeEndpoint = mkOption { + description = "Endpoint at which to find the container runtime api interface/socket"; + type = str; + default = "unix:///run/containerd/containerd.sock"; + }; + enable = mkEnableOption "Kubernetes kubelet."; extraOpts = mkOption { description = "Kubernetes kubelet extra command line options."; default = ""; - type = str; + type = separatedString " "; }; featureGates = mkOption { @@ -235,17 +247,39 @@ in ###### implementation config = mkMerge [ (mkIf cfg.enable { + + environment.etc."cni/net.d".source = cniConfig; + services.kubernetes.kubelet.seedDockerImages = [infraContainer]; + boot.kernel.sysctl = { + "net.bridge.bridge-nf-call-iptables" = 1; + "net.ipv4.ip_forward" = 1; + "net.bridge.bridge-nf-call-ip6tables" = 1; + }; + systemd.services.kubelet = { description = "Kubernetes Kubelet Service"; wantedBy = [ "kubernetes.target" ]; - after = [ "network.target" "docker.service" "kube-apiserver.service" ]; - path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path; + after = [ "containerd.service" "network.target" "kube-apiserver.service" ]; + path = with pkgs; [ + gitMinimal + openssh + util-linux + iproute2 + ethtool + thin-provisioning-tools + iptables + socat + ] ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package ++ top.path; preStart = '' ${concatMapStrings (img: '' - echo "Seeding docker image: ${img}" - docker load <${img} + echo "Seeding container image: ${img}" + ${if (lib.hasSuffix "gz" img) then + ''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import --all-platforms -'' + else + ''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import --all-platforms -'' + } '') cfg.seedDockerImages} rm /opt/cni/bin/* || true @@ -296,6 +330,9 @@ in ${optionalString (cfg.tlsKeyFile != null) "--tls-private-key-file=${cfg.tlsKeyFile}"} \ ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ + --container-runtime=${cfg.containerRuntime} \ + --container-runtime-endpoint=${cfg.containerRuntimeEndpoint} \ + --cgroup-driver=systemd \ ${cfg.extraOpts} ''; WorkingDirectory = top.dataDir; @@ -305,7 +342,7 @@ in # Allways include cni plugins services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins]; - boot.kernelModules = ["br_netfilter"]; + boot.kernelModules = ["br_netfilter" "overlay"]; services.kubernetes.kubelet.hostname = with config.networking; mkDefault (hostName + optionalString (domain != null) ".${domain}"); diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 4275563f1a3..d9311d3e3a0 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -20,7 +20,7 @@ let size = 2048; }; CN = top.masterAddress; - hosts = cfg.cfsslAPIExtraSANs; + hosts = [top.masterAddress] ++ cfg.cfsslAPIExtraSANs; }); cfsslAPITokenBaseName = "apitoken.secret"; @@ -189,6 +189,7 @@ in # manually paste it in place. Just symlink. # otherwise, create the target file, ready for users to insert the token + mkdir -p $(dirname ${certmgrAPITokenPath}) if [ -f "${cfsslAPITokenPath}" ]; then ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}" else @@ -228,7 +229,8 @@ in }; private_key = cert.privateKeyOptions; request = { - inherit (cert) CN hosts; + hosts = [cert.CN] ++ cert.hosts; + inherit (cert) CN; key = { algo = "rsa"; size = 2048; @@ -360,6 +362,7 @@ in tlsCertFile = mkDefault cert; tlsKeyFile = mkDefault key; serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert; + serviceAccountSigningKeyFile = mkDefault cfg.certs.serviceAccount.key; kubeletClientCaFile = mkDefault caCert; kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert; kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key; diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix index 86d1dc2439b..42729f54643 100644 --- a/nixos/modules/services/cluster/kubernetes/proxy.nix +++ b/nixos/modules/services/cluster/kubernetes/proxy.nix @@ -25,7 +25,7 @@ in extraOpts = mkOption { description = "Kubernetes proxy extra command line options."; default = ""; - type = str; + type = separatedString " "; }; featureGates = mkOption { @@ -59,7 +59,7 @@ in description = "Kubernetes Proxy Service"; wantedBy = [ "kubernetes.target" ]; after = [ "kube-apiserver.service" ]; - path = with pkgs; [ iptables conntrack_tools ]; + path = with pkgs; [ iptables conntrack-tools ]; serviceConfig = { Slice = "kubernetes.slice"; ExecStart = ''${top.package}/bin/kube-proxy \ diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix index 5f6113227d9..454c689759d 100644 --- a/nixos/modules/services/cluster/kubernetes/scheduler.nix +++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix @@ -21,7 +21,7 @@ in extraOpts = mkOption { description = "Kubernetes scheduler extra command line options."; default = ""; - type = str; + type = separatedString " "; }; featureGates = mkOption { |