diff options
author | Frederik Rietdijk <fridh@fridh.nl> | 2019-02-21 07:54:33 +0100 |
---|---|---|
committer | Frederik Rietdijk <fridh@fridh.nl> | 2019-02-21 07:54:33 +0100 |
commit | e2cd07b9978198696681776405c4e23526d483a7 (patch) | |
tree | 3986a58bee29938ea6fd1ef205fcd526ba7dec67 /nixos/modules | |
parent | c9b14043f0e66fff8facd866c0786b414e0765c1 (diff) | |
parent | c9d8f5235bd9201355a7817cab79d38a8bc6e731 (diff) | |
download | nixpkgs-e2cd07b9978198696681776405c4e23526d483a7.tar nixpkgs-e2cd07b9978198696681776405c4e23526d483a7.tar.gz nixpkgs-e2cd07b9978198696681776405c4e23526d483a7.tar.bz2 nixpkgs-e2cd07b9978198696681776405c4e23526d483a7.tar.lz nixpkgs-e2cd07b9978198696681776405c4e23526d483a7.tar.xz nixpkgs-e2cd07b9978198696681776405c4e23526d483a7.tar.zst nixpkgs-e2cd07b9978198696681776405c4e23526d483a7.zip |
Merge staging-next into staging
Diffstat (limited to 'nixos/modules')
23 files changed, 2169 insertions, 1091 deletions
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 08422d911eb..7af6e117c51 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -82,6 +82,7 @@ ./misc/version.nix ./programs/adb.nix ./programs/atop.nix + ./programs/autojump.nix ./programs/bash/bash.nix ./programs/bcc.nix ./programs/blcr.nix @@ -195,9 +196,17 @@ ./services/backup/tarsnap.nix ./services/backup/znapzend.nix ./services/cluster/hadoop/default.nix + ./services/cluster/kubernetes/addons/dns.nix + ./services/cluster/kubernetes/addons/dashboard.nix + ./services/cluster/kubernetes/addon-manager.nix + ./services/cluster/kubernetes/apiserver.nix + ./services/cluster/kubernetes/controller-manager.nix ./services/cluster/kubernetes/default.nix - ./services/cluster/kubernetes/dns.nix - ./services/cluster/kubernetes/dashboard.nix + ./services/cluster/kubernetes/flannel.nix + ./services/cluster/kubernetes/kubelet.nix + ./services/cluster/kubernetes/pki.nix + ./services/cluster/kubernetes/proxy.nix + ./services/cluster/kubernetes/scheduler.nix ./services/computing/boinc/client.nix ./services/computing/torque/server.nix ./services/computing/torque/mom.nix @@ -283,6 +292,7 @@ ./services/hardware/acpid.nix ./services/hardware/actkbd.nix ./services/hardware/bluetooth.nix + ./services/hardware/bolt.nix ./services/hardware/brltty.nix ./services/hardware/freefall.nix ./services/hardware/fwupd.nix diff --git a/nixos/modules/programs/autojump.nix b/nixos/modules/programs/autojump.nix new file mode 100644 index 00000000000..3a8feec4bb4 --- /dev/null +++ b/nixos/modules/programs/autojump.nix @@ -0,0 +1,33 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.programs.autojump; + prg = config.programs; +in +{ + options = { + programs.autojump = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable autojump. + ''; + }; + }; + }; + + ###### implementation + + config = mkIf cfg.enable { + environment.pathsToLink = [ "/share/autojump" ]; + environment.systemPackages = [ pkgs.autojump ]; + + programs.bash.interactiveShellInit = "source ${pkgs.autojump}/share/autojump/autojump.bash"; + programs.zsh.interactiveShellInit = mkIf prg.zsh.enable "source ${pkgs.autojump}/share/autojump/autojump.zsh"; + programs.fish.interactiveShellInit = mkIf prg.fish.enable "source ${pkgs.autojump}/share/autojump/autojump.fish"; + }; +} diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix index 24ab963f718..1e6557e1f0e 100644 --- a/nixos/modules/rename.nix +++ b/nixos/modules/rename.nix @@ -40,9 +40,19 @@ with lib; (mkRenamedOptionModule [ "services" "kibana" "host" ] [ "services" "kibana" "listenAddress" ]) (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "admissionControl" ] [ "services" "kubernetes" "apiserver" "enableAdmissionPlugins" ]) (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "address" ] ["services" "kubernetes" "apiserver" "bindAddress"]) + (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "port" ] ["services" "kubernetes" "apiserver" "insecurePort"]) (mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "publicAddress" ] "") (mkRenamedOptionModule [ "services" "kubernetes" "addons" "dashboard" "enableRBAC" ] [ "services" "kubernetes" "addons" "dashboard" "rbac" "enable" ]) + (mkRenamedOptionModule [ "services" "kubernetes" "controllerManager" "address" ] ["services" "kubernetes" "controllerManager" "bindAddress"]) + (mkRenamedOptionModule [ "services" "kubernetes" "controllerManager" "port" ] ["services" "kubernetes" "controllerManager" "insecurePort"]) + (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "servers" ] [ "services" "kubernetes" "apiserver" "etcd" "servers" ]) + (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "keyFile" ] [ "services" "kubernetes" "apiserver" "etcd" "keyFile" ]) + (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "certFile" ] [ "services" "kubernetes" "apiserver" "etcd" "certFile" ]) + (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "caFile" ] [ "services" "kubernetes" "apiserver" "etcd" "caFile" ]) + (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "") (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "") + (mkRenamedOptionModule [ "services" "kubernetes" "proxy" "address" ] ["services" "kubernetes" "proxy" "bindAddress"]) + (mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "") (mkRenamedOptionModule [ "services" "logstash" "address" ] [ "services" "logstash" "listenAddress" ]) (mkRenamedOptionModule [ "services" "mpd" "network" "host" ] [ "services" "mpd" "network" "listenAddress" ]) (mkRenamedOptionModule [ "services" "neo4j" "host" ] [ "services" "neo4j" "defaultListenAddress" ]) @@ -59,6 +69,7 @@ with lib; (mkRenamedOptionModule [ "services" "statsd" "host" ] [ "services" "statsd" "listenAddress" ]) (mkRenamedOptionModule [ "services" "subsonic" "host" ] [ "services" "subsonic" "listenAddress" ]) (mkRenamedOptionModule [ "services" "tor" "relay" "portSpec" ] [ "services" "tor" "relay" "port" ]) + (mkRenamedOptionModule [ "services" "vmwareGuest" ] [ "virtualisation" "vmware" "guest" ]) (mkRenamedOptionModule [ "jobs" ] [ "systemd" "services" ]) (mkRenamedOptionModule [ "services" "gitlab" "stateDir" ] [ "services" "gitlab" "statePath" ]) diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix new file mode 100644 index 00000000000..17f2dde31a7 --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix @@ -0,0 +1,167 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + top = config.services.kubernetes; + cfg = top.addonManager; + + isRBACEnabled = elem "RBAC" top.apiserver.authorizationMode; + + addons = pkgs.runCommand "kubernetes-addons" { } '' + mkdir -p $out + # since we are mounting the addons to the addon manager, they need to be copied + ${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon: + pkgs.writeTextDir "${name}.json" (builtins.toJSON addon) + ) (cfg.addons))} + ''; +in +{ + ###### interface + options.services.kubernetes.addonManager = with lib.types; { + + bootstrapAddons = mkOption { + description = '' + Bootstrap addons are like regular addons, but they are applied with cluster-admin rigths. + They are applied at addon-manager startup only. + ''; + default = { }; + type = attrsOf attrs; + example = literalExample '' + { + "my-service" = { + "apiVersion" = "v1"; + "kind" = "Service"; + "metadata" = { + "name" = "my-service"; + "namespace" = "default"; + }; + "spec" = { ... }; + }; + } + ''; + }; + + addons = mkOption { + description = "Kubernetes addons (any kind of Kubernetes resource can be an addon)."; + default = { }; + type = attrsOf (either attrs (listOf attrs)); + example = literalExample '' + { + "my-service" = { + "apiVersion" = "v1"; + "kind" = "Service"; + "metadata" = { + "name" = "my-service"; + "namespace" = "default"; + }; + "spec" = { ... }; + }; + } + // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; }; + ''; + }; + + enable = mkEnableOption "Whether to enable Kubernetes addon manager."; + }; + + ###### implementation + config = mkIf cfg.enable { + environment.etc."kubernetes/addons".source = "${addons}/"; + + systemd.services.kube-addon-manager = { + description = "Kubernetes addon manager"; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + environment.ADDON_PATH = "/etc/kubernetes/addons/"; + path = [ pkgs.gawk ]; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = "${top.package}/bin/kube-addons"; + WorkingDirectory = top.dataDir; + User = "kubernetes"; + Group = "kubernetes"; + Restart = "on-failure"; + RestartSec = 10; + }; + }; + + services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled + (let + name = system:kube-addon-manager; + namespace = "kube-system"; + in + { + + kube-addon-manager-r = { + apiVersion = "rbac.authorization.k8s.io/v1"; + kind = "Role"; + metadata = { + inherit name namespace; + }; + rules = [{ + apiGroups = ["*"]; + resources = ["*"]; + verbs = ["*"]; + }]; + }; + + kube-addon-manager-rb = { + apiVersion = "rbac.authorization.k8s.io/v1"; + kind = "RoleBinding"; + metadata = { + inherit name namespace; + }; + roleRef = { + apiGroup = "rbac.authorization.k8s.io"; + kind = "Role"; + inherit name; + }; + subjects = [{ + apiGroup = "rbac.authorization.k8s.io"; + kind = "User"; + inherit name; + }]; + }; + + kube-addon-manager-cluster-lister-cr = { + apiVersion = "rbac.authorization.k8s.io/v1"; + kind = "ClusterRole"; + metadata = { + name = "${name}:cluster-lister"; + }; + rules = [{ + apiGroups = ["*"]; + resources = ["*"]; + verbs = ["list"]; + }]; + }; + + kube-addon-manager-cluster-lister-crb = { + apiVersion = "rbac.authorization.k8s.io/v1"; + kind = "ClusterRoleBinding"; + metadata = { + name = "${name}:cluster-lister"; + }; + roleRef = { + apiGroup = "rbac.authorization.k8s.io"; + kind = "ClusterRole"; + name = "${name}:cluster-lister"; + }; + subjects = [{ + kind = "User"; + inherit name; + }]; + }; + }); + + services.kubernetes.pki.certs = { + addonManager = top.lib.mkCert { + name = "kube-addon-manager"; + CN = "system:kube-addon-manager"; + action = "systemctl restart kube-addon-manager.service"; + }; + }; + }; + +} diff --git a/nixos/modules/services/cluster/kubernetes/dashboard.nix b/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix index cbd6e8f7bf7..454e7d35bc0 100644 --- a/nixos/modules/services/cluster/kubernetes/dashboard.nix +++ b/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix @@ -8,6 +8,13 @@ in { options.services.kubernetes.addons.dashboard = { enable = mkEnableOption "kubernetes dashboard addon"; + extraArgs = mkOption { + description = "Extra arguments to append to the dashboard cmdline"; + type = types.listOf types.str; + default = []; + example = ["--enable-skip-login"]; + }; + rbac = mkOption { description = "Role-based access control (RBAC) options"; default = {}; @@ -31,7 +38,7 @@ in { version = mkOption { description = "Which version of the kubernetes dashboard to deploy"; type = types.str; - default = "v1.8.3"; + default = "v1.10.1"; }; image = mkOption { @@ -39,9 +46,9 @@ in { type = types.attrs; default = { imageName = "k8s.gcr.io/kubernetes-dashboard-amd64"; - imageDigest = "sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0"; + imageDigest = "sha256:0ae6b69432e78069c5ce2bcde0fe409c5c4d6f0f4d9cd50a17974fea38898747"; finalImageTag = cfg.version; - sha256 = "18ajcg0q1vignfjk2sm4xj4wzphfz8wah69ps8dklqfvv0164mc8"; + sha256 = "01xrr4pwgr2hcjrjsi3d14ifpzdfbxzqpzxbk2fkbjb9zkv38zxy"; }; }; }; @@ -99,7 +106,7 @@ in { memory = "100Mi"; }; }; - args = ["--auto-generate-certificates"]; + args = ["--auto-generate-certificates"] ++ cfg.extraArgs; volumeMounts = [{ name = "tmp-volume"; mountPath = "/tmp"; diff --git a/nixos/modules/services/cluster/kubernetes/dns.nix b/nixos/modules/services/cluster/kubernetes/addons/dns.nix index 5a3e281ea69..8f3234bfc70 100644 --- a/nixos/modules/services/cluster/kubernetes/dns.nix +++ b/nixos/modules/services/cluster/kubernetes/addons/dns.nix @@ -3,7 +3,7 @@ with lib; let - version = "1.2.5"; + version = "1.3.1"; cfg = config.services.kubernetes.addons.dns; ports = { dns = 10053; @@ -43,9 +43,9 @@ in { type = types.attrs; default = { imageName = "coredns/coredns"; - imageDigest = "sha256:33c8da20b887ae12433ec5c40bfddefbbfa233d5ce11fb067122e68af30291d6"; + imageDigest = "sha256:02382353821b12c21b062c59184e227e001079bb13ebd01f9d3270ba0fcbf1e4"; finalImageTag = version; - sha256 = "13q19rgwapv27xcs664dw502254yw4zw63insf6g2danidv2mg6i"; + sha256 = "0vbylgyxv2jm2mnzk6f28jbsj305zsxmx3jr6ngjq461czcl5fi5"; }; }; }; @@ -54,21 +54,7 @@ in { services.kubernetes.kubelet.seedDockerImages = singleton (pkgs.dockerTools.pullImage cfg.coredns); - services.kubernetes.addonManager.addons = { - coredns-sa = { - apiVersion = "v1"; - kind = "ServiceAccount"; - metadata = { - labels = { - "addonmanager.kubernetes.io/mode" = "Reconcile"; - "k8s-app" = "kube-dns"; - "kubernetes.io/cluster-service" = "true"; - }; - name = "coredns"; - namespace = "kube-system"; - }; - }; - + services.kubernetes.addonManager.bootstrapAddons = { coredns-cr = { apiVersion = "rbac.authorization.k8s.io/v1beta1"; kind = "ClusterRole"; @@ -123,6 +109,22 @@ in { } ]; }; + }; + + services.kubernetes.addonManager.addons = { + coredns-sa = { + apiVersion = "v1"; + kind = "ServiceAccount"; + metadata = { + labels = { + "addonmanager.kubernetes.io/mode" = "Reconcile"; + "k8s-app" = "kube-dns"; + "kubernetes.io/cluster-service" = "true"; + }; + name = "coredns"; + namespace = "kube-system"; + }; + }; coredns-cm = { apiVersion = "v1"; diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix new file mode 100644 index 00000000000..81e45b417de --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix @@ -0,0 +1,428 @@ + { config, lib, pkgs, ... }: + +with lib; + +let + top = config.services.kubernetes; + cfg = top.apiserver; + + isRBACEnabled = elem "RBAC" cfg.authorizationMode; + + apiserverServiceIP = (concatStringsSep "." ( + take 3 (splitString "." cfg.serviceClusterIpRange + )) + ".1"); +in +{ + ###### interface + options.services.kubernetes.apiserver = with lib.types; { + + advertiseAddress = mkOption { + description = '' + Kubernetes apiserver IP address on which to advertise the apiserver + to members of the cluster. This address must be reachable by the rest + of the cluster. + ''; + default = null; + type = nullOr str; + }; + + allowPrivileged = mkOption { + description = "Whether to allow privileged containers on Kubernetes."; + default = false; + type = bool; + }; + + authorizationMode = mkOption { + description = '' + Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See + <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/> + ''; + default = ["RBAC" "Node"]; # Enabling RBAC by default, although kubernetes default is AllowAllow + type = listOf (enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "Webhook" "RBAC" "Node"]); + }; + + authorizationPolicy = mkOption { + description = '' + Kubernetes apiserver authorization policy file. See + <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/> + ''; + default = []; + type = listOf attrs; + }; + + basicAuthFile = mkOption { + description = '' + Kubernetes apiserver basic authentication file. See + <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/> + ''; + default = null; + type = nullOr path; + }; + + bindAddress = mkOption { + description = '' + The IP address on which to listen for the --secure-port port. + The associated interface(s) must be reachable by the rest + of the cluster, and by CLI/web clients. + ''; + default = "0.0.0.0"; + type = str; + }; + + clientCaFile = mkOption { + description = "Kubernetes apiserver CA file for client auth."; + default = top.caFile; + type = nullOr path; + }; + + disableAdmissionPlugins = mkOption { + description = '' + Kubernetes admission control plugins to disable. See + <link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/> + ''; + default = []; + type = listOf str; + }; + + enable = mkEnableOption "Kubernetes apiserver"; + + enableAdmissionPlugins = mkOption { + description = '' + Kubernetes admission control plugins to enable. See + <link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/> + ''; + default = [ + "NamespaceLifecycle" "LimitRanger" "ServiceAccount" + "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" + "NodeRestriction" + ]; + example = [ + "NamespaceLifecycle" "NamespaceExists" "LimitRanger" + "SecurityContextDeny" "ServiceAccount" "ResourceQuota" + "PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass" + ]; + type = listOf str; + }; + + etcd = { + servers = mkOption { + description = "List of etcd servers."; + default = ["http://127.0.0.1:2379"]; + type = types.listOf types.str; + }; + + keyFile = mkOption { + description = "Etcd key file."; + default = null; + type = types.nullOr types.path; + }; + + certFile = mkOption { + description = "Etcd cert file."; + default = null; + type = types.nullOr types.path; + }; + + caFile = mkOption { + description = "Etcd ca file."; + default = top.caFile; + type = types.nullOr types.path; + }; + }; + + extraOpts = mkOption { + description = "Kubernetes apiserver extra command line options."; + default = ""; + type = str; + }; + + extraSANs = mkOption { + description = "Extra x509 Subject Alternative Names to be added to the kubernetes apiserver tls cert."; + default = []; + type = listOf str; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = top.featureGates; + type = listOf str; + }; + + insecureBindAddress = mkOption { + description = "The IP address on which to serve the --insecure-port."; + default = "127.0.0.1"; + type = str; + }; + + insecurePort = mkOption { + description = "Kubernetes apiserver insecure listening port. (0 = disabled)"; + default = 0; + type = int; + }; + + kubeletClientCaFile = mkOption { + description = "Path to a cert file for connecting to kubelet."; + default = top.caFile; + type = nullOr path; + }; + + kubeletClientCertFile = mkOption { + description = "Client certificate to use for connections to kubelet."; + default = null; + type = nullOr path; + }; + + kubeletClientKeyFile = mkOption { + description = "Key to use for connections to kubelet."; + default = null; + type = nullOr path; + }; + + kubeletHttps = mkOption { + description = "Whether to use https for connections to kubelet."; + default = true; + type = bool; + }; + + runtimeConfig = mkOption { + description = '' + Api runtime configuration. See + <link xlink:href="https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/"/> + ''; + default = "authentication.k8s.io/v1beta1=true"; + example = "api/all=false,api/v1=true"; + type = str; + }; + + storageBackend = mkOption { + description = '' + Kubernetes apiserver storage backend. + ''; + default = "etcd3"; + type = enum ["etcd2" "etcd3"]; + }; + + securePort = mkOption { + description = "Kubernetes apiserver secure port."; + default = 6443; + type = int; + }; + + serviceAccountKeyFile = mkOption { + description = '' + Kubernetes apiserver PEM-encoded x509 RSA private or public key file, + used to verify ServiceAccount tokens. By default tls private key file + is used. + ''; + default = null; + type = nullOr path; + }; + + serviceClusterIpRange = mkOption { + description = '' + A CIDR notation IP range from which to assign service cluster IPs. + This must not overlap with any IP ranges assigned to nodes for pods. + ''; + default = "10.0.0.0/24"; + type = str; + }; + + tlsCertFile = mkOption { + description = "Kubernetes apiserver certificate file."; + default = null; + type = nullOr path; + }; + + tlsKeyFile = mkOption { + description = "Kubernetes apiserver private key file."; + default = null; + type = nullOr path; + }; + + tokenAuthFile = mkOption { + description = '' + Kubernetes apiserver token authentication file. See + <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/> + ''; + default = null; + type = nullOr path; + }; + + verbosity = mkOption { + description = '' + Optional glog verbosity level for logging statements. See + <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> + ''; + default = null; + type = nullOr int; + }; + + webhookConfig = mkOption { + description = '' + Kubernetes apiserver Webhook config file. It uses the kubeconfig file format. + See <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/webhook/"/> + ''; + default = null; + type = nullOr path; + }; + + }; + + + ###### implementation + config = mkMerge [ + + (mkIf cfg.enable { + systemd.services.kube-apiserver = { + description = "Kubernetes APIServer Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "network.target" ]; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = ''${top.package}/bin/kube-apiserver \ + --allow-privileged=${boolToString cfg.allowPrivileged} \ + --authorization-mode=${concatStringsSep "," cfg.authorizationMode} \ + ${optionalString (elem "ABAC" cfg.authorizationMode) + "--authorization-policy-file=${ + pkgs.writeText "kube-auth-policy.jsonl" + (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.authorizationPolicy) + }" + } \ + ${optionalString (elem "Webhook" cfg.authorizationMode) + "--authorization-webhook-config-file=${cfg.webhookConfig}" + } \ + --bind-address=${cfg.bindAddress} \ + ${optionalString (cfg.advertiseAddress != null) + "--advertise-address=${cfg.advertiseAddress}"} \ + ${optionalString (cfg.clientCaFile != null) + "--client-ca-file=${cfg.clientCaFile}"} \ + --disable-admission-plugins=${concatStringsSep "," cfg.disableAdmissionPlugins} \ + --enable-admission-plugins=${concatStringsSep "," cfg.enableAdmissionPlugins} \ + --etcd-servers=${concatStringsSep "," cfg.etcd.servers} \ + ${optionalString (cfg.etcd.caFile != null) + "--etcd-cafile=${cfg.etcd.caFile}"} \ + ${optionalString (cfg.etcd.certFile != null) + "--etcd-certfile=${cfg.etcd.certFile}"} \ + ${optionalString (cfg.etcd.keyFile != null) + "--etcd-keyfile=${cfg.etcd.keyFile}"} \ + ${optionalString (cfg.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ + ${optionalString (cfg.basicAuthFile != null) + "--basic-auth-file=${cfg.basicAuthFile}"} \ + --kubelet-https=${boolToString cfg.kubeletHttps} \ + ${optionalString (cfg.kubeletClientCaFile != null) + "--kubelet-certificate-authority=${cfg.kubeletClientCaFile}"} \ + ${optionalString (cfg.kubeletClientCertFile != null) + "--kubelet-client-certificate=${cfg.kubeletClientCertFile}"} \ + ${optionalString (cfg.kubeletClientKeyFile != null) + "--kubelet-client-key=${cfg.kubeletClientKeyFile}"} \ + --insecure-bind-address=${cfg.insecureBindAddress} \ + --insecure-port=${toString cfg.insecurePort} \ + ${optionalString (cfg.runtimeConfig != "") + "--runtime-config=${cfg.runtimeConfig}"} \ + --secure-port=${toString cfg.securePort} \ + ${optionalString (cfg.serviceAccountKeyFile!=null) + "--service-account-key-file=${cfg.serviceAccountKeyFile}"} \ + --service-cluster-ip-range=${cfg.serviceClusterIpRange} \ + --storage-backend=${cfg.storageBackend} \ + ${optionalString (cfg.tlsCertFile != null) + "--tls-cert-file=${cfg.tlsCertFile}"} \ + ${optionalString (cfg.tlsKeyFile != null) + "--tls-private-key-file=${cfg.tlsKeyFile}"} \ + ${optionalString (cfg.tokenAuthFile != null) + "--token-auth-file=${cfg.tokenAuthFile}"} \ + ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ + ${cfg.extraOpts} + ''; + WorkingDirectory = top.dataDir; + User = "kubernetes"; + Group = "kubernetes"; + AmbientCapabilities = "cap_net_bind_service"; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + services.etcd = { + clientCertAuth = mkDefault true; + peerClientCertAuth = mkDefault true; + listenClientUrls = mkDefault ["https://0.0.0.0:2379"]; + listenPeerUrls = mkDefault ["https://0.0.0.0:2380"]; + advertiseClientUrls = mkDefault ["https://${top.masterAddress}:2379"]; + initialCluster = mkDefault ["${top.masterAddress}=https://${top.masterAddress}:2380"]; + name = top.masterAddress; + initialAdvertisePeerUrls = mkDefault ["https://${top.masterAddress}:2380"]; + }; + + services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled { + + apiserver-kubelet-api-admin-crb = { + apiVersion = "rbac.authorization.k8s.io/v1"; + kind = "ClusterRoleBinding"; + metadata = { + name = "system:kube-apiserver:kubelet-api-admin"; + }; + roleRef = { + apiGroup = "rbac.authorization.k8s.io"; + kind = "ClusterRole"; + name = "system:kubelet-api-admin"; + }; + subjects = [{ + kind = "User"; + name = "system:kube-apiserver"; + }]; + }; + + }; + + services.kubernetes.pki.certs = with top.lib; { + apiServer = mkCert { + name = "kube-apiserver"; + CN = "kubernetes"; + hosts = [ + "kubernetes.default.svc" + "kubernetes.default.svc.${top.addons.dns.clusterDomain}" + cfg.advertiseAddress + top.masterAddress + apiserverServiceIP + "127.0.0.1" + ] ++ cfg.extraSANs; + action = "systemctl restart kube-apiserver.service"; + }; + apiserverKubeletClient = mkCert { + name = "kube-apiserver-kubelet-client"; + CN = "system:kube-apiserver"; + action = "systemctl restart kube-apiserver.service"; + }; + apiserverEtcdClient = mkCert { + name = "kube-apiserver-etcd-client"; + CN = "etcd-client"; + action = "systemctl restart kube-apiserver.service"; + }; + clusterAdmin = mkCert { + name = "cluster-admin"; + CN = "cluster-admin"; + fields = { + O = "system:masters"; + }; + privateKeyOwner = "root"; + }; + etcd = mkCert { + name = "etcd"; + CN = top.masterAddress; + hosts = [ + "etcd.local" + "etcd.${top.addons.dns.clusterDomain}" + top.masterAddress + cfg.advertiseAddress + ]; + privateKeyOwner = "etcd"; + action = "systemctl restart etcd.service"; + }; + }; + + }) + + ]; + +} diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix new file mode 100644 index 00000000000..dff97f144d5 --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix @@ -0,0 +1,162 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + top = config.services.kubernetes; + cfg = top.controllerManager; +in +{ + ###### interface + options.services.kubernetes.controllerManager = with lib.types; { + + allocateNodeCIDRs = mkOption { + description = "Whether to automatically allocate CIDR ranges for cluster nodes."; + default = true; + type = bool; + }; + + bindAddress = mkOption { + description = "Kubernetes controller manager listening address."; + default = "127.0.0.1"; + type = str; + }; + + clusterCidr = mkOption { + description = "Kubernetes CIDR Range for Pods in cluster."; + default = top.clusterCidr; + type = str; + }; + + enable = mkEnableOption "Kubernetes controller manager."; + + extraOpts = mkOption { + description = "Kubernetes controller manager extra command line options."; + default = ""; + type = str; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = top.featureGates; + type = listOf str; + }; + + insecurePort = mkOption { + description = "Kubernetes controller manager insecure listening port."; + default = 0; + type = int; + }; + + kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes controller manager"; + + leaderElect = mkOption { + description = "Whether to start leader election before executing main loop."; + type = bool; + default = true; + }; + + rootCaFile = mkOption { + description = '' + Kubernetes controller manager certificate authority file included in + service account's token secret. + ''; + default = top.caFile; + type = nullOr path; + }; + + securePort = mkOption { + description = "Kubernetes controller manager secure listening port."; + default = 10252; + type = int; + }; + + serviceAccountKeyFile = mkOption { + description = '' + Kubernetes controller manager PEM-encoded private RSA key file used to + sign service account tokens + ''; + default = null; + type = nullOr path; + }; + + tlsCertFile = mkOption { + description = "Kubernetes controller-manager certificate file."; + default = null; + type = nullOr path; + }; + + tlsKeyFile = mkOption { + description = "Kubernetes controller-manager private key file."; + default = null; + type = nullOr path; + }; + + verbosity = mkOption { + description = '' + Optional glog verbosity level for logging statements. See + <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> + ''; + default = null; + type = nullOr int; + }; + + }; + + ###### implementation + config = mkIf cfg.enable { + systemd.services.kube-controller-manager = { + description = "Kubernetes Controller Manager Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + serviceConfig = { + RestartSec = "30s"; + Restart = "on-failure"; + Slice = "kubernetes.slice"; + ExecStart = ''${top.package}/bin/kube-controller-manager \ + --allocate-node-cidrs=${boolToString cfg.allocateNodeCIDRs} \ + --bind-address=${cfg.bindAddress} \ + ${optionalString (cfg.clusterCidr!=null) + "--cluster-cidr=${cfg.clusterCidr}"} \ + ${optionalString (cfg.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ + --kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \ + --leader-elect=${boolToString cfg.leaderElect} \ + ${optionalString (cfg.rootCaFile!=null) + "--root-ca-file=${cfg.rootCaFile}"} \ + --port=${toString cfg.insecurePort} \ + --secure-port=${toString cfg.securePort} \ + ${optionalString (cfg.serviceAccountKeyFile!=null) + "--service-account-private-key-file=${cfg.serviceAccountKeyFile}"} \ + ${optionalString (cfg.tlsCertFile!=null) + "--tls-cert-file=${cfg.tlsCertFile}"} \ + ${optionalString (cfg.tlsKeyFile!=null) + "--tls-key-file=${cfg.tlsKeyFile}"} \ + ${optionalString (elem "RBAC" top.apiserver.authorizationMode) + "--use-service-account-credentials"} \ + ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ + ${cfg.extraOpts} + ''; + WorkingDirectory = top.dataDir; + User = "kubernetes"; + Group = "kubernetes"; + }; + path = top.path; + }; + + services.kubernetes.pki.certs = with top.lib; { + controllerManager = mkCert { + name = "kube-controller-manager"; + CN = "kube-controller-manager"; + action = "systemctl restart kube-controller-manager.service"; + }; + controllerManagerClient = mkCert { + name = "kube-controller-manager-client"; + CN = "system:kube-controller-manager"; + action = "systemctl restart kube-controller-manager.service"; + }; + }; + + services.kubernetes.controllerManager.kubeconfig.server = mkDefault top.apiserverAddress; + }; +} diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index 6f3c45b29bf..375e33e91b5 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -5,74 +5,52 @@ with lib; let cfg = config.services.kubernetes; - # YAML config; see: - # https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ - # https://github.com/kubernetes/kubernetes/blob/release-1.10/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go - # - # TODO: migrate the following flags to this config file - # - # --pod-manifest-path - # --address - # --port - # --tls-cert-file - # --tls-private-key-file - # --client-ca-file - # --authentication-token-webhook - # --authentication-token-webhook-cache-ttl - # --authorization-mode - # --healthz-bind-address - # --healthz-port - # --allow-privileged - # --cluster-dns - # --cluster-domain - # --hairpin-mode - # --feature-gates - kubeletConfig = pkgs.runCommand "kubelet-config.yaml" { } '' - echo > $out ${pkgs.lib.escapeShellArg (builtins.toJSON { - kind = "KubeletConfiguration"; - apiVersion = "kubelet.config.k8s.io/v1beta1"; - ${if cfg.kubelet.applyManifests then "staticPodPath" else null} = - manifests; - })} - ''; - - infraContainer = pkgs.dockerTools.buildImage { - name = "pause"; - tag = "latest"; - contents = cfg.package.pause; - config.Cmd = "/bin/pause"; - }; - - mkKubeConfig = name: cfg: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON { + mkKubeConfig = name: conf: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON { apiVersion = "v1"; kind = "Config"; clusters = [{ name = "local"; cluster.certificate-authority = cfg.caFile; - cluster.server = cfg.server; + cluster.server = conf.server; }]; users = [{ - name = "kubelet"; + inherit name; user = { - client-certificate = cfg.certFile; - client-key = cfg.keyFile; + client-certificate = conf.certFile; + client-key = conf.keyFile; }; }]; contexts = [{ context = { cluster = "local"; - user = "kubelet"; + user = name; }; - current-context = "kubelet-context"; + current-context = "local"; }]; }); + caCert = secret "ca"; + + etcdEndpoints = ["https://${cfg.masterAddress}:2379"]; + + mkCert = { name, CN, hosts ? [], fields ? {}, action ? "", + privateKeyOwner ? "kubernetes" }: rec { + inherit name caCert CN hosts fields action; + cert = secret name; + key = secret "${name}-key"; + privateKeyOptions = { + owner = privateKeyOwner; + group = "nogroup"; + mode = "0600"; + path = key; + }; + }; + + secret = name: "${cfg.secretsPath}/${name}.pem"; + mkKubeConfigOptions = prefix: { server = mkOption { description = "${prefix} kube-apiserver server address."; - default = "http://${if cfg.apiserver.advertiseAddress != null - then cfg.apiserver.advertiseAddress - else "127.0.0.1"}:${toString cfg.apiserver.port}"; type = types.str; }; @@ -101,66 +79,6 @@ let certFile = mkDefault cfg.kubeconfig.certFile; keyFile = mkDefault cfg.kubeconfig.keyFile; }; - - cniConfig = - if cfg.kubelet.cni.config != [] && !(isNull cfg.kubelet.cni.configDir) then - throw "Verbatim CNI-config and CNI configDir cannot both be set." - else if !(isNull cfg.kubelet.cni.configDir) then - cfg.kubelet.cni.configDir - else - (pkgs.buildEnv { - name = "kubernetes-cni-config"; - paths = imap (i: entry: - pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry) - ) cfg.kubelet.cni.config; - }); - - manifests = pkgs.buildEnv { - name = "kubernetes-manifests"; - paths = mapAttrsToList (name: manifest: - pkgs.writeTextDir "${name}.json" (builtins.toJSON manifest) - ) cfg.kubelet.manifests; - }; - - addons = pkgs.runCommand "kubernetes-addons" { } '' - mkdir -p $out - # since we are mounting the addons to the addon manager, they need to be copied - ${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon: - pkgs.writeTextDir "${name}.json" (builtins.toJSON addon) - ) (cfg.addonManager.addons))} - ''; - - taintOptions = { name, ... }: { - options = { - key = mkOption { - description = "Key of taint."; - default = name; - type = types.str; - }; - value = mkOption { - description = "Value of taint."; - type = types.str; - }; - effect = mkOption { - description = "Effect of taint."; - example = "NoSchedule"; - type = types.enum ["NoSchedule" "PreferNoSchedule" "NoExecute"]; - }; - }; - }; - - taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.kubelet.taints); - - # needed for flannel to pass options to docker - mkDockerOpts = pkgs.runCommand "mk-docker-opts" { - buildInputs = [ pkgs.makeWrapper ]; - } '' - mkdir -p $out - cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh - - # bashInteractive needed for `compgen` - makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh" - ''; in { ###### interface @@ -170,8 +88,9 @@ in { description = '' Kubernetes role that this machine should take. - Master role will enable etcd, apiserver, scheduler and controller manager - services. Node role will enable etcd, docker, kubelet and proxy services. + Master role will enable etcd, apiserver, scheduler, controller manager + addon manager, flannel and proxy services. + Node role will enable flannel, docker, kubelet and proxy services. ''; default = []; type = types.listOf (types.enum ["master" "node"]); @@ -184,40 +103,17 @@ in { defaultText = "pkgs.kubernetes"; }; - verbose = mkOption { - description = "Kubernetes enable verbose mode for debugging."; - default = false; - type = types.bool; - }; - - etcd = { - servers = mkOption { - description = "List of etcd servers. By default etcd is started, except if this option is changed."; - default = ["http://127.0.0.1:2379"]; - type = types.listOf types.str; - }; - - keyFile = mkOption { - description = "Etcd key file."; - default = null; - type = types.nullOr types.path; - }; - - certFile = mkOption { - description = "Etcd cert file."; - default = null; - type = types.nullOr types.path; - }; + kubeconfig = mkKubeConfigOptions "Default kubeconfig"; - caFile = mkOption { - description = "Etcd ca file."; - default = cfg.caFile; - type = types.nullOr types.path; - }; + apiserverAddress = mkOption { + description = '' + Clusterwide accessible address for the kubernetes apiserver, + including protocol and optional port. + ''; + example = "https://kubernetes-apiserver.example.com:6443"; + type = types.str; }; - kubeconfig = mkKubeConfigOptions "Default kubeconfig"; - caFile = mkOption { description = "Default kubernetes certificate authority"; type = types.nullOr types.path; @@ -230,549 +126,22 @@ in { type = types.path; }; + easyCerts = mkOption { + description = "Automatically setup x509 certificates and keys for the entire cluster."; + default = false; + type = types.bool; + }; + featureGates = mkOption { - description = "List set of feature gates"; + description = "List set of feature gates."; default = []; type = types.listOf types.str; }; - apiserver = { - enable = mkOption { - description = "Whether to enable Kubernetes apiserver."; - default = false; - type = types.bool; - }; - - featureGates = mkOption { - description = "List set of feature gates"; - default = cfg.featureGates; - type = types.listOf types.str; - }; - - bindAddress = mkOption { - description = '' - The IP address on which to listen for the --secure-port port. - The associated interface(s) must be reachable by the rest - of the cluster, and by CLI/web clients. - ''; - default = "0.0.0.0"; - type = types.str; - }; - - advertiseAddress = mkOption { - description = '' - Kubernetes apiserver IP address on which to advertise the apiserver - to members of the cluster. This address must be reachable by the rest - of the cluster. - ''; - default = null; - type = types.nullOr types.str; - }; - - storageBackend = mkOption { - description = '' - Kubernetes apiserver storage backend. - ''; - default = "etcd3"; - type = types.enum ["etcd2" "etcd3"]; - }; - - port = mkOption { - description = "Kubernetes apiserver listening port."; - default = 8080; - type = types.int; - }; - - securePort = mkOption { - description = "Kubernetes apiserver secure port."; - default = 443; - type = types.int; - }; - - tlsCertFile = mkOption { - description = "Kubernetes apiserver certificate file."; - default = null; - type = types.nullOr types.path; - }; - - tlsKeyFile = mkOption { - description = "Kubernetes apiserver private key file."; - default = null; - type = types.nullOr types.path; - }; - - clientCaFile = mkOption { - description = "Kubernetes apiserver CA file for client auth."; - default = cfg.caFile; - type = types.nullOr types.path; - }; - - tokenAuthFile = mkOption { - description = '' - Kubernetes apiserver token authentication file. See - <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/> - ''; - default = null; - type = types.nullOr types.path; - }; - - basicAuthFile = mkOption { - description = '' - Kubernetes apiserver basic authentication file. See - <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/> - ''; - default = pkgs.writeText "users" '' - kubernetes,admin,0 - ''; - type = types.nullOr types.path; - }; - - authorizationMode = mkOption { - description = '' - Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See - <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/> - ''; - default = ["RBAC" "Node"]; - type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "Webhook" "RBAC" "Node"]); - }; - - authorizationPolicy = mkOption { - description = '' - Kubernetes apiserver authorization policy file. See - <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/> - ''; - default = []; - type = types.listOf types.attrs; - }; - - webhookConfig = mkOption { - description = '' - Kubernetes apiserver Webhook config file. It uses the kubeconfig file format. - See <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/webhook/"/> - ''; - default = null; - type = types.nullOr types.path; - }; - - allowPrivileged = mkOption { - description = "Whether to allow privileged containers on Kubernetes."; - default = true; - type = types.bool; - }; - - serviceClusterIpRange = mkOption { - description = '' - A CIDR notation IP range from which to assign service cluster IPs. - This must not overlap with any IP ranges assigned to nodes for pods. - ''; - default = "10.0.0.0/24"; - type = types.str; - }; - - runtimeConfig = mkOption { - description = '' - Api runtime configuration. See - <link xlink:href="https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/"/> - ''; - default = "authentication.k8s.io/v1beta1=true"; - example = "api/all=false,api/v1=true"; - type = types.str; - }; - - enableAdmissionPlugins = mkOption { - description = '' - Kubernetes admission control plugins to enable. See - <link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/> - ''; - default = ["NamespaceLifecycle" "LimitRanger" "ServiceAccount" "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" "NodeRestriction"]; - example = [ - "NamespaceLifecycle" "NamespaceExists" "LimitRanger" - "SecurityContextDeny" "ServiceAccount" "ResourceQuota" - "PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass" - ]; - type = types.listOf types.str; - }; - - disableAdmissionPlugins = mkOption { - description = '' - Kubernetes admission control plugins to disable. See - <link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/> - ''; - default = []; - type = types.listOf types.str; - }; - - serviceAccountKeyFile = mkOption { - description = '' - Kubernetes apiserver PEM-encoded x509 RSA private or public key file, - used to verify ServiceAccount tokens. By default tls private key file - is used. - ''; - default = null; - type = types.nullOr types.path; - }; - - kubeletClientCaFile = mkOption { - description = "Path to a cert file for connecting to kubelet."; - default = cfg.caFile; - type = types.nullOr types.path; - }; - - kubeletClientCertFile = mkOption { - description = "Client certificate to use for connections to kubelet."; - default = null; - type = types.nullOr types.path; - }; - - kubeletClientKeyFile = mkOption { - description = "Key to use for connections to kubelet."; - default = null; - type = types.nullOr types.path; - }; - - kubeletHttps = mkOption { - description = "Whether to use https for connections to kubelet."; - default = true; - type = types.bool; - }; - - extraOpts = mkOption { - description = "Kubernetes apiserver extra command line options."; - default = ""; - type = types.str; - }; - }; - - scheduler = { - enable = mkOption { - description = "Whether to enable Kubernetes scheduler."; - default = false; - type = types.bool; - }; - - featureGates = mkOption { - description = "List set of feature gates"; - default = cfg.featureGates; - type = types.listOf types.str; - }; - - address = mkOption { - description = "Kubernetes scheduler listening address."; - default = "127.0.0.1"; - type = types.str; - }; - - port = mkOption { - description = "Kubernetes scheduler listening port."; - default = 10251; - type = types.int; - }; - - leaderElect = mkOption { - description = "Whether to start leader election before executing main loop."; - type = types.bool; - default = true; - }; - - kubeconfig = mkKubeConfigOptions "Kubernetes scheduler"; - - extraOpts = mkOption { - description = "Kubernetes scheduler extra command line options."; - default = ""; - type = types.str; - }; - }; - - controllerManager = { - enable = mkOption { - description = "Whether to enable Kubernetes controller manager."; - default = false; - type = types.bool; - }; - - featureGates = mkOption { - description = "List set of feature gates"; - default = cfg.featureGates; - type = types.listOf types.str; - }; - - address = mkOption { - description = "Kubernetes controller manager listening address."; - default = "127.0.0.1"; - type = types.str; - }; - - port = mkOption { - description = "Kubernetes controller manager listening port."; - default = 10252; - type = types.int; - }; - - leaderElect = mkOption { - description = "Whether to start leader election before executing main loop."; - type = types.bool; - default = true; - }; - - serviceAccountKeyFile = mkOption { - description = '' - Kubernetes controller manager PEM-encoded private RSA key file used to - sign service account tokens - ''; - default = null; - type = types.nullOr types.path; - }; - - rootCaFile = mkOption { - description = '' - Kubernetes controller manager certificate authority file included in - service account's token secret. - ''; - default = cfg.caFile; - type = types.nullOr types.path; - }; - - kubeconfig = mkKubeConfigOptions "Kubernetes controller manager"; - - extraOpts = mkOption { - description = "Kubernetes controller manager extra command line options."; - default = ""; - type = types.str; - }; - }; - - kubelet = { - enable = mkOption { - description = "Whether to enable Kubernetes kubelet."; - default = false; - type = types.bool; - }; - - featureGates = mkOption { - description = "List set of feature gates"; - default = cfg.featureGates; - type = types.listOf types.str; - }; - - seedDockerImages = mkOption { - description = "List of docker images to preload on system"; - default = []; - type = types.listOf types.package; - }; - - registerNode = mkOption { - description = "Whether to auto register kubelet with API server."; - default = true; - type = types.bool; - }; - - address = mkOption { - description = "Kubernetes kubelet info server listening address."; - default = "0.0.0.0"; - type = types.str; - }; - - port = mkOption { - description = "Kubernetes kubelet info server listening port."; - default = 10250; - type = types.int; - }; - - tlsCertFile = mkOption { - description = "File containing x509 Certificate for HTTPS."; - default = null; - type = types.nullOr types.path; - }; - - tlsKeyFile = mkOption { - description = "File containing x509 private key matching tlsCertFile."; - default = null; - type = types.nullOr types.path; - }; - - clientCaFile = mkOption { - description = "Kubernetes apiserver CA file for client authentication."; - default = cfg.caFile; - type = types.nullOr types.path; - }; - - healthz = { - bind = mkOption { - description = "Kubernetes kubelet healthz listening address."; - default = "127.0.0.1"; - type = types.str; - }; - - port = mkOption { - description = "Kubernetes kubelet healthz port."; - default = 10248; - type = types.int; - }; - }; - - hostname = mkOption { - description = "Kubernetes kubelet hostname override."; - default = config.networking.hostName; - type = types.str; - }; - - allowPrivileged = mkOption { - description = "Whether to allow Kubernetes containers to request privileged mode."; - default = true; - type = types.bool; - }; - - clusterDns = mkOption { - description = "Use alternative DNS."; - default = "10.1.0.1"; - type = types.str; - }; - - clusterDomain = mkOption { - description = "Use alternative domain."; - default = config.services.kubernetes.addons.dns.clusterDomain; - type = types.str; - }; - - networkPlugin = mkOption { - description = "Network plugin to use by Kubernetes."; - type = types.nullOr (types.enum ["cni" "kubenet"]); - default = "kubenet"; - }; - - cni = { - packages = mkOption { - description = "List of network plugin packages to install."; - type = types.listOf types.package; - default = []; - }; - - config = mkOption { - description = "Kubernetes CNI configuration."; - type = types.listOf types.attrs; - default = []; - example = literalExample '' - [{ - "cniVersion": "0.2.0", - "name": "mynet", - "type": "bridge", - "bridge": "cni0", - "isGateway": true, - "ipMasq": true, - "ipam": { - "type": "host-local", - "subnet": "10.22.0.0/16", - "routes": [ - { "dst": "0.0.0.0/0" } - ] - } - } { - "cniVersion": "0.2.0", - "type": "loopback" - }] - ''; - }; - - configDir = mkOption { - description = "Path to Kubernetes CNI configuration directory."; - type = types.nullOr types.path; - default = null; - }; - }; - - manifests = mkOption { - description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)"; - type = types.attrsOf types.attrs; - default = {}; - }; - - applyManifests = mkOption { - description = "Whether to apply manifests (this is true for master node)."; - default = false; - type = types.bool; - }; - - unschedulable = mkOption { - description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role."; - default = false; - type = types.bool; - }; - - taints = mkOption { - description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)."; - default = {}; - type = types.attrsOf (types.submodule [ taintOptions ]); - }; - - nodeIp = mkOption { - description = "IP address of the node. If set, kubelet will use this IP address for the node."; - default = null; - type = types.nullOr types.str; - }; - - kubeconfig = mkKubeConfigOptions "Kubelet"; - - extraOpts = mkOption { - description = "Kubernetes kubelet extra command line options."; - default = ""; - type = types.str; - }; - }; - - proxy = { - enable = mkOption { - description = "Whether to enable Kubernetes proxy."; - default = false; - type = types.bool; - }; - - featureGates = mkOption { - description = "List set of feature gates"; - default = cfg.featureGates; - type = types.listOf types.str; - }; - - address = mkOption { - description = "Kubernetes proxy listening address."; - default = "0.0.0.0"; - type = types.str; - }; - - kubeconfig = mkKubeConfigOptions "Kubernetes proxy"; - - extraOpts = mkOption { - description = "Kubernetes proxy extra command line options."; - default = ""; - type = types.str; - }; - }; - - addonManager = { - enable = mkOption { - description = "Whether to enable Kubernetes addon manager."; - default = false; - type = types.bool; - }; - - addons = mkOption { - description = "Kubernetes addons (any kind of Kubernetes resource can be an addon)."; - default = { }; - type = types.attrsOf (types.either types.attrs (types.listOf types.attrs)); - example = literalExample '' - { - "my-service" = { - "apiVersion" = "v1"; - "kind" = "Service"; - "metadata" = { - "name" = "my-service"; - "namespace" = "default"; - }; - "spec" = { ... }; - }; - } - // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; }; - ''; - }; + masterAddress = mkOption { + description = "Clusterwide available network address or hostname for the kubernetes master server."; + example = "master.example.com"; + type = types.str; }; path = mkOption { @@ -787,304 +156,75 @@ in { type = types.nullOr types.str; }; - flannel.enable = mkOption { - description = "Whether to enable flannel networking"; - default = false; - type = types.bool; + lib = mkOption { + description = "Common functions for the kubernetes modules."; + default = { + inherit mkCert; + inherit mkKubeConfig; + inherit mkKubeConfigOptions; + }; + type = types.attrs; }; + secretsPath = mkOption { + description = "Default location for kubernetes secrets. Not a store location."; + type = types.path; + default = cfg.dataDir + "/secrets"; + }; }; ###### implementation config = mkMerge [ - (mkIf cfg.kubelet.enable { - services.kubernetes.kubelet.seedDockerImages = [infraContainer]; - - systemd.services.kubelet-bootstrap = { - description = "Boostrap Kubelet"; - wantedBy = ["kubernetes.target"]; - after = ["docker.service" "network.target"]; - path = with pkgs; [ docker ]; - script = '' - ${concatMapStrings (img: '' - echo "Seeding docker image: ${img}" - docker load <${img} - '') cfg.kubelet.seedDockerImages} - - rm /opt/cni/bin/* || true - ${concatMapStrings (package: '' - echo "Linking cni package: ${package}" - ln -fs ${package}/bin/* /opt/cni/bin - '') cfg.kubelet.cni.packages} - ''; - serviceConfig = { - Slice = "kubernetes.slice"; - Type = "oneshot"; - }; - }; - systemd.services.kubelet = { - description = "Kubernetes Kubelet Service"; - wantedBy = [ "kubernetes.target" ]; - after = [ "network.target" "docker.service" "kube-apiserver.service" "kubelet-bootstrap.service" ]; - path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ cfg.path; - serviceConfig = { - Slice = "kubernetes.slice"; - CPUAccounting = true; - MemoryAccounting = true; - ExecStart = ''${cfg.package}/bin/kubelet \ - ${optionalString (taints != "") - "--register-with-taints=${taints}"} \ - --kubeconfig=${mkKubeConfig "kubelet" cfg.kubelet.kubeconfig} \ - --config=${kubeletConfig} \ - --address=${cfg.kubelet.address} \ - --port=${toString cfg.kubelet.port} \ - --register-node=${boolToString cfg.kubelet.registerNode} \ - ${optionalString (cfg.kubelet.tlsCertFile != null) - "--tls-cert-file=${cfg.kubelet.tlsCertFile}"} \ - ${optionalString (cfg.kubelet.tlsKeyFile != null) - "--tls-private-key-file=${cfg.kubelet.tlsKeyFile}"} \ - ${optionalString (cfg.kubelet.clientCaFile != null) - "--client-ca-file=${cfg.kubelet.clientCaFile}"} \ - --authentication-token-webhook \ - --authentication-token-webhook-cache-ttl="10s" \ - --authorization-mode=Webhook \ - --healthz-bind-address=${cfg.kubelet.healthz.bind} \ - --healthz-port=${toString cfg.kubelet.healthz.port} \ - --hostname-override=${cfg.kubelet.hostname} \ - --allow-privileged=${boolToString cfg.kubelet.allowPrivileged} \ - --root-dir=${cfg.dataDir} \ - ${optionalString (cfg.kubelet.clusterDns != "") - "--cluster-dns=${cfg.kubelet.clusterDns}"} \ - ${optionalString (cfg.kubelet.clusterDomain != "") - "--cluster-domain=${cfg.kubelet.clusterDomain}"} \ - --pod-infra-container-image=pause \ - ${optionalString (cfg.kubelet.networkPlugin != null) - "--network-plugin=${cfg.kubelet.networkPlugin}"} \ - --cni-conf-dir=${cniConfig} \ - --hairpin-mode=hairpin-veth \ - ${optionalString (cfg.kubelet.nodeIp != null) - "--node-ip=${cfg.kubelet.nodeIp}"} \ - ${optionalString (cfg.kubelet.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \ - ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \ - ${cfg.kubelet.extraOpts} - ''; - WorkingDirectory = cfg.dataDir; - }; - }; - - # Allways include cni plugins - services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins]; - - boot.kernelModules = ["br_netfilter"]; - - services.kubernetes.kubelet.kubeconfig = kubeConfigDefaults; - }) - - (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) { - environment.etc = mapAttrs' (name: manifest: - nameValuePair "kubernetes/manifests/${name}.json" { - text = builtins.toJSON manifest; - mode = "0755"; - } - ) cfg.kubelet.manifests; - }) - - (mkIf (cfg.kubelet.unschedulable && cfg.kubelet.enable) { - services.kubernetes.kubelet.taints.unschedulable = { - value = "true"; - effect = "NoSchedule"; - }; - }) - - (mkIf cfg.apiserver.enable { - systemd.services.kube-apiserver = { - description = "Kubernetes APIServer Service"; - wantedBy = [ "kubernetes.target" ]; - after = [ "network.target" "docker.service" ]; - serviceConfig = { - Slice = "kubernetes.slice"; - ExecStart = ''${cfg.package}/bin/kube-apiserver \ - --etcd-servers=${concatStringsSep "," cfg.etcd.servers} \ - ${optionalString (cfg.etcd.caFile != null) - "--etcd-cafile=${cfg.etcd.caFile}"} \ - ${optionalString (cfg.etcd.certFile != null) - "--etcd-certfile=${cfg.etcd.certFile}"} \ - ${optionalString (cfg.etcd.keyFile != null) - "--etcd-keyfile=${cfg.etcd.keyFile}"} \ - --insecure-port=${toString cfg.apiserver.port} \ - --bind-address=${cfg.apiserver.bindAddress} \ - ${optionalString (cfg.apiserver.advertiseAddress != null) - "--advertise-address=${cfg.apiserver.advertiseAddress}"} \ - --allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\ - ${optionalString (cfg.apiserver.tlsCertFile != null) - "--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \ - ${optionalString (cfg.apiserver.tlsKeyFile != null) - "--tls-private-key-file=${cfg.apiserver.tlsKeyFile}"} \ - ${optionalString (cfg.apiserver.tokenAuthFile != null) - "--token-auth-file=${cfg.apiserver.tokenAuthFile}"} \ - ${optionalString (cfg.apiserver.basicAuthFile != null) - "--basic-auth-file=${cfg.apiserver.basicAuthFile}"} \ - --kubelet-https=${if cfg.apiserver.kubeletHttps then "true" else "false"} \ - ${optionalString (cfg.apiserver.kubeletClientCaFile != null) - "--kubelet-certificate-authority=${cfg.apiserver.kubeletClientCaFile}"} \ - ${optionalString (cfg.apiserver.kubeletClientCertFile != null) - "--kubelet-client-certificate=${cfg.apiserver.kubeletClientCertFile}"} \ - ${optionalString (cfg.apiserver.kubeletClientKeyFile != null) - "--kubelet-client-key=${cfg.apiserver.kubeletClientKeyFile}"} \ - ${optionalString (cfg.apiserver.clientCaFile != null) - "--client-ca-file=${cfg.apiserver.clientCaFile}"} \ - --authorization-mode=${concatStringsSep "," cfg.apiserver.authorizationMode} \ - ${optionalString (elem "ABAC" cfg.apiserver.authorizationMode) - "--authorization-policy-file=${ - pkgs.writeText "kube-auth-policy.jsonl" - (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy) - }" - } \ - ${optionalString (elem "Webhook" cfg.apiserver.authorizationMode) - "--authorization-webhook-config-file=${cfg.apiserver.webhookConfig}" - } \ - --secure-port=${toString cfg.apiserver.securePort} \ - --service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \ - ${optionalString (cfg.apiserver.runtimeConfig != "") - "--runtime-config=${cfg.apiserver.runtimeConfig}"} \ - --enable-admission-plugins=${concatStringsSep "," cfg.apiserver.enableAdmissionPlugins} \ - --disable-admission-plugins=${concatStringsSep "," cfg.apiserver.disableAdmissionPlugins} \ - ${optionalString (cfg.apiserver.serviceAccountKeyFile!=null) - "--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \ - ${optionalString cfg.verbose "--v=6"} \ - ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ - --storage-backend=${cfg.apiserver.storageBackend} \ - ${optionalString (cfg.kubelet.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \ - ${cfg.apiserver.extraOpts} - ''; - WorkingDirectory = cfg.dataDir; - User = "kubernetes"; - Group = "kubernetes"; - AmbientCapabilities = "cap_net_bind_service"; - Restart = "on-failure"; - RestartSec = 5; - }; - }; + (mkIf cfg.easyCerts { + services.kubernetes.pki.enable = mkDefault true; + services.kubernetes.caFile = caCert; }) - (mkIf cfg.scheduler.enable { - systemd.services.kube-scheduler = { - description = "Kubernetes Scheduler Service"; - wantedBy = [ "kubernetes.target" ]; - after = [ "kube-apiserver.service" ]; - serviceConfig = { - Slice = "kubernetes.slice"; - ExecStart = ''${cfg.package}/bin/kube-scheduler \ - --address=${cfg.scheduler.address} \ - --port=${toString cfg.scheduler.port} \ - --leader-elect=${boolToString cfg.scheduler.leaderElect} \ - --kubeconfig=${mkKubeConfig "kube-scheduler" cfg.scheduler.kubeconfig} \ - ${optionalString cfg.verbose "--v=6"} \ - ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ - ${optionalString (cfg.scheduler.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.scheduler.featureGates}"} \ - ${cfg.scheduler.extraOpts} - ''; - WorkingDirectory = cfg.dataDir; - User = "kubernetes"; - Group = "kubernetes"; - }; - }; - - services.kubernetes.scheduler.kubeconfig = kubeConfigDefaults; - }) - - (mkIf cfg.controllerManager.enable { - systemd.services.kube-controller-manager = { - description = "Kubernetes Controller Manager Service"; - wantedBy = [ "kubernetes.target" ]; - after = [ "kube-apiserver.service" ]; - serviceConfig = { - RestartSec = "30s"; - Restart = "on-failure"; - Slice = "kubernetes.slice"; - ExecStart = ''${cfg.package}/bin/kube-controller-manager \ - --address=${cfg.controllerManager.address} \ - --port=${toString cfg.controllerManager.port} \ - --kubeconfig=${mkKubeConfig "kube-controller-manager" cfg.controllerManager.kubeconfig} \ - --leader-elect=${boolToString cfg.controllerManager.leaderElect} \ - ${if (cfg.controllerManager.serviceAccountKeyFile!=null) - then "--service-account-private-key-file=${cfg.controllerManager.serviceAccountKeyFile}" - else "--service-account-private-key-file=/var/run/kubernetes/apiserver.key"} \ - ${if (cfg.controllerManager.rootCaFile!=null) - then "--root-ca-file=${cfg.controllerManager.rootCaFile}" - else "--root-ca-file=/var/run/kubernetes/apiserver.crt"} \ - ${if (cfg.clusterCidr!=null) - then "--cluster-cidr=${cfg.clusterCidr} --allocate-node-cidrs=true" - else "--allocate-node-cidrs=false"} \ - ${optionalString (cfg.controllerManager.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.controllerManager.featureGates}"} \ - ${optionalString cfg.verbose "--v=6"} \ - ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ - ${cfg.controllerManager.extraOpts} - ''; - WorkingDirectory = cfg.dataDir; - User = "kubernetes"; - Group = "kubernetes"; + (mkIf (elem "master" cfg.roles) { + services.kubernetes.apiserver.enable = mkDefault true; + services.kubernetes.scheduler.enable = mkDefault true; + services.kubernetes.controllerManager.enable = mkDefault true; + services.kubernetes.addonManager.enable = mkDefault true; + services.kubernetes.proxy.enable = mkDefault true; + services.etcd.enable = true; # Cannot mkDefault because of flannel default options + services.kubernetes.kubelet = { + enable = mkDefault true; + taints = mkIf (!(elem "node" cfg.roles)) { + master = { + key = "node-role.kubernetes.io/master"; + value = "true"; + effect = "NoSchedule"; + }; }; - path = cfg.path; }; - - services.kubernetes.controllerManager.kubeconfig = kubeConfigDefaults; }) - (mkIf cfg.proxy.enable { - systemd.services.kube-proxy = { - description = "Kubernetes Proxy Service"; - wantedBy = [ "kubernetes.target" ]; - after = [ "kube-apiserver.service" ]; - path = [pkgs.iptables pkgs.conntrack_tools]; - serviceConfig = { - Slice = "kubernetes.slice"; - ExecStart = ''${cfg.package}/bin/kube-proxy \ - --kubeconfig=${mkKubeConfig "kube-proxy" cfg.proxy.kubeconfig} \ - --bind-address=${cfg.proxy.address} \ - ${optionalString (cfg.proxy.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.proxy.featureGates}"} \ - ${optionalString cfg.verbose "--v=6"} \ - ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ - ${optionalString (cfg.clusterCidr!=null) - "--cluster-cidr=${cfg.clusterCidr}"} \ - ${cfg.proxy.extraOpts} - ''; - WorkingDirectory = cfg.dataDir; - }; - }; - - # kube-proxy needs iptables - networking.firewall.enable = mkDefault true; - services.kubernetes.proxy.kubeconfig = kubeConfigDefaults; + (mkIf (all (el: el == "master") cfg.roles) { + # if this node is only a master make it unschedulable by default + services.kubernetes.kubelet.unschedulable = mkDefault true; }) - (mkIf (any (el: el == "master") cfg.roles) { - virtualisation.docker.enable = mkDefault true; + (mkIf (elem "node" cfg.roles) { services.kubernetes.kubelet.enable = mkDefault true; - services.kubernetes.kubelet.allowPrivileged = mkDefault true; - services.kubernetes.kubelet.applyManifests = mkDefault true; - services.kubernetes.apiserver.enable = mkDefault true; - services.kubernetes.scheduler.enable = mkDefault true; - services.kubernetes.controllerManager.enable = mkDefault true; - services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]); - services.kubernetes.addonManager.enable = mkDefault true; services.kubernetes.proxy.enable = mkDefault true; }) - # if this node is only a master make it unschedulable by default - (mkIf (all (el: el == "master") cfg.roles) { - services.kubernetes.kubelet.unschedulable = mkDefault true; + # Using "services.kubernetes.roles" will automatically enable easyCerts and flannel + (mkIf (cfg.roles != []) { + services.kubernetes.flannel.enable = mkDefault true; + services.flannel.etcd.endpoints = mkDefault etcdEndpoints; + services.kubernetes.easyCerts = mkDefault true; + }) + + (mkIf cfg.apiserver.enable { + services.kubernetes.pki.etcClusterAdminKubeconfig = mkDefault "kubernetes/cluster-admin.kubeconfig"; + services.kubernetes.apiserver.etcd.servers = mkDefault etcdEndpoints; }) - (mkIf (any (el: el == "node") cfg.roles) { + (mkIf cfg.kubelet.enable { virtualisation.docker = { enable = mkDefault true; @@ -1094,26 +234,18 @@ in { # iptables must be disabled for kubernetes extraOptions = "--iptables=false --ip-masq=false"; }; - - services.kubernetes.kubelet.enable = mkDefault true; - services.kubernetes.proxy.enable = mkDefault true; }) - (mkIf cfg.addonManager.enable { - environment.etc."kubernetes/addons".source = "${addons}/"; - - systemd.services.kube-addon-manager = { - description = "Kubernetes addon manager"; - wantedBy = [ "kubernetes.target" ]; - after = [ "kube-apiserver.service" ]; - environment.ADDON_PATH = "/etc/kubernetes/addons/"; - path = [ pkgs.gawk ]; - serviceConfig = { - Slice = "kubernetes.slice"; - ExecStart = "${cfg.package}/bin/kube-addons"; - WorkingDirectory = cfg.dataDir; - User = "kubernetes"; - Group = "kubernetes"; + (mkIf (cfg.apiserver.enable || cfg.controllerManager.enable) { + services.kubernetes.pki.certs = { + serviceAccount = mkCert { + name = "service-account"; + CN = "system:service-account-signer"; + action = '' + systemctl reload \ + kube-apiserver.service \ + kube-controller-manager.service + ''; }; }; }) @@ -1123,7 +255,8 @@ in { cfg.scheduler.enable || cfg.controllerManager.enable || cfg.kubelet.enable || - cfg.proxy.enable + cfg.proxy.enable || + cfg.addonManager.enable ) { systemd.targets.kubernetes = { description = "Kubernetes"; @@ -1132,11 +265,10 @@ in { systemd.tmpfiles.rules = [ "d /opt/cni/bin 0755 root root -" - "d /var/run/kubernetes 0755 kubernetes kubernetes -" + "d /run/kubernetes 0755 kubernetes kubernetes -" "d /var/lib/kubernetes 0755 kubernetes kubernetes -" ]; - environment.systemPackages = [ cfg.package ]; users.users = singleton { name = "kubernetes"; uid = config.ids.uids.kubernetes; @@ -1148,53 +280,12 @@ in { }; users.groups.kubernetes.gid = config.ids.gids.kubernetes; - # dns addon is enabled by default + # dns addon is enabled by default services.kubernetes.addons.dns.enable = mkDefault true; - }) - (mkIf cfg.flannel.enable { - services.flannel = { - enable = mkDefault true; - network = mkDefault cfg.clusterCidr; - etcd = mkDefault { - endpoints = cfg.etcd.servers; - inherit (cfg.etcd) caFile certFile keyFile; - }; - }; - - services.kubernetes.kubelet = { - networkPlugin = mkDefault "cni"; - cni.config = mkDefault [{ - name = "mynet"; - type = "flannel"; - delegate = { - isDefaultGateway = true; - bridge = "docker0"; - }; - }]; - }; - - systemd.services."mk-docker-opts" = { - description = "Pre-Docker Actions"; - wantedBy = [ "flannel.service" ]; - before = [ "docker.service" ]; - after = [ "flannel.service" ]; - path = [ pkgs.gawk pkgs.gnugrep ]; - script = '' - mkdir -p /run/flannel - ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker - ''; - serviceConfig.Type = "oneshot"; - }; - systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker"; - - # read environment variables generated by mk-docker-opts - virtualisation.docker.extraOptions = "$DOCKER_OPTS"; - - networking.firewall.allowedUDPPorts = [ - 8285 # flannel udp - 8472 # flannel vxlan - ]; + services.kubernetes.apiserverAddress = mkDefault ("https://${if cfg.apiserver.advertiseAddress != null + then cfg.apiserver.advertiseAddress + else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}"); }) ]; } diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix new file mode 100644 index 00000000000..93ee2fd65ee --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/flannel.nix @@ -0,0 +1,134 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + top = config.services.kubernetes; + cfg = top.flannel; + + # we want flannel to use kubernetes itself as configuration backend, not direct etcd + storageBackend = "kubernetes"; + + # needed for flannel to pass options to docker + mkDockerOpts = pkgs.runCommand "mk-docker-opts" { + buildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out + cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh + + # bashInteractive needed for `compgen` + makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh" + ''; +in +{ + ###### interface + options.services.kubernetes.flannel = { + enable = mkEnableOption "enable flannel networking"; + }; + + ###### implementation + config = mkIf cfg.enable { + services.flannel = { + + enable = mkDefault true; + network = mkDefault top.clusterCidr; + inherit storageBackend; + nodeName = config.services.kubernetes.kubelet.hostname; + }; + + services.kubernetes.kubelet = { + networkPlugin = mkDefault "cni"; + cni.config = mkDefault [{ + name = "mynet"; + type = "flannel"; + delegate = { + isDefaultGateway = true; + bridge = "docker0"; + }; + }]; + }; + + systemd.services."mk-docker-opts" = { + description = "Pre-Docker Actions"; + path = with pkgs; [ gawk gnugrep ]; + script = '' + ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker + systemctl restart docker + ''; + serviceConfig.Type = "oneshot"; + }; + + systemd.paths."flannel-subnet-env" = { + wantedBy = [ "flannel.service" ]; + pathConfig = { + PathModified = "/run/flannel/subnet.env"; + Unit = "mk-docker-opts.service"; + }; + }; + + systemd.services.docker = { + environment.DOCKER_OPTS = "-b none"; + serviceConfig.EnvironmentFile = "-/run/flannel/docker"; + }; + + # read environment variables generated by mk-docker-opts + virtualisation.docker.extraOptions = "$DOCKER_OPTS"; + + networking = { + firewall.allowedUDPPorts = [ + 8285 # flannel udp + 8472 # flannel vxlan + ]; + dhcpcd.denyInterfaces = [ "docker*" "flannel*" ]; + }; + + services.kubernetes.pki.certs = { + flannelClient = top.lib.mkCert { + name = "flannel-client"; + CN = "flannel-client"; + action = "systemctl restart flannel.service"; + }; + }; + + # give flannel som kubernetes rbac permissions if applicable + services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) { + + flannel-cr = { + apiVersion = "rbac.authorization.k8s.io/v1beta1"; + kind = "ClusterRole"; + metadata = { name = "flannel"; }; + rules = [{ + apiGroups = [ "" ]; + resources = [ "pods" ]; + verbs = [ "get" ]; + } + { + apiGroups = [ "" ]; + resources = [ "nodes" ]; + verbs = [ "list" "watch" ]; + } + { + apiGroups = [ "" ]; + resources = [ "nodes/status" ]; + verbs = [ "patch" ]; + }]; + }; + + flannel-crb = { + apiVersion = "rbac.authorization.k8s.io/v1beta1"; + kind = "ClusterRoleBinding"; + metadata = { name = "flannel"; }; + roleRef = { + apiGroup = "rbac.authorization.k8s.io"; + kind = "ClusterRole"; + name = "flannel"; + }; + subjects = [{ + kind = "User"; + name = "flannel-client"; + }]; + }; + + }; + }; +} diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix new file mode 100644 index 00000000000..c94bb28bf7f --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix @@ -0,0 +1,358 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + top = config.services.kubernetes; + cfg = top.kubelet; + + cniConfig = + if cfg.cni.config != [] && !(isNull cfg.cni.configDir) then + throw "Verbatim CNI-config and CNI configDir cannot both be set." + else if !(isNull cfg.cni.configDir) then + cfg.cni.configDir + else + (pkgs.buildEnv { + name = "kubernetes-cni-config"; + paths = imap (i: entry: + pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry) + ) cfg.cni.config; + }); + + infraContainer = pkgs.dockerTools.buildImage { + name = "pause"; + tag = "latest"; + contents = top.package.pause; + config.Cmd = "/bin/pause"; + }; + + kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig; + + manifests = pkgs.buildEnv { + name = "kubernetes-manifests"; + paths = mapAttrsToList (name: manifest: + pkgs.writeTextDir "${name}.json" (builtins.toJSON manifest) + ) cfg.manifests; + }; + + manifestPath = "kubernetes/manifests"; + + taintOptions = with lib.types; { name, ... }: { + options = { + key = mkOption { + description = "Key of taint."; + default = name; + type = str; + }; + value = mkOption { + description = "Value of taint."; + type = str; + }; + effect = mkOption { + description = "Effect of taint."; + example = "NoSchedule"; + type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"]; + }; + }; + }; + + taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints); +in +{ + ###### interface + options.services.kubernetes.kubelet = with lib.types; { + + address = mkOption { + description = "Kubernetes kubelet info server listening address."; + default = "0.0.0.0"; + type = str; + }; + + allowPrivileged = mkOption { + description = "Whether to allow Kubernetes containers to request privileged mode."; + default = false; + type = bool; + }; + + clusterDns = mkOption { + description = "Use alternative DNS."; + default = "10.1.0.1"; + type = str; + }; + + clusterDomain = mkOption { + description = "Use alternative domain."; + default = config.services.kubernetes.addons.dns.clusterDomain; + type = str; + }; + + clientCaFile = mkOption { + description = "Kubernetes apiserver CA file for client authentication."; + default = top.caFile; + type = nullOr path; + }; + + cni = { + packages = mkOption { + description = "List of network plugin packages to install."; + type = listOf package; + default = []; + }; + + config = mkOption { + description = "Kubernetes CNI configuration."; + type = listOf attrs; + default = []; + example = literalExample '' + [{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "10.22.0.0/16", + "routes": [ + { "dst": "0.0.0.0/0" } + ] + } + } { + "cniVersion": "0.2.0", + "type": "loopback" + }] + ''; + }; + + configDir = mkOption { + description = "Path to Kubernetes CNI configuration directory."; + type = nullOr path; + default = null; + }; + }; + + enable = mkEnableOption "Kubernetes kubelet."; + + extraOpts = mkOption { + description = "Kubernetes kubelet extra command line options."; + default = ""; + type = str; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = top.featureGates; + type = listOf str; + }; + + healthz = { + bind = mkOption { + description = "Kubernetes kubelet healthz listening address."; + default = "127.0.0.1"; + type = str; + }; + + port = mkOption { + description = "Kubernetes kubelet healthz port."; + default = 10248; + type = int; + }; + }; + + hostname = mkOption { + description = "Kubernetes kubelet hostname override."; + default = config.networking.hostName; + type = str; + }; + + kubeconfig = top.lib.mkKubeConfigOptions "Kubelet"; + + manifests = mkOption { + description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)"; + type = attrsOf attrs; + default = {}; + }; + + networkPlugin = mkOption { + description = "Network plugin to use by Kubernetes."; + type = nullOr (enum ["cni" "kubenet"]); + default = "kubenet"; + }; + + nodeIp = mkOption { + description = "IP address of the node. If set, kubelet will use this IP address for the node."; + default = null; + type = nullOr str; + }; + + registerNode = mkOption { + description = "Whether to auto register kubelet with API server."; + default = true; + type = bool; + }; + + port = mkOption { + description = "Kubernetes kubelet info server listening port."; + default = 10250; + type = int; + }; + + seedDockerImages = mkOption { + description = "List of docker images to preload on system"; + default = []; + type = listOf package; + }; + + taints = mkOption { + description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)."; + default = {}; + type = attrsOf (submodule [ taintOptions ]); + }; + + tlsCertFile = mkOption { + description = "File containing x509 Certificate for HTTPS."; + default = null; + type = nullOr path; + }; + + tlsKeyFile = mkOption { + description = "File containing x509 private key matching tlsCertFile."; + default = null; + type = nullOr path; + }; + + unschedulable = mkOption { + description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role."; + default = false; + type = bool; + }; + + verbosity = mkOption { + description = '' + Optional glog verbosity level for logging statements. See + <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> + ''; + default = null; + type = nullOr int; + }; + + }; + + ###### implementation + config = mkMerge [ + (mkIf cfg.enable { + services.kubernetes.kubelet.seedDockerImages = [infraContainer]; + + systemd.services.kubelet = { + description = "Kubernetes Kubelet Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "network.target" "docker.service" "kube-apiserver.service" ]; + path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path; + preStart = '' + ${concatMapStrings (img: '' + echo "Seeding docker image: ${img}" + docker load <${img} + '') cfg.seedDockerImages} + + rm /opt/cni/bin/* || true + ${concatMapStrings (package: '' + echo "Linking cni package: ${package}" + ln -fs ${package}/bin/* /opt/cni/bin + '') cfg.cni.packages} + ''; + serviceConfig = { + Slice = "kubernetes.slice"; + CPUAccounting = true; + MemoryAccounting = true; + Restart = "on-failure"; + RestartSec = "1000ms"; + ExecStart = ''${top.package}/bin/kubelet \ + --address=${cfg.address} \ + --allow-privileged=${boolToString cfg.allowPrivileged} \ + --authentication-token-webhook \ + --authentication-token-webhook-cache-ttl="10s" \ + --authorization-mode=Webhook \ + ${optionalString (cfg.clientCaFile != null) + "--client-ca-file=${cfg.clientCaFile}"} \ + ${optionalString (cfg.clusterDns != "") + "--cluster-dns=${cfg.clusterDns}"} \ + ${optionalString (cfg.clusterDomain != "") + "--cluster-domain=${cfg.clusterDomain}"} \ + --cni-conf-dir=${cniConfig} \ + ${optionalString (cfg.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ + --hairpin-mode=hairpin-veth \ + --healthz-bind-address=${cfg.healthz.bind} \ + --healthz-port=${toString cfg.healthz.port} \ + --hostname-override=${cfg.hostname} \ + --kubeconfig=${kubeconfig} \ + ${optionalString (cfg.networkPlugin != null) + "--network-plugin=${cfg.networkPlugin}"} \ + ${optionalString (cfg.nodeIp != null) + "--node-ip=${cfg.nodeIp}"} \ + --pod-infra-container-image=pause \ + ${optionalString (cfg.manifests != {}) + "--pod-manifest-path=/etc/${manifestPath}"} \ + --port=${toString cfg.port} \ + --register-node=${boolToString cfg.registerNode} \ + ${optionalString (taints != "") + "--register-with-taints=${taints}"} \ + --root-dir=${top.dataDir} \ + ${optionalString (cfg.tlsCertFile != null) + "--tls-cert-file=${cfg.tlsCertFile}"} \ + ${optionalString (cfg.tlsKeyFile != null) + "--tls-private-key-file=${cfg.tlsKeyFile}"} \ + ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ + ${cfg.extraOpts} + ''; + WorkingDirectory = top.dataDir; + }; + }; + + # Allways include cni plugins + services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins]; + + boot.kernelModules = ["br_netfilter"]; + + services.kubernetes.kubelet.hostname = with config.networking; + mkDefault (hostName + optionalString (!isNull domain) ".${domain}"); + + services.kubernetes.pki.certs = with top.lib; { + kubelet = mkCert { + name = "kubelet"; + CN = top.kubelet.hostname; + action = "systemctl restart kubelet.service"; + + }; + kubeletClient = mkCert { + name = "kubelet-client"; + CN = "system:node:${top.kubelet.hostname}"; + fields = { + O = "system:nodes"; + }; + action = "systemctl restart kubelet.service"; + }; + }; + + services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress; + }) + + (mkIf (cfg.enable && cfg.manifests != {}) { + environment.etc = mapAttrs' (name: manifest: + nameValuePair "${manifestPath}/${name}.json" { + text = builtins.toJSON manifest; + mode = "0755"; + } + ) cfg.manifests; + }) + + (mkIf (cfg.unschedulable && cfg.enable) { + services.kubernetes.kubelet.taints.unschedulable = { + value = "true"; + effect = "NoSchedule"; + }; + }) + + ]; +} diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix new file mode 100644 index 00000000000..38deca23a99 --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -0,0 +1,388 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + top = config.services.kubernetes; + cfg = top.pki; + + csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON { + key = { + algo = "rsa"; + size = 2048; + }; + names = singleton cfg.caSpec; + }); + + csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON { + key = { + algo = "rsa"; + size = 2048; + }; + CN = top.masterAddress; + }); + + cfsslAPITokenBaseName = "apitoken.secret"; + cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}"; + certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}"; + cfsslAPITokenLength = 32; + + clusterAdminKubeconfig = with cfg.certs.clusterAdmin; + top.lib.mkKubeConfig "cluster-admin" { + server = top.apiserverAddress; + certFile = cert; + keyFile = key; + }; + + remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}"; +in +{ + ###### interface + options.services.kubernetes.pki = with lib.types; { + + enable = mkEnableOption "Whether to enable easyCert issuer service."; + + certs = mkOption { + description = "List of certificate specs to feed to cert generator."; + default = {}; + type = attrs; + }; + + genCfsslCACert = mkOption { + description = '' + Whether to automatically generate cfssl CA certificate and key, + if they don't exist. + ''; + default = true; + type = bool; + }; + + genCfsslAPICerts = mkOption { + description = '' + Whether to automatically generate cfssl API webserver TLS cert and key, + if they don't exist. + ''; + default = true; + type = bool; + }; + + genCfsslAPIToken = mkOption { + description = '' + Whether to automatically generate cfssl API-token secret, + if they doesn't exist. + ''; + default = true; + type = bool; + }; + + pkiTrustOnBootstrap = mkOption { + description = "Whether to always trust remote cfssl server upon initial PKI bootstrap."; + default = true; + type = bool; + }; + + caCertPathPrefix = mkOption { + description = '' + Path-prefrix for the CA-certificate to be used for cfssl signing. + Suffixes ".pem" and "-key.pem" will be automatically appended for + the public and private keys respectively. + ''; + default = "${config.services.cfssl.dataDir}/ca"; + type = str; + }; + + caSpec = mkOption { + description = "Certificate specification for the auto-generated CAcert."; + default = { + CN = "kubernetes-cluster-ca"; + O = "NixOS"; + OU = "services.kubernetes.pki.caSpec"; + L = "auto-generated"; + }; + type = attrs; + }; + + etcClusterAdminKubeconfig = mkOption { + description = '' + Symlink a kubeconfig with cluster-admin privileges to environment path + (/etc/<path>). + ''; + default = null; + type = nullOr str; + }; + + }; + + ###### implementation + config = mkIf cfg.enable + (let + cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl"; + cfsslCert = "${cfsslCertPathPrefix}.pem"; + cfsslKey = "${cfsslCertPathPrefix}-key.pem"; + in + { + + services.cfssl = mkIf (top.apiserver.enable) { + enable = true; + address = "0.0.0.0"; + tlsCert = cfsslCert; + tlsKey = cfsslKey; + configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON { + signing = { + profiles = { + default = { + usages = ["digital signature"]; + auth_key = "default"; + expiry = "720h"; + }; + }; + }; + auth_keys = { + default = { + type = "standard"; + key = "file:${cfsslAPITokenPath}"; + }; + }; + })); + }; + + systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable) + (concatStringsSep "\n" [ + "set -e" + (optionalString cfg.genCfsslCACert '' + if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then + ${cfssl}/bin/cfssl genkey -initca ${csrCA} | \ + ${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix} + fi + '') + (optionalString cfg.genCfsslAPICerts '' + if [ ! -f "${dataDir}/cfssl.pem" ]; then + ${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \ + ${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix} + fi + '') + (optionalString cfg.genCfsslAPIToken '' + if [ ! -f "${cfsslAPITokenPath}" ]; then + head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ' >"${cfsslAPITokenPath}" + fi + chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}" + '')]); + + systemd.services.kube-certmgr-bootstrap = { + description = "Kubernetes certmgr bootstrapper"; + wantedBy = [ "certmgr.service" ]; + after = [ "cfssl.target" ]; + script = concatStringsSep "\n" ['' + set -e + + # If there's a cfssl (cert issuer) running locally, then don't rely on user to + # manually paste it in place. Just symlink. + # otherwise, create the target file, ready for users to insert the token + + if [ -f "${cfsslAPITokenPath}" ]; then + ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}" + else + touch "${certmgrAPITokenPath}" && chmod 600 "${certmgrAPITokenPath}" + fi + '' + (optionalString (cfg.pkiTrustOnBootstrap) '' + if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then + ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \ + ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile} + fi + '') + ]; + serviceConfig = { + RestartSec = "10s"; + Restart = "on-failure"; + }; + }; + + services.certmgr = { + enable = true; + package = pkgs.certmgr-selfsigned; + svcManager = "command"; + specs = + let + mkSpec = _: cert: { + inherit (cert) action; + authority = { + inherit remote; + file.path = cert.caCert; + root_ca = cert.caCert; + profile = "default"; + auth_key_file = certmgrAPITokenPath; + }; + certificate = { + path = cert.cert; + }; + private_key = cert.privateKeyOptions; + request = { + inherit (cert) CN hosts; + key = { + algo = "rsa"; + size = 2048; + }; + names = [ cert.fields ]; + }; + }; + in + mapAttrs mkSpec cfg.certs; + }; + + #TODO: Get rid of kube-addon-manager in the future for the following reasons + # - it is basically just a shell script wrapped around kubectl + # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount + # - it is designed to be used with k8s system components only + # - it would be better with a more Nix-oriented way of managing addons + systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{ + environment.KUBECONFIG = with cfg.certs.addonManager; + top.lib.mkKubeConfig "addon-manager" { + server = top.apiserverAddress; + certFile = cert; + keyFile = key; + }; + } + + (optionalAttrs (top.addonManager.bootstrapAddons != {}) { + serviceConfig.PermissionsStartOnly = true; + preStart = with pkgs; + let + files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v)) + top.addonManager.bootstrapAddons; + in + '' + export KUBECONFIG=${clusterAdminKubeconfig} + ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files} + ''; + })]); + + environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig) + clusterAdminKubeconfig; + + environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [ + (pkgs.writeScriptBin "nixos-kubernetes-node-join" '' + set -e + exec 1>&2 + + if [ $# -gt 0 ]; then + echo "Usage: $(basename $0)" + echo "" + echo "No args. Apitoken must be provided on stdin." + echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node." + exit 1 + fi + + if [ $(id -u) != 0 ]; then + echo "Run as root please." + exit 1 + fi + + read -r token + if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then + echo "Token must be of length ${toString cfsslAPITokenLength}." + exit 1 + fi + + echo $token > ${certmgrAPITokenPath} + chmod 600 ${certmgrAPITokenPath} + + echo "Restarting certmgr..." >&1 + systemctl restart certmgr + + echo "Waiting for certs to appear..." >&1 + + ${optionalString top.kubelet.enable '' + while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done + echo "Restarting kubelet..." >&1 + systemctl restart kubelet + ''} + + ${optionalString top.proxy.enable '' + while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done + echo "Restarting kube-proxy..." >&1 + systemctl restart kube-proxy + ''} + + ${optionalString top.flannel.enable '' + while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done + echo "Restarting flannel..." >&1 + systemctl restart flannel + ''} + + echo "Node joined succesfully" + '')]; + + # isolate etcd on loopback at the master node + # easyCerts doesn't support multimaster clusters anyway atm. + services.etcd = with cfg.certs.etcd; { + listenClientUrls = ["https://127.0.0.1:2379"]; + listenPeerUrls = ["https://127.0.0.1:2380"]; + advertiseClientUrls = ["https://etcd.local:2379"]; + initialCluster = ["${top.masterAddress}=https://etcd.local:2380"]; + initialAdvertisePeerUrls = ["https://etcd.local:2380"]; + certFile = mkDefault cert; + keyFile = mkDefault key; + trustedCaFile = mkDefault caCert; + }; + networking.extraHosts = mkIf (config.services.etcd.enable) '' + 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local + ''; + + services.flannel = with cfg.certs.flannelClient; { + kubeconfig = top.lib.mkKubeConfig "flannel" { + server = top.apiserverAddress; + certFile = cert; + keyFile = key; + }; + }; + + services.kubernetes = { + + apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; { + etcd = with cfg.certs.apiserverEtcdClient; { + servers = ["https://etcd.local:2379"]; + certFile = mkDefault cert; + keyFile = mkDefault key; + caFile = mkDefault caCert; + }; + clientCaFile = mkDefault caCert; + tlsCertFile = mkDefault cert; + tlsKeyFile = mkDefault key; + serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert; + kubeletClientCaFile = mkDefault caCert; + kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert; + kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key; + }); + controllerManager = mkIf top.controllerManager.enable { + serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key; + rootCaFile = cfg.certs.controllerManagerClient.caCert; + kubeconfig = with cfg.certs.controllerManagerClient; { + certFile = mkDefault cert; + keyFile = mkDefault key; + }; + }; + scheduler = mkIf top.scheduler.enable { + kubeconfig = with cfg.certs.schedulerClient; { + certFile = mkDefault cert; + keyFile = mkDefault key; + }; + }; + kubelet = mkIf top.kubelet.enable { + clientCaFile = mkDefault cfg.certs.kubelet.caCert; + tlsCertFile = mkDefault cfg.certs.kubelet.cert; + tlsKeyFile = mkDefault cfg.certs.kubelet.key; + kubeconfig = with cfg.certs.kubeletClient; { + certFile = mkDefault cert; + keyFile = mkDefault key; + }; + }; + proxy = mkIf top.proxy.enable { + kubeconfig = with cfg.certs.kubeProxyClient; { + certFile = mkDefault cert; + keyFile = mkDefault key; + }; + }; + }; + }); +} diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix new file mode 100644 index 00000000000..6bcf2eaca82 --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/proxy.nix @@ -0,0 +1,80 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + top = config.services.kubernetes; + cfg = top.proxy; +in +{ + + ###### interface + options.services.kubernetes.proxy = with lib.types; { + + bindAddress = mkOption { + description = "Kubernetes proxy listening address."; + default = "0.0.0.0"; + type = str; + }; + + enable = mkEnableOption "Whether to enable Kubernetes proxy."; + + extraOpts = mkOption { + description = "Kubernetes proxy extra command line options."; + default = ""; + type = str; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = top.featureGates; + type = listOf str; + }; + + kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes proxy"; + + verbosity = mkOption { + description = '' + Optional glog verbosity level for logging statements. See + <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> + ''; + default = null; + type = nullOr int; + }; + + }; + + ###### implementation + config = mkIf cfg.enable { + systemd.services.kube-proxy = { + description = "Kubernetes Proxy Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + path = with pkgs; [ iptables conntrack_tools ]; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = ''${top.package}/bin/kube-proxy \ + --bind-address=${cfg.bindAddress} \ + ${optionalString (top.clusterCidr!=null) + "--cluster-cidr=${top.clusterCidr}"} \ + ${optionalString (cfg.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ + --kubeconfig=${top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \ + ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ + ${cfg.extraOpts} + ''; + WorkingDirectory = top.dataDir; + }; + }; + + services.kubernetes.pki.certs = { + kubeProxyClient = top.lib.mkCert { + name = "kube-proxy-client"; + CN = "system:kube-proxy"; + action = "systemctl restart kube-proxy.service"; + }; + }; + + services.kubernetes.proxy.kubeconfig.server = mkDefault top.apiserverAddress; + }; +} diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix new file mode 100644 index 00000000000..655e6f8b6e2 --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix @@ -0,0 +1,92 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + top = config.services.kubernetes; + cfg = top.scheduler; +in +{ + ###### interface + options.services.kubernetes.scheduler = with lib.types; { + + address = mkOption { + description = "Kubernetes scheduler listening address."; + default = "127.0.0.1"; + type = str; + }; + + enable = mkEnableOption "Whether to enable Kubernetes scheduler."; + + extraOpts = mkOption { + description = "Kubernetes scheduler extra command line options."; + default = ""; + type = str; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = top.featureGates; + type = listOf str; + }; + + kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes scheduler"; + + leaderElect = mkOption { + description = "Whether to start leader election before executing main loop."; + type = bool; + default = true; + }; + + port = mkOption { + description = "Kubernetes scheduler listening port."; + default = 10251; + type = int; + }; + + verbosity = mkOption { + description = '' + Optional glog verbosity level for logging statements. See + <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> + ''; + default = null; + type = nullOr int; + }; + + }; + + ###### implementation + config = mkIf cfg.enable { + systemd.services.kube-scheduler = { + description = "Kubernetes Scheduler Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = ''${top.package}/bin/kube-scheduler \ + --address=${cfg.address} \ + ${optionalString (cfg.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ + --kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \ + --leader-elect=${boolToString cfg.leaderElect} \ + --port=${toString cfg.port} \ + ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ + ${cfg.extraOpts} + ''; + WorkingDirectory = top.dataDir; + User = "kubernetes"; + Group = "kubernetes"; + }; + }; + + services.kubernetes.pki.certs = { + schedulerClient = top.lib.mkCert { + name = "kube-scheduler-client"; + CN = "system:kube-scheduler"; + action = "systemctl restart kube-scheduler.service"; + }; + }; + + services.kubernetes.scheduler.kubeconfig.server = mkDefault top.apiserverAddress; + }; +} diff --git a/nixos/modules/services/hardware/bolt.nix b/nixos/modules/services/hardware/bolt.nix new file mode 100644 index 00000000000..32b60af0603 --- /dev/null +++ b/nixos/modules/services/hardware/bolt.nix @@ -0,0 +1,34 @@ +# Thunderbolt 3 device manager + +{ config, lib, pkgs, ...}: + +with lib; + +{ + options = { + + services.hardware.bolt = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable Bolt, a userspace daemon to enable + security levels for Thunderbolt 3 on GNU/Linux. + + Bolt is used by GNOME 3 to handle Thunderbolt settings. + ''; + }; + + }; + + }; + + config = mkIf config.services.hardware.bolt.enable { + + environment.systemPackages = [ pkgs.bolt ]; + services.udev.packages = [ pkgs.bolt ]; + systemd.packages = [ pkgs.bolt ]; + + }; +} diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index 25c258ebe13..b8617e48d8e 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -22,7 +22,8 @@ let password = cfg.databasePassword; username = cfg.databaseUsername; encoding = "utf8"; - }; + pool = cfg.databasePool; + } // cfg.extraDatabaseConfig; }; gitalyToml = pkgs.writeText "gitaly.toml" '' @@ -253,6 +254,18 @@ in { description = "Gitlab database user."; }; + databasePool = mkOption { + type = types.int; + default = 5; + description = "Database connection pool size."; + }; + + extraDatabaseConfig = mkOption { + type = types.attrs; + default = {}; + description = "Extra configuration in config/database.yml."; + }; + host = mkOption { type = types.str; default = config.networking.hostName; diff --git a/nixos/modules/services/misc/jackett.nix b/nixos/modules/services/misc/jackett.nix index 8d1b3d225a4..b18ce2b1f81 100644 --- a/nixos/modules/services/misc/jackett.nix +++ b/nixos/modules/services/misc/jackett.nix @@ -4,11 +4,36 @@ with lib; let cfg = config.services.jackett; + in { options = { services.jackett = { enable = mkEnableOption "Jackett"; + + dataDir = mkOption { + type = types.str; + default = "/var/lib/jackett/.config/Jackett"; + description = "The directory where Jackett stores its data files."; + }; + + openFirewall = mkOption { + type = types.bool; + default = false; + description = "Open ports in the firewall for the Jackett web interface."; + }; + + user = mkOption { + type = types.str; + default = "jackett"; + description = "User account under which Jackett runs."; + }; + + group = mkOption { + type = types.str; + default = "jackett"; + description = "Group under which Jackett runs."; + }; }; }; @@ -18,30 +43,38 @@ in after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; preStart = '' - test -d /var/lib/jackett/ || { - echo "Creating jackett data directory in /var/lib/jackett/" - mkdir -p /var/lib/jackett/ + test -d ${cfg.dataDir} || { + echo "Creating jackett data directory in ${cfg.dataDir}" + mkdir -p ${cfg.dataDir} } - chown -R jackett:jackett /var/lib/jackett/ - chmod 0700 /var/lib/jackett/ + chown -R ${cfg.user}:${cfg.group} ${cfg.dataDir} + chmod 0700 ${cfg.dataDir} ''; serviceConfig = { Type = "simple"; - User = "jackett"; - Group = "jackett"; + User = cfg.user; + Group = cfg.group; PermissionsStartOnly = "true"; - ExecStart = "${pkgs.jackett}/bin/Jackett"; + ExecStart = "${pkgs.jackett}/bin/Jackett --NoUpdates --DataFolder '${cfg.dataDir}'"; Restart = "on-failure"; }; }; - users.users.jackett = { - uid = config.ids.uids.jackett; - home = "/var/lib/jackett"; - group = "jackett"; + networking.firewall = mkIf cfg.openFirewall { + allowedTCPPorts = [ 9117 ]; + }; + + users.users = mkIf (cfg.user == "jackett") { + jackett = { + group = cfg.group; + home = cfg.dataDir; + uid = config.ids.uids.jackett; + }; }; - users.groups.jackett.gid = config.ids.gids.jackett; + users.groups = mkIf (cfg.group == "jackett") { + jackett.gid = config.ids.gids.jackett; + }; }; } diff --git a/nixos/modules/services/misc/radarr.nix b/nixos/modules/services/misc/radarr.nix index 1a9fad3883c..9ab26d84832 100644 --- a/nixos/modules/services/misc/radarr.nix +++ b/nixos/modules/services/misc/radarr.nix @@ -4,11 +4,36 @@ with lib; let cfg = config.services.radarr; + in { options = { services.radarr = { enable = mkEnableOption "Radarr"; + + dataDir = mkOption { + type = types.str; + default = "/var/lib/radarr/.config/Radarr"; + description = "The directory where Radarr stores its data files."; + }; + + openFirewall = mkOption { + type = types.bool; + default = false; + description = "Open ports in the firewall for the Radarr web interface."; + }; + + user = mkOption { + type = types.str; + default = "radarr"; + description = "User account under which Radarr runs."; + }; + + group = mkOption { + type = types.str; + default = "radarr"; + description = "Group under which Radarr runs."; + }; }; }; @@ -18,30 +43,38 @@ in after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; preStart = '' - test -d /var/lib/radarr/ || { - echo "Creating radarr data directory in /var/lib/radarr/" - mkdir -p /var/lib/radarr/ + test -d ${cfg.dataDir} || { + echo "Creating radarr data directory in ${cfg.dataDir}" + mkdir -p ${cfg.dataDir} } - chown -R radarr:radarr /var/lib/radarr/ - chmod 0700 /var/lib/radarr/ + chown -R ${cfg.user}:${cfg.group} ${cfg.dataDir} + chmod 0700 ${cfg.dataDir} ''; serviceConfig = { Type = "simple"; - User = "radarr"; - Group = "radarr"; + User = cfg.user; + Group = cfg.group; PermissionsStartOnly = "true"; - ExecStart = "${pkgs.radarr}/bin/Radarr"; + ExecStart = "${pkgs.radarr}/bin/Radarr -nobrowser -data='${cfg.dataDir}'"; Restart = "on-failure"; }; }; - users.users.radarr = { - uid = config.ids.uids.radarr; - home = "/var/lib/radarr"; - group = "radarr"; + networking.firewall = mkIf cfg.openFirewall { + allowedTCPPorts = [ 7878 ]; + }; + + users.users = mkIf (cfg.user == "radarr") { + radarr = { + group = cfg.group; + home = cfg.dataDir; + uid = config.ids.uids.radarr; + }; }; - users.groups.radarr.gid = config.ids.gids.radarr; + users.groups = mkIf (cfg.group == "radarr") { + radarr.gid = config.ids.gids.radarr; + }; }; } diff --git a/nixos/modules/services/networking/flannel.nix b/nixos/modules/services/networking/flannel.nix index 6c43573851b..ec702cdc6ff 100644 --- a/nixos/modules/services/networking/flannel.nix +++ b/nixos/modules/services/networking/flannel.nix @@ -161,7 +161,10 @@ in { FLANNELD_KUBECONFIG_FILE = cfg.kubeconfig; NODE_NAME = cfg.nodeName; }; - preStart = mkIf (cfg.storageBackend == "etcd") '' + preStart = '' + mkdir -p /run/flannel + touch /run/flannel/docker + '' + optionalString (cfg.storageBackend == "etcd") '' echo "setting network configuration" until ${pkgs.etcdctl.bin}/bin/etcdctl set /coreos.com/network/config '${builtins.toJSON networkConfig}' do @@ -169,13 +172,11 @@ in { sleep 1 done ''; - postStart = '' - while [ ! -f /run/flannel/subnet.env ] - do - sleep 1 - done - ''; - serviceConfig.ExecStart = "${cfg.package}/bin/flannel"; + serviceConfig = { + ExecStart = "${cfg.package}/bin/flannel"; + Restart = "always"; + RestartSec = "10s"; + }; }; services.etcd.enable = mkDefault (cfg.storageBackend == "etcd" && cfg.etcd.endpoints == ["http://127.0.0.1:2379"]); diff --git a/nixos/modules/services/web-apps/tt-rss.nix b/nixos/modules/services/web-apps/tt-rss.nix index fa42ce81234..f7a3daa5fdd 100644 --- a/nixos/modules/services/web-apps/tt-rss.nix +++ b/nixos/modules/services/web-apps/tt-rss.nix @@ -40,7 +40,7 @@ let else if (cfg.database.passwordFile != null) then "file_get_contents('${cfg.database.passwordFile}')" else - "" + "''" }); define('DB_PORT', '${toString dbPort}'); diff --git a/nixos/modules/services/web-servers/apache-httpd/default.nix b/nixos/modules/services/web-servers/apache-httpd/default.nix index eebb1c413fa..3fd19d425c7 100644 --- a/nixos/modules/services/web-servers/apache-httpd/default.nix +++ b/nixos/modules/services/web-servers/apache-httpd/default.nix @@ -497,8 +497,8 @@ in default = false; description = '' If enabled, each virtual host gets its own - <filename>access_log</filename> and - <filename>error_log</filename>, namely suffixed by the + <filename>access.log</filename> and + <filename>error.log</filename>, namely suffixed by the <option>hostName</option> of the virtual host. ''; }; diff --git a/nixos/modules/services/x11/desktop-managers/gnome3.nix b/nixos/modules/services/x11/desktop-managers/gnome3.nix index 7544ba4638a..31ff60019ae 100644 --- a/nixos/modules/services/x11/desktop-managers/gnome3.nix +++ b/nixos/modules/services/x11/desktop-managers/gnome3.nix @@ -151,6 +151,7 @@ in { services.colord.enable = mkDefault true; services.packagekit.enable = mkDefault true; hardware.bluetooth.enable = mkDefault true; + services.hardware.bolt.enable = mkDefault true; services.xserver.libinput.enable = mkDefault true; # for controlling touchpad settings via gnome control center services.udev.packages = [ pkgs.gnome3.gnome-settings-daemon ]; systemd.packages = [ pkgs.gnome3.vino ]; diff --git a/nixos/modules/virtualisation/vmware-guest.nix b/nixos/modules/virtualisation/vmware-guest.nix index 15c78f14c52..d18778f8158 100644 --- a/nixos/modules/virtualisation/vmware-guest.nix +++ b/nixos/modules/virtualisation/vmware-guest.nix @@ -3,19 +3,17 @@ with lib; let - cfg = config.services.vmwareGuest; + cfg = config.virtualisation.vmware.guest; open-vm-tools = if cfg.headless then pkgs.open-vm-tools-headless else pkgs.open-vm-tools; xf86inputvmmouse = pkgs.xorg.xf86inputvmmouse; in { - options = { - services.vmwareGuest = { - enable = mkEnableOption "VMWare Guest Support"; - headless = mkOption { - type = types.bool; - default = false; - description = "Whether to disable X11-related features."; - }; + options.virtualisation.vmware.guest = { + enable = mkEnableOption "VMWare Guest Support"; + headless = mkOption { + type = types.bool; + default = false; + description = "Whether to disable X11-related features."; }; }; @@ -25,6 +23,8 @@ in message = "VMWare guest is not currently supported on ${pkgs.stdenv.hostPlatform.system}"; } ]; + boot.initrd.kernelModules = [ "vmw_pvscsi" ]; + environment.systemPackages = [ open-vm-tools ]; systemd.services.vmware = |