From 35f7a3347c3844dc2f41d66a6fb898d9d45d15b9 Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Mon, 31 Aug 2020 13:07:58 +0200 Subject: kubernetes: fix certificate generation Signed-off-by: Sascha Grunert --- nixos/modules/services/cluster/kubernetes/pki.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'nixos/modules/services/cluster/kubernetes/pki.nix') diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 4275563f1a3..933ae481e96 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -20,7 +20,7 @@ let size = 2048; }; CN = top.masterAddress; - hosts = cfg.cfsslAPIExtraSANs; + hosts = [top.masterAddress] ++ cfg.cfsslAPIExtraSANs; }); cfsslAPITokenBaseName = "apitoken.secret"; @@ -228,7 +228,8 @@ in }; private_key = cert.privateKeyOptions; request = { - inherit (cert) CN hosts; + hosts = [cert.CN] ++ cert.hosts; + inherit (cert) CN; key = { algo = "rsa"; size = 2048; -- cgit 1.4.1 From 7da62867be079bb5f6412fb12a76dbb68f9bad4b Mon Sep 17 00:00:00 2001 From: Yurii Matsiuk Date: Wed, 13 Jan 2021 20:10:04 +0100 Subject: nixos/kubernetes: adapt module and test cases to fit kubernetes v1.20.X as well as coredns v1.7.X --- .../services/cluster/kubernetes/addons/dns.nix | 7 ++-- .../services/cluster/kubernetes/apiserver.nix | 44 ++++++++++++++++++---- .../services/cluster/kubernetes/kubelet.nix | 16 +++++++- nixos/modules/services/cluster/kubernetes/pki.nix | 1 + .../networking/cluster/kubernetes/default.nix | 11 +++++- .../kubernetes/fixup-addonmanager-lib-path.patch | 23 +++++++++++ 6 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 pkgs/applications/networking/cluster/kubernetes/fixup-addonmanager-lib-path.patch (limited to 'nixos/modules/services/cluster/kubernetes/pki.nix') diff --git a/nixos/modules/services/cluster/kubernetes/addons/dns.nix b/nixos/modules/services/cluster/kubernetes/addons/dns.nix index f12e866930d..24d86628b21 100644 --- a/nixos/modules/services/cluster/kubernetes/addons/dns.nix +++ b/nixos/modules/services/cluster/kubernetes/addons/dns.nix @@ -3,7 +3,7 @@ with lib; let - version = "1.6.4"; + version = "1.7.1"; cfg = config.services.kubernetes.addons.dns; ports = { dns = 10053; @@ -55,9 +55,9 @@ in { type = types.attrs; default = { imageName = "coredns/coredns"; - imageDigest = "sha256:493ee88e1a92abebac67cbd4b5658b4730e0f33512461442d8d9214ea6734a9b"; + imageDigest = "sha256:4a6e0769130686518325b21b0c1d0688b54e7c79244d48e1b15634e98e40c6ef"; finalImageTag = version; - sha256 = "0fm9zdjavpf5hni8g7fkdd3csjbhd7n7py7llxjc66sbii087028"; + sha256 = "02r440xcdsgi137k5lmmvp0z5w5fmk8g9mysq5pnysq1wl8sj6mw"; }; }; }; @@ -156,7 +156,6 @@ in { health :${toString ports.health} kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa { pods insecure - upstream fallthrough in-addr.arpa ip6.arpa } prometheus :${toString ports.metrics} diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix index 95bdb4c0d14..616389dfaac 100644 --- a/nixos/modules/services/cluster/kubernetes/apiserver.nix +++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix @@ -238,14 +238,42 @@ in type = int; }; + apiAudiences = mkOption { + description = '' + Kubernetes apiserver ServiceAccount issuer. + ''; + default = "api,https://kubernetes.default.svc"; + type = str; + }; + + serviceAccountIssuer = mkOption { + description = '' + Kubernetes apiserver ServiceAccount issuer. + ''; + default = "https://kubernetes.default.svc"; + type = str; + }; + + serviceAccountSigningKeyFile = mkOption { + description = '' + Path to the file that contains the current private key of the service + account token issuer. The issuer will sign issued ID tokens with this + private key. + ''; + default = top.serviceAccountSigningKeyFile; + type = path; + }; + serviceAccountKeyFile = mkOption { description = '' - Kubernetes apiserver PEM-encoded x509 RSA private or public key file, - used to verify ServiceAccount tokens. By default tls private key file - is used. + File containing PEM-encoded x509 RSA or ECDSA private or public keys, + used to verify ServiceAccount tokens. The specified file can contain + multiple keys, and the flag can be specified multiple times with + different files. If unspecified, --tls-private-key-file is used. + Must be specified when --service-account-signing-key is provided ''; - default = null; - type = nullOr path; + default = top.serviceAccountKeyFile; + type = path; }; serviceClusterIpRange = mkOption { @@ -357,8 +385,10 @@ in ${optionalString (cfg.runtimeConfig != "") "--runtime-config=${cfg.runtimeConfig}"} \ --secure-port=${toString cfg.securePort} \ - ${optionalString (cfg.serviceAccountKeyFile!=null) - "--service-account-key-file=${cfg.serviceAccountKeyFile}"} \ + --api-audiences=${toString cfg.apiAudiences} \ + --service-account-issuer=${toString cfg.serviceAccountIssuer} \ + --service-account-signing-key-file=${cfg.serviceAccountSigningKeyFile} \ + --service-account-key-file=${cfg.serviceAccountKeyFile} \ --service-cluster-ip-range=${cfg.serviceClusterIpRange} \ --storage-backend=${cfg.storageBackend} \ ${optionalString (cfg.tlsCertFile != null) diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix index 479027f1b27..4da6efca535 100644 --- a/nixos/modules/services/cluster/kubernetes/kubelet.nix +++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix @@ -125,6 +125,18 @@ in }; }; + containerRuntime = mkOption { + description = "Which container runtime type to use"; + type = enum ["docker" "remote"]; + default = "remote"; + }; + + containerRuntimeEndpoint = mkOption { + description = "Endpoint at which to find the container runtime api interface/socket"; + type = str; + default = "unix:///var/run/docker/containerd/containerd.sock"; + }; + enable = mkEnableOption "Kubernetes kubelet."; extraOpts = mkOption { @@ -240,7 +252,7 @@ in systemd.services.kubelet = { description = "Kubernetes Kubelet Service"; wantedBy = [ "kubernetes.target" ]; - after = [ "network.target" "docker.service" "kube-apiserver.service" ]; + after = [ "network.target" "kube-apiserver.service" "sockets.target" ]; path = with pkgs; [ gitMinimal openssh @@ -306,6 +318,8 @@ in ${optionalString (cfg.tlsKeyFile != null) "--tls-private-key-file=${cfg.tlsKeyFile}"} \ ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ + --container-runtime=${cfg.containerRuntime} \ + --container-runtime-endpoint=${cfg.containerRuntimeEndpoint} \ ${cfg.extraOpts} ''; WorkingDirectory = top.dataDir; diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 933ae481e96..8de6a3ba0d8 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -361,6 +361,7 @@ in tlsCertFile = mkDefault cert; tlsKeyFile = mkDefault key; serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert; + serviceAccountSigningKeyFile = mkDefault cfg.certs.serviceAccount.key; kubeletClientCaFile = mkDefault caCert; kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert; kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key; diff --git a/pkgs/applications/networking/cluster/kubernetes/default.nix b/pkgs/applications/networking/cluster/kubernetes/default.nix index 184f36c69f7..cb669615f63 100644 --- a/pkgs/applications/networking/cluster/kubernetes/default.nix +++ b/pkgs/applications/networking/cluster/kubernetes/default.nix @@ -33,6 +33,8 @@ stdenv.mkDerivation rec { outputs = [ "out" "man" "pause" ]; + patches = [ ./fixup-addonmanager-lib-path.patch ]; + postPatch = '' # go env breaks the sandbox substituteInPlace "hack/lib/golang.sh" \ @@ -64,10 +66,17 @@ stdenv.mkDerivation rec { install -D build/pause/linux/pause -t $pause/bin installManPage docs/man/man1/*.[1-9] - cp cluster/addons/addon-manager/kube-addons.sh $out/bin/kube-addons + # Unfortunately, kube-addons-main.sh only looks for the lib file in either the current working dir + # or in /opt. We have to patch this for now. + substitute cluster/addons/addon-manager/kube-addons-main.sh $out/bin/kube-addons \ + --subst-var out + + chmod +x $out/bin/kube-addons patchShebangs $out/bin/kube-addons wrapProgram $out/bin/kube-addons --set "KUBECTL_BIN" "$out/bin/kubectl" + cp cluster/addons/addon-manager/kube-addons.sh $out/bin/kube-addons-lib.sh + cp ${./mk-docker-opts.sh} $out/bin/mk-docker-opts.sh for tool in kubeadm kubectl; do diff --git a/pkgs/applications/networking/cluster/kubernetes/fixup-addonmanager-lib-path.patch b/pkgs/applications/networking/cluster/kubernetes/fixup-addonmanager-lib-path.patch new file mode 100644 index 00000000000..ef2904bdcfe --- /dev/null +++ b/pkgs/applications/networking/cluster/kubernetes/fixup-addonmanager-lib-path.patch @@ -0,0 +1,23 @@ +diff --git a/cluster/addons/addon-manager/kube-addons-main.sh b/cluster/addons/addon-manager/kube-addons-main.sh +index 849973470d1..e4fef30eaea 100755 +--- a/cluster/addons/addon-manager/kube-addons-main.sh ++++ b/cluster/addons/addon-manager/kube-addons-main.sh +@@ -17,17 +17,7 @@ + # Import required functions. The addon manager is installed to /opt in + # production use (see the Dockerfile) + # Disabling shellcheck following files as the full path would be required. +-if [ -f "kube-addons.sh" ]; then +- # shellcheck disable=SC1091 +- source "kube-addons.sh" +-elif [ -f "/opt/kube-addons.sh" ]; then +- # shellcheck disable=SC1091 +- source "/opt/kube-addons.sh" +-else +- # If the required source is missing, we have to fail. +- log ERR "== Could not find kube-addons.sh (not in working directory or /opt) at $(date -Is) ==" +- exit 1 +-fi ++source "@out@/bin/kube-addons-lib.sh" + + # The business logic for whether a given object should be created + # was already enforced by salt, and /etc/kubernetes/addons is the -- cgit 1.4.1 From ab1567e8121dfcdb4f7a395b39a3a345f41c7474 Mon Sep 17 00:00:00 2001 From: Alexandru Scvortov Date: Mon, 28 Jun 2021 20:33:17 +0100 Subject: kubernetes: fix conntrack-tools package name, missing dir, and tests --- nixos/modules/services/cluster/kubernetes/pki.nix | 1 + nixos/modules/services/cluster/kubernetes/proxy.nix | 2 +- nixos/tests/kubernetes/base.nix | 9 ++++++++- 3 files changed, 10 insertions(+), 2 deletions(-) (limited to 'nixos/modules/services/cluster/kubernetes/pki.nix') diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 8de6a3ba0d8..d9311d3e3a0 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -189,6 +189,7 @@ in # manually paste it in place. Just symlink. # otherwise, create the target file, ready for users to insert the token + mkdir -p $(dirname ${certmgrAPITokenPath}) if [ -f "${cfsslAPITokenPath}" ]; then ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}" else diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix index 7aa449f9aa2..42729f54643 100644 --- a/nixos/modules/services/cluster/kubernetes/proxy.nix +++ b/nixos/modules/services/cluster/kubernetes/proxy.nix @@ -59,7 +59,7 @@ in description = "Kubernetes Proxy Service"; wantedBy = [ "kubernetes.target" ]; after = [ "kube-apiserver.service" ]; - path = with pkgs; [ iptables conntrack_tools ]; + path = with pkgs; [ iptables conntrack-tools ]; serviceConfig = { Slice = "kubernetes.slice"; ExecStart = ''${top.package}/bin/kube-proxy \ diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix index 8cfac10b6dc..1f23ca55fb2 100644 --- a/nixos/tests/kubernetes/base.nix +++ b/nixos/tests/kubernetes/base.nix @@ -40,7 +40,7 @@ let allowedTCPPorts = [ 10250 # kubelet ]; - trustedInterfaces = ["docker0"]; + trustedInterfaces = ["mynet"]; extraCommands = concatMapStrings (node: '' iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT @@ -61,6 +61,13 @@ let advertiseAddress = master.ip; }; masterAddress = "${masterName}.${config.networking.domain}"; + # workaround for: + # https://github.com/kubernetes/kubernetes/issues/102676 + # (workaround from) https://github.com/kubernetes/kubernetes/issues/95488 + kubelet.extraOpts = ''\ + --cgroups-per-qos=false \ + --enforce-node-allocatable="" \ + ''; }; } (optionalAttrs (any (role: role == "master") machine.roles) { -- cgit 1.4.1