diff options
Diffstat (limited to 'nixos/modules/services/cluster/kubernetes/pki.nix')
| -rw-r--r-- | nixos/modules/services/cluster/kubernetes/pki.nix | 167 |
1 files changed, 93 insertions, 74 deletions
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 38deca23a99..32eacad9025 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -27,12 +27,11 @@ let certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}"; cfsslAPITokenLength = 32; - clusterAdminKubeconfig = with cfg.certs.clusterAdmin; - top.lib.mkKubeConfig "cluster-admin" { - server = top.apiserverAddress; - certFile = cert; - keyFile = key; - }; + clusterAdminKubeconfig = with cfg.certs.clusterAdmin; { + server = top.apiserverAddress; + certFile = cert; + keyFile = key; + }; remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}"; in @@ -40,7 +39,7 @@ in ###### interface options.services.kubernetes.pki = with lib.types; { - enable = mkEnableOption "Whether to enable easyCert issuer service."; + enable = mkEnableOption "easyCert issuer service"; certs = mkOption { description = "List of certificate specs to feed to cert generator."; @@ -119,6 +118,12 @@ in cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl"; cfsslCert = "${cfsslCertPathPrefix}.pem"; cfsslKey = "${cfsslCertPathPrefix}-key.pem"; + cfsslPort = toString config.services.cfssl.port; + + certmgrPaths = [ + top.caFile + certmgrAPITokenPath + ]; in { @@ -168,13 +173,40 @@ in chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}" '')]); + systemd.targets.cfssl-online = { + wantedBy = [ "network-online.target" ]; + after = [ "cfssl.service" "network-online.target" "cfssl-online.service" ]; + }; + + systemd.services.cfssl-online = { + description = "Wait for ${remote} to be reachable."; + wantedBy = [ "cfssl-online.target" ]; + before = [ "cfssl-online.target" ]; + path = [ pkgs.curl ]; + preStart = '' + until curl --fail-early -fskd '{}' ${remote}/api/v1/cfssl/info -o /dev/null; do + echo curl ${remote}/api/v1/cfssl/info: exit status $? + sleep 2 + done + ''; + script = "echo Ok"; + serviceConfig = { + TimeoutSec = "300"; + }; + }; + systemd.services.kube-certmgr-bootstrap = { description = "Kubernetes certmgr bootstrapper"; - wantedBy = [ "certmgr.service" ]; - after = [ "cfssl.target" ]; + wantedBy = [ "cfssl-online.target" ]; + after = [ "cfssl-online.target" ]; + before = [ "certmgr.service" ]; + path = with pkgs; [ curl cfssl ]; script = concatStringsSep "\n" ['' set -e + mkdir -p $(dirname ${certmgrAPITokenPath}) + mkdir -p $(dirname ${top.caFile}) + # If there's a cfssl (cert issuer) running locally, then don't rely on user to # manually paste it in place. Just symlink. # otherwise, create the target file, ready for users to insert the token @@ -186,15 +218,18 @@ in fi '' (optionalString (cfg.pkiTrustOnBootstrap) '' - if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then - ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \ - ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile} + if [ ! -s "${top.caFile}" ]; then + until test -s ${top.caFile}.json; do + sleep 2 + curl --fail-early -fskd '{}' ${remote}/api/v1/cfssl/info -o ${top.caFile}.json + done + cfssljson -f ${top.caFile}.json -stdout >${top.caFile} + rm ${top.caFile}.json fi '') ]; serviceConfig = { - RestartSec = "10s"; - Restart = "on-failure"; + TimeoutSec = "500"; }; }; @@ -230,35 +265,28 @@ in mapAttrs mkSpec cfg.certs; }; - #TODO: Get rid of kube-addon-manager in the future for the following reasons - # - it is basically just a shell script wrapped around kubectl - # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount - # - it is designed to be used with k8s system components only - # - it would be better with a more Nix-oriented way of managing addons - systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{ - environment.KUBECONFIG = with cfg.certs.addonManager; - top.lib.mkKubeConfig "addon-manager" { - server = top.apiserverAddress; - certFile = cert; - keyFile = key; - }; - } - - (optionalAttrs (top.addonManager.bootstrapAddons != {}) { - serviceConfig.PermissionsStartOnly = true; - preStart = with pkgs; - let - files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v)) - top.addonManager.bootstrapAddons; - in - '' - export KUBECONFIG=${clusterAdminKubeconfig} - ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files} - ''; - })]); + systemd.services.certmgr = { + wantedBy = [ "cfssl-online.target" ]; + after = [ "cfssl-online.target" "kube-certmgr-bootstrap.service" ]; + preStart = '' + while ! test -s ${certmgrAPITokenPath} ; do + sleep 1 + echo Waiting for ${certmgrAPITokenPath} + done + ''; + unitConfig.ConditionPathExists = certmgrPaths; + }; + + systemd.paths.certmgr = { + wantedBy = [ "certmgr.service" ]; + pathConfig = { + PathExists = certmgrPaths; + PathChanged = certmgrPaths; + }; + }; environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig) - clusterAdminKubeconfig; + (top.lib.mkKubeConfig "cluster-admin" clusterAdminKubeconfig); environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [ (pkgs.writeScriptBin "nixos-kubernetes-node-join" '' @@ -284,38 +312,22 @@ in exit 1 fi + do_restart=$(test -s ${certmgrAPITokenPath} && echo -n y || echo -n n) + echo $token > ${certmgrAPITokenPath} chmod 600 ${certmgrAPITokenPath} - echo "Restarting certmgr..." >&1 - systemctl restart certmgr - - echo "Waiting for certs to appear..." >&1 - - ${optionalString top.kubelet.enable '' - while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done - echo "Restarting kubelet..." >&1 - systemctl restart kubelet - ''} - - ${optionalString top.proxy.enable '' - while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done - echo "Restarting kube-proxy..." >&1 - systemctl restart kube-proxy - ''} - - ${optionalString top.flannel.enable '' - while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done - echo "Restarting flannel..." >&1 - systemctl restart flannel - ''} + if [ y = $do_restart ]; then + echo "Restarting certmgr..." >&1 + systemctl restart certmgr + fi - echo "Node joined succesfully" + echo "Node joined succesfully" >&1 '')]; # isolate etcd on loopback at the master node # easyCerts doesn't support multimaster clusters anyway atm. - services.etcd = with cfg.certs.etcd; { + services.etcd = mkIf top.apiserver.enable (with cfg.certs.etcd; { listenClientUrls = ["https://127.0.0.1:2379"]; listenPeerUrls = ["https://127.0.0.1:2380"]; advertiseClientUrls = ["https://etcd.local:2379"]; @@ -324,19 +336,11 @@ in certFile = mkDefault cert; keyFile = mkDefault key; trustedCaFile = mkDefault caCert; - }; + }); networking.extraHosts = mkIf (config.services.etcd.enable) '' 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local ''; - services.flannel = with cfg.certs.flannelClient; { - kubeconfig = top.lib.mkKubeConfig "flannel" { - server = top.apiserverAddress; - certFile = cert; - keyFile = key; - }; - }; - services.kubernetes = { apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; { @@ -353,7 +357,16 @@ in kubeletClientCaFile = mkDefault caCert; kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert; kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key; + proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert; + proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key; }); + addonManager = mkIf top.addonManager.enable { + kubeconfig = with cfg.certs.addonManager; { + certFile = mkDefault cert; + keyFile = mkDefault key; + }; + bootstrapAddonsKubeconfig = clusterAdminKubeconfig; + }; controllerManager = mkIf top.controllerManager.enable { serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key; rootCaFile = cfg.certs.controllerManagerClient.caCert; @@ -362,6 +375,12 @@ in keyFile = mkDefault key; }; }; + flannel = mkIf top.flannel.enable { + kubeconfig = with cfg.certs.flannelClient; { + certFile = cert; + keyFile = key; + }; + }; scheduler = mkIf top.scheduler.enable { kubeconfig = with cfg.certs.schedulerClient; { certFile = mkDefault cert; |