Cleanup pki: addon-manager

2 files changed, 76 insertions, 59 deletions

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index 406b20b0d8d..b9a56811d2b 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -63,24 +63,48 @@ in
     };
 
     enable = mkEnableOption "Whether to enable Kubernetes addon manager.";
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager";
+    bootstrapAddonsKubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager bootstrap";
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = let
+
+    addonManagerPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+    bootstrapAddonsPaths = filter (a: a != null) [
+      cfg.bootstrapAddonsKubeconfig.caFile
+      cfg.bootstrapAddonsKubeconfig.certFile
+      cfg.bootstrapAddonsKubeconfig.keyFile
+    ];
+
+  in mkIf cfg.enable {
     environment.etc."kubernetes/addons".source = "${addons}/";
 
+    #TODO: Get rid of kube-addon-manager in the future for the following reasons
+    # - it is basically just a shell script wrapped around kubectl
+    # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
+    # - it is designed to be used with k8s system components only
+    # - it would be better with a more Nix-oriented way of managing addons
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
-      wantedBy = [ "kube-control-plane-online.target" ];
-      after = [ "kube-addon-manager-bootstrap.service" ];
-      before = [ "kube-control-plane-online.target" ];
-      environment.ADDON_PATH = "/etc/kubernetes/addons/";
-      path = [ pkgs.gawk ];
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-node-online.target" ];
+      before = [ "kubernetes.target" ];
+      environment = {
+        ADDON_PATH = "/etc/kubernetes/addons/";
+        KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" cfg.kubeconfig;
+      };
+      path = with pkgs; [ gawk kubectl ];
       preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager; {
-          path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl -n kube-system get serviceaccounts/default 2>/dev/null; do
+          echo kubectl -n kube-system get serviceaccounts/default: exit status $?
+          sleep 2
+        done
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
@@ -91,27 +115,52 @@ in
         Restart = "on-failure";
         RestartSec = 10;
       };
+      unitConfig.ConditionPathExists = addonManagerPaths;
+    };
+
+    systemd.paths.kube-addon-manager = {
+      wantedBy = [ "kube-addon-manager.service" ];
+      pathConfig = {
+        PathExists = addonManagerPaths;
+        PathChanged = addonManagerPaths;
+      };
     };
 
+    services.kubernetes.addonManager.kubeconfig.server = mkDefault top.apiserverAddress;
+
     systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
       wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
       path = [ pkgs.kubectl ];
+      environment = {
+        KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager-bootstrap" cfg.bootstrapAddonsKubeconfig;
+      };
       preStart = with pkgs; let
         files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
           cfg.bootstrapAddons;
       in ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager-bootstrap; {
-          path = "/api";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl auth can-i '*' '*' -q 2>/dev/null; do
+          echo kubectl auth can-i '*' '*': exit status $?
+          sleep 2
+        done
 
         kubectl apply -f ${concatStringsSep " \\\n -f " files}
       '';
       script = "echo Ok";
+      unitConfig.ConditionPathExists = bootstrapAddonsPaths;
+    };
+
+    systemd.paths.kube-addon-manager-bootstrap = {
+      wantedBy = [ "kube-addon-manager-bootstrap.service" ];
+      pathConfig = {
+        PathExists = bootstrapAddonsPaths;
+        PathChanged = bootstrapAddonsPaths;
+      };
     };
 
+    services.kubernetes.addonManager.bootstrapAddonsKubeconfig.server = mkDefault top.apiserverAddress;
+
     services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled
     (let
       name = system:kube-addon-manager;
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 8bacc07b008..90b40dd4c1f 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -27,12 +27,11 @@ let
   certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
   cfsslAPITokenLength = 32;
 
-  clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
-    top.lib.mkKubeConfig "cluster-admin" {
-        server = top.apiserverAddress;
-        certFile = cert;
-        keyFile = key;
-    };
+  clusterAdminKubeconfig = with cfg.certs.clusterAdmin; {
+    server = top.apiserverAddress;
+    certFile = cert;
+    keyFile = key;
+  };
 
   remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
 in
@@ -142,12 +141,6 @@ in
       config.services.etcd.keyFile
       config.services.etcd.trustedCaFile
     ];
-    addonManagerPaths = mkIf top.addonManager.enable [
-      cfg.certs.addonManager.cert
-      cfg.certs.addonManager.key
-      cfg.certs.clusterAdmin.cert
-      cfg.certs.clusterAdmin.key
-    ];
     flannelPaths = [
       cfg.certs.flannelClient.cert
       cfg.certs.flannelClient.key
@@ -331,38 +324,6 @@ in
         };
       };
 
-      systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
-        environment = {
-          KUBECONFIG = clusterAdminKubeconfig;
-          inherit (cfg.certs.clusterAdmin) cert key;
-        };
-      };
-
-      #TODO: Get rid of kube-addon-manager in the future for the following reasons
-      # - it is basically just a shell script wrapped around kubectl
-      # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
-      # - it is designed to be used with k8s system components only
-      # - it would be better with a more Nix-oriented way of managing addons
-      systemd.services.kube-addon-manager = mkIf top.addonManager.enable {
-        environment = with cfg.certs.addonManager; {
-          KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" {
-              server = top.apiserverAddress;
-              certFile = cert;
-              keyFile = key;
-          };
-          inherit cert key;
-        };
-        unitConfig.ConditionPathExists = addonManagerPaths;
-      };
-
-      systemd.paths.kube-addon-manager = mkIf top.addonManager.enable {
-        wantedBy = [ "kube-addon-manager.service" ];
-        pathConfig = {
-          PathExists = addonManagerPaths;
-          PathChanged = addonManagerPaths;
-        };
-      };
-
       systemd.services.kube-controller-manager = mkIf top.controllerManager.enable {
         environment = { inherit (cfg.certs.controllerManagerClient) cert key; };
         unitConfig.ConditionPathExists = controllerManagerPaths;
@@ -396,7 +357,7 @@ in
       };
 
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
-        clusterAdminKubeconfig;
+        (top.lib.mkKubeConfig "cluster-admin" clusterAdminKubeconfig);
 
       environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
       (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
@@ -538,6 +499,13 @@ in
           kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
           kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
         });
+        addonManager = mkIf top.addonManager.enable {
+          kubeconfig = with cfg.certs.addonManager; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+          bootstrapAddonsKubeconfig = clusterAdminKubeconfig;
+        };
         controllerManager = mkIf top.controllerManager.enable {
           serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
           rootCaFile = cfg.certs.controllerManagerClient.caCert;