From f9e2f76a590d11cbeaa10e3953ddc96110bf1b3b Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 07:56:59 +0100
Subject: nixos/kubernetes: Add systemd path units

to protect services from crashing and clobbering the logs when
certificates are not in place yet and make sure services are activated
when certificates are ready.

To prevent errors similar to "kube-controller-manager.path: Failed to
enter waiting state: Too many open files"
fs.inotify.max_user_instances has to be increased.
---
 .../services/cluster/kubernetes/apiserver.nix      | 41 ++++++++++-
 .../cluster/kubernetes/controller-manager.nix      | 22 +++++-
 .../services/cluster/kubernetes/flannel.nix        |  8 ++-
 .../services/cluster/kubernetes/kubelet.nix        | 19 ++++-
 nixos/modules/services/cluster/kubernetes/pki.nix  | 84 +++++++++++++++++++++-
 nixos/tests/kubernetes/base.nix                    |  5 +-
 6 files changed, 168 insertions(+), 11 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 81e45b417de..08f929060aa 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -272,7 +272,25 @@ in
   ###### implementation
   config = mkMerge [
 
-    (mkIf cfg.enable {
+    (mkIf cfg.enable (let
+      apiserverPaths = [
+        cfg.clientCaFile
+        cfg.etcd.caFile
+        cfg.etcd.certFile
+        cfg.etcd.keyFile
+        cfg.kubeletClientCaFile
+        cfg.kubeletClientCertFile
+        cfg.kubeletClientKeyFile
+        cfg.serviceAccountKeyFile
+        cfg.tlsCertFile
+        cfg.tlsKeyFile
+      ];
+      etcdPaths = [
+        config.services.etcd.certFile
+        config.services.etcd.keyFile
+        config.services.etcd.trustedCaFile
+      ];
+    in {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
           wantedBy = [ "kubernetes.target" ];
@@ -341,6 +359,25 @@ in
             Restart = "on-failure";
             RestartSec = 5;
           };
+          unitConfig.ConditionPathExists = apiserverPaths;
+        };
+
+        systemd.paths.kube-apiserver = {
+          wantedBy = [ "kube-apiserver.service" ];
+          pathConfig = {
+            PathExists = apiserverPaths;
+            PathChanged = apiserverPaths;
+          };
+        };
+
+        systemd.services.etcd.unitConfig.ConditionPathExists = etcdPaths;
+
+        systemd.paths.etcd = {
+          wantedBy = [ "etcd.service" ];
+          pathConfig = {
+            PathExists = etcdPaths;
+            PathChanged = etcdPaths;
+          };
         };
 
         services.etcd = {
@@ -421,7 +458,7 @@ in
         };
       };
 
-    })
+    }))
 
   ];
 
diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index dff97f144d5..27b28311adb 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -104,7 +104,16 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = mkIf cfg.enable (let
+    controllerManagerPaths = [
+      cfg.rootCaFile
+      cfg.tlsCertFile
+      cfg.tlsKeyFile
+      top.pki.certs.controllerManagerClient.cert
+      top.pki.certs.controllerManagerClient.key
+    ];
+  in {
+
     systemd.services.kube-controller-manager = {
       description = "Kubernetes Controller Manager Service";
       wantedBy = [ "kubernetes.target" ];
@@ -142,6 +151,15 @@ in
         Group = "kubernetes";
       };
       path = top.path;
+      unitConfig.ConditionPathExists = controllerManagerPaths;
+    };
+
+    systemd.paths.kube-controller-manager = {
+      wantedBy = [ "kube-controller-manager.service" ];
+      pathConfig = {
+        PathExists = controllerManagerPaths;
+        PathChanged = controllerManagerPaths;
+      };
     };
 
     services.kubernetes.pki.certs = with top.lib; {
@@ -158,5 +176,5 @@ in
     };
 
     services.kubernetes.controllerManager.kubeconfig.server = mkDefault top.apiserverAddress;
-  };
+  });
 }
diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index 93ee2fd65ee..ef06acb6de3 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -55,13 +55,15 @@ in
         ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
         systemctl restart docker
       '';
+      unitConfig.ConditionPathExists = [ "/run/flannel/subnet.env" ];
       serviceConfig.Type = "oneshot";
     };
 
-    systemd.paths."flannel-subnet-env" = {
-      wantedBy = [ "flannel.service" ];
+    systemd.paths.flannel-subnet-env = {
+      wantedBy = [ "mk-docker-opts.service" ];
       pathConfig = {
-        PathModified = "/run/flannel/subnet.env";
+        PathExists = [ "/run/flannel/subnet.env" ];
+        PathChanged = [ "/run/flannel/subnet.env" ];
         Unit = "mk-docker-opts.service";
       };
     };
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index c94bb28bf7f..86402cba7c4 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -241,7 +241,13 @@ in
 
   ###### implementation
   config = mkMerge [
-    (mkIf cfg.enable {
+    (mkIf cfg.enable (let
+      kubeletPaths = [
+        cfg.clientCaFile
+        cfg.tlsCertFile
+        cfg.tlsKeyFile
+      ];
+    in {
       services.kubernetes.kubelet.seedDockerImages = [infraContainer];
 
       systemd.services.kubelet = {
@@ -308,6 +314,15 @@ in
           '';
           WorkingDirectory = top.dataDir;
         };
+        unitConfig.ConditionPathExists = kubeletPaths;
+      };
+
+      systemd.paths.kubelet = {
+        wantedBy =  [ "kubelet.service" ];
+        pathConfig = {
+          PathExists = kubeletPaths;
+          PathChanged = kubeletPaths;
+        };
       };
 
       # Allways include cni plugins
@@ -336,7 +351,7 @@ in
       };
 
       services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
-    })
+    }))
 
     (mkIf (cfg.enable && cfg.manifests != {}) {
       environment.etc = mapAttrs' (name: manifest:
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 38deca23a99..8ad17d4dfb4 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -119,6 +119,29 @@ in
     cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
     cfsslCert = "${cfsslCertPathPrefix}.pem";
     cfsslKey = "${cfsslCertPathPrefix}-key.pem";
+
+    certmgrPaths = [
+      top.caFile
+      certmgrAPITokenPath
+    ];
+    addonManagerPaths = mkIf top.addonManager.enable [
+      cfg.certs.addonManager.cert
+      cfg.certs.addonManager.key
+      cfg.certs.clusterAdmin.cert
+      cfg.certs.clusterAdmin.key
+    ];
+    flannelPaths = [
+      cfg.certs.flannelClient.cert
+      cfg.certs.flannelClient.key
+    ];
+    proxyPaths = mkIf top.proxy.enable [
+      cfg.certs.kubeProxyClient.cert
+      cfg.certs.kubeProxyClient.key
+    ];
+    schedulerPaths = mkIf top.scheduler.enable [
+      cfg.certs.schedulerClient.cert
+      cfg.certs.schedulerClient.key
+    ];
   in
   {
 
@@ -230,6 +253,18 @@ in
           mapAttrs mkSpec cfg.certs;
       };
 
+      systemd.services.certmgr = {
+        unitConfig.ConditionPathExists = certmgrPaths;
+      };
+
+      systemd.paths.certmgr = {
+        wantedBy = [ "certmgr.service" ];
+        pathConfig = {
+          PathExists = certmgrPaths;
+          PathChanged = certmgrPaths;
+        };
+      };
+
       #TODO: Get rid of kube-addon-manager in the future for the following reasons
       # - it is basically just a shell script wrapped around kubectl
       # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
@@ -255,7 +290,18 @@ in
             export KUBECONFIG=${clusterAdminKubeconfig}
             ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
           '';
-        })]);
+        })
+        {
+          unitConfig.ConditionPathExists = addonManagerPaths;
+        }]);
+
+      systemd.paths.kube-addon-manager = mkIf top.addonManager.enable {
+        wantedBy = [ "kube-addon-manager.service" ];
+        pathConfig = {
+          PathExists = addonManagerPaths;
+          PathChanged = addonManagerPaths;
+        };
+      };
 
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
         clusterAdminKubeconfig;
@@ -337,6 +383,42 @@ in
         };
       };
 
+      systemd.services.flannel = {
+        unitConfig.ConditionPathExists = flannelPaths;
+      };
+
+      systemd.paths.flannel = {
+        wantedBy = [ "flannel.service" ];
+        pathConfig = {
+          PathExists = flannelPaths;
+          PathChanged = flannelPaths;
+        };
+      };
+
+      systemd.services.kube-proxy = mkIf top.proxy.enable {
+        unitConfig.ConditionPathExists = proxyPaths;
+      };
+
+      systemd.paths.kube-proxy = mkIf top.proxy.enable {
+        wantedBy = [ "kube-proxy.service" ];
+        pathConfig = {
+          PathExists = proxyPaths;
+          PathChanged = proxyPaths;
+        };
+      };
+
+      systemd.services.kube-scheduler = mkIf top.scheduler.enable {
+        unitConfig.ConditionPathExists = schedulerPaths;
+      };
+
+      systemd.paths.kube-scheduler = mkIf top.scheduler.enable {
+        wantedBy = [ "kube-scheduler.service" ];
+        pathConfig = {
+          PathExists = schedulerPaths;
+          PathChanged = schedulerPaths;
+        };
+      };
+
       services.kubernetes = {
 
         apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix
index ec1a75e74c4..212023859f6 100644
--- a/nixos/tests/kubernetes/base.nix
+++ b/nixos/tests/kubernetes/base.nix
@@ -30,7 +30,10 @@ let
         { config, pkgs, lib, nodes, ... }:
           mkMerge [
             {
-              boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+              boot = {
+                postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+                kernel.sysctl = { "fs.inotify.max_user_instances" = 256; };
+              };
               virtualisation.memorySize = mkDefault 1536;
               virtualisation.diskSize = mkDefault 4096;
               networking = {
-- 
cgit 1.4.1


From 62f03750e48ae7658ea18d7ac75833279da02a5a Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 08:44:45 +0100
Subject: nixos/kubernetes: Stabilize services startup across machines
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

by adding targets and curl wait loops to services to ensure services
are not started before their depended services are reachable.

Extra targets cfssl-online.target and kube-apiserver-online.target
syncronize starts across machines and node-online.target ensures
docker is restarted and ready to deploy containers on after flannel
has discussed the network cidr with apiserver.

Since flannel needs to be started before addon-manager to configure
the docker interface, it has to have its own rbac bootstrap service.

The curl wait loops within the other services exists to ensure that when
starting the service it is able to do its work immediately without
clobbering the log about failing conditions.

By ensuring kubernetes.target is only reached after starting the
cluster it can be used in the tests as a wait condition.

In kube-certmgr-bootstrap mkdir is needed for it to not fail to start.

The following is the relevant part of systemctl list-dependencies

default.target
● ├─certmgr.service
● ├─cfssl.service
● ├─docker.service
● ├─etcd.service
● ├─flannel.service
● ├─kubernetes.target
● │ ├─kube-addon-manager.service
● │ ├─kube-proxy.service
● │ ├─kube-apiserver-online.target
● │ │ ├─flannel-rbac-bootstrap.service
● │ │ ├─kube-apiserver-online.service
● │ │ ├─kube-apiserver.service
● │ │ ├─kube-controller-manager.service
● │ │ └─kube-scheduler.service
● │ └─node-online.target
● │   ├─node-online.service
● │   ├─flannel.target
● │   │ ├─flannel.service
● │   │ └─mk-docker-opts.service
● │   └─kubelet.target
● │     └─kubelet.service
● ├─network-online.target
● │ └─cfssl-online.target
● │   ├─certmgr.service
● │   ├─cfssl-online.service
● │   └─kube-certmgr-bootstrap.service
---
 .../services/cluster/kubernetes/addon-manager.nix  |   2 +-
 .../services/cluster/kubernetes/apiserver.nix      |  26 ++++-
 .../cluster/kubernetes/controller-manager.nix      |  11 +-
 .../services/cluster/kubernetes/default.nix        |  13 +++
 .../services/cluster/kubernetes/flannel.nix        | 121 ++++++++++++++-------
 .../services/cluster/kubernetes/kubelet.nix        |  40 ++++++-
 nixos/modules/services/cluster/kubernetes/pki.nix  |  65 ++++++++++-
 .../modules/services/cluster/kubernetes/proxy.nix  |  10 +-
 .../services/cluster/kubernetes/scheduler.nix      |  11 +-
 nixos/tests/kubernetes/dns.nix                     |   3 +
 nixos/tests/kubernetes/rbac.nix                    |   4 +
 11 files changed, 253 insertions(+), 53 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index 17f2dde31a7..abd9e99ba02 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -72,7 +72,7 @@ in
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
       wantedBy = [ "kubernetes.target" ];
-      after = [ "kube-apiserver.service" ];
+      after = [ "kube-apiserver-online.target" "node-online.target" ];
       environment.ADDON_PATH = "/etc/kubernetes/addons/";
       path = [ pkgs.gawk ];
       serviceConfig = {
diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 08f929060aa..567d31f06ef 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -293,8 +293,9 @@ in
     in {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
-          wantedBy = [ "kubernetes.target" ];
-          after = [ "network.target" ];
+          wantedBy = [ "kube-apiserver-online.target" ];
+          after = [ "certmgr.service" ];
+          before = [ "kube-apiserver-online.target" ];
           serviceConfig = {
             Slice = "kubernetes.slice";
             ExecStart = ''${top.package}/bin/kube-apiserver \
@@ -459,7 +460,28 @@ in
       };
 
     }))
+    {
+      systemd.targets.kube-apiserver-online = {
+        wantedBy = [ "kubernetes.target" ];
+        before = [ "kubernetes.target" ];
+      };
 
+      systemd.services.kube-apiserver-online = mkIf top.flannel.enable {
+        description = "apiserver control plane is online";
+        wantedBy = [ "kube-apiserver-online.target" ];
+        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
+        before = [ "kube-apiserver-online.target" ];
+        preStart = ''
+          ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
+            sleep = 3;
+            path = "/healthz";
+            cacert = top.caFile;
+            inherit cert key;
+          })}
+        '';
+        script = "echo apiserver control plane is online";
+      };
+    }
   ];
 
 }
diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index 27b28311adb..20f471215db 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -116,8 +116,17 @@ in
 
     systemd.services.kube-controller-manager = {
       description = "Kubernetes Controller Manager Service";
-      wantedBy = [ "kubernetes.target" ];
+      wantedBy = [ "kube-apiserver-online.target" ];
       after = [ "kube-apiserver.service" ];
+      before = [ "kube-apiserver-online.target" ];
+      preStart = ''
+        ${top.lib.mkWaitCurl (with top.pki.certs.controllerManagerClient; {
+          sleep = 1;
+          path = "/api";
+          cacert = top.caFile;
+          inherit cert key;
+        })}
+      '';
       serviceConfig = {
         RestartSec = "30s";
         Restart = "on-failure";
diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index 375e33e91b5..f1f544afc4d 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -73,6 +73,18 @@ let
     };
   };
 
+  mkWaitCurl = { address ? cfg.apiserverAddress, sleep ? 2, path ? "", args ? "-o /dev/null",
+                 cacert ? null, cert ? null, key ? null, }: ''
+    while ! ${pkgs.curl}/bin/curl --fail-early -fs \
+      ${if cacert != null then "--cacert ${cacert}" else ""} \
+      ${if cert != null then "--cert ${cert}" else ""} \
+      ${if key != null then "--key ${key}" else ""} \
+      ${address}${path} ${args} ; do
+        sleep ${toString sleep}
+        echo Waiting to be able to reach ${address}${path}
+    done
+  '';
+
   kubeConfigDefaults = {
     server = mkDefault cfg.kubeconfig.server;
     caFile = mkDefault cfg.kubeconfig.caFile;
@@ -162,6 +174,7 @@ in {
         inherit mkCert;
         inherit mkKubeConfig;
         inherit mkKubeConfigOptions;
+        inherit mkWaitCurl;
       };
       type = types.attrs;
     };
diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index ef06acb6de3..4aa547c9d3e 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -27,7 +27,12 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = mkIf cfg.enable (let
+    flannelBootstrapPaths = mkIf top.apiserver.enable [
+      top.pki.certs.clusterAdmin.cert
+      top.pki.certs.clusterAdmin.key
+    ];
+  in {
     services.flannel = {
 
       enable = mkDefault true;
@@ -48,8 +53,10 @@ in
       }];
     };
 
-    systemd.services."mk-docker-opts" = {
+    systemd.services.mk-docker-opts = {
       description = "Pre-Docker Actions";
+      wantedBy = [ "flannel.target" ];
+      before = [ "flannel.target" ];
       path = with pkgs; [ gawk gnugrep ];
       script = ''
         ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
@@ -68,6 +75,17 @@ in
       };
     };
 
+    systemd.targets.flannel = {
+      wantedBy = [ "node-online.target" ];
+      before = [ "node-online.target" ];
+    };
+
+    systemd.services.flannel = {
+      wantedBy = [ "flannel.target" ];
+      after = [ "kubelet.target" ];
+      before = [ "flannel.target" ];
+    };
+
     systemd.services.docker = {
       environment.DOCKER_OPTS = "-b none";
       serviceConfig.EnvironmentFile = "-/run/flannel/docker";
@@ -93,44 +111,69 @@ in
     };
 
     # give flannel som kubernetes rbac permissions if applicable
-    services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) {
-
-      flannel-cr = {
-        apiVersion = "rbac.authorization.k8s.io/v1beta1";
-        kind = "ClusterRole";
-        metadata = { name = "flannel"; };
-        rules = [{
-          apiGroups = [ "" ];
-          resources = [ "pods" ];
-          verbs = [ "get" ];
-        }
-        {
-          apiGroups = [ "" ];
-          resources = [ "nodes" ];
-          verbs = [ "list" "watch" ];
-        }
-        {
-          apiGroups = [ "" ];
-          resources = [ "nodes/status" ];
-          verbs = [ "patch" ];
-        }];
-      };
-
-      flannel-crb = {
-        apiVersion = "rbac.authorization.k8s.io/v1beta1";
-        kind = "ClusterRoleBinding";
-        metadata = { name = "flannel"; };
-        roleRef = {
-          apiGroup = "rbac.authorization.k8s.io";
-          kind = "ClusterRole";
-          name = "flannel";
+    systemd.services.flannel-rbac-bootstrap = mkIf (top.apiserver.enable && (elem "RBAC" top.apiserver.authorizationMode)) {
+
+      wantedBy = [ "kube-apiserver-online.target" ];
+      after = [ "kube-apiserver-online.target" ];
+      before = [ "flannel.service" ];
+      path = with pkgs; [ kubectl ];
+      preStart = let
+        files = mapAttrsToList (n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)) {
+          flannel-cr = {
+            apiVersion = "rbac.authorization.k8s.io/v1beta1";
+            kind = "ClusterRole";
+            metadata = { name = "flannel"; };
+            rules = [{
+              apiGroups = [ "" ];
+              resources = [ "pods" ];
+              verbs = [ "get" ];
+            }
+            {
+              apiGroups = [ "" ];
+              resources = [ "nodes" ];
+              verbs = [ "list" "watch" ];
+            }
+            {
+              apiGroups = [ "" ];
+              resources = [ "nodes/status" ];
+              verbs = [ "patch" ];
+            }];
+          };
+
+          flannel-crb = {
+            apiVersion = "rbac.authorization.k8s.io/v1beta1";
+            kind = "ClusterRoleBinding";
+            metadata = { name = "flannel"; };
+            roleRef = {
+              apiGroup = "rbac.authorization.k8s.io";
+              kind = "ClusterRole";
+              name = "flannel";
+            };
+            subjects = [{
+              kind = "User";
+              name = "flannel-client";
+            }];
+          };
         };
-        subjects = [{
-          kind = "User";
-          name = "flannel-client";
-        }];
-      };
+      in ''
+        ${top.lib.mkWaitCurl (with top.pki.certs.clusterAdmin; {
+          path = "/";
+          cacert = top.caFile;
+          inherit cert key;
+        })}
+
+        kubectl -s ${top.apiserverAddress} --certificate-authority=${top.caFile} --client-certificate=${top.pki.certs.clusterAdmin.cert} --client-key=${top.pki.certs.clusterAdmin.key} apply -f ${concatStringsSep " \\\n -f " files}
+      '';
+      script = "echo Ok";
+      unitConfig.ConditionPathExists = flannelBootstrapPaths;
+    };
 
+    systemd.paths.flannel-rbac-bootstrap = mkIf top.apiserver.enable {
+      wantedBy = [ "flannel-rbac-bootstrap.service" ];
+      pathConfig = {
+        PathExists = flannelBootstrapPaths;
+        PathChanged = flannelBootstrapPaths;
+      };
     };
-  };
+  });
 }
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index 86402cba7c4..b3f3c036564 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -252,8 +252,9 @@ in
 
       systemd.services.kubelet = {
         description = "Kubernetes Kubelet Service";
-        wantedBy = [ "kubernetes.target" ];
-        after = [ "network.target" "docker.service" "kube-apiserver.service" ];
+        wantedBy = [ "kubelet.target" ];
+        after = [ "kube-apiserver-online.target" ];
+        before = [ "kubelet.target" ];
         path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
         preStart = ''
           ${concatMapStrings (img: ''
@@ -325,6 +326,30 @@ in
         };
       };
 
+      systemd.services.docker.before = [ "kubelet.service" ];
+
+      systemd.services.node-online = {
+        wantedBy = [ "node-online.target" ];
+        after = [ "flannel.target" "kubelet.target" ];
+        before = [ "node-online.target" ];
+        # it is complicated. flannel needs kubelet to run the pause container before
+        # it discusses the node CIDR with apiserver and afterwards configures and restarts
+        # dockerd. Until then prevent creating any pods because they have to be recreated anyway
+        # because the network of docker0 has been changed by flannel.
+        script = let
+          docker-env = "/run/flannel/docker";
+          flannel-date = "stat --print=%Y ${docker-env}";
+          docker-date = "systemctl show --property=ActiveEnterTimestamp --value docker";
+        in ''
+          while ! test -f ${docker-env} ; do sleep 1 ; done
+          while test `${flannel-date}` -gt `date +%s --date="$(${docker-date})"` ; do
+            sleep 1
+          done
+        '';
+        serviceConfig.Type = "oneshot";
+        serviceConfig.Slice = "kubernetes.slice";
+      };
+
       # Allways include cni plugins
       services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
 
@@ -369,5 +394,16 @@ in
       };
     })
 
+    {
+      systemd.targets.kubelet = {
+        wantedBy = [ "node-online.target" ];
+        before = [ "node-online.target" ];
+      };
+
+      systemd.targets.node-online = {
+        wantedBy = [ "kubernetes.target" ];
+        before = [ "kubernetes.target" ];
+      };
+    }
   ];
 }
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 8ad17d4dfb4..d08d7892bb5 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -119,6 +119,7 @@ in
     cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
     cfsslCert = "${cfsslCertPathPrefix}.pem";
     cfsslKey = "${cfsslCertPathPrefix}-key.pem";
+    cfsslPort = toString config.services.cfssl.port;
 
     certmgrPaths = [
       top.caFile
@@ -191,13 +192,39 @@ in
         chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}"
       '')]);
 
+    systemd.targets.cfssl-online = {
+      wantedBy = [ "network-online.target" ];
+      after = [ "cfssl.service" "network-online.target" "cfssl-online.service" ];
+    };
+
+    systemd.services.cfssl-online = {
+      description = "Wait for ${remote} to be reachable.";
+      wantedBy = [ "cfssl-online.target" ];
+      before = [ "cfssl-online.target" ];
+      preStart = ''
+        ${top.lib.mkWaitCurl {
+          address = remote;
+          path = "/api/v1/cfssl/info";
+          args = "-kd '{}' -o /dev/null";
+        }}
+      '';
+      script = "echo Ok";
+      serviceConfig = {
+        TimeoutSec = "300";
+      };
+    };
+
     systemd.services.kube-certmgr-bootstrap = {
       description = "Kubernetes certmgr bootstrapper";
-      wantedBy = [ "certmgr.service" ];
-      after = [ "cfssl.target" ];
+      wantedBy = [ "cfssl-online.target" ];
+      after = [ "cfssl-online.target" ];
+      before = [ "certmgr.service" ];
       script = concatStringsSep "\n" [''
         set -e
 
+        mkdir -p $(dirname ${certmgrAPITokenPath})
+        mkdir -p $(dirname ${top.caFile})
+
         # If there's a cfssl (cert issuer) running locally, then don't rely on user to
         # manually paste it in place. Just symlink.
         # otherwise, create the target file, ready for users to insert the token
@@ -209,14 +236,18 @@ in
         fi
       ''
       (optionalString (cfg.pkiTrustOnBootstrap) ''
-        if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
-          ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
-            ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
+        if [ ! -s "${top.caFile}" ]; then
+          ${top.lib.mkWaitCurl {
+            address = "https://${top.masterAddress}:${cfsslPort}";
+            path = "/api/v1/cfssl/info";
+            args = "-kd '{}' -o - | ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}";
+          }}
         fi
       '')
       ];
       serviceConfig = {
-        RestartSec = "10s";
+        TimeoutSec = "300";
+        RestartSec = "1s";
         Restart = "on-failure";
       };
     };
@@ -254,6 +285,14 @@ in
       };
 
       systemd.services.certmgr = {
+        wantedBy = [ "cfssl-online.target" ];
+        after = [ "cfssl-online.target" "kube-certmgr-bootstrap.service" ];
+        preStart = ''
+          while ! test -s ${certmgrAPITokenPath} ; do
+            sleep 1
+            echo Waiting for ${certmgrAPITokenPath}
+          done
+        '';
         unitConfig.ConditionPathExists = certmgrPaths;
       };
 
@@ -289,6 +328,12 @@ in
           ''
             export KUBECONFIG=${clusterAdminKubeconfig}
             ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
+
+            ${top.lib.mkWaitCurl (with top.pki.certs.addonManager; {
+              path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
+              cacert = top.caFile;
+              inherit cert key;
+            })}
           '';
         })
         {
@@ -384,6 +429,14 @@ in
       };
 
       systemd.services.flannel = {
+        preStart = ''
+          ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
+            path = "/api/v1/nodes";
+            cacert = top.caFile;
+            inherit cert key;
+            args = "-o - | grep podCIDR >/dev/null";
+          })}
+        '';
         unitConfig.ConditionPathExists = flannelPaths;
       };
 
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index 83cd3e23100..073756d58ab 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -49,8 +49,16 @@ in
     systemd.services.kube-proxy = {
       description = "Kubernetes Proxy Service";
       wantedBy = [ "kubernetes.target" ];
-      after = [ "kube-apiserver.service" ];
+      after = [ "node-online.target" ];
+      before = [ "kubernetes.target" ];
       path = with pkgs; [ iptables conntrack_tools ];
+      preStart = ''
+        ${top.lib.mkWaitCurl (with top.pki.certs.kubeProxyClient; {
+          path = "/api/v1/nodes/${top.kubelet.hostname}";
+          cacert = top.caFile;
+          inherit cert key;
+        })}
+      '';
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = ''${top.package}/bin/kube-proxy \
diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix
index 0305b9aefe5..d3302a15402 100644
--- a/nixos/modules/services/cluster/kubernetes/scheduler.nix
+++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix
@@ -59,8 +59,17 @@ in
   config = mkIf cfg.enable {
     systemd.services.kube-scheduler = {
       description = "Kubernetes Scheduler Service";
-      wantedBy = [ "kubernetes.target" ];
+      wantedBy = [ "kube-apiserver-online.target" ];
       after = [ "kube-apiserver.service" ];
+      before = [ "kube-apiserver-online.target" ];
+      preStart = ''
+        ${top.lib.mkWaitCurl (with top.pki.certs.schedulerClient; {
+          sleep = 1;
+          path = "/api";
+          cacert = top.caFile;
+          inherit cert key;
+        })}
+      '';
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = ''${top.package}/bin/kube-scheduler \
diff --git a/nixos/tests/kubernetes/dns.nix b/nixos/tests/kubernetes/dns.nix
index 46bcb01a526..e7db0a58ab6 100644
--- a/nixos/tests/kubernetes/dns.nix
+++ b/nixos/tests/kubernetes/dns.nix
@@ -77,6 +77,7 @@ let
   singleNodeTest = {
     test = ''
       # prepare machine1 for test
+      $machine1->waitForUnit("kubernetes.target");
       $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
       $machine1->waitUntilSucceeds("docker load < ${redisImage}");
       $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
@@ -102,6 +103,8 @@ let
       # Node token exchange
       $machine1->waitUntilSucceeds("cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret");
       $machine2->waitUntilSucceeds("cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join");
+      $machine1->waitForUnit("kubernetes.target");
+      $machine2->waitForUnit("kubernetes.target");
 
       # prepare machines for test
       $machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready");
diff --git a/nixos/tests/kubernetes/rbac.nix b/nixos/tests/kubernetes/rbac.nix
index 3ce7adcd0d7..967fe506004 100644
--- a/nixos/tests/kubernetes/rbac.nix
+++ b/nixos/tests/kubernetes/rbac.nix
@@ -94,6 +94,8 @@ let
 
   singlenode = base // {
     test = ''
+      $machine1->waitForUnit("kubernetes.target");
+
       $machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
 
       $machine1->waitUntilSucceeds("docker load < ${kubectlImage}");
@@ -116,6 +118,8 @@ let
       # Node token exchange
       $machine1->waitUntilSucceeds("cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret");
       $machine2->waitUntilSucceeds("cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join");
+      $machine1->waitForUnit("kubernetes.target");
+      $machine2->waitForUnit("kubernetes.target");
 
       $machine1->waitUntilSucceeds("kubectl get node machine2.my.zyx | grep -w Ready");
 
-- 
cgit 1.4.1


From 51aeaaffc2a108268f8f9b633f3291115f15e72d Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Sun, 3 Mar 2019 19:06:46 +0100
Subject: nixos/kubernetes: flannel needs iptables in service path

---
 nixos/modules/services/cluster/kubernetes/flannel.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index 4aa547c9d3e..a5b4f7103dc 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -84,6 +84,7 @@ in
       wantedBy = [ "flannel.target" ];
       after = [ "kubelet.target" ];
       before = [ "flannel.target" ];
+      path = [ pkgs.iptables ];
     };
 
     systemd.services.docker = {
-- 
cgit 1.4.1


From cf8389c9048b9e46dadc47142f85a6086098e064 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Sun, 3 Mar 2019 19:07:22 +0100
Subject: nixos/kubernetes: Add longer timeouts for waiting services

---
 nixos/modules/services/cluster/kubernetes/apiserver.nix | 3 +++
 nixos/modules/services/cluster/kubernetes/pki.nix       | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 567d31f06ef..2e7e2a6ec3e 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -480,6 +480,9 @@ in
           })}
         '';
         script = "echo apiserver control plane is online";
+        serviceConfig = {
+          TimeoutSec = "500";
+        };
       };
     }
   ];
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index d08d7892bb5..6f3f4107207 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -246,7 +246,7 @@ in
       '')
       ];
       serviceConfig = {
-        TimeoutSec = "300";
+        TimeoutSec = "500";
         RestartSec = "1s";
         Restart = "on-failure";
       };
-- 
cgit 1.4.1


From fd28c0a82a56da127ad8e77d34941b57e9e46065 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 10:23:34 +0100
Subject: nixos/kubernetes: Seed docker images before kubelet service start

to speed up startup time because it can be parallelized.
---
 .../services/cluster/kubernetes/kubelet.nix        | 23 ++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index b3f3c036564..0b1d2a67565 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -257,12 +257,7 @@ in
         before = [ "kubelet.target" ];
         path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
         preStart = ''
-          ${concatMapStrings (img: ''
-            echo "Seeding docker image: ${img}"
-            docker load <${img}
-          '') cfg.seedDockerImages}
-
-          rm /opt/cni/bin/* || true
+          rm -f /opt/cni/bin/* || true
           ${concatMapStrings (package: ''
             echo "Linking cni package: ${package}"
             ln -fs ${package}/bin/* /opt/cni/bin
@@ -328,6 +323,22 @@ in
 
       systemd.services.docker.before = [ "kubelet.service" ];
 
+      systemd.services.docker-seed-images = {
+        wantedBy = [ "docker.service" ];
+        after = [ "docker.service" ];
+        before = [ "kubelet.service" ];
+        path = with pkgs; [ docker ];
+        preStart = ''
+          ${concatMapStrings (img: ''
+            echo "Seeding docker image: ${img}"
+            docker load <${img}
+          '') cfg.seedDockerImages}
+        '';
+        script = "echo Ok";
+        serviceConfig.Type = "oneshot";
+        serviceConfig.Slice = "kubernetes.slice";
+      };
+
       systemd.services.node-online = {
         wantedBy = [ "node-online.target" ];
         after = [ "flannel.target" "kubelet.target" ];
-- 
cgit 1.4.1


From 7df88bd802c939cb1118bd9c3208999796dc0795 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 10:28:29 +0100
Subject: nixos/kubernetes: Put dashboard service account into bootstrapAddons

to prevent errors in log about missing permissions when
addon manager starts the dashboard.
---
 .../cluster/kubernetes/addons/dashboard.nix        | 36 +++++++++++++---------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix b/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix
index 454e7d35bc0..2295694ffc7 100644
--- a/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix
+++ b/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix
@@ -169,6 +169,23 @@ in {
         };
       };
 
+      kubernetes-dashboard-cm = {
+        apiVersion = "v1";
+        kind = "ConfigMap";
+        metadata = {
+          labels = {
+            k8s-app = "kubernetes-dashboard";
+            # Allows editing resource and makes sure it is created first.
+            "addonmanager.kubernetes.io/mode" = "EnsureExists";
+          };
+          name = "kubernetes-dashboard-settings";
+          namespace = "kube-system";
+        };
+      };
+    };
+
+    services.kubernetes.addonManager.bootstrapAddons = mkMerge [{
+
       kubernetes-dashboard-sa = {
         apiVersion = "v1";
         kind = "ServiceAccount";
@@ -210,20 +227,9 @@ in {
         };
         type = "Opaque";
       };
-      kubernetes-dashboard-cm = {
-        apiVersion = "v1";
-        kind = "ConfigMap";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            # Allows editing resource and makes sure it is created first.
-            "addonmanager.kubernetes.io/mode" = "EnsureExists";
-          };
-          name = "kubernetes-dashboard-settings";
-          namespace = "kube-system";
-        };
-      };
-    } // (optionalAttrs cfg.rbac.enable
+    }
+
+    (optionalAttrs cfg.rbac.enable
       (let
         subjects = [{
           kind = "ServiceAccount";
@@ -323,6 +329,6 @@ in {
             inherit subjects;
           };
         })
-    ));
+    ))];
   };
 }
-- 
cgit 1.4.1


From 74962bf767b67ca8b92e82fb0d6f6f96927601d0 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 10:43:26 +0100
Subject: nixos/kubernetes: No need to restart services besides certmgr

within the node join script, since certmgr is taking care of
restarting services.
---
 nixos/modules/services/cluster/kubernetes/pki.nix | 30 ++++++-----------------
 1 file changed, 7 insertions(+), 23 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 6f3f4107207..329278e375c 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -375,33 +375,17 @@ in
           exit 1
         fi
 
+        do_restart=$(test -s ${certmgrAPITokenPath} && echo -n y || echo -n n)
+
         echo $token > ${certmgrAPITokenPath}
         chmod 600 ${certmgrAPITokenPath}
 
-        echo "Restarting certmgr..." >&1
-        systemctl restart certmgr
-
-        echo "Waiting for certs to appear..." >&1
-
-        ${optionalString top.kubelet.enable ''
-          while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
-          echo "Restarting kubelet..." >&1
-          systemctl restart kubelet
-        ''}
-
-        ${optionalString top.proxy.enable ''
-          while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
-          echo "Restarting kube-proxy..." >&1
-          systemctl restart kube-proxy
-        ''}
-
-        ${optionalString top.flannel.enable ''
-          while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
-          echo "Restarting flannel..." >&1
-          systemctl restart flannel
-        ''}
+        if [ y = $do_restart ]; then
+          echo "Restarting certmgr..." >&1
+          systemctl restart certmgr
+        fi
 
-        echo "Node joined succesfully"
+        echo "Node joined succesfully" >&1
       '')];
 
       # isolate etcd on loopback at the master node
-- 
cgit 1.4.1


From ff91d5818cf4703e01670251096da301cc2c7c54 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:40:27 +0100
Subject: nixos/kubernetes: Address review: Rename targets and move proxy to
 node-online.target

---
 .../services/cluster/kubernetes/addon-manager.nix        |  4 ++--
 nixos/modules/services/cluster/kubernetes/apiserver.nix  | 16 ++++++++--------
 .../services/cluster/kubernetes/controller-manager.nix   |  4 ++--
 nixos/modules/services/cluster/kubernetes/kubelet.nix    |  4 ++--
 nixos/modules/services/cluster/kubernetes/proxy.nix      |  6 +++---
 nixos/modules/services/cluster/kubernetes/scheduler.nix  |  4 ++--
 6 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index abd9e99ba02..46f5b68b2a5 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -71,8 +71,8 @@ in
 
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
-      wantedBy = [ "kubernetes.target" ];
-      after = [ "kube-apiserver-online.target" "node-online.target" ];
+      wantedBy = [ "kube-control-plane-online.target" ];
+      before = [ "kube-control-plane-online.target" ];
       environment.ADDON_PATH = "/etc/kubernetes/addons/";
       path = [ pkgs.gawk ];
       serviceConfig = {
diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 2e7e2a6ec3e..e4650c12cff 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -293,9 +293,9 @@ in
     in {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
-          wantedBy = [ "kube-apiserver-online.target" ];
+          wantedBy = [ "kube-control-plane-online.target" ];
           after = [ "certmgr.service" ];
-          before = [ "kube-apiserver-online.target" ];
+          before = [ "kube-control-plane-online.target" ];
           serviceConfig = {
             Slice = "kubernetes.slice";
             ExecStart = ''${top.package}/bin/kube-apiserver \
@@ -461,16 +461,16 @@ in
 
     }))
     {
-      systemd.targets.kube-apiserver-online = {
+      systemd.targets.kube-control-plane-online = {
         wantedBy = [ "kubernetes.target" ];
         before = [ "kubernetes.target" ];
       };
 
-      systemd.services.kube-apiserver-online = mkIf top.flannel.enable {
-        description = "apiserver control plane is online";
-        wantedBy = [ "kube-apiserver-online.target" ];
+      systemd.services.kube-control-plane-online = rec {
+        description = "Kubernetes control plane is online";
+        wantedBy = [ "kube-control-plane-online.target" ];
         after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
-        before = [ "kube-apiserver-online.target" ];
+        before = [ "kube-control-plane-online.target" ];
         preStart = ''
           ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
             sleep = 3;
@@ -479,7 +479,7 @@ in
             inherit cert key;
           })}
         '';
-        script = "echo apiserver control plane is online";
+        script = "echo Ok";
         serviceConfig = {
           TimeoutSec = "500";
         };
diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index 20f471215db..8e82db36425 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -116,9 +116,9 @@ in
 
     systemd.services.kube-controller-manager = {
       description = "Kubernetes Controller Manager Service";
-      wantedBy = [ "kube-apiserver-online.target" ];
+      wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
-      before = [ "kube-apiserver-online.target" ];
+      before = [ "kube-control-plane-online.target" ];
       preStart = ''
         ${top.lib.mkWaitCurl (with top.pki.certs.controllerManagerClient; {
           sleep = 1;
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index 0b1d2a67565..c4ecb2417e2 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -253,7 +253,7 @@ in
       systemd.services.kubelet = {
         description = "Kubernetes Kubelet Service";
         wantedBy = [ "kubelet.target" ];
-        after = [ "kube-apiserver-online.target" ];
+        after = [ "kube-control-plane-online.target" ];
         before = [ "kubelet.target" ];
         path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
         preStart = ''
@@ -339,7 +339,7 @@ in
         serviceConfig.Slice = "kubernetes.slice";
       };
 
-      systemd.services.node-online = {
+      systemd.services.kubelet-online = {
         wantedBy = [ "node-online.target" ];
         after = [ "flannel.target" "kubelet.target" ];
         before = [ "node-online.target" ];
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index 073756d58ab..d13d23e997b 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -48,9 +48,9 @@ in
   config = mkIf cfg.enable {
     systemd.services.kube-proxy = {
       description = "Kubernetes Proxy Service";
-      wantedBy = [ "kubernetes.target" ];
-      after = [ "node-online.target" ];
-      before = [ "kubernetes.target" ];
+      wantedBy = [ "node-online.target" ];
+      after = [ "kubelet-online.service" ];
+      before = [ "node-online.target" ];
       path = with pkgs; [ iptables conntrack_tools ];
       preStart = ''
         ${top.lib.mkWaitCurl (with top.pki.certs.kubeProxyClient; {
diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix
index d3302a15402..4aea9e9b6bd 100644
--- a/nixos/modules/services/cluster/kubernetes/scheduler.nix
+++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix
@@ -59,9 +59,9 @@ in
   config = mkIf cfg.enable {
     systemd.services.kube-scheduler = {
       description = "Kubernetes Scheduler Service";
-      wantedBy = [ "kube-apiserver-online.target" ];
+      wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
-      before = [ "kube-apiserver-online.target" ];
+      before = [ "kube-control-plane-online.target" ];
       preStart = ''
         ${top.lib.mkWaitCurl (with top.pki.certs.schedulerClient; {
           sleep = 1;
-- 
cgit 1.4.1


From 6e9037fed0ce0b55ef37188ec1a58e18e196a780 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:44:38 +0100
Subject: nixos/kubernetes: Address review: Move bootstrapping addons into own
 service

---
 .../services/cluster/kubernetes/addon-manager.nix  |  26 ++++++
 .../services/cluster/kubernetes/flannel.nix        | 104 +++++++--------------
 nixos/modules/services/cluster/kubernetes/pki.nix  |  46 ++++-----
 3 files changed, 79 insertions(+), 97 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index 46f5b68b2a5..406b20b0d8d 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -72,9 +72,16 @@ in
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
       wantedBy = [ "kube-control-plane-online.target" ];
+      after = [ "kube-addon-manager-bootstrap.service" ];
       before = [ "kube-control-plane-online.target" ];
       environment.ADDON_PATH = "/etc/kubernetes/addons/";
       path = [ pkgs.gawk ];
+      preStart = ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager; {
+          path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
+          cacert = top.caFile;
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+      '';
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = "${top.package}/bin/kube-addons";
@@ -86,6 +93,25 @@ in
       };
     };
 
+    systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
+      wantedBy = [ "kube-control-plane-online.target" ];
+      after = [ "kube-apiserver.service" ];
+      before = [ "kube-control-plane-online.target" ];
+      path = [ pkgs.kubectl ];
+      preStart = with pkgs; let
+        files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
+          cfg.bootstrapAddons;
+      in ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager-bootstrap; {
+          path = "/api";
+          cacert = top.caFile;
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+
+        kubectl apply -f ${concatStringsSep " \\\n -f " files}
+      '';
+      script = "echo Ok";
+    };
+
     services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled
     (let
       name = system:kube-addon-manager;
diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index a5b4f7103dc..fba70e3b920 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -27,12 +27,7 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable (let
-    flannelBootstrapPaths = mkIf top.apiserver.enable [
-      top.pki.certs.clusterAdmin.cert
-      top.pki.certs.clusterAdmin.key
-    ];
-  in {
+  config = mkIf cfg.enable {
     services.flannel = {
 
       enable = mkDefault true;
@@ -112,69 +107,42 @@ in
     };
 
     # give flannel som kubernetes rbac permissions if applicable
-    systemd.services.flannel-rbac-bootstrap = mkIf (top.apiserver.enable && (elem "RBAC" top.apiserver.authorizationMode)) {
-
-      wantedBy = [ "kube-apiserver-online.target" ];
-      after = [ "kube-apiserver-online.target" ];
-      before = [ "flannel.service" ];
-      path = with pkgs; [ kubectl ];
-      preStart = let
-        files = mapAttrsToList (n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)) {
-          flannel-cr = {
-            apiVersion = "rbac.authorization.k8s.io/v1beta1";
-            kind = "ClusterRole";
-            metadata = { name = "flannel"; };
-            rules = [{
-              apiGroups = [ "" ];
-              resources = [ "pods" ];
-              verbs = [ "get" ];
-            }
-            {
-              apiGroups = [ "" ];
-              resources = [ "nodes" ];
-              verbs = [ "list" "watch" ];
-            }
-            {
-              apiGroups = [ "" ];
-              resources = [ "nodes/status" ];
-              verbs = [ "patch" ];
-            }];
-          };
-
-          flannel-crb = {
-            apiVersion = "rbac.authorization.k8s.io/v1beta1";
-            kind = "ClusterRoleBinding";
-            metadata = { name = "flannel"; };
-            roleRef = {
-              apiGroup = "rbac.authorization.k8s.io";
-              kind = "ClusterRole";
-              name = "flannel";
-            };
-            subjects = [{
-              kind = "User";
-              name = "flannel-client";
-            }];
-          };
-        };
-      in ''
-        ${top.lib.mkWaitCurl (with top.pki.certs.clusterAdmin; {
-          path = "/";
-          cacert = top.caFile;
-          inherit cert key;
-        })}
-
-        kubectl -s ${top.apiserverAddress} --certificate-authority=${top.caFile} --client-certificate=${top.pki.certs.clusterAdmin.cert} --client-key=${top.pki.certs.clusterAdmin.key} apply -f ${concatStringsSep " \\\n -f " files}
-      '';
-      script = "echo Ok";
-      unitConfig.ConditionPathExists = flannelBootstrapPaths;
-    };
+    services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) {
+      flannel-cr = {
+        apiVersion = "rbac.authorization.k8s.io/v1beta1";
+        kind = "ClusterRole";
+        metadata = { name = "flannel"; };
+        rules = [{
+          apiGroups = [ "" ];
+          resources = [ "pods" ];
+          verbs = [ "get" ];
+        }
+        {
+          apiGroups = [ "" ];
+          resources = [ "nodes" ];
+          verbs = [ "list" "watch" ];
+        }
+        {
+          apiGroups = [ "" ];
+          resources = [ "nodes/status" ];
+          verbs = [ "patch" ];
+        }];
+      };
 
-    systemd.paths.flannel-rbac-bootstrap = mkIf top.apiserver.enable {
-      wantedBy = [ "flannel-rbac-bootstrap.service" ];
-      pathConfig = {
-        PathExists = flannelBootstrapPaths;
-        PathChanged = flannelBootstrapPaths;
+      flannel-crb = {
+        apiVersion = "rbac.authorization.k8s.io/v1beta1";
+        kind = "ClusterRoleBinding";
+        metadata = { name = "flannel"; };
+        roleRef = {
+          apiGroup = "rbac.authorization.k8s.io";
+          kind = "ClusterRole";
+          name = "flannel";
+        };
+        subjects = [{
+          kind = "User";
+          name = "flannel-client";
+        }];
       };
     };
-  });
+  };
 }
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 329278e375c..4d97d8322cd 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -304,41 +304,29 @@ in
         };
       };
 
+      systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
+        environment = {
+          KUBECONFIG = clusterAdminKubeconfig;
+          inherit (cfg.certs.clusterAdmin) cert key;
+        };
+      };
+
       #TODO: Get rid of kube-addon-manager in the future for the following reasons
       # - it is basically just a shell script wrapped around kubectl
       # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
       # - it is designed to be used with k8s system components only
       # - it would be better with a more Nix-oriented way of managing addons
-      systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
-        environment.KUBECONFIG = with cfg.certs.addonManager;
-          top.lib.mkKubeConfig "addon-manager" {
-            server = top.apiserverAddress;
-            certFile = cert;
-            keyFile = key;
+      systemd.services.kube-addon-manager = mkIf top.addonManager.enable {
+        environment = with cfg.certs.addonManager; {
+          KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" {
+              server = top.apiserverAddress;
+              certFile = cert;
+              keyFile = key;
           };
-        }
-
-        (optionalAttrs (top.addonManager.bootstrapAddons != {}) {
-          serviceConfig.PermissionsStartOnly = true;
-          preStart = with pkgs;
-          let
-            files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
-              top.addonManager.bootstrapAddons;
-          in
-          ''
-            export KUBECONFIG=${clusterAdminKubeconfig}
-            ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
-
-            ${top.lib.mkWaitCurl (with top.pki.certs.addonManager; {
-              path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
-              cacert = top.caFile;
-              inherit cert key;
-            })}
-          '';
-        })
-        {
-          unitConfig.ConditionPathExists = addonManagerPaths;
-        }]);
+          inherit cert key;
+        };
+        unitConfig.ConditionPathExists = addonManagerPaths;
+      };
 
       systemd.paths.kube-addon-manager = mkIf top.addonManager.enable {
         wantedBy = [ "kube-addon-manager.service" ];
-- 
cgit 1.4.1


From 52fe1d2e7a9c154fe962f7b47ce008bf06cfe746 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:50:35 +0100
Subject: nixos/kubernetes: Address review: Move controller manager paths into
 pki

---
 .../cluster/kubernetes/controller-manager.nix      | 22 ++--------------------
 nixos/modules/services/cluster/kubernetes/pki.nix  | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index 8e82db36425..cc43a243df1 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -104,16 +104,7 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable (let
-    controllerManagerPaths = [
-      cfg.rootCaFile
-      cfg.tlsCertFile
-      cfg.tlsKeyFile
-      top.pki.certs.controllerManagerClient.cert
-      top.pki.certs.controllerManagerClient.key
-    ];
-  in {
-
+  config = mkIf cfg.enable {
     systemd.services.kube-controller-manager = {
       description = "Kubernetes Controller Manager Service";
       wantedBy = [ "kube-control-plane-online.target" ];
@@ -160,15 +151,6 @@ in
         Group = "kubernetes";
       };
       path = top.path;
-      unitConfig.ConditionPathExists = controllerManagerPaths;
-    };
-
-    systemd.paths.kube-controller-manager = {
-      wantedBy = [ "kube-controller-manager.service" ];
-      pathConfig = {
-        PathExists = controllerManagerPaths;
-        PathChanged = controllerManagerPaths;
-      };
     };
 
     services.kubernetes.pki.certs = with top.lib; {
@@ -185,5 +167,5 @@ in
     };
 
     services.kubernetes.controllerManager.kubeconfig.server = mkDefault top.apiserverAddress;
-  });
+  };
 }
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 4d97d8322cd..98284fba12a 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -143,6 +143,13 @@ in
       cfg.certs.schedulerClient.cert
       cfg.certs.schedulerClient.key
     ];
+    controllerManagerPaths = [
+      top.controllerManager.rootCaFile
+      top.controllerManager.tlsCertFile
+      top.controllerManager.tlsKeyFile
+      cfg.certs.controllerManagerClient.cert
+      cfg.certs.controllerManagerClient.key
+    ];
   in
   {
 
@@ -336,6 +343,18 @@ in
         };
       };
 
+      systemd.services.kube-controller-manager = mkIf top.controllerManager.enable {
+        unitConfig.ConditionPathExists = controllerManagerPaths;
+      };
+
+      systemd.paths.kube-controller-manager = mkIf top.controllerManager.enable {
+        wantedBy = [ "kube-controller-manager.service" ];
+        pathConfig = {
+          PathExists = controllerManagerPaths;
+          PathChanged = controllerManagerPaths;
+        };
+      };
+
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
         clusterAdminKubeconfig;
 
-- 
cgit 1.4.1


From 7323b77435f69362b0b4cc7edcb0915e9ab1ff48 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:52:27 +0100
Subject: nixos/kubernetes: Address review: Separate preStart from certificates

---
 .../services/cluster/kubernetes/apiserver.nix      |  5 +--
 .../cluster/kubernetes/controller-manager.nix      |  5 +--
 .../services/cluster/kubernetes/flannel.nix        |  7 ++++
 nixos/modules/services/cluster/kubernetes/pki.nix  | 46 +++++++++++-----------
 .../modules/services/cluster/kubernetes/proxy.nix  |  5 +--
 .../services/cluster/kubernetes/scheduler.nix      |  5 +--
 6 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index e4650c12cff..72fb9535832 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -472,12 +472,11 @@ in
         after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
         before = [ "kube-control-plane-online.target" ];
         preStart = ''
-          ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
+          ${top.lib.mkWaitCurl ( with config.systemd.services.kube-control-plane-online; {
             sleep = 3;
             path = "/healthz";
             cacert = top.caFile;
-            inherit cert key;
-          })}
+          } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
         '';
         script = "echo Ok";
         serviceConfig = {
diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index cc43a243df1..a39fd62c689 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -111,12 +111,11 @@ in
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
       preStart = ''
-        ${top.lib.mkWaitCurl (with top.pki.certs.controllerManagerClient; {
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-controller-manager; {
           sleep = 1;
           path = "/api";
           cacert = top.caFile;
-          inherit cert key;
-        })}
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
       '';
       serviceConfig = {
         RestartSec = "30s";
diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index fba70e3b920..f85ebdafa31 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -80,6 +80,13 @@ in
       after = [ "kubelet.target" ];
       before = [ "flannel.target" ];
       path = [ pkgs.iptables ];
+      preStart = ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.flannel; {
+          path = "/api/v1/nodes";
+          cacert = top.caFile;
+          args = "-o - | grep podCIDR >/dev/null";
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+      '';
     };
 
     systemd.services.docker = {
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 98284fba12a..14af3840eee 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -344,6 +344,7 @@ in
       };
 
       systemd.services.kube-controller-manager = mkIf top.controllerManager.enable {
+        environment = { inherit (cfg.certs.controllerManagerClient) cert key; };
         unitConfig.ConditionPathExists = controllerManagerPaths;
       };
 
@@ -355,6 +356,25 @@ in
         };
       };
 
+      systemd.services.kube-scheduler = mkIf top.scheduler.enable {
+        environment = { inherit (top.pki.certs.schedulerClient) cert key; };
+        unitConfig.ConditionPathExists = schedulerPaths;
+      };
+
+      systemd.paths.kube-scheduler = mkIf top.scheduler.enable {
+        wantedBy = [ "kube-scheduler.service" ];
+        pathConfig = {
+          PathExists = schedulerPaths;
+          PathChanged = schedulerPaths;
+        };
+      };
+
+      systemd.services.kube-control-plane-online.environment = let
+        client = with cfg.certs; if top.apiserver.enable then clusterAdmin else kubelet;
+      in {
+        inherit (client) cert key;
+      };
+
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
         clusterAdminKubeconfig;
 
@@ -419,19 +439,12 @@ in
         };
       };
 
-      systemd.services.flannel = {
-        preStart = ''
-          ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
-            path = "/api/v1/nodes";
-            cacert = top.caFile;
-            inherit cert key;
-            args = "-o - | grep podCIDR >/dev/null";
-          })}
-        '';
+      systemd.services.flannel = mkIf top.flannel.enable {
+        environment = { inherit (top.pki.certs.flannelClient) cert key; };
         unitConfig.ConditionPathExists = flannelPaths;
       };
 
-      systemd.paths.flannel = {
+      systemd.paths.flannel = mkIf top.flannel.enable {
         wantedBy = [ "flannel.service" ];
         pathConfig = {
           PathExists = flannelPaths;
@@ -440,6 +453,7 @@ in
       };
 
       systemd.services.kube-proxy = mkIf top.proxy.enable {
+        environment = { inherit (top.pki.certs.kubeProxyClient) cert key; };
         unitConfig.ConditionPathExists = proxyPaths;
       };
 
@@ -451,18 +465,6 @@ in
         };
       };
 
-      systemd.services.kube-scheduler = mkIf top.scheduler.enable {
-        unitConfig.ConditionPathExists = schedulerPaths;
-      };
-
-      systemd.paths.kube-scheduler = mkIf top.scheduler.enable {
-        wantedBy = [ "kube-scheduler.service" ];
-        pathConfig = {
-          PathExists = schedulerPaths;
-          PathChanged = schedulerPaths;
-        };
-      };
-
       services.kubernetes = {
 
         apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index d13d23e997b..01d59e9ac88 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -53,11 +53,10 @@ in
       before = [ "node-online.target" ];
       path = with pkgs; [ iptables conntrack_tools ];
       preStart = ''
-        ${top.lib.mkWaitCurl (with top.pki.certs.kubeProxyClient; {
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-proxy; {
           path = "/api/v1/nodes/${top.kubelet.hostname}";
           cacert = top.caFile;
-          inherit cert key;
-        })}
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix
index 4aea9e9b6bd..32a84563076 100644
--- a/nixos/modules/services/cluster/kubernetes/scheduler.nix
+++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix
@@ -63,12 +63,11 @@ in
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
       preStart = ''
-        ${top.lib.mkWaitCurl (with top.pki.certs.schedulerClient; {
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-scheduler; {
           sleep = 1;
           path = "/api";
           cacert = top.caFile;
-          inherit cert key;
-        })}
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
-- 
cgit 1.4.1


From 5684034693fc4d22ae212bdfbb1e33f930de1cc1 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:53:33 +0100
Subject: nixos/kubernetes: Address review: Remove restart from certmgr
 bootstrap service

---
 nixos/modules/services/cluster/kubernetes/pki.nix | 2 --
 1 file changed, 2 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 14af3840eee..0b43f2034c2 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -254,8 +254,6 @@ in
       ];
       serviceConfig = {
         TimeoutSec = "500";
-        RestartSec = "1s";
-        Restart = "on-failure";
       };
     };
 
-- 
cgit 1.4.1


From e148cb040b84a55229e097cb9c6b1af3c2a5484f Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 17:17:20 +0100
Subject: nixos/kubernetes: Address review: rename node-online target

---
 nixos/modules/services/cluster/kubernetes/flannel.nix |  4 ++--
 nixos/modules/services/cluster/kubernetes/kubelet.nix | 10 +++++-----
 nixos/modules/services/cluster/kubernetes/proxy.nix   |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index f85ebdafa31..4c5fe7559eb 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -71,8 +71,8 @@ in
     };
 
     systemd.targets.flannel = {
-      wantedBy = [ "node-online.target" ];
-      before = [ "node-online.target" ];
+      wantedBy = [ "kube-node-online.target" ];
+      before = [ "kube-node-online.target" ];
     };
 
     systemd.services.flannel = {
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index c4ecb2417e2..01cdfccccf9 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -340,9 +340,9 @@ in
       };
 
       systemd.services.kubelet-online = {
-        wantedBy = [ "node-online.target" ];
+        wantedBy = [ "kube-node-online.target" ];
         after = [ "flannel.target" "kubelet.target" ];
-        before = [ "node-online.target" ];
+        before = [ "kube-node-online.target" ];
         # it is complicated. flannel needs kubelet to run the pause container before
         # it discusses the node CIDR with apiserver and afterwards configures and restarts
         # dockerd. Until then prevent creating any pods because they have to be recreated anyway
@@ -407,11 +407,11 @@ in
 
     {
       systemd.targets.kubelet = {
-        wantedBy = [ "node-online.target" ];
-        before = [ "node-online.target" ];
+        wantedBy = [ "kube-node-online.target" ];
+        before = [ "kube-node-online.target" ];
       };
 
-      systemd.targets.node-online = {
+      systemd.targets.kube-node-online = {
         wantedBy = [ "kubernetes.target" ];
         before = [ "kubernetes.target" ];
       };
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index 01d59e9ac88..65d4f9ccbfc 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -48,9 +48,9 @@ in
   config = mkIf cfg.enable {
     systemd.services.kube-proxy = {
       description = "Kubernetes Proxy Service";
-      wantedBy = [ "node-online.target" ];
+      wantedBy = [ "kube-node-online.target" ];
       after = [ "kubelet-online.service" ];
-      before = [ "node-online.target" ];
+      before = [ "kube-node-online.target" ];
       path = with pkgs; [ iptables conntrack_tools ];
       preStart = ''
         ${top.lib.mkWaitCurl ( with config.systemd.services.kube-proxy; {
-- 
cgit 1.4.1


From ff382c18c8f8e3eba1fc3ff331b7146bcb3af674 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 17:56:28 +0100
Subject: nixos/kubernetes: Address review: Move remaining paths to pki

---
 .../services/cluster/kubernetes/apiserver.nix      | 41 +-------------
 .../services/cluster/kubernetes/kubelet.nix        | 19 +------
 nixos/modules/services/cluster/kubernetes/pki.nix  | 62 +++++++++++++++++++++-
 3 files changed, 64 insertions(+), 58 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 72fb9535832..63b485c43b8 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -272,25 +272,7 @@ in
   ###### implementation
   config = mkMerge [
 
-    (mkIf cfg.enable (let
-      apiserverPaths = [
-        cfg.clientCaFile
-        cfg.etcd.caFile
-        cfg.etcd.certFile
-        cfg.etcd.keyFile
-        cfg.kubeletClientCaFile
-        cfg.kubeletClientCertFile
-        cfg.kubeletClientKeyFile
-        cfg.serviceAccountKeyFile
-        cfg.tlsCertFile
-        cfg.tlsKeyFile
-      ];
-      etcdPaths = [
-        config.services.etcd.certFile
-        config.services.etcd.keyFile
-        config.services.etcd.trustedCaFile
-      ];
-    in {
+    (mkIf cfg.enable {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
           wantedBy = [ "kube-control-plane-online.target" ];
@@ -360,25 +342,6 @@ in
             Restart = "on-failure";
             RestartSec = 5;
           };
-          unitConfig.ConditionPathExists = apiserverPaths;
-        };
-
-        systemd.paths.kube-apiserver = {
-          wantedBy = [ "kube-apiserver.service" ];
-          pathConfig = {
-            PathExists = apiserverPaths;
-            PathChanged = apiserverPaths;
-          };
-        };
-
-        systemd.services.etcd.unitConfig.ConditionPathExists = etcdPaths;
-
-        systemd.paths.etcd = {
-          wantedBy = [ "etcd.service" ];
-          pathConfig = {
-            PathExists = etcdPaths;
-            PathChanged = etcdPaths;
-          };
         };
 
         services.etcd = {
@@ -459,7 +422,7 @@ in
         };
       };
 
-    }))
+    })
     {
       systemd.targets.kube-control-plane-online = {
         wantedBy = [ "kubernetes.target" ];
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index 01cdfccccf9..8eb212b41ec 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -241,13 +241,7 @@ in
 
   ###### implementation
   config = mkMerge [
-    (mkIf cfg.enable (let
-      kubeletPaths = [
-        cfg.clientCaFile
-        cfg.tlsCertFile
-        cfg.tlsKeyFile
-      ];
-    in {
+    (mkIf cfg.enable {
       services.kubernetes.kubelet.seedDockerImages = [infraContainer];
 
       systemd.services.kubelet = {
@@ -310,15 +304,6 @@ in
           '';
           WorkingDirectory = top.dataDir;
         };
-        unitConfig.ConditionPathExists = kubeletPaths;
-      };
-
-      systemd.paths.kubelet = {
-        wantedBy =  [ "kubelet.service" ];
-        pathConfig = {
-          PathExists = kubeletPaths;
-          PathChanged = kubeletPaths;
-        };
       };
 
       systemd.services.docker.before = [ "kubelet.service" ];
@@ -387,7 +372,7 @@ in
       };
 
       services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
-    }))
+    })
 
     (mkIf (cfg.enable && cfg.manifests != {}) {
       environment.etc = mapAttrs' (name: manifest:
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 0b43f2034c2..8bacc07b008 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -125,6 +125,23 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
+    apiserverPaths = [
+      top.apiserver.clientCaFile
+      top.apiserver.etcd.caFile
+      top.apiserver.etcd.certFile
+      top.apiserver.etcd.keyFile
+      top.apiserver.kubeletClientCaFile
+      top.apiserver.kubeletClientCertFile
+      top.apiserver.kubeletClientKeyFile
+      top.apiserver.serviceAccountKeyFile
+      top.apiserver.tlsCertFile
+      top.apiserver.tlsKeyFile
+    ];
+    etcdPaths = [
+      config.services.etcd.certFile
+      config.services.etcd.keyFile
+      config.services.etcd.trustedCaFile
+    ];
     addonManagerPaths = mkIf top.addonManager.enable [
       cfg.certs.addonManager.cert
       cfg.certs.addonManager.key
@@ -150,6 +167,11 @@ in
       cfg.certs.controllerManagerClient.cert
       cfg.certs.controllerManagerClient.key
     ];
+    kubeletPaths = [
+      top.kubelet.clientCaFile
+      top.kubelet.tlsCertFile
+      top.kubelet.tlsKeyFile
+    ];
   in
   {
 
@@ -415,7 +437,7 @@ in
 
       # isolate etcd on loopback at the master node
       # easyCerts doesn't support multimaster clusters anyway atm.
-      services.etcd = with cfg.certs.etcd; {
+      services.etcd = mkIf top.apiserver.enable (with cfg.certs.etcd; {
         listenClientUrls = ["https://127.0.0.1:2379"];
         listenPeerUrls = ["https://127.0.0.1:2380"];
         advertiseClientUrls = ["https://etcd.local:2379"];
@@ -424,11 +446,35 @@ in
         certFile = mkDefault cert;
         keyFile = mkDefault key;
         trustedCaFile = mkDefault caCert;
-      };
+      });
       networking.extraHosts = mkIf (config.services.etcd.enable) ''
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
+      systemd.services.kube-apiserver = mkIf top.apiserver.enable {
+        unitConfig.ConditionPathExists = apiserverPaths;
+      };
+
+      systemd.paths.kube-apiserver = mkIf top.apiserver.enable {
+        wantedBy = [ "kube-apiserver.service" ];
+        pathConfig = {
+          PathExists = apiserverPaths;
+          PathChanged = apiserverPaths;
+        };
+      };
+
+      systemd.services.etcd = mkIf top.apiserver.enable {
+        unitConfig.ConditionPathExists = etcdPaths;
+      };
+
+      systemd.paths.etcd = mkIf top.apiserver.enable {
+        wantedBy = [ "etcd.service" ];
+        pathConfig = {
+          PathExists = etcdPaths;
+          PathChanged = etcdPaths;
+        };
+      };
+
       services.flannel = with cfg.certs.flannelClient; {
         kubeconfig = top.lib.mkKubeConfig "flannel" {
           server = top.apiserverAddress;
@@ -455,6 +501,18 @@ in
         unitConfig.ConditionPathExists = proxyPaths;
       };
 
+      systemd.services.kubelet = mkIf top.kubelet.enable {
+        unitConfig.ConditionPathExists = kubeletPaths;
+      };
+
+      systemd.paths.kubelet = mkIf top.kubelet.enable {
+        wantedBy =  [ "kubelet.service" ];
+        pathConfig = {
+          PathExists = kubeletPaths;
+          PathChanged = kubeletPaths;
+        };
+      };
+
       systemd.paths.kube-proxy = mkIf top.proxy.enable {
         wantedBy = [ "kube-proxy.service" ];
         pathConfig = {
-- 
cgit 1.4.1


From 154356d820179a04c073ceadad0a4594ec18bf7d Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Thu, 7 Mar 2019 14:34:15 +0100
Subject: nixos/kubernetes: Fix kube-control-plane-online must not be present

outside kubernetes module.
---
 .../services/cluster/kubernetes/apiserver.nix      | 23 ----------------------
 .../services/cluster/kubernetes/default.nix        | 23 ++++++++++++++++++++++
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 63b485c43b8..28784407459 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -423,30 +423,7 @@ in
       };
 
     })
-    {
-      systemd.targets.kube-control-plane-online = {
-        wantedBy = [ "kubernetes.target" ];
-        before = [ "kubernetes.target" ];
-      };
 
-      systemd.services.kube-control-plane-online = rec {
-        description = "Kubernetes control plane is online";
-        wantedBy = [ "kube-control-plane-online.target" ];
-        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
-        before = [ "kube-control-plane-online.target" ];
-        preStart = ''
-          ${top.lib.mkWaitCurl ( with config.systemd.services.kube-control-plane-online; {
-            sleep = 3;
-            path = "/healthz";
-            cacert = top.caFile;
-          } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
-        '';
-        script = "echo Ok";
-        serviceConfig = {
-          TimeoutSec = "500";
-        };
-      };
-    }
   ];
 
 }
diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index f1f544afc4d..6560cff6328 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -299,6 +299,29 @@ in {
       services.kubernetes.apiserverAddress = mkDefault ("https://${if cfg.apiserver.advertiseAddress != null
                           then cfg.apiserver.advertiseAddress
                           else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}");
+
+      systemd.targets.kube-control-plane-online = {
+        wantedBy = [ "kubernetes.target" ];
+        before = [ "kubernetes.target" ];
+      };
+
+      systemd.services.kube-control-plane-online = rec {
+        description = "Kubernetes control plane is online";
+        wantedBy = [ "kube-control-plane-online.target" ];
+        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
+        before = [ "kube-control-plane-online.target" ];
+        preStart = ''
+          ${cfg.lib.mkWaitCurl ( with config.systemd.services.kube-control-plane-online; {
+            sleep = 3;
+            path = "/healthz";
+            cacert = cfg.caFile;
+          } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        '';
+        script = "echo Ok";
+        serviceConfig = {
+          TimeoutSec = "500";
+        };
+      };
     })
   ];
 }
-- 
cgit 1.4.1


From ee9dd4386a061594ad69ff5a3a683f899f9f8c93 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:44:24 +0100
Subject: Cleanup pki: addon-manager

---
 .../services/cluster/kubernetes/addon-manager.nix  | 77 ++++++++++++++++++----
 nixos/modules/services/cluster/kubernetes/pki.nix  | 58 ++++------------
 2 files changed, 76 insertions(+), 59 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index 406b20b0d8d..b9a56811d2b 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -63,24 +63,48 @@ in
     };
 
     enable = mkEnableOption "Whether to enable Kubernetes addon manager.";
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager";
+    bootstrapAddonsKubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager bootstrap";
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = let
+
+    addonManagerPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+    bootstrapAddonsPaths = filter (a: a != null) [
+      cfg.bootstrapAddonsKubeconfig.caFile
+      cfg.bootstrapAddonsKubeconfig.certFile
+      cfg.bootstrapAddonsKubeconfig.keyFile
+    ];
+
+  in mkIf cfg.enable {
     environment.etc."kubernetes/addons".source = "${addons}/";
 
+    #TODO: Get rid of kube-addon-manager in the future for the following reasons
+    # - it is basically just a shell script wrapped around kubectl
+    # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
+    # - it is designed to be used with k8s system components only
+    # - it would be better with a more Nix-oriented way of managing addons
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
-      wantedBy = [ "kube-control-plane-online.target" ];
-      after = [ "kube-addon-manager-bootstrap.service" ];
-      before = [ "kube-control-plane-online.target" ];
-      environment.ADDON_PATH = "/etc/kubernetes/addons/";
-      path = [ pkgs.gawk ];
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-node-online.target" ];
+      before = [ "kubernetes.target" ];
+      environment = {
+        ADDON_PATH = "/etc/kubernetes/addons/";
+        KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" cfg.kubeconfig;
+      };
+      path = with pkgs; [ gawk kubectl ];
       preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager; {
-          path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl -n kube-system get serviceaccounts/default 2>/dev/null; do
+          echo kubectl -n kube-system get serviceaccounts/default: exit status $?
+          sleep 2
+        done
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
@@ -91,27 +115,52 @@ in
         Restart = "on-failure";
         RestartSec = 10;
       };
+      unitConfig.ConditionPathExists = addonManagerPaths;
+    };
+
+    systemd.paths.kube-addon-manager = {
+      wantedBy = [ "kube-addon-manager.service" ];
+      pathConfig = {
+        PathExists = addonManagerPaths;
+        PathChanged = addonManagerPaths;
+      };
     };
 
+    services.kubernetes.addonManager.kubeconfig.server = mkDefault top.apiserverAddress;
+
     systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
       wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
       path = [ pkgs.kubectl ];
+      environment = {
+        KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager-bootstrap" cfg.bootstrapAddonsKubeconfig;
+      };
       preStart = with pkgs; let
         files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
           cfg.bootstrapAddons;
       in ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager-bootstrap; {
-          path = "/api";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl auth can-i '*' '*' -q 2>/dev/null; do
+          echo kubectl auth can-i '*' '*': exit status $?
+          sleep 2
+        done
 
         kubectl apply -f ${concatStringsSep " \\\n -f " files}
       '';
       script = "echo Ok";
+      unitConfig.ConditionPathExists = bootstrapAddonsPaths;
+    };
+
+    systemd.paths.kube-addon-manager-bootstrap = {
+      wantedBy = [ "kube-addon-manager-bootstrap.service" ];
+      pathConfig = {
+        PathExists = bootstrapAddonsPaths;
+        PathChanged = bootstrapAddonsPaths;
+      };
     };
 
+    services.kubernetes.addonManager.bootstrapAddonsKubeconfig.server = mkDefault top.apiserverAddress;
+
     services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled
     (let
       name = system:kube-addon-manager;
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 8bacc07b008..90b40dd4c1f 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -27,12 +27,11 @@ let
   certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
   cfsslAPITokenLength = 32;
 
-  clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
-    top.lib.mkKubeConfig "cluster-admin" {
-        server = top.apiserverAddress;
-        certFile = cert;
-        keyFile = key;
-    };
+  clusterAdminKubeconfig = with cfg.certs.clusterAdmin; {
+    server = top.apiserverAddress;
+    certFile = cert;
+    keyFile = key;
+  };
 
   remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
 in
@@ -142,12 +141,6 @@ in
       config.services.etcd.keyFile
       config.services.etcd.trustedCaFile
     ];
-    addonManagerPaths = mkIf top.addonManager.enable [
-      cfg.certs.addonManager.cert
-      cfg.certs.addonManager.key
-      cfg.certs.clusterAdmin.cert
-      cfg.certs.clusterAdmin.key
-    ];
     flannelPaths = [
       cfg.certs.flannelClient.cert
       cfg.certs.flannelClient.key
@@ -331,38 +324,6 @@ in
         };
       };
 
-      systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
-        environment = {
-          KUBECONFIG = clusterAdminKubeconfig;
-          inherit (cfg.certs.clusterAdmin) cert key;
-        };
-      };
-
-      #TODO: Get rid of kube-addon-manager in the future for the following reasons
-      # - it is basically just a shell script wrapped around kubectl
-      # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
-      # - it is designed to be used with k8s system components only
-      # - it would be better with a more Nix-oriented way of managing addons
-      systemd.services.kube-addon-manager = mkIf top.addonManager.enable {
-        environment = with cfg.certs.addonManager; {
-          KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" {
-              server = top.apiserverAddress;
-              certFile = cert;
-              keyFile = key;
-          };
-          inherit cert key;
-        };
-        unitConfig.ConditionPathExists = addonManagerPaths;
-      };
-
-      systemd.paths.kube-addon-manager = mkIf top.addonManager.enable {
-        wantedBy = [ "kube-addon-manager.service" ];
-        pathConfig = {
-          PathExists = addonManagerPaths;
-          PathChanged = addonManagerPaths;
-        };
-      };
-
       systemd.services.kube-controller-manager = mkIf top.controllerManager.enable {
         environment = { inherit (cfg.certs.controllerManagerClient) cert key; };
         unitConfig.ConditionPathExists = controllerManagerPaths;
@@ -396,7 +357,7 @@ in
       };
 
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
-        clusterAdminKubeconfig;
+        (top.lib.mkKubeConfig "cluster-admin" clusterAdminKubeconfig);
 
       environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
       (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
@@ -538,6 +499,13 @@ in
           kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
           kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
         });
+        addonManager = mkIf top.addonManager.enable {
+          kubeconfig = with cfg.certs.addonManager; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+          bootstrapAddonsKubeconfig = clusterAdminKubeconfig;
+        };
         controllerManager = mkIf top.controllerManager.enable {
           serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
           rootCaFile = cfg.certs.controllerManagerClient.caCert;
-- 
cgit 1.4.1


From 8ab50cb239e4aaeb88c372171a79f1fd874dfe50 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:47:58 +0100
Subject: Cleanup pki: apiserver and etcd

---
 .../services/cluster/kubernetes/apiserver.nix      | 43 +++++++++++++++++++++-
 nixos/modules/services/cluster/kubernetes/pki.nix  | 41 ---------------------
 2 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 28784407459..677738b4ec5 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -272,7 +272,27 @@ in
   ###### implementation
   config = mkMerge [
 
-    (mkIf cfg.enable {
+    (let
+
+      apiserverPaths = filter (a: a != null) [
+        cfg.clientCaFile
+        cfg.etcd.caFile
+        cfg.etcd.certFile
+        cfg.etcd.keyFile
+        cfg.kubeletClientCaFile
+        cfg.kubeletClientCertFile
+        cfg.kubeletClientKeyFile
+        cfg.serviceAccountKeyFile
+        cfg.tlsCertFile
+        cfg.tlsKeyFile
+      ];
+      etcdPaths = filter (a: a != null) [
+        config.services.etcd.trustedCaFile
+        config.services.etcd.certFile
+        config.services.etcd.keyFile
+      ];
+
+    in mkIf cfg.enable {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
           wantedBy = [ "kube-control-plane-online.target" ];
@@ -342,6 +362,15 @@ in
             Restart = "on-failure";
             RestartSec = 5;
           };
+          unitConfig.ConditionPathExists = apiserverPaths;
+        };
+
+        systemd.paths.kube-apiserver = mkIf top.apiserver.enable {
+          wantedBy = [ "kube-apiserver.service" ];
+          pathConfig = {
+            PathExists = apiserverPaths;
+            PathChanged = apiserverPaths;
+          };
         };
 
         services.etcd = {
@@ -355,6 +384,18 @@ in
           initialAdvertisePeerUrls = mkDefault ["https://${top.masterAddress}:2380"];
         };
 
+        systemd.services.etcd = {
+          unitConfig.ConditionPathExists = etcdPaths;
+        };
+
+        systemd.paths.etcd = {
+          wantedBy = [ "etcd.service" ];
+          pathConfig = {
+            PathExists = etcdPaths;
+            PathChanged = etcdPaths;
+          };
+        };
+
         services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled {
 
           apiserver-kubelet-api-admin-crb = {
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 90b40dd4c1f..85e1fc9671c 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -124,23 +124,6 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
-    apiserverPaths = [
-      top.apiserver.clientCaFile
-      top.apiserver.etcd.caFile
-      top.apiserver.etcd.certFile
-      top.apiserver.etcd.keyFile
-      top.apiserver.kubeletClientCaFile
-      top.apiserver.kubeletClientCertFile
-      top.apiserver.kubeletClientKeyFile
-      top.apiserver.serviceAccountKeyFile
-      top.apiserver.tlsCertFile
-      top.apiserver.tlsKeyFile
-    ];
-    etcdPaths = [
-      config.services.etcd.certFile
-      config.services.etcd.keyFile
-      config.services.etcd.trustedCaFile
-    ];
     flannelPaths = [
       cfg.certs.flannelClient.cert
       cfg.certs.flannelClient.key
@@ -412,30 +395,6 @@ in
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
-      systemd.services.kube-apiserver = mkIf top.apiserver.enable {
-        unitConfig.ConditionPathExists = apiserverPaths;
-      };
-
-      systemd.paths.kube-apiserver = mkIf top.apiserver.enable {
-        wantedBy = [ "kube-apiserver.service" ];
-        pathConfig = {
-          PathExists = apiserverPaths;
-          PathChanged = apiserverPaths;
-        };
-      };
-
-      systemd.services.etcd = mkIf top.apiserver.enable {
-        unitConfig.ConditionPathExists = etcdPaths;
-      };
-
-      systemd.paths.etcd = mkIf top.apiserver.enable {
-        wantedBy = [ "etcd.service" ];
-        pathConfig = {
-          PathExists = etcdPaths;
-          PathChanged = etcdPaths;
-        };
-      };
-
       services.flannel = with cfg.certs.flannelClient; {
         kubeconfig = top.lib.mkKubeConfig "flannel" {
           server = top.apiserverAddress;
-- 
cgit 1.4.1


From ce83dc2c52dae33330d868abd358166f9a4cb77a Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:50:32 +0100
Subject: Cleanup pki: controller-manager

---
 .../cluster/kubernetes/controller-manager.nix      | 39 +++++++++++++++++-----
 nixos/modules/services/cluster/kubernetes/pki.nix  | 20 -----------
 2 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index a39fd62c689..a28679dbb9a 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -104,18 +104,30 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
-    systemd.services.kube-controller-manager = {
+  config = let
+
+    controllerManagerPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+      cfg.rootCaFile
+      cfg.serviceAccountKeyFile
+      cfg.tlsCertFile
+      cfg.tlsKeyFile
+    ];
+
+  in mkIf cfg.enable {
+    systemd.services.kube-controller-manager = rec {
       description = "Kubernetes Controller Manager Service";
       wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
+      environment.KUBECONFIG = top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig;
       preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-controller-manager; {
-          sleep = 1;
-          path = "/api";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl auth can-i get /api -q 2>/dev/null; do
+          echo kubectl auth can-i get /api: exit status $?
+          sleep 2
+        done
       '';
       serviceConfig = {
         RestartSec = "30s";
@@ -128,7 +140,7 @@ in
             "--cluster-cidr=${cfg.clusterCidr}"} \
           ${optionalString (cfg.featureGates != [])
             "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
-          --kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \
+          --kubeconfig=${environment.KUBECONFIG} \
           --leader-elect=${boolToString cfg.leaderElect} \
           ${optionalString (cfg.rootCaFile!=null)
             "--root-ca-file=${cfg.rootCaFile}"} \
@@ -149,7 +161,16 @@ in
         User = "kubernetes";
         Group = "kubernetes";
       };
-      path = top.path;
+      path = top.path ++ [ pkgs.kubectl ];
+      unitConfig.ConditionPathExists = controllerManagerPaths;
+    };
+
+    systemd.paths.kube-controller-manager = {
+      wantedBy = [ "kube-controller-manager.service" ];
+      pathConfig = {
+        PathExists = controllerManagerPaths;
+        PathChanged = controllerManagerPaths;
+      };
     };
 
     services.kubernetes.pki.certs = with top.lib; {
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 85e1fc9671c..3c7af73e0b9 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -136,13 +136,6 @@ in
       cfg.certs.schedulerClient.cert
       cfg.certs.schedulerClient.key
     ];
-    controllerManagerPaths = [
-      top.controllerManager.rootCaFile
-      top.controllerManager.tlsCertFile
-      top.controllerManager.tlsKeyFile
-      cfg.certs.controllerManagerClient.cert
-      cfg.certs.controllerManagerClient.key
-    ];
     kubeletPaths = [
       top.kubelet.clientCaFile
       top.kubelet.tlsCertFile
@@ -307,19 +300,6 @@ in
         };
       };
 
-      systemd.services.kube-controller-manager = mkIf top.controllerManager.enable {
-        environment = { inherit (cfg.certs.controllerManagerClient) cert key; };
-        unitConfig.ConditionPathExists = controllerManagerPaths;
-      };
-
-      systemd.paths.kube-controller-manager = mkIf top.controllerManager.enable {
-        wantedBy = [ "kube-controller-manager.service" ];
-        pathConfig = {
-          PathExists = controllerManagerPaths;
-          PathChanged = controllerManagerPaths;
-        };
-      };
-
       systemd.services.kube-scheduler = mkIf top.scheduler.enable {
         environment = { inherit (top.pki.certs.schedulerClient) cert key; };
         unitConfig.ConditionPathExists = schedulerPaths;
-- 
cgit 1.4.1


From ea6985ffc1af67fe11581c0e4c0a28e852f26c0b Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:53:59 +0100
Subject: Cleanup pki: flannel

---
 .../services/cluster/kubernetes/flannel.nix        | 49 +++++++++++++++++-----
 nixos/modules/services/cluster/kubernetes/pki.nix  | 31 +++-----------
 2 files changed, 45 insertions(+), 35 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index 4c5fe7559eb..e79fbcb6200 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -24,16 +24,26 @@ in
   ###### interface
   options.services.kubernetes.flannel = {
     enable = mkEnableOption "enable flannel networking";
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes flannel";
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = let
+
+    flannelPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+    kubeconfig = top.lib.mkKubeConfig "flannel" cfg.kubeconfig;
+
+  in mkIf cfg.enable {
     services.flannel = {
 
       enable = mkDefault true;
       network = mkDefault top.clusterCidr;
-      inherit storageBackend;
-      nodeName = config.services.kubernetes.kubelet.hostname;
+      inherit storageBackend kubeconfig;
+      nodeName = top.kubelet.hostname;
     };
 
     services.kubernetes.kubelet = {
@@ -79,16 +89,35 @@ in
       wantedBy = [ "flannel.target" ];
       after = [ "kubelet.target" ];
       before = [ "flannel.target" ];
-      path = [ pkgs.iptables ];
-      preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.flannel; {
-          path = "/api/v1/nodes";
-          cacert = top.caFile;
-          args = "-o - | grep podCIDR >/dev/null";
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+      path = with pkgs; [ iptables kubectl ];
+      environment.KUBECONFIG = kubeconfig;
+      preStart = let
+        args = [
+          "--selector=kubernetes.io/hostname=${top.kubelet.hostname}"
+          # flannel exits if node is not registered yet, before that there is no podCIDR
+          "--output=jsonpath={.items[0].spec.podCIDR}"
+          # if jsonpath cannot be resolved exit with status 1
+          "--allow-missing-template-keys=false"
+        ];
+      in ''
+        until kubectl get nodes ${concatStringsSep " " args} 2>/dev/null; do
+          echo Waiting for ${top.kubelet.hostname} to be RegisteredNode
+          sleep 1
+        done
       '';
+      unitConfig.ConditionPathExists = flannelPaths;
     };
 
+    systemd.paths.flannel = {
+      wantedBy = [ "flannel.service" ];
+      pathConfig = {
+        PathExists = flannelPaths;
+        PathChanged = flannelPaths;
+      };
+    };
+
+    services.kubernetes.flannel.kubeconfig.server = mkDefault top.apiserverAddress;
+
     systemd.services.docker = {
       environment.DOCKER_OPTS = "-b none";
       serviceConfig.EnvironmentFile = "-/run/flannel/docker";
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 3c7af73e0b9..2e79e7590e3 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -124,10 +124,6 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
-    flannelPaths = [
-      cfg.certs.flannelClient.cert
-      cfg.certs.flannelClient.key
-    ];
     proxyPaths = mkIf top.proxy.enable [
       cfg.certs.kubeProxyClient.cert
       cfg.certs.kubeProxyClient.key
@@ -375,27 +371,6 @@ in
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
-      services.flannel = with cfg.certs.flannelClient; {
-        kubeconfig = top.lib.mkKubeConfig "flannel" {
-          server = top.apiserverAddress;
-          certFile = cert;
-          keyFile = key;
-        };
-      };
-
-      systemd.services.flannel = mkIf top.flannel.enable {
-        environment = { inherit (top.pki.certs.flannelClient) cert key; };
-        unitConfig.ConditionPathExists = flannelPaths;
-      };
-
-      systemd.paths.flannel = mkIf top.flannel.enable {
-        wantedBy = [ "flannel.service" ];
-        pathConfig = {
-          PathExists = flannelPaths;
-          PathChanged = flannelPaths;
-        };
-      };
-
       systemd.services.kube-proxy = mkIf top.proxy.enable {
         environment = { inherit (top.pki.certs.kubeProxyClient) cert key; };
         unitConfig.ConditionPathExists = proxyPaths;
@@ -453,6 +428,12 @@ in
             keyFile = mkDefault key;
           };
         };
+        flannel = mkIf top.flannel.enable {
+          kubeconfig = with cfg.certs.flannelClient; {
+            certFile = cert;
+            keyFile = key;
+          };
+        };
         scheduler = mkIf top.scheduler.enable {
           kubeconfig = with cfg.certs.schedulerClient; {
             certFile = mkDefault cert;
-- 
cgit 1.4.1


From 73657b7fcfe8ad87af70a1b6186a355971da6c97 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:58:57 +0100
Subject: Cleanup pki: kubelet

---
 .../services/cluster/kubernetes/kubelet.nix        | 25 ++++++++++++++++++++--
 nixos/modules/services/cluster/kubernetes/pki.nix  | 17 ---------------
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index 8eb212b41ec..2a4a0624555 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -241,7 +241,18 @@ in
 
   ###### implementation
   config = mkMerge [
-    (mkIf cfg.enable {
+    (let
+
+      kubeletPaths = filter (a: a != null) [
+        cfg.kubeconfig.caFile
+        cfg.kubeconfig.certFile
+        cfg.kubeconfig.keyFile
+        cfg.clientCaFile
+        cfg.tlsCertFile
+        cfg.tlsKeyFile
+      ];
+
+    in mkIf cfg.enable {
       services.kubernetes.kubelet.seedDockerImages = [infraContainer];
 
       systemd.services.kubelet = {
@@ -304,6 +315,15 @@ in
           '';
           WorkingDirectory = top.dataDir;
         };
+        unitConfig.ConditionPathExists = kubeletPaths;
+      };
+
+      systemd.paths.kubelet = {
+        wantedBy =  [ "kubelet.service" ];
+        pathConfig = {
+          PathExists = kubeletPaths;
+          PathChanged = kubeletPaths;
+        };
       };
 
       systemd.services.docker.before = [ "kubelet.service" ];
@@ -321,6 +341,7 @@ in
         '';
         script = "echo Ok";
         serviceConfig.Type = "oneshot";
+        serviceConfig.RemainAfterExit = true;
         serviceConfig.Slice = "kubernetes.slice";
       };
 
@@ -337,7 +358,7 @@ in
           flannel-date = "stat --print=%Y ${docker-env}";
           docker-date = "systemctl show --property=ActiveEnterTimestamp --value docker";
         in ''
-          while ! test -f ${docker-env} ; do sleep 1 ; done
+          until test -f ${docker-env} ; do sleep 1 ; done
           while test `${flannel-date}` -gt `date +%s --date="$(${docker-date})"` ; do
             sleep 1
           done
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 2e79e7590e3..92eefae5bda 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -132,11 +132,6 @@ in
       cfg.certs.schedulerClient.cert
       cfg.certs.schedulerClient.key
     ];
-    kubeletPaths = [
-      top.kubelet.clientCaFile
-      top.kubelet.tlsCertFile
-      top.kubelet.tlsKeyFile
-    ];
   in
   {
 
@@ -376,18 +371,6 @@ in
         unitConfig.ConditionPathExists = proxyPaths;
       };
 
-      systemd.services.kubelet = mkIf top.kubelet.enable {
-        unitConfig.ConditionPathExists = kubeletPaths;
-      };
-
-      systemd.paths.kubelet = mkIf top.kubelet.enable {
-        wantedBy =  [ "kubelet.service" ];
-        pathConfig = {
-          PathExists = kubeletPaths;
-          PathChanged = kubeletPaths;
-        };
-      };
-
       systemd.paths.kube-proxy = mkIf top.proxy.enable {
         wantedBy = [ "kube-proxy.service" ];
         pathConfig = {
-- 
cgit 1.4.1


From 46653f84c94ab9190a6841dbbfd874bb25d2a7f6 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 11:01:54 +0100
Subject: Cleanup pki: proxy

---
 nixos/modules/services/cluster/kubernetes/pki.nix  | 17 -----------
 .../modules/services/cluster/kubernetes/proxy.nix  | 34 +++++++++++++++++-----
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 92eefae5bda..be0b50e9329 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -124,10 +124,6 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
-    proxyPaths = mkIf top.proxy.enable [
-      cfg.certs.kubeProxyClient.cert
-      cfg.certs.kubeProxyClient.key
-    ];
     schedulerPaths = mkIf top.scheduler.enable [
       cfg.certs.schedulerClient.cert
       cfg.certs.schedulerClient.key
@@ -366,19 +362,6 @@ in
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
-      systemd.services.kube-proxy = mkIf top.proxy.enable {
-        environment = { inherit (top.pki.certs.kubeProxyClient) cert key; };
-        unitConfig.ConditionPathExists = proxyPaths;
-      };
-
-      systemd.paths.kube-proxy = mkIf top.proxy.enable {
-        wantedBy = [ "kube-proxy.service" ];
-        pathConfig = {
-          PathExists = proxyPaths;
-          PathChanged = proxyPaths;
-        };
-      };
-
       services.kubernetes = {
 
         apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index 65d4f9ccbfc..8a90542fe63 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -45,18 +45,27 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
-    systemd.services.kube-proxy = {
+  config = let
+
+    proxyPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+
+  in mkIf cfg.enable {
+    systemd.services.kube-proxy = rec {
       description = "Kubernetes Proxy Service";
       wantedBy = [ "kube-node-online.target" ];
       after = [ "kubelet-online.service" ];
       before = [ "kube-node-online.target" ];
-      path = with pkgs; [ iptables conntrack_tools ];
+      environment.KUBECONFIG = top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig;
+      path = with pkgs; [ iptables conntrack_tools kubectl ];
       preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-proxy; {
-          path = "/api/v1/nodes/${top.kubelet.hostname}";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl auth can-i get nodes/${top.kubelet.hostname} -q 2>/dev/null; do
+          echo kubectl auth can-i get nodes/${top.kubelet.hostname}: exit status $?
+          sleep 2
+        done
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
@@ -66,7 +75,7 @@ in
             "--cluster-cidr=${top.clusterCidr}"} \
           ${optionalString (cfg.featureGates != [])
             "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
-          --kubeconfig=${top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \
+          --kubeconfig=${environment.KUBECONFIG} \
           ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
           ${cfg.extraOpts}
         '';
@@ -74,6 +83,15 @@ in
         Restart = "on-failure";
         RestartSec = 5;
       };
+      unitConfig.ConditionPathExists = proxyPaths;
+    };
+
+    systemd.paths.kube-proxy = {
+      wantedBy = [ "kube-proxy.service" ];
+      pathConfig = {
+        PathExists = proxyPaths;
+        PathChanged = proxyPaths;
+      };
     };
 
     services.kubernetes.pki.certs = {
-- 
cgit 1.4.1


From 50c5f489ef4d9a1273860a5f5eaa9810f2c9d2ce Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 11:03:40 +0100
Subject: Cleanup pki: scheduler

---
 nixos/modules/services/cluster/kubernetes/pki.nix  | 17 -----------
 .../services/cluster/kubernetes/scheduler.nix      | 34 +++++++++++++++++-----
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index be0b50e9329..6396ec22907 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -124,10 +124,6 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
-    schedulerPaths = mkIf top.scheduler.enable [
-      cfg.certs.schedulerClient.cert
-      cfg.certs.schedulerClient.key
-    ];
   in
   {
 
@@ -287,19 +283,6 @@ in
         };
       };
 
-      systemd.services.kube-scheduler = mkIf top.scheduler.enable {
-        environment = { inherit (top.pki.certs.schedulerClient) cert key; };
-        unitConfig.ConditionPathExists = schedulerPaths;
-      };
-
-      systemd.paths.kube-scheduler = mkIf top.scheduler.enable {
-        wantedBy = [ "kube-scheduler.service" ];
-        pathConfig = {
-          PathExists = schedulerPaths;
-          PathChanged = schedulerPaths;
-        };
-      };
-
       systemd.services.kube-control-plane-online.environment = let
         client = with cfg.certs; if top.apiserver.enable then clusterAdmin else kubelet;
       in {
diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix
index 32a84563076..d5852825954 100644
--- a/nixos/modules/services/cluster/kubernetes/scheduler.nix
+++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix
@@ -56,18 +56,27 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
-    systemd.services.kube-scheduler = {
+  config =  let
+
+    schedulerPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+
+  in mkIf cfg.enable {
+    systemd.services.kube-scheduler = rec {
       description = "Kubernetes Scheduler Service";
       wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
+      environment.KUBECONFIG = top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig;
+      path = [ pkgs.kubectl ];
       preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-scheduler; {
-          sleep = 1;
-          path = "/api";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl auth can-i get /api -q 2>/dev/null; do
+          echo kubectl auth can-i get /api: exit status $?
+          sleep 2
+        done
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
@@ -75,7 +84,7 @@ in
           --address=${cfg.address} \
           ${optionalString (cfg.featureGates != [])
             "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
-          --kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \
+          --kubeconfig=${environment.KUBECONFIG} \
           --leader-elect=${boolToString cfg.leaderElect} \
           --port=${toString cfg.port} \
           ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
@@ -87,6 +96,15 @@ in
         Restart = "on-failure";
         RestartSec = 5;
       };
+      unitConfig.ConditionPathExists = schedulerPaths;
+    };
+
+    systemd.paths.kube-scheduler = {
+      wantedBy = [ "kube-scheduler.service" ];
+      pathConfig = {
+        PathExists = schedulerPaths;
+        PathChanged = schedulerPaths;
+      };
     };
 
     services.kubernetes.pki.certs = {
-- 
cgit 1.4.1


From 45e683fbd6bc2b8ccf57b6425f4877deed618569 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 11:42:12 +0100
Subject: Cleanup pki: control-plane-online

---
 .../services/cluster/kubernetes/default.nix        | 47 ++++++++++++----------
 nixos/modules/services/cluster/kubernetes/pki.nix  |  6 ---
 2 files changed, 25 insertions(+), 28 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index 6560cff6328..7cc172f1255 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -276,6 +276,30 @@ in {
         wantedBy = [ "multi-user.target" ];
       };
 
+      systemd.targets.kube-control-plane-online = {
+        wantedBy = [ "kubernetes.target" ];
+        before = [ "kubernetes.target" ];
+      };
+
+      systemd.services.kube-control-plane-online = rec {
+        description = "Kubernetes control plane is online";
+        wantedBy = [ "kube-control-plane-online.target" ];
+        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
+        before = [ "kube-control-plane-online.target" ];
+        environment.KUBECONFIG = cfg.lib.mkKubeConfig "default" cfg.kubeconfig;
+        path = [ pkgs.kubectl ];
+        preStart = ''
+          until kubectl get --raw=/healthz 2>/dev/null; do
+            echo kubectl get --raw=/healthz: exit status $?
+            sleep 3
+          done
+        '';
+        script = "echo Ok";
+        serviceConfig = {
+          TimeoutSec = "500";
+        };
+      };
+
       systemd.tmpfiles.rules = [
         "d /opt/cni/bin 0755 root root -"
         "d /run/kubernetes 0755 kubernetes kubernetes -"
@@ -300,28 +324,7 @@ in {
                           then cfg.apiserver.advertiseAddress
                           else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}");
 
-      systemd.targets.kube-control-plane-online = {
-        wantedBy = [ "kubernetes.target" ];
-        before = [ "kubernetes.target" ];
-      };
-
-      systemd.services.kube-control-plane-online = rec {
-        description = "Kubernetes control plane is online";
-        wantedBy = [ "kube-control-plane-online.target" ];
-        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
-        before = [ "kube-control-plane-online.target" ];
-        preStart = ''
-          ${cfg.lib.mkWaitCurl ( with config.systemd.services.kube-control-plane-online; {
-            sleep = 3;
-            path = "/healthz";
-            cacert = cfg.caFile;
-          } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
-        '';
-        script = "echo Ok";
-        serviceConfig = {
-          TimeoutSec = "500";
-        };
-      };
+      services.kubernetes.kubeconfig.server = mkDefault cfg.apiserverAddress;
     })
   ];
 }
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 6396ec22907..1d0232fa235 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -283,12 +283,6 @@ in
         };
       };
 
-      systemd.services.kube-control-plane-online.environment = let
-        client = with cfg.certs; if top.apiserver.enable then clusterAdmin else kubelet;
-      in {
-        inherit (client) cert key;
-      };
-
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
         (top.lib.mkKubeConfig "cluster-admin" clusterAdminKubeconfig);
 
-- 
cgit 1.4.1


From e3a80ebc40c9ce68db32a20dc806710b36393080 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 11:42:48 +0100
Subject: Cleanup pki: remove mkWaitCurl

---
 .../services/cluster/kubernetes/default.nix        | 13 -------------
 nixos/modules/services/cluster/kubernetes/pki.nix  | 22 ++++++++++++----------
 2 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index 7cc172f1255..84ed6821692 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -73,18 +73,6 @@ let
     };
   };
 
-  mkWaitCurl = { address ? cfg.apiserverAddress, sleep ? 2, path ? "", args ? "-o /dev/null",
-                 cacert ? null, cert ? null, key ? null, }: ''
-    while ! ${pkgs.curl}/bin/curl --fail-early -fs \
-      ${if cacert != null then "--cacert ${cacert}" else ""} \
-      ${if cert != null then "--cert ${cert}" else ""} \
-      ${if key != null then "--key ${key}" else ""} \
-      ${address}${path} ${args} ; do
-        sleep ${toString sleep}
-        echo Waiting to be able to reach ${address}${path}
-    done
-  '';
-
   kubeConfigDefaults = {
     server = mkDefault cfg.kubeconfig.server;
     caFile = mkDefault cfg.kubeconfig.caFile;
@@ -174,7 +162,6 @@ in {
         inherit mkCert;
         inherit mkKubeConfig;
         inherit mkKubeConfigOptions;
-        inherit mkWaitCurl;
       };
       type = types.attrs;
     };
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 1d0232fa235..4cf3269e18f 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -182,12 +182,12 @@ in
       description = "Wait for ${remote} to be reachable.";
       wantedBy = [ "cfssl-online.target" ];
       before = [ "cfssl-online.target" ];
+      path = [ pkgs.curl ];
       preStart = ''
-        ${top.lib.mkWaitCurl {
-          address = remote;
-          path = "/api/v1/cfssl/info";
-          args = "-kd '{}' -o /dev/null";
-        }}
+        until curl --fail-early -fskd '{}' ${remote}/api/v1/cfssl/info -o /dev/null; do
+          echo curl ${remote}/api/v1/cfssl/info: exit status $?
+          sleep 2
+        done
       '';
       script = "echo Ok";
       serviceConfig = {
@@ -200,6 +200,7 @@ in
       wantedBy = [ "cfssl-online.target" ];
       after = [ "cfssl-online.target" ];
       before = [ "certmgr.service" ];
+      path = with pkgs; [ curl cfssl ];
       script = concatStringsSep "\n" [''
         set -e
 
@@ -218,11 +219,12 @@ in
       ''
       (optionalString (cfg.pkiTrustOnBootstrap) ''
         if [ ! -s "${top.caFile}" ]; then
-          ${top.lib.mkWaitCurl {
-            address = "https://${top.masterAddress}:${cfsslPort}";
-            path = "/api/v1/cfssl/info";
-            args = "-kd '{}' -o - | ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}";
-          }}
+          until test -s ${top.caFile}.json; do
+            sleep 2
+            curl --fail-early -fskd '{}' ${remote}/api/v1/cfssl/info -o ${top.caFile}.json
+          done
+          cfssljson -f ${top.caFile}.json -stdout >${top.caFile}
+          rm ${top.caFile}.json
         fi
       '')
       ];
-- 
cgit 1.4.1