diff options
| -rw-r--r-- | nixos/modules/services/cluster/kubernetes/apiserver.nix | 1 | ||||
| -rw-r--r-- | nixos/modules/services/cluster/kubernetes/default.nix | 10 | ||||
| -rw-r--r-- | nixos/modules/services/cluster/kubernetes/flannel.nix | 52 | ||||
| -rw-r--r-- | nixos/modules/services/cluster/kubernetes/pki.nix | 23 | ||||
| -rw-r--r-- | nixos/tests/kubernetes/base.nix | 1 |
5 files changed, 78 insertions, 9 deletions
diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix index 465d74d83c8..81e45b417de 100644 --- a/nixos/modules/services/cluster/kubernetes/apiserver.nix +++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix @@ -411,6 +411,7 @@ in name = "etcd"; CN = top.masterAddress; hosts = [ + "etcd.local" "etcd.${top.addons.dns.clusterDomain}" top.masterAddress cfg.advertiseAddress diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index a5a59f4a5fc..375e33e91b5 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -189,6 +189,16 @@ in { services.kubernetes.addonManager.enable = mkDefault true; services.kubernetes.proxy.enable = mkDefault true; services.etcd.enable = true; # Cannot mkDefault because of flannel default options + services.kubernetes.kubelet = { + enable = mkDefault true; + taints = mkIf (!(elem "node" cfg.roles)) { + master = { + key = "node-role.kubernetes.io/master"; + value = "true"; + effect = "NoSchedule"; + }; + }; + }; }) diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix index 38dc1e2b47d..fb70e6513d3 100644 --- a/nixos/modules/services/cluster/kubernetes/flannel.nix +++ b/nixos/modules/services/cluster/kubernetes/flannel.nix @@ -6,6 +6,9 @@ let top = config.services.kubernetes; cfg = top.flannel; + # we want flannel to use kubernetes itself as configuration backend, not direct etcd + storageBackend = "kubernetes"; + # needed for flannel to pass options to docker mkDockerOpts = pkgs.runCommand "mk-docker-opts" { buildInputs = [ pkgs.makeWrapper ]; @@ -29,6 +32,8 @@ in enable = mkDefault true; network = mkDefault top.clusterCidr; + inherit storageBackend; + nodeName = config.services.kubernetes.kubelet.hostname; }; services.kubernetes.kubelet = { @@ -69,11 +74,52 @@ in }; services.kubernetes.pki.certs = { - flannelEtcdClient = top.lib.mkCert { - name = "flannel-etcd-client"; - CN = "flannel-etcd-client"; + flannelClient = top.lib.mkCert { + name = "flannel-client"; + CN = "flannel-client"; action = "systemctl restart flannel.service"; }; }; + + # give flannel som kubernetes rbac permissions if applicable + services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) { + + flannel-cr = { + apiVersion = "rbac.authorization.k8s.io/v1beta1"; + kind = "ClusterRole"; + metadata = { name = "flannel"; }; + rules = [{ + apiGroups = [ "" ]; + resources = [ "pods" ]; + verbs = [ "get" ]; + } + { + apiGroups = [ "" ]; + resources = [ "nodes" ]; + verbs = [ "list" "watch" ]; + } + { + apiGroups = [ "" ]; + resources = [ "nodes/status" ]; + verbs = [ "patch" ]; + }]; + }; + + flannel-crb = { + apiVersion = "rbac.authorization.k8s.io/v1beta1"; + kind = "ClusterRoleBinding"; + metadata = { name = "flannel"; }; + roleRef = { + apiGroup = "rbac.authorization.k8s.io"; + kind = "ClusterRole"; + name = "flannel"; + }; + subjects = [{ + kind = "User"; + name = "flannel-client"; + }]; + }; + + }; }; } diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 4587373d519..38deca23a99 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -305,7 +305,7 @@ in ''} ${optionalString top.flannel.enable '' - while [ ! -f ${cfg.certs.flannelEtcdClient.cert} ]; do sleep 1; done + while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done echo "Restarting flannel..." >&1 systemctl restart flannel ''} @@ -313,22 +313,35 @@ in echo "Node joined succesfully" '')]; + # isolate etcd on loopback at the master node + # easyCerts doesn't support multimaster clusters anyway atm. services.etcd = with cfg.certs.etcd; { + listenClientUrls = ["https://127.0.0.1:2379"]; + listenPeerUrls = ["https://127.0.0.1:2380"]; + advertiseClientUrls = ["https://etcd.local:2379"]; + initialCluster = ["${top.masterAddress}=https://etcd.local:2380"]; + initialAdvertisePeerUrls = ["https://etcd.local:2380"]; certFile = mkDefault cert; keyFile = mkDefault key; trustedCaFile = mkDefault caCert; }; + networking.extraHosts = mkIf (config.services.etcd.enable) '' + 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local + ''; - services.flannel.etcd = with cfg.certs.flannelEtcdClient; { - certFile = mkDefault cert; - keyFile = mkDefault key; - caFile = mkDefault caCert; + services.flannel = with cfg.certs.flannelClient; { + kubeconfig = top.lib.mkKubeConfig "flannel" { + server = top.apiserverAddress; + certFile = cert; + keyFile = key; + }; }; services.kubernetes = { apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; { etcd = with cfg.certs.apiserverEtcdClient; { + servers = ["https://etcd.local:2379"]; certFile = mkDefault cert; keyFile = mkDefault key; caFile = mkDefault caCert; diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix index 3529f35f60e..ec1a75e74c4 100644 --- a/nixos/tests/kubernetes/base.nix +++ b/nixos/tests/kubernetes/base.nix @@ -65,7 +65,6 @@ let } (optionalAttrs (any (role: role == "master") machine.roles) { networking.firewall.allowedTCPPorts = [ - 2379 2380 # etcd 443 # kubernetes apiserver ]; }) |