summary refs log tree commit diff
path: root/nixos/tests/kubernetes
diff options
context:
space:
mode:
Diffstat (limited to 'nixos/tests/kubernetes')
-rw-r--r--nixos/tests/kubernetes/base.nix38
-rw-r--r--nixos/tests/kubernetes/certs.nix219
-rw-r--r--nixos/tests/kubernetes/dns.nix7
-rw-r--r--nixos/tests/kubernetes/kubernetes-common.nix57
-rw-r--r--nixos/tests/kubernetes/rbac.nix9
5 files changed, 28 insertions, 302 deletions
diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix
index 9d77be13175..3529f35f60e 100644
--- a/nixos/tests/kubernetes/base.nix
+++ b/nixos/tests/kubernetes/base.nix
@@ -10,7 +10,6 @@ let
   mkKubernetesBaseTest =
     { name, domain ? "my.zyx", test, machines
     , pkgs ? import <nixpkgs> { inherit system; }
-    , certs ? import ./certs.nix { inherit pkgs; externalDomain = domain; kubelets = attrNames machines; }
     , extraConfiguration ? null }:
     let
       masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
@@ -20,6 +19,10 @@ let
         ${master.ip}  api.${domain}
         ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip}  ${machineName}.${domain}") (attrNames machines)}
       '';
+      kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
+        mkdir -p $out/bin
+        makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
+      '';
     in makeTest {
       inherit name;
 
@@ -27,6 +30,7 @@ let
         { config, pkgs, lib, nodes, ... }:
           mkMerge [
             {
+              boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
               virtualisation.memorySize = mkDefault 1536;
               virtualisation.diskSize = mkDefault 4096;
               networking = {
@@ -45,34 +49,26 @@ let
                 };
               };
               programs.bash.enableCompletion = true;
-              environment.variables = {
-                ETCDCTL_CERT_FILE = "${certs.worker}/etcd-client.pem";
-                ETCDCTL_KEY_FILE = "${certs.worker}/etcd-client-key.pem";
-                ETCDCTL_CA_FILE = "${certs.worker}/ca.pem";
-                ETCDCTL_PEERS = "https://etcd.${domain}:2379";
-              };
+              environment.systemPackages = [ kubectl ];
               services.flannel.iface = "eth1";
-              services.kubernetes.apiserver.advertiseAddress = master.ip;
+              services.kubernetes = {
+                addons.dashboard.enable = true;
+
+                easyCerts = true;
+                inherit (machine) roles;
+                apiserver = {
+                  securePort = 443;
+                  advertiseAddress = master.ip;
+                };
+                masterAddress = "${masterName}.${config.networking.domain}";
+              };
             }
             (optionalAttrs (any (role: role == "master") machine.roles) {
               networking.firewall.allowedTCPPorts = [
                 2379 2380  # etcd
                 443 # kubernetes apiserver
               ];
-              services.etcd = {
-                enable = true;
-                certFile = "${certs.master}/etcd.pem";
-                keyFile = "${certs.master}/etcd-key.pem";
-                trustedCaFile = "${certs.master}/ca.pem";
-                peerClientCertAuth = true;
-                listenClientUrls = ["https://0.0.0.0:2379"];
-                listenPeerUrls = ["https://0.0.0.0:2380"];
-                advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"];
-                initialCluster = ["${masterName}=https://etcd.${config.networking.domain}:2380"];
-                initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"];
-              };
             })
-            (import ./kubernetes-common.nix { inherit (machine) roles; inherit pkgs config certs; })
             (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
             (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
           ]
diff --git a/nixos/tests/kubernetes/certs.nix b/nixos/tests/kubernetes/certs.nix
deleted file mode 100644
index 85e92f6330c..00000000000
--- a/nixos/tests/kubernetes/certs.nix
+++ /dev/null
@@ -1,219 +0,0 @@
-{
-  pkgs ? import <nixpkgs> {},
-  externalDomain ? "myawesomecluster.cluster.yourdomain.net",
-  serviceClusterIp ? "10.0.0.1",
-  kubelets,
-  ...
-}:
-let
-   runWithCFSSL = name: cmd:
-     let secrets = pkgs.runCommand "${name}-cfss.json" {
-         buildInputs = [ pkgs.cfssl pkgs.jq ];
-         outputs = [ "out" "cert" "key" "csr" ];
-       }
-       ''
-         (
-           echo "${cmd}"
-           cfssl ${cmd} > tmp
-           cat tmp | jq -r .key > $key
-           cat tmp | jq -r .cert > $cert
-           cat tmp | jq -r .csr > $csr
-
-           touch $out
-         ) 2>&1 | fold -w 80 -s
-       '';
-     in {
-       key = secrets.key;
-       cert = secrets.cert;
-       csr = secrets.csr;
-     };
-
-   writeCFSSL = content:
-     pkgs.runCommand content.name {
-      buildInputs = [ pkgs.cfssl pkgs.jq ];
-     } ''
-       mkdir -p $out
-       cd $out
-
-       json=${pkgs.lib.escapeShellArg (builtins.toJSON content)}
-
-       # for a given $field in the $json, treat the associated value as a
-       # file path and substitute the contents thereof into the $json
-       # object.
-       expandFileField() {
-         local field=$1
-         if jq -e --arg field "$field" 'has($field)'; then
-           local path="$(echo "$json" | jq -r ".$field")"
-           json="$(echo "$json" | jq --arg val "$(cat "$path")" ".$field = \$val")"
-         fi
-       }
-
-       expandFileField key
-       expandFileField ca
-       expandFileField cert
-
-       echo "$json" | cfssljson -bare ${content.name}
-     '';
-
-  noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;
-  noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;
-
-  writeFile = content:
-    if pkgs.lib.isDerivation content
-    then content
-    else pkgs.writeText "content" (builtins.toJSON content);
-
-  createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:
-    noCSR (
-      (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=server -config=${writeFile ca.config} ${writeFile {
-        CN = cn;
-        hosts = hosts;
-        key = { algo = "rsa"; inherit size; };
-      }}") // { inherit name; }
-    );
-
-  createClientCertKey = { ca, cn, groups ? [], size ? 2048, name ? cn }:
-    noCSR (
-      (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=client -config=${writeFile ca.config} ${writeFile {
-        CN = cn;
-        names = map (group: {O = group;}) groups;
-        hosts = [""];
-        key = { algo = "rsa"; inherit size; };
-      }}") // { inherit name; }
-    );
-
-  createSigningCertKey = { C ? "xx", ST ? "x", L ? "x", O ? "x", OU ? "x", CN ? "ca", emailAddress ? "x", expiry ? "43800h", size ? 2048, name ? CN }:
-    (noCSR (runWithCFSSL CN "genkey -initca ${writeFile {
-      key = { algo = "rsa"; inherit size; };
-      names = [{ inherit C ST L O OU CN emailAddress; }];
-    }}")) // {
-      inherit name;
-      config.signing = {
-        default.expiry = expiry;
-        profiles = {
-          server = {
-            inherit expiry;
-            usages = [
-              "signing"
-              "key encipherment"
-              "server auth"
-            ];
-          };
-          client = {
-            inherit expiry;
-            usages = [
-              "signing"
-              "key encipherment"
-              "client auth"
-            ];
-          };
-          peer = {
-            inherit expiry;
-            usages = [
-              "signing"
-              "key encipherment"
-              "server auth"
-              "client auth"
-            ];
-          };
-        };
-      };
-    };
-
-  ca = createSigningCertKey {};
-
-  kube-apiserver = createServingCertKey {
-    inherit ca;
-    cn = "kube-apiserver";
-    hosts = ["kubernetes.default" "kubernetes.default.svc" "localhost" "api.${externalDomain}" serviceClusterIp];
-  };
-
-  kubelet = createServingCertKey {
-    inherit ca;
-    cn = "kubelet";
-    hosts = ["*.${externalDomain}"];
-  };
-
-  service-accounts = createServingCertKey {
-    inherit ca;
-    cn = "kube-service-accounts";
-  };
-
-  etcd = createServingCertKey {
-    inherit ca;
-    cn = "etcd";
-    hosts = ["etcd.${externalDomain}"];
-  };
-
-  etcd-client = createClientCertKey {
-    inherit ca;
-    cn = "etcd-client";
-  };
-
-  kubelet-client = createClientCertKey {
-    inherit ca;
-    cn = "kubelet-client";
-    groups = ["system:masters"];
-  };
-
-  apiserver-client = {
-    kubelet = hostname: createClientCertKey {
-      inherit ca;
-      name = "apiserver-client-kubelet-${hostname}";
-      cn = "system:node:${hostname}.${externalDomain}";
-      groups = ["system:nodes"];
-    };
-
-    kube-proxy = createClientCertKey {
-      inherit ca;
-      name = "apiserver-client-kube-proxy";
-      cn = "system:kube-proxy";
-      groups = ["system:kube-proxy" "system:nodes"];
-    };
-
-    kube-controller-manager = createClientCertKey {
-      inherit ca;
-      name = "apiserver-client-kube-controller-manager";
-      cn = "system:kube-controller-manager";
-      groups = ["system:masters"];
-    };
-
-    kube-scheduler = createClientCertKey {
-      inherit ca;
-      name = "apiserver-client-kube-scheduler";
-      cn = "system:kube-scheduler";
-      groups = ["system:kube-scheduler"];
-    };
-
-    admin = createClientCertKey {
-      inherit ca;
-      cn = "admin";
-      groups = ["system:masters"];
-    };
-  };
-in {
-  master = pkgs.buildEnv {
-    name = "master-keys";
-    paths = [
-      (writeCFSSL (noKey ca))
-      (writeCFSSL kube-apiserver)
-      (writeCFSSL kubelet-client)
-      (writeCFSSL apiserver-client.kube-controller-manager)
-      (writeCFSSL apiserver-client.kube-scheduler)
-      (writeCFSSL service-accounts)
-      (writeCFSSL etcd)
-    ];
-  };
-
-  worker = pkgs.buildEnv {
-    name = "worker-keys";
-    paths = [
-      (writeCFSSL (noKey ca))
-      (writeCFSSL kubelet)
-      (writeCFSSL apiserver-client.kube-proxy)
-      (writeCFSSL etcd-client)
-    ] ++ map (hostname: writeCFSSL (apiserver-client.kubelet hostname)) kubelets;
-  };
-
-  admin = writeCFSSL apiserver-client.admin;
-}
diff --git a/nixos/tests/kubernetes/dns.nix b/nixos/tests/kubernetes/dns.nix
index f25ea5b9ed8..42eafcfc195 100644
--- a/nixos/tests/kubernetes/dns.nix
+++ b/nixos/tests/kubernetes/dns.nix
@@ -71,7 +71,7 @@ let
 
   base = {
     name = "dns";
-    inherit domain certs extraConfiguration;
+    inherit domain extraConfiguration;
   };
 
   singleNodeTest = {
@@ -99,8 +99,11 @@ let
 
   multiNodeTest = {
     test = ''
+      # Node token exchange
+      $machine1->waitUntilSucceeds("cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret");
+      $machine2->waitUntilSucceeds("cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join");
+
       # prepare machines for test
-      $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
       $machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready");
       $machine2->execute("docker load < ${redisImage}");
       $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
diff --git a/nixos/tests/kubernetes/kubernetes-common.nix b/nixos/tests/kubernetes/kubernetes-common.nix
deleted file mode 100644
index 87c65b88365..00000000000
--- a/nixos/tests/kubernetes/kubernetes-common.nix
+++ /dev/null
@@ -1,57 +0,0 @@
-{ roles, config, pkgs, certs }:
-with pkgs.lib;
-let
-  base = {
-    inherit roles;
-    flannel.enable = true;
-    addons.dashboard.enable = true;
-
-    caFile = "${certs.master}/ca.pem";
-    apiserver = {
-      tlsCertFile = "${certs.master}/kube-apiserver.pem";
-      tlsKeyFile = "${certs.master}/kube-apiserver-key.pem";
-      kubeletClientCertFile = "${certs.master}/kubelet-client.pem";
-      kubeletClientKeyFile = "${certs.master}/kubelet-client-key.pem";
-      serviceAccountKeyFile = "${certs.master}/kube-service-accounts.pem";
-    };
-    etcd = {
-      servers = ["https://etcd.${config.networking.domain}:2379"];
-      certFile = "${certs.worker}/etcd-client.pem";
-      keyFile = "${certs.worker}/etcd-client-key.pem";
-    };
-    kubeconfig = {
-      server = "https://api.${config.networking.domain}";
-    };
-    kubelet = {
-      tlsCertFile = "${certs.worker}/kubelet.pem";
-      tlsKeyFile = "${certs.worker}/kubelet-key.pem";
-      hostname = "${config.networking.hostName}.${config.networking.domain}";
-      kubeconfig = {
-        certFile = "${certs.worker}/apiserver-client-kubelet-${config.networking.hostName}.pem";
-        keyFile = "${certs.worker}/apiserver-client-kubelet-${config.networking.hostName}-key.pem";
-      };
-    };
-    controllerManager = {
-      serviceAccountKeyFile = "${certs.master}/kube-service-accounts-key.pem";
-      kubeconfig = {
-        certFile = "${certs.master}/apiserver-client-kube-controller-manager.pem";
-        keyFile = "${certs.master}/apiserver-client-kube-controller-manager-key.pem";
-      };
-    };
-    scheduler = {
-      kubeconfig = {
-        certFile = "${certs.master}/apiserver-client-kube-scheduler.pem";
-        keyFile = "${certs.master}/apiserver-client-kube-scheduler-key.pem";
-      };
-    };
-    proxy = {
-      kubeconfig = {
-        certFile = "${certs.worker}/apiserver-client-kube-proxy.pem";
-        keyFile = "${certs.worker}//apiserver-client-kube-proxy-key.pem";
-      };
-    };
-  };
-
-in {
-  services.kubernetes = base;
-}
diff --git a/nixos/tests/kubernetes/rbac.nix b/nixos/tests/kubernetes/rbac.nix
index 226808c4b26..91f97bed681 100644
--- a/nixos/tests/kubernetes/rbac.nix
+++ b/nixos/tests/kubernetes/rbac.nix
@@ -105,7 +105,7 @@ let
 
       $machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
 
-      $machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
+      $machine1->waitUntilSucceeds("kubectl exec -ti kubectl -- kubectl get pods");
       $machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
       $machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
     '';
@@ -113,7 +113,10 @@ let
 
   multinode = base // {
     test = ''
-      $machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
+      # Node token exchange
+      $machine1->waitUntilSucceeds("cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret");
+      $machine2->waitUntilSucceeds("cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join");
+
       $machine1->waitUntilSucceeds("kubectl get node machine2.my.zyx | grep -w Ready");
 
       $machine2->execute("docker load < ${kubectlImage}");
@@ -125,7 +128,7 @@ let
 
       $machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
 
-      $machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
+      $machine1->waitUntilSucceeds("kubectl exec -ti kubectl -- kubectl get pods");
       $machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
       $machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
     '';
generated by cgit-pink 1.4.1 (git 2.36.1) at 2024-06-27 16:46:58 +0000