kubernetes: fix tests

10 files changed, 536 insertions, 688 deletions

diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix
new file mode 100644
index 00000000000..acf2e025081
--- /dev/null
+++ b/nixos/tests/kubernetes/base.nix
@@ -0,0 +1,113 @@
+{ system ? builtins.currentSystem }:
+
+with import ../../lib/testing.nix { inherit system; };
+with import ../../lib/qemu-flags.nix;
+with pkgs.lib;
+
+let
+  mkKubernetesBaseTest =
+    { name, domain ? "my.zyx", test, machines
+    , pkgs ? import <nixpkgs> { inherit system; }
+    , certs ? import ./certs.nix { inherit pkgs; externalDomain = domain; }
+    , extraConfiguration ? null }:
+    let
+      masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
+      master = machines.${masterName};
+      extraHosts = ''
+        ${master.ip}  etcd.${domain}
+        ${master.ip}  api.${domain}
+        ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip}  ${machineName}.${domain}") (attrNames machines)}
+      '';
+    in makeTest {
+      inherit name;
+
+      nodes = mapAttrs (machineName: machine:
+        { config, pkgs, lib, nodes, ... }:
+          mkMerge [
+            {
+              virtualisation.memorySize = mkDefault 768;
+              virtualisation.diskSize = mkDefault 4096;
+              networking = {
+                inherit domain extraHosts;
+                primaryIPAddress = mkForce machine.ip;
+
+                firewall = {
+                  allowedTCPPorts = [
+                    10250 # kubelet
+                  ];
+                  trustedInterfaces = ["docker0"];
+
+                  extraCommands = concatMapStrings  (node: ''
+                    iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
+                  '') (attrValues nodes);
+                };
+              };
+              programs.bash.enableCompletion = true;
+              environment.variables = {
+                ETCDCTL_CERT_FILE = "${certs.worker}/etcd-client.pem";
+                ETCDCTL_KEY_FILE = "${certs.worker}/etcd-client-key.pem";
+                ETCDCTL_CA_FILE = "${certs.worker}/ca.pem";
+                ETCDCTL_PEERS = "https://etcd.${domain}:2379";
+              };
+              services.flannel.iface = "eth1";
+              services.kubernetes.apiserver.advertiseAddress = master.ip;
+            }
+            (optionalAttrs (any (role: role == "master") machine.roles) {
+              networking.firewall.allowedTCPPorts = [
+                2379 2380  # etcd
+                443 # kubernetes apiserver
+              ];
+              services.etcd = {
+                enable = true;
+                certFile = "${certs.master}/etcd.pem";
+                keyFile = "${certs.master}/etcd-key.pem";
+                trustedCaFile = "${certs.master}/ca.pem";
+                peerClientCertAuth = true;
+                listenClientUrls = ["https://0.0.0.0:2379"];
+                listenPeerUrls = ["https://0.0.0.0:2380"];
+                advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"];
+                initialCluster = ["${masterName}=https://etcd.${config.networking.domain}:2380"];
+                initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"];
+              };
+            })
+            (import ./kubernetes-common.nix { inherit (machine) roles; inherit pkgs config certs; })
+            (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
+            (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
+          ]
+      ) machines;
+
+      testScript = ''
+        startAll;
+
+        ${test}
+      '';
+    };
+
+  mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
+    machines = {
+      machine1 = {
+        roles = ["master"];
+        ip = "192.168.1.1";
+      };
+      machine2 = {
+        roles = ["node"];
+        ip = "192.168.1.2";
+      };
+    };
+  } // attrs // {
+    name = "kubernetes-${attrs.name}-multinode";
+  });
+
+  mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
+    machines = {
+      machine1 = {
+        roles = ["master" "node"];
+        ip = "192.168.1.1";
+      };
+    };
+  } // attrs // {
+    name = "kubernetes-${attrs.name}-singlenode";
+  });
+in {
+  inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
+}
diff --git a/nixos/tests/kubernetes/certs.nix b/nixos/tests/kubernetes/certs.nix
index f167224e549..f108e35b98c 100644
--- a/nixos/tests/kubernetes/certs.nix
+++ b/nixos/tests/kubernetes/certs.nix
@@ -1,229 +1,185 @@
 {
   pkgs ? import <nixpkgs> {},
-  servers ? {test = "1.2.3.4";},
-  internalDomain ? "cluster.local",
-  externalDomain ? "nixos.xyz"
+  internalDomain ? "cloud.yourdomain.net",
+  externalDomain ? "myawesomecluster.cluster.yourdomain.net",
+  serviceClusterIp ? "10.0.0.1"
 }:
 let
-  mkAltNames = ipFrom: dnsFrom:
-    pkgs.lib.concatImapStringsSep "\n" (i: v: "IP.${toString (i+ipFrom)} = ${v.ip}\nDNS.${toString (i+dnsFrom)} = ${v.name}.${externalDomain}") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers);
-
-  runWithOpenSSL = file: cmd: pkgs.runCommand file {
-    buildInputs = [ pkgs.openssl ];
-    passthru = { inherit file; };
-  } cmd;
-
-  ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
-  ca_pem = runWithOpenSSL "ca.pem" ''
-    openssl req \
-      -x509 -new -nodes -key ${ca_key} \
-      -days 10000 -out $out -subj "/CN=etcd-ca"
-  '';
-
-  etcd_cnf = pkgs.writeText "openssl.cnf" ''
-    [req]
-    req_extensions = v3_req
-    distinguished_name = req_distinguished_name
-    [req_distinguished_name]
-    [ v3_req ]
-    basicConstraints = CA:FALSE
-    keyUsage = digitalSignature, keyEncipherment
-    extendedKeyUsage = serverAuth
-    subjectAltName = @alt_names
-    [alt_names]
-    DNS.1 = etcd.kubernetes.${externalDomain}
-    IP.1 = 127.0.0.1
-  '';
-  etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
-  etcd_csr = runWithOpenSSL "etcd.csr" ''
-    openssl req \
-      -new -key ${etcd_key} \
-      -out $out -subj "/CN=etcd" \
-      -config ${etcd_cnf}
-  '';
-  etcd_cert = runWithOpenSSL "etcd.pem" ''
-    openssl x509 \
-      -req -in ${etcd_csr} \
-      -CA ${ca_pem} -CAkey ${ca_key} \
-      -CAcreateserial -out $out \
-      -days 3650 -extensions v3_req \
-      -extfile ${etcd_cnf}
-  '';
-
-  etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
-    "openssl genrsa -out $out 2048";
-
-  etcd_client_csr = runWithOpenSSL "etcd-client.csr" ''
-    openssl req \
-      -new -key ${etcd_client_key} \
-      -out $out -subj "/CN=etcd-client" \
-      -config ${client_openssl_cnf}
-  '';
-
-  etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''
-    openssl x509 \
-      -req -in ${etcd_client_csr} \
-      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
-      -out $out -days 3650 -extensions v3_req \
-      -extfile ${client_openssl_cnf}
-  '';
-
-  admin_key = runWithOpenSSL "admin-key.pem"
-    "openssl genrsa -out $out 2048";
-
-  admin_csr = runWithOpenSSL "admin.csr" ''
-    openssl req \
-      -new -key ${admin_key} \
-      -out $out -subj "/CN=admin/O=system:masters" \
-      -config ${client_openssl_cnf}
-  '';
-
-  admin_cert = runWithOpenSSL "admin.crt" ''
-    openssl x509 \
-      -req -in ${admin_csr} \
-      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
-      -out $out -days 3650 -extensions v3_req \
-      -extfile ${client_openssl_cnf}
-  '';
-
-  apiserver_key = runWithOpenSSL "apiserver-key.pem" "openssl genrsa -out $out 2048";
-
-  apiserver_csr = runWithOpenSSL "apiserver.csr" ''
-    openssl req \
-      -new -key ${apiserver_key} \
-      -out $out -subj "/CN=kube-apiserver" \
-      -config ${apiserver_cnf}
-  '';
-
-  apiserver_cert = runWithOpenSSL "apiserver.pem" ''
-    openssl x509 \
-      -req -in ${apiserver_csr} \
-      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
-      -out $out -days 3650 -extensions v3_req \
-      -extfile ${apiserver_cnf}
-  '';
-
-  worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048";
-
-  worker_csr = runWithOpenSSL "worker.csr" ''
-    openssl req \
-      -new -key ${worker_key} \
-      -out $out -subj "/CN=kube-worker/O=system:authenticated" \
-      -config ${worker_cnf}
-  '';
-
-  worker_cert = runWithOpenSSL "worker.pem" ''
-    openssl x509 \
-      -req -in ${worker_csr} \
-      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
-      -out $out -days 3650 -extensions v3_req \
-      -extfile ${worker_cnf}
-  '';
-
-  openssl_cnf = pkgs.writeText "openssl.cnf" ''
-    [req]
-    req_extensions = v3_req
-    distinguished_name = req_distinguished_name
-    [req_distinguished_name]
-    [ v3_req ]
-    basicConstraints = CA:FALSE
-    keyUsage = digitalSignature, keyEncipherment
-    extendedKeyUsage = serverAuth
-    subjectAltName = @alt_names
-    [alt_names]
-    DNS.1 = *.cluster.${externalDomain}
-    IP.1 = 127.0.0.1
-    ${mkAltNames 1 1}
-  '';
-
-  client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''
-    [req]
-    req_extensions = v3_req
-    distinguished_name = req_distinguished_name
-    [req_distinguished_name]
-    [ v3_req ]
-    basicConstraints = CA:FALSE
-    keyUsage = digitalSignature, keyEncipherment
-    extendedKeyUsage = clientAuth
-    subjectAltName = @alt_names
-    [alt_names]
-    DNS.1 = kubernetes
-    DNS.2 = kubernetes.default
-    DNS.3 = kubernetes.default.svc
-    DNS.4 = kubernetes.default.svc.${internalDomain}
-    DNS.5 = kubernetes.${externalDomain}
-    DNS.6 = *.cluster.${externalDomain}
-    IP.1 = 10.1.10.1
-    ${mkAltNames 1 6}
-  '';
-
-  apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" ''
-    [req]
-    req_extensions = v3_req
-    distinguished_name = req_distinguished_name
-    [req_distinguished_name]
-    [ v3_req ]
-    basicConstraints = CA:FALSE
-    keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-    extendedKeyUsage = serverAuth
-    subjectAltName = @alt_names
-    [alt_names]
-    DNS.1 = kubernetes
-    DNS.2 = kubernetes.default
-    DNS.3 = kubernetes.default.svc
-    DNS.4 = kubernetes.default.svc.${internalDomain}
-    DNS.5 = kubernetes.${externalDomain}
-    DNS.6 = *.cluster.${externalDomain}
-    IP.1 = 10.1.10.1
-    ${mkAltNames 1 6}
-  '';
-
-  worker_cnf = pkgs.writeText "worker-openssl.cnf" ''
-    [req]
-    req_extensions = v3_req
-    distinguished_name = req_distinguished_name
-    [req_distinguished_name]
-    [ v3_req ]
-    basicConstraints = CA:FALSE
-    keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-    subjectAltName = @alt_names
-    [alt_names]
-    DNS.1 = *.cluster.${externalDomain}
-    IP.1 = 10.1.10.1
-    ${mkAltNames 1 1}
-  '';
-
-  ln = cert: target: ''
-    cp -v ${cert} ${target}/${cert.file}
-  '';
-in
-  pkgs.stdenv.mkDerivation rec {
-    name = "kubernetes-certs";
-    unpackPhase = "true";
-    installPhase = ''
-      set -xe
+  runWithCFSSL = name: cmd:
+    builtins.fromJSON (builtins.readFile (
+      pkgs.runCommand "${name}-cfss.json" {
+        buildInputs = [ pkgs.cfssl ];
+      } "cfssl ${cmd} > $out"
+    ));
+
+  writeCFSSL = content:
+    pkgs.runCommand content.name {
+      buildInputs = [ pkgs.cfssl ];
+    } ''
       mkdir -p $out
-      ${ln ca_key "$out"}
-      ${ln ca_pem "$out"}
-
-      ${ln etcd_key "$out"}
-      ${ln etcd_csr "$out"}
-      ${ln etcd_cert "$out"}
-
-      ${ln etcd_client_key "$out"}
-      ${ln etcd_client_csr "$out"}
-      ${ln etcd_client_cert "$out"}
-
-      ${ln apiserver_key "$out"}
-      ${ln apiserver_csr "$out"}
-      ${ln apiserver_cert "$out"}
-
-      ${ln worker_key "$out"}
-      ${ln worker_csr "$out"}
-      ${ln worker_cert "$out"}
-
-      ${ln admin_key "$out"}
-      ${ln admin_csr "$out"}
-      ${ln admin_cert "$out"}
+      cd $out
+      cat ${writeFile content} | cfssljson -bare ${content.name}
     '';
-  }
+
+  noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;
+  noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;
+
+  writeFile = content: pkgs.writeText "content" (
+    if pkgs.lib.isAttrs content then builtins.toJSON content
+    else toString content
+  );
+
+  createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:
+    noCSR (
+      (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=server -config=${writeFile ca.config} ${writeFile {
+        CN = cn;
+        hosts = hosts;
+        key = { algo = "rsa"; inherit size; };
+      }}") // { inherit name; }
+    );
+
+  createClientCertKey = { ca, cn, groups ? [], size ? 2048, name ? cn }:
+    noCSR (
+      (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=client -config=${writeFile ca.config} ${writeFile {
+        CN = cn;
+        names = map (group: {O = group;}) groups;
+        hosts = [""];
+        key = { algo = "rsa"; inherit size; };
+      }}") // { inherit name; }
+    );
+
+  createSigningCertKey = { C ? "xx", ST ? "x", L ? "x", O ? "x", OU ? "x", CN ? "ca", emailAddress ? "x", expiry ? "43800h", size ? 2048, name ? CN }:
+    (noCSR (runWithCFSSL CN "genkey -initca ${writeFile {
+      key = { algo = "rsa"; inherit size; };
+      names = [{ inherit C ST L O OU CN emailAddress; }];
+    }}")) // {
+      inherit name;
+      config.signing = {
+        default.expiry = expiry;
+        profiles = {
+          server = {
+            inherit expiry;
+            usages = [
+              "signing"
+              "key encipherment"
+              "server auth"
+            ];
+          };
+          client = {
+            inherit expiry;
+            usages = [
+              "signing"
+              "key encipherment"
+              "client auth"
+            ];
+          };
+          peer = {
+            inherit expiry;
+            usages = [
+              "signing"
+              "key encipherment"
+              "server auth"
+              "client auth"
+            ];
+          };
+        };
+      };
+    };
+
+  ca = createSigningCertKey {};
+
+  kube-apiserver = createServingCertKey {
+    inherit ca;
+    cn = "kube-apiserver";
+    hosts = ["kubernetes.default" "kubernetes.default.svc" "localhost" "api.${externalDomain}" serviceClusterIp];
+  };
+
+  kubelet = createServingCertKey {
+    inherit ca;
+    cn = "kubelet";
+    hosts = ["*.${externalDomain}"];
+  };
+
+  service-accounts = createServingCertKey {
+    inherit ca;
+    cn = "kube-service-accounts";
+  };
+
+  etcd = createServingCertKey {
+    inherit ca;
+    cn = "etcd";
+    hosts = ["etcd.${externalDomain}"];
+  };
+
+  etcd-client = createClientCertKey {
+    inherit ca;
+    cn = "etcd-client";
+  };
+
+  kubelet-client = createClientCertKey {
+    inherit ca;
+    cn = "kubelet-client";
+    groups = ["system:masters"];
+  };
+
+  apiserver-client = {
+    kubelet = createClientCertKey {
+      inherit ca;
+      cn = "apiserver-client-kubelet";
+      groups = ["system:nodes"];
+    };
+
+    kube-proxy = createClientCertKey {
+      inherit ca;
+      name = "apiserver-client-kube-proxy";
+      cn = "system:kube-proxy";
+      groups = ["system:kube-proxy" "system:nodes"];
+    };
+
+    kube-controller-manager = createClientCertKey {
+      inherit ca;
+      name = "apiserver-client-kube-controller-manager";
+      cn = "system:kube-controller-manager";
+      groups = ["system:masters"];
+    };
+
+    kube-scheduler = createClientCertKey {
+      inherit ca;
+      name = "apiserver-client-kube-scheduler";
+      cn = "system:kube-scheduler";
+      groups = ["system:kube-scheduler"];
+    };
+
+    admin = createClientCertKey {
+      inherit ca;
+      cn = "admin";
+      groups = ["system:masters"];
+    };
+  };
+in {
+  master = pkgs.buildEnv {
+    name = "master-keys";
+    paths = [
+      (writeCFSSL (noKey ca))
+      (writeCFSSL kube-apiserver)
+      (writeCFSSL kubelet-client)
+      (writeCFSSL apiserver-client.kube-controller-manager)
+      (writeCFSSL apiserver-client.kube-scheduler)
+      (writeCFSSL service-accounts)
+      (writeCFSSL etcd)
+    ];
+  };
+
+  worker = pkgs.buildEnv {
+    name = "worker-keys";
+    paths = [
+      (writeCFSSL (noKey ca))
+      (writeCFSSL kubelet)
+      (writeCFSSL apiserver-client.kubelet)
+      (writeCFSSL apiserver-client.kube-proxy)
+      (writeCFSSL etcd-client)
+    ];
+  };
+
+  admin = writeCFSSL apiserver-client.admin;
+}
diff --git a/nixos/tests/kubernetes/default.nix b/nixos/tests/kubernetes/default.nix
index 2b61980349e..a801759bf58 100644
--- a/nixos/tests/kubernetes/default.nix
+++ b/nixos/tests/kubernetes/default.nix
@@ -1,7 +1,7 @@
 { system ? builtins.currentSystem }:
 {
-    kubernetes-singlenode = import ./singlenode.nix { inherit system; };
-    kubernetes-multinode-kubectl = import ./multinode-kubectl.nix { inherit system; };
-    kubernetes-rbac = import ./rbac.nix { inherit system; };
-    kubernetes-dns = import ./dns.nix { inherit system; };
+  dns = import ./dns.nix { inherit system; };
+  # e2e = import ./e2e.nix { inherit system; };  # TODO: make it pass
+  # the following test(s) can be removed when e2e is working:
+  rbac = import ./rbac.nix { inherit system; };
 }
diff --git a/nixos/tests/kubernetes/dns.nix b/nixos/tests/kubernetes/dns.nix
index 4659c520dee..74d98dabec8 100644
--- a/nixos/tests/kubernetes/dns.nix
+++ b/nixos/tests/kubernetes/dns.nix
@@ -1,16 +1,11 @@
-{ system ? builtins.currentSystem }:
-
-with import ../../lib/testing.nix { inherit system; };
-with import ../../lib/qemu-flags.nix;
-with pkgs.lib;
-
+{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
+with import ./base.nix { inherit system; };
 let
-  servers.master = "192.168.1.1";
-  servers.one = "192.168.1.10";
+  domain = "my.zyx";
 
-  certs = import ./certs.nix { inherit servers; };
+  certs = import ./certs.nix { externalDomain = domain; };
 
-  redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {
+  redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON {
     kind = "Pod";
     apiVersion = "v1";
     metadata.name = "redis";
@@ -40,64 +35,93 @@ let
   redisImage = pkgs.dockerTools.buildImage {
     name = "redis";
     tag = "latest";
-    contents = [ pkgs.redis pkgs.bind.dnsutils pkgs.coreutils pkgs.inetutils pkgs.nmap ];
+    contents = [ pkgs.redis pkgs.bind.host ];
     config.Entrypoint = "/bin/redis-server";
   };
 
-  test = ''
-    $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
-    $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
-
-    $one->execute("docker load < ${redisImage}");
-
-    $master->waitUntilSucceeds("kubectl create -f ${redisPod} || kubectl apply -f ${redisPod}");
-    $master->waitUntilSucceeds("kubectl create -f ${redisService} || kubectl apply -f ${redisService}");
-
-    $master->waitUntilSucceeds("kubectl get pod redis | grep Running");
-
-    $master->succeed("dig \@192.168.1.1 redis.default.svc.cluster.local");
-    $one->succeed("dig \@192.168.1.10 redis.default.svc.cluster.local");
-
-
-    $master->succeed("kubectl exec -ti redis -- cat /etc/resolv.conf | grep 'nameserver 192.168.1.10'");
-
-    $master->succeed("kubectl exec -ti redis -- dig \@192.168.1.10 redis.default.svc.cluster.local");
-  '';
-
-in makeTest {
-  name = "kubernetes-dns";
-
-  nodes = {
-    master =
-      { config, pkgs, lib, nodes, ... }:
-        mkMerge [
-          {
-            virtualisation.memorySize = 768;
-            virtualisation.diskSize = 4096;
-            networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
-            networking.primaryIPAddress = mkForce servers.master;
-          }
-          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
-          (import ./kubernetes-master.nix { inherit pkgs config certs; })
-        ];
-
-    one =
-      { config, pkgs, lib, nodes, ... }:
-        mkMerge [
-          {
-            virtualisation.memorySize = 768;
-            virtualisation.diskSize = 4096;
-            networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
-            networking.primaryIPAddress = mkForce servers.one;
-            services.kubernetes.roles = ["node"];
-          }
-          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
-        ];
+  probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {
+    kind = "Pod";
+    apiVersion = "v1";
+    metadata.name = "probe";
+    metadata.labels.name = "probe";
+    spec.containers = [{
+      name = "probe";
+      image = "probe";
+      args = [ "-f" ];
+      tty = true;
+      imagePullPolicy = "Never";
+    }];
+  });
+
+  probeImage = pkgs.dockerTools.buildImage {
+    name = "probe";
+    tag = "latest";
+    contents = [ pkgs.bind.host pkgs.busybox ];
+    config.Entrypoint = "/bin/tail";
+  };
+
+  extraConfiguration = { config, pkgs, lib, nodes, ... }: {
+    environment.systemPackages = [ pkgs.bind.host ];
+    # virtualisation.docker.extraOptions = "--dns=${config.services.kubernetes.addons.dns.clusterIp}";
+    services.dnsmasq.enable = true;
+    services.dnsmasq.servers = [
+      "/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
+    ];
   };
 
-  testScript = ''
-    startAll;
+  base = {
+    name = "dns";
+    inherit domain certs extraConfiguration;
+  };
+
+  singleNodeTest = {
+    test = ''
+      # prepare machine1 for test
+      $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
+      $machine1->execute("docker load < ${redisImage}");
+      $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
+      $machine1->waitUntilSucceeds("kubectl create -f ${redisService}");
+      $machine1->execute("docker load < ${probeImage}");
+      $machine1->waitUntilSucceeds("kubectl create -f ${probePod}");
+
+      # check if pods are running
+      $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
+      $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
+      $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");
+
+      # check dns on host (dnsmasq)
+      $machine1->succeed("host redis.default.svc.cluster.local");
+
+      # check dns inside the container
+      $machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");
+    '';
+  };
 
-    ${test}
-  '';
+  multiNodeTest = {
+    test = ''
+      # prepare machines for test
+      $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
+      $machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready");
+      $machine2->execute("docker load < ${redisImage}");
+      $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
+      $machine1->waitUntilSucceeds("kubectl create -f ${redisService}");
+      $machine2->execute("docker load < ${probeImage}");
+      $machine1->waitUntilSucceeds("kubectl create -f ${probePod}");
+
+      # check if pods are running
+      $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
+      $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
+      $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");
+
+      # check dns on hosts (dnsmasq)
+      $machine1->succeed("host redis.default.svc.cluster.local");
+      $machine2->succeed("host redis.default.svc.cluster.local");
+
+      # check dns inside the container
+      $machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");
+    '';
+  };
+in {
+  singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest);
+  multinode = mkKubernetesMultiNodeTest (base // multiNodeTest);
 }
diff --git a/nixos/tests/kubernetes/e2e.nix b/nixos/tests/kubernetes/e2e.nix
new file mode 100644
index 00000000000..d9d7ba9bb2c
--- /dev/null
+++ b/nixos/tests/kubernetes/e2e.nix
@@ -0,0 +1,40 @@
+{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
+with import ./base.nix { inherit system; };
+let
+  domain = "my.zyx";
+  certs = import ./certs.nix { externalDomain = domain; };
+  kubeconfig = pkgs.writeText "kubeconfig.json" (builtins.toJSON {
+    apiVersion = "v1";
+    kind = "Config";
+    clusters = [{
+      name = "local";
+      cluster.certificate-authority = "${certs.master}/ca.pem";
+      cluster.server = "https://api.${domain}";
+    }];
+    users = [{
+      name = "kubelet";
+      user = {
+        client-certificate = "${certs.admin}/admin.pem";
+        client-key = "${certs.admin}/admin-key.pem";
+      };
+    }];
+    contexts = [{
+      context = {
+        cluster = "local";
+        user = "kubelet";
+      };
+      current-context = "kubelet-context";
+    }];
+  });
+
+  base = {
+    name = "e2e";
+    inherit domain certs;
+    test = ''
+      $machine1->succeed("e2e.test -kubeconfig ${kubeconfig} -provider local -ginkgo.focus '\\[Conformance\\]' -ginkgo.skip '\\[Flaky\\]|\\[Serial\\]'");
+    '';
+  };
+in {
+  singlenode = mkKubernetesSingleNodeTest base;
+  multinode = mkKubernetesMultiNodeTest base;
+}
diff --git a/nixos/tests/kubernetes/kubernetes-common.nix b/nixos/tests/kubernetes/kubernetes-common.nix
index 9f9e730fa65..00a5c9aba4e 100644
--- a/nixos/tests/kubernetes/kubernetes-common.nix
+++ b/nixos/tests/kubernetes/kubernetes-common.nix
@@ -1,72 +1,59 @@
-{ config, pkgs, certs, servers }:
-
+{ roles, config, pkgs, certs }:
+with pkgs.lib;
 let
-  etcd_key = "${certs}/etcd-key.pem";
-  etcd_cert = "${certs}/etcd.pem";
-  ca_pem = "${certs}/ca.pem";
-  etcd_client_cert = "${certs}/etcd-client.crt";
-  etcd_client_key = "${certs}/etcd-client-key.pem";
-
-  worker_key = "${certs}/worker-key.pem";
-  worker_cert = "${certs}/worker.pem";
-
-  rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
-    ${pkgs.lib.readFile "${certs}/ca.pem"}
-
-    ${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")}
-  '';
-
-  mkHosts =
-    pkgs.lib.concatMapStringsSep "\n" (v: "${v.ip} ${v.name}.nixos.xyz") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers);
-
-in
-{
-  programs.bash.enableCompletion = true;
-  environment.systemPackages = with pkgs; [ netcat bind etcd.bin ];
-
-  networking = {
-    firewall.allowedTCPPorts = [
-      10250 # kubelet
-    ];
-    extraHosts = ''
-      # register "external" domains
-      ${servers.master} etcd.kubernetes.nixos.xyz
-      ${servers.master} kubernetes.nixos.xyz
-      ${mkHosts}
-    '';
-  };
-  services.flannel.iface = "eth1";
-  environment.variables = {
-    ETCDCTL_CERT_FILE = "${etcd_client_cert}";
-    ETCDCTL_KEY_FILE = "${etcd_client_key}";
-    ETCDCTL_CA_FILE = "${rootCaFile}";
-    ETCDCTL_PEERS = "https://etcd.kubernetes.nixos.xyz:2379";
-  };
-
-  services.kubernetes = {
-    kubelet = {
-      tlsKeyFile = worker_key;
-      tlsCertFile = worker_cert;
-      hostname = "${config.networking.hostName}.nixos.xyz";
-      nodeIp = config.networking.primaryIPAddress;
+  base = {
+    inherit roles;
+    featureGates = ["AllAlpha"];
+    flannel.enable = true;
+    addons.dashboard.enable = true;
+    verbose = true;
+
+    caFile = "${certs.master}/ca.pem";
+    apiserver = {
+      tlsCertFile = "${certs.master}/kube-apiserver.pem";
+      tlsKeyFile = "${certs.master}/kube-apiserver-key.pem";
+      kubeletClientCertFile = "${certs.master}/kubelet-client.pem";
+      kubeletClientKeyFile = "${certs.master}/kubelet-client-key.pem";
+      serviceAccountKeyFile = "${certs.master}/kube-service-accounts.pem";
     };
     etcd = {
-      servers = ["https://etcd.kubernetes.nixos.xyz:2379"];
-      keyFile = etcd_client_key;
-      certFile = etcd_client_cert;
-      caFile = ca_pem;
+      servers = ["https://etcd.${config.networking.domain}:2379"];
+      certFile = "${certs.worker}/etcd-client.pem";
+      keyFile = "${certs.worker}/etcd-client-key.pem";
     };
     kubeconfig = {
-      server = "https://kubernetes.nixos.xyz";
-      caFile = rootCaFile;
-      certFile = worker_cert;
-      keyFile = worker_key;
+      server = "https://api.${config.networking.domain}";
+    };
+    kubelet = {
+      tlsCertFile = "${certs.worker}/kubelet.pem";
+      tlsKeyFile = "${certs.worker}/kubelet-key.pem";
+      hostname = "${config.networking.hostName}.${config.networking.domain}";
+      kubeconfig = {
+        certFile = "${certs.worker}/apiserver-client-kubelet.pem";
+        keyFile = "${certs.worker}/apiserver-client-kubelet-key.pem";
+      };
+    };
+    controllerManager = {
+      serviceAccountKeyFile = "${certs.master}/kube-service-accounts-key.pem";
+      kubeconfig = {
+        certFile = "${certs.master}/apiserver-client-kube-controller-manager.pem";
+        keyFile = "${certs.master}/apiserver-client-kube-controller-manager-key.pem";
+      };
+    };
+    scheduler = {
+      kubeconfig = {
+        certFile = "${certs.master}/apiserver-client-kube-scheduler.pem";
+        keyFile = "${certs.master}/apiserver-client-kube-scheduler-key.pem";
+      };
+    };
+    proxy = {
+      kubeconfig = {
+        certFile = "${certs.worker}/apiserver-client-kube-proxy.pem";
+        keyFile = "${certs.worker}//apiserver-client-kube-proxy-key.pem";
+      };
     };
-    flannel.enable = true;
-
-    dns.port = 4453;
   };
 
-  services.dnsmasq.enable = true;
-  services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
+in {
+  services.kubernetes = base;
 }
diff --git a/nixos/tests/kubernetes/kubernetes-master.nix b/nixos/tests/kubernetes/kubernetes-master.nix
deleted file mode 100644
index 28cce6ea653..00000000000
--- a/nixos/tests/kubernetes/kubernetes-master.nix
+++ /dev/null
@@ -1,63 +0,0 @@
-{ config, pkgs, certs }:
-let
-  etcd_key = "${certs}/etcd-key.pem";
-  etcd_cert = "${certs}/etcd.pem";
-  ca_pem = "${certs}/ca.pem";
-  etcd_client_cert = "${certs}/etcd-client.crt";
-  etcd_client_key = "${certs}/etcd-client-key.pem";
-
-  apiserver_key = "${certs}/apiserver-key.pem";
-  apiserver_cert = "${certs}/apiserver.pem";
-  worker_key = "${certs}/worker-key.pem";
-  worker_cert = "${certs}/worker.pem";
-
-
-  rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
-    ${pkgs.lib.readFile "${certs}/ca.pem"}
-
-    ${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")}
-  '';
-in
-{
-  networking = {
-    firewall = {
-      enable = true;
-      allowPing = true;
-      allowedTCPPorts = [
-        2379 2380  # etcd
-        443 # kubernetes apiserver
-      ];
-    };
-  };
-
-  services.etcd = {
-    enable = pkgs.lib.mkForce true;
-    keyFile = etcd_key;
-    certFile = etcd_cert;
-    trustedCaFile = rootCaFile;
-    peerClientCertAuth = true;
-    listenClientUrls = ["https://0.0.0.0:2379"];
-    listenPeerUrls = ["https://0.0.0.0:2380"];
-
-    advertiseClientUrls = ["https://etcd.kubernetes.nixos.xyz:2379"];
-    initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"];
-    initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"];
-  };
-
-  services.kubernetes = {
-    roles = ["master"];
-    scheduler.leaderElect = true;
-    controllerManager.rootCaFile = rootCaFile;
-    controllerManager.serviceAccountKeyFile = apiserver_key;
-    apiserver = {
-      publicAddress = "192.168.1.1";
-      advertiseAddress = "192.168.1.1";
-      tlsKeyFile = apiserver_key;
-      tlsCertFile = apiserver_cert;
-      clientCaFile = rootCaFile;
-      kubeletClientCaFile = rootCaFile;
-      kubeletClientKeyFile = worker_key;
-      kubeletClientCertFile = worker_cert;
-    };
-  };
-}
diff --git a/nixos/tests/kubernetes/multinode-kubectl.nix b/nixos/tests/kubernetes/multinode-kubectl.nix
deleted file mode 100644
index c5dd999a01e..00000000000
--- a/nixos/tests/kubernetes/multinode-kubectl.nix
+++ /dev/null
@@ -1,123 +0,0 @@
-{ system ? builtins.currentSystem }:
-
-with import ../../lib/testing.nix { inherit system; };
-with import ../../lib/qemu-flags.nix;
-with pkgs.lib;
-
-let
-  servers.master = "192.168.1.1";
-  servers.one = "192.168.1.10";
-  servers.two = "192.168.1.20";
-
-  certs = import ./certs.nix { inherit servers; };
-
-  kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON {
-    kind = "Pod";
-    apiVersion = "v1";
-    metadata.name = "kubectl";
-    metadata.labels.name = "kubectl";
-    spec.containers = [{
-      name = "kubectl";
-      image = "kubectl:latest";
-      command = ["${pkgs.busybox}/bin/tail" "-f"];
-      imagePullPolicy = "Never";
-      tty = true;
-    }];
-  });
-
-  kubectlImage = pkgs.dockerTools.buildImage {
-    name = "kubectl";
-    tag = "latest";
-    contents = [ pkgs.kubernetes pkgs.busybox certs kubeconfig ];
-    config.Entrypoint = "${pkgs.busybox}/bin/sh";
-  };
-
-  kubeconfig = pkgs.writeTextDir "kubeconfig.json" (builtins.toJSON {
-    apiVersion = "v1";
-    kind = "Config";
-    clusters = [{
-      name = "local";
-      cluster.certificate-authority = "/ca.pem";
-      cluster.server = "https://${servers.master}";
-    }];
-    users = [{
-      name = "kubelet";
-      user = {
-        client-certificate = "/admin.crt";
-        client-key = "/admin-key.pem";
-      };
-    }];
-    contexts = [{
-      context = {
-        cluster = "local";
-        user = "kubelet";
-      };
-      current-context = "kubelet-context";
-    }];
-  });
-
-  test = ''
-    $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
-    $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
-    $master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready");
-
-    $one->execute("docker load < ${kubectlImage}");
-    $two->execute("docker load < ${kubectlImage}");
-
-    $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
-
-    $master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
-
-    $master->succeed("kubectl exec -ti kubectl -- kubectl --kubeconfig=/kubeconfig.json version");
-  '';
-
-in makeTest {
-  name = "kubernetes-multinode-kubectl";
-
-  nodes = {
-    master =
-      { config, pkgs, lib, nodes, ... }:
-        mkMerge [
-          {
-            virtualisation.memorySize = 768;
-            virtualisation.diskSize = 4096;
-            networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
-            networking.primaryIPAddress = mkForce servers.master;
-          }
-          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
-          (import ./kubernetes-master.nix { inherit pkgs config certs; })
-        ];
-
-    one =
-      { config, pkgs, lib, nodes, ... }:
-        mkMerge [
-          {
-            virtualisation.memorySize = 768;
-            virtualisation.diskSize = 4096;
-            networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
-            networking.primaryIPAddress = mkForce servers.one;
-            services.kubernetes.roles = ["node"];
-          }
-          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
-        ];
-
-    two =
-      { config, pkgs, lib, nodes, ... }:
-        mkMerge [
-          {
-            virtualisation.memorySize = 768;
-            virtualisation.diskSize = 4096;
-            networking.interfaces.eth1.ip4 = mkForce [{address = servers.two; prefixLength = 24;}];
-            networking.primaryIPAddress = mkForce servers.two;
-            services.kubernetes.roles = ["node"];
-          }
-          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
-        ];
-  };
-
-  testScript = ''
-    startAll;
-
-    ${test}
-  '';
-}
diff --git a/nixos/tests/kubernetes/rbac.nix b/nixos/tests/kubernetes/rbac.nix
index dfb55e7e058..1966fed3a5f 100644
--- a/nixos/tests/kubernetes/rbac.nix
+++ b/nixos/tests/kubernetes/rbac.nix
@@ -1,14 +1,6 @@
-{ system ? builtins.currentSystem }:
-
-with import ../../lib/testing.nix { inherit system; };
-with import ../../lib/qemu-flags.nix;
-with pkgs.lib;
-
+{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
+with import ./base.nix { inherit system; };
 let
-  servers.master = "192.168.1.1";
-  servers.one = "192.168.1.10";
-
-  certs = import ./certs.nix { inherit servers; };
 
   roServiceAccount = pkgs.writeText "ro-service-account.json" (builtins.toJSON {
     kind = "ServiceAccount";
@@ -20,21 +12,21 @@ let
   });
 
   roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON {
-    "apiVersion" = "rbac.authorization.k8s.io/v1beta1";
-    "kind" = "RoleBinding";
-    "metadata" = {
-      "name" = "read-pods";
-      "namespace" = "default";
+    apiVersion = "rbac.authorization.k8s.io/v1beta1";
+    kind = "RoleBinding";
+    metadata = {
+      name = "read-pods";
+      namespace = "default";
     };
-    "roleRef" = {
-      "apiGroup" = "rbac.authorization.k8s.io";
-      "kind" = "Role";
-      "name" = "pod-reader";
+    roleRef = {
+      apiGroup = "rbac.authorization.k8s.io";
+      kind = "Role";
+      name = "pod-reader";
     };
-    "subjects" = [{
-      "kind" = "ServiceAccount";
-      "name" = "read-only";
-      "namespace" = "default";
+    subjects = [{
+      kind = "ServiceAccount";
+      name = "read-only";
+      namespace = "default";
     }];
   });
 
@@ -62,7 +54,7 @@ let
     spec.containers = [{
       name = "kubectl";
       image = "kubectl:latest";
-      command = ["${pkgs.busybox}/bin/tail" "-f"];
+      command = ["/bin/tail" "-f"];
       imagePullPolicy = "Never";
       tty = true;
     }];
@@ -78,71 +70,68 @@ let
     spec.containers = [{
       name = "kubectl-2";
       image = "kubectl:latest";
-      command = ["${pkgs.busybox}/bin/tail" "-f"];
+      command = ["/bin/tail" "-f"];
       imagePullPolicy = "Never";
       tty = true;
     }];
   });
 
+  kubectl = pkgs.runCommand "copy-kubectl" { buildInputs = [ pkgs.kubernetes ]; } ''
+    mkdir -p $out/bin
+    cp ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl
+  '';
+
   kubectlImage = pkgs.dockerTools.buildImage {
     name = "kubectl";
     tag = "latest";
-    contents = [ pkgs.kubernetes pkgs.busybox kubectlPod2 ];  # certs kubeconfig
-    config.Entrypoint = "${pkgs.busybox}/bin/sh";
+    contents = [ kubectl pkgs.busybox kubectlPod2 ];
+    config.Entrypoint = "/bin/sh";
   };
 
-  test = ''
-    $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
-    $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
+  base = {
+    name = "rbac";
+  };
 
-    $one->execute("docker load < ${kubectlImage}");
+  singlenode = base // {
+    test = ''
+      $machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
 
-    $master->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
-    $master->waitUntilSucceeds("kubectl apply -f ${roRole}");
-    $master->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
-    $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
+      $machine1->execute("docker load < ${kubectlImage}");
 
-    $master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
+      $machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
+      $machine1->waitUntilSucceeds("kubectl apply -f ${roRole}");
+      $machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
+      $machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}");
 
-    $master->succeed("kubectl exec -ti kubectl -- kubectl get pods");
-    $master->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
-    $master->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
-  '';
+      $machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
 
-in makeTest {
-  name = "kubernetes-rbac";
-
-  nodes = {
-    master =
-      { config, pkgs, lib, nodes, ... }:
-        mkMerge [
-          {
-            virtualisation.memorySize = 768;
-            virtualisation.diskSize = 4096;
-            networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
-            networking.primaryIPAddress = mkForce servers.master;
-          }
-          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
-          (import ./kubernetes-master.nix { inherit pkgs config certs; })
-        ];
-
-    one =
-      { config, pkgs, lib, nodes, ... }:
-        mkMerge [
-          {
-            virtualisation.memorySize = 768;
-            virtualisation.diskSize = 4096;
-            networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
-            networking.primaryIPAddress = mkForce servers.one;
-            services.kubernetes.roles = ["node"];
-          }
-          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
-        ];
+      $machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
+      $machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
+      $machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
+    '';
   };
 
-  testScript = ''
-    startAll;
+  multinode = base // {
+    test = ''
+      $machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
+      $machine1->waitUntilSucceeds("kubectl get node machine2.my.zyx | grep -w Ready");
 
-    ${test}
-  '';
+      $machine2->execute("docker load < ${kubectlImage}");
+
+      $machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
+      $machine1->waitUntilSucceeds("kubectl apply -f ${roRole}");
+      $machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
+      $machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}");
+
+      $machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
+
+      $machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
+      $machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
+      $machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
+    '';
+  };
+
+in {
+  singlenode = mkKubernetesSingleNodeTest singlenode;
+  multinode = mkKubernetesMultiNodeTest multinode;
 }
diff --git a/nixos/tests/kubernetes/singlenode.nix b/nixos/tests/kubernetes/singlenode.nix
deleted file mode 100644
index d924da155bd..00000000000
--- a/nixos/tests/kubernetes/singlenode.nix
+++ /dev/null
@@ -1,75 +0,0 @@
-{ system ? builtins.currentSystem }:
-
-with import ../../lib/testing.nix { inherit system; };
-with import ../../lib/qemu-flags.nix;
-with pkgs.lib;
-
-let
-  redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {
-    kind = "Pod";
-    apiVersion = "v1";
-    metadata.name = "redis";
-    metadata.labels.name = "redis";
-    spec.containers = [{
-      name = "redis";
-      image = "redis";
-      args = ["--bind" "0.0.0.0"];
-      imagePullPolicy = "Never";
-      ports = [{
-        name = "redis-server";
-        containerPort = 6379;
-      }];
-    }];
-  });
-
-  redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {
-    kind = "Service";
-    apiVersion = "v1";
-    metadata.name = "redis";
-    spec = {
-      ports = [{port = 6379; targetPort = 6379;}];
-      selector = {name = "redis";};
-    };
-  });
-
-  redisImage = pkgs.dockerTools.buildImage {
-    name = "redis";
-    tag = "latest";
-    contents = pkgs.redis;
-    config.Entrypoint = "/bin/redis-server";
-  };
-
-  testSimplePod = ''
-    $kubernetes->execute("docker load < ${redisImage}");
-    $kubernetes->waitUntilSucceeds("kubectl create -f ${redisPod}");
-    $kubernetes->succeed("kubectl create -f ${redisService}");
-    $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running");
-    $kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379");
-  '';
-in makeTest {
-  name = "kubernetes-singlenode";
-
-  nodes = {
-    kubernetes =
-      { config, pkgs, lib, nodes, ... }:
-        {
-          virtualisation.memorySize = 768;
-          virtualisation.diskSize = 2048;
-
-          programs.bash.enableCompletion = true;
-          environment.systemPackages = with pkgs; [ netcat bind ];
-
-          services.kubernetes.roles = ["master" "node"];
-          services.kubernetes.dns.port = 4453;
-
-          services.dnsmasq.enable = true;
-          services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
-        };
-  };
-
-  testScript = ''
-    startAll;
-
-    ${testSimplePod}
-  '';
-}