diff options
author | Matej Cotman <cotman.matej@gmail.com> | 2017-05-03 01:20:32 +0200 |
---|---|---|
committer | Robin Gloster <mail@glob.in> | 2017-09-24 11:44:25 +0200 |
commit | 7f9d1a7aafc76f506f25c9608a68980161d43d66 (patch) | |
tree | 5b65ce0426a7959e607c9a547cca02d73621c880 | |
parent | ed322f42357e1822560fd1fd80f56a9fbf89a672 (diff) | |
download | nixpkgs-7f9d1a7aafc76f506f25c9608a68980161d43d66.tar nixpkgs-7f9d1a7aafc76f506f25c9608a68980161d43d66.tar.gz nixpkgs-7f9d1a7aafc76f506f25c9608a68980161d43d66.tar.bz2 nixpkgs-7f9d1a7aafc76f506f25c9608a68980161d43d66.tar.lz nixpkgs-7f9d1a7aafc76f506f25c9608a68980161d43d66.tar.xz nixpkgs-7f9d1a7aafc76f506f25c9608a68980161d43d66.tar.zst nixpkgs-7f9d1a7aafc76f506f25c9608a68980161d43d66.zip |
kubernetes: add tests
-rw-r--r-- | nixos/tests/kubernetes/certs.nix | 229 | ||||
-rw-r--r-- | nixos/tests/kubernetes/default.nix | 7 | ||||
-rw-r--r-- | nixos/tests/kubernetes/dns.nix | 103 | ||||
-rw-r--r-- | nixos/tests/kubernetes/kubernetes-common.nix | 131 | ||||
-rw-r--r-- | nixos/tests/kubernetes/kubernetes-master.nix | 148 | ||||
-rw-r--r-- | nixos/tests/kubernetes/multinode-kubectl.nix | 147 | ||||
-rw-r--r-- | nixos/tests/kubernetes/rbac.nix | 206 | ||||
-rw-r--r-- | nixos/tests/kubernetes/singlenode-kubectl.nix | 97 | ||||
-rw-r--r-- | nixos/tests/kubernetes/singlenode.nix | 82 |
9 files changed, 1150 insertions, 0 deletions
diff --git a/nixos/tests/kubernetes/certs.nix b/nixos/tests/kubernetes/certs.nix new file mode 100644 index 00000000000..f167224e549 --- /dev/null +++ b/nixos/tests/kubernetes/certs.nix @@ -0,0 +1,229 @@ +{ + pkgs ? import <nixpkgs> {}, + servers ? {test = "1.2.3.4";}, + internalDomain ? "cluster.local", + externalDomain ? "nixos.xyz" +}: +let + mkAltNames = ipFrom: dnsFrom: + pkgs.lib.concatImapStringsSep "\n" (i: v: "IP.${toString (i+ipFrom)} = ${v.ip}\nDNS.${toString (i+dnsFrom)} = ${v.name}.${externalDomain}") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers); + + runWithOpenSSL = file: cmd: pkgs.runCommand file { + buildInputs = [ pkgs.openssl ]; + passthru = { inherit file; }; + } cmd; + + ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048"; + ca_pem = runWithOpenSSL "ca.pem" '' + openssl req \ + -x509 -new -nodes -key ${ca_key} \ + -days 10000 -out $out -subj "/CN=etcd-ca" + ''; + + etcd_cnf = pkgs.writeText "openssl.cnf" '' + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = digitalSignature, keyEncipherment + extendedKeyUsage = serverAuth + subjectAltName = @alt_names + [alt_names] + DNS.1 = etcd.kubernetes.${externalDomain} + IP.1 = 127.0.0.1 + ''; + etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048"; + etcd_csr = runWithOpenSSL "etcd.csr" '' + openssl req \ + -new -key ${etcd_key} \ + -out $out -subj "/CN=etcd" \ + -config ${etcd_cnf} + ''; + etcd_cert = runWithOpenSSL "etcd.pem" '' + openssl x509 \ + -req -in ${etcd_csr} \ + -CA ${ca_pem} -CAkey ${ca_key} \ + -CAcreateserial -out $out \ + -days 3650 -extensions v3_req \ + -extfile ${etcd_cnf} + ''; + + etcd_client_key = runWithOpenSSL "etcd-client-key.pem" + "openssl genrsa -out $out 2048"; + + etcd_client_csr = runWithOpenSSL "etcd-client.csr" '' + openssl req \ + -new -key ${etcd_client_key} \ + -out $out -subj "/CN=etcd-client" \ + -config ${client_openssl_cnf} + ''; + + etcd_client_cert = runWithOpenSSL "etcd-client.crt" '' + openssl x509 \ + -req -in ${etcd_client_csr} \ + -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ + -out $out -days 3650 -extensions v3_req \ + -extfile ${client_openssl_cnf} + ''; + + admin_key = runWithOpenSSL "admin-key.pem" + "openssl genrsa -out $out 2048"; + + admin_csr = runWithOpenSSL "admin.csr" '' + openssl req \ + -new -key ${admin_key} \ + -out $out -subj "/CN=admin/O=system:masters" \ + -config ${client_openssl_cnf} + ''; + + admin_cert = runWithOpenSSL "admin.crt" '' + openssl x509 \ + -req -in ${admin_csr} \ + -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ + -out $out -days 3650 -extensions v3_req \ + -extfile ${client_openssl_cnf} + ''; + + apiserver_key = runWithOpenSSL "apiserver-key.pem" "openssl genrsa -out $out 2048"; + + apiserver_csr = runWithOpenSSL "apiserver.csr" '' + openssl req \ + -new -key ${apiserver_key} \ + -out $out -subj "/CN=kube-apiserver" \ + -config ${apiserver_cnf} + ''; + + apiserver_cert = runWithOpenSSL "apiserver.pem" '' + openssl x509 \ + -req -in ${apiserver_csr} \ + -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ + -out $out -days 3650 -extensions v3_req \ + -extfile ${apiserver_cnf} + ''; + + worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048"; + + worker_csr = runWithOpenSSL "worker.csr" '' + openssl req \ + -new -key ${worker_key} \ + -out $out -subj "/CN=kube-worker/O=system:authenticated" \ + -config ${worker_cnf} + ''; + + worker_cert = runWithOpenSSL "worker.pem" '' + openssl x509 \ + -req -in ${worker_csr} \ + -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ + -out $out -days 3650 -extensions v3_req \ + -extfile ${worker_cnf} + ''; + + openssl_cnf = pkgs.writeText "openssl.cnf" '' + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = digitalSignature, keyEncipherment + extendedKeyUsage = serverAuth + subjectAltName = @alt_names + [alt_names] + DNS.1 = *.cluster.${externalDomain} + IP.1 = 127.0.0.1 + ${mkAltNames 1 1} + ''; + + client_openssl_cnf = pkgs.writeText "client-openssl.cnf" '' + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = digitalSignature, keyEncipherment + extendedKeyUsage = clientAuth + subjectAltName = @alt_names + [alt_names] + DNS.1 = kubernetes + DNS.2 = kubernetes.default + DNS.3 = kubernetes.default.svc + DNS.4 = kubernetes.default.svc.${internalDomain} + DNS.5 = kubernetes.${externalDomain} + DNS.6 = *.cluster.${externalDomain} + IP.1 = 10.1.10.1 + ${mkAltNames 1 6} + ''; + + apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" '' + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + extendedKeyUsage = serverAuth + subjectAltName = @alt_names + [alt_names] + DNS.1 = kubernetes + DNS.2 = kubernetes.default + DNS.3 = kubernetes.default.svc + DNS.4 = kubernetes.default.svc.${internalDomain} + DNS.5 = kubernetes.${externalDomain} + DNS.6 = *.cluster.${externalDomain} + IP.1 = 10.1.10.1 + ${mkAltNames 1 6} + ''; + + worker_cnf = pkgs.writeText "worker-openssl.cnf" '' + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + subjectAltName = @alt_names + [alt_names] + DNS.1 = *.cluster.${externalDomain} + IP.1 = 10.1.10.1 + ${mkAltNames 1 1} + ''; + + ln = cert: target: '' + cp -v ${cert} ${target}/${cert.file} + ''; +in + pkgs.stdenv.mkDerivation rec { + name = "kubernetes-certs"; + unpackPhase = "true"; + installPhase = '' + set -xe + mkdir -p $out + ${ln ca_key "$out"} + ${ln ca_pem "$out"} + + ${ln etcd_key "$out"} + ${ln etcd_csr "$out"} + ${ln etcd_cert "$out"} + + ${ln etcd_client_key "$out"} + ${ln etcd_client_csr "$out"} + ${ln etcd_client_cert "$out"} + + ${ln apiserver_key "$out"} + ${ln apiserver_csr "$out"} + ${ln apiserver_cert "$out"} + + ${ln worker_key "$out"} + ${ln worker_csr "$out"} + ${ln worker_cert "$out"} + + ${ln admin_key "$out"} + ${ln admin_csr "$out"} + ${ln admin_cert "$out"} + ''; + } diff --git a/nixos/tests/kubernetes/default.nix b/nixos/tests/kubernetes/default.nix new file mode 100644 index 00000000000..6ba4f1904ea --- /dev/null +++ b/nixos/tests/kubernetes/default.nix @@ -0,0 +1,7 @@ +{ }: +{ + kubernetes-singlenode = import ./singlenode.nix {}; + kubernetes-multinode-kubectl = import ./multinode-kubectl.nix {}; + kubernetes-rbac = import ./rbac.nix {}; + kubernetes-dns = import ./dns.nix {}; +} diff --git a/nixos/tests/kubernetes/dns.nix b/nixos/tests/kubernetes/dns.nix new file mode 100644 index 00000000000..4659c520dee --- /dev/null +++ b/nixos/tests/kubernetes/dns.nix @@ -0,0 +1,103 @@ +{ system ? builtins.currentSystem }: + +with import ../../lib/testing.nix { inherit system; }; +with import ../../lib/qemu-flags.nix; +with pkgs.lib; + +let + servers.master = "192.168.1.1"; + servers.one = "192.168.1.10"; + + certs = import ./certs.nix { inherit servers; }; + + redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON { + kind = "Pod"; + apiVersion = "v1"; + metadata.name = "redis"; + metadata.labels.name = "redis"; + spec.containers = [{ + name = "redis"; + image = "redis"; + args = ["--bind" "0.0.0.0"]; + imagePullPolicy = "Never"; + ports = [{ + name = "redis-server"; + containerPort = 6379; + }]; + }]; + }); + + redisService = pkgs.writeText "redis-service.json" (builtins.toJSON { + kind = "Service"; + apiVersion = "v1"; + metadata.name = "redis"; + spec = { + ports = [{port = 6379; targetPort = 6379;}]; + selector = {name = "redis";}; + }; + }); + + redisImage = pkgs.dockerTools.buildImage { + name = "redis"; + tag = "latest"; + contents = [ pkgs.redis pkgs.bind.dnsutils pkgs.coreutils pkgs.inetutils pkgs.nmap ]; + config.Entrypoint = "/bin/redis-server"; + }; + + test = '' + $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready"); + $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready"); + + $one->execute("docker load < ${redisImage}"); + + $master->waitUntilSucceeds("kubectl create -f ${redisPod} || kubectl apply -f ${redisPod}"); + $master->waitUntilSucceeds("kubectl create -f ${redisService} || kubectl apply -f ${redisService}"); + + $master->waitUntilSucceeds("kubectl get pod redis | grep Running"); + + $master->succeed("dig \@192.168.1.1 redis.default.svc.cluster.local"); + $one->succeed("dig \@192.168.1.10 redis.default.svc.cluster.local"); + + + $master->succeed("kubectl exec -ti redis -- cat /etc/resolv.conf | grep 'nameserver 192.168.1.10'"); + + $master->succeed("kubectl exec -ti redis -- dig \@192.168.1.10 redis.default.svc.cluster.local"); + ''; + +in makeTest { + name = "kubernetes-dns"; + + nodes = { + master = + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 4096; + networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}]; + networking.primaryIPAddress = mkForce servers.master; + } + (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) + (import ./kubernetes-master.nix { inherit pkgs config certs; }) + ]; + + one = + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 4096; + networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}]; + networking.primaryIPAddress = mkForce servers.one; + services.kubernetes.roles = ["node"]; + } + (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) + ]; + }; + + testScript = '' + startAll; + + ${test} + ''; +} diff --git a/nixos/tests/kubernetes/kubernetes-common.nix b/nixos/tests/kubernetes/kubernetes-common.nix new file mode 100644 index 00000000000..bc28244ad5b --- /dev/null +++ b/nixos/tests/kubernetes/kubernetes-common.nix @@ -0,0 +1,131 @@ +{ config, pkgs, certs, servers }: +let + etcd_key = "${certs}/etcd-key.pem"; + etcd_cert = "${certs}/etcd.pem"; + ca_pem = "${certs}/ca.pem"; + etcd_client_cert = "${certs}/etcd-client.crt"; + etcd_client_key = "${certs}/etcd-client-key.pem"; + + worker_key = "${certs}/worker-key.pem"; + worker_cert = "${certs}/worker.pem"; + + mkDockerOpts = "${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh"; + + rootCaFile = pkgs.writeScript "rootCaFile.pem" '' + ${pkgs.lib.readFile "${certs}/ca.pem"} + + ${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")} + ''; + + mkHosts = + pkgs.lib.concatMapStringsSep "\n" (v: "${v.ip} ${v.name}.nixos.xyz") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers); + +in +{ + programs.bash.enableCompletion = true; + environment.systemPackages = with pkgs; [ netcat bind etcd.bin ]; + + networking = { + firewall = { + enable = true; + allowedTCPPorts = [ + 10250 80 443 + ]; + allowedUDPPorts = [ + 8285 # flannel udp + 8472 # flannel vxlan + ]; + }; + extraHosts = '' + # register "external" domains + ${servers.master} etcd.kubernetes.nixos.xyz + ${servers.master} kubernetes.nixos.xyz + ${mkHosts} + ''; + }; + virtualisation.docker.extraOptions = '' + --iptables=false $DOCKER_OPTS + ''; + + # lets create environment file for docker startup - network stuff + systemd.services."pre-docker" = { + description = "Pre-Docker Actions"; + wantedBy = [ "flannel.service" ]; + before = [ "docker.service" ]; + after = [ "flannel.service" ]; + path = [ pkgs.gawk pkgs.gnugrep ]; + script = '' + mkdir -p /run/flannel + # bashInteractive needed for `compgen` + ${pkgs.bashInteractive}/bin/bash ${mkDockerOpts} -d /run/flannel/docker + cat /run/flannel/docker # just for debugging + + # allow container to host communication for DNS traffic + ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -i docker0 --dport 53 -j nixos-fw-accept + ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -i docker0 --dport 53 -j nixos-fw-accept + ''; + serviceConfig.Type = "simple"; + }; + systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker"; + + services.flannel = { + enable = true; + network = "10.2.0.0/16"; + iface = "eth1"; + etcd = { + endpoints = ["https://etcd.kubernetes.nixos.xyz:2379"]; + keyFile = etcd_client_key; + certFile = etcd_client_cert; + caFile = ca_pem; + }; + }; + environment.variables = { + ETCDCTL_CERT_FILE = "${etcd_client_cert}"; + ETCDCTL_KEY_FILE = "${etcd_client_key}"; + ETCDCTL_CA_FILE = "${rootCaFile}"; + ETCDCTL_PEERS = "https://etcd.kubernetes.nixos.xyz:2379"; + }; + + services.kubernetes = { + kubelet = { + networkPlugin = "cni"; + cni.config = [{ + name = "mynet"; + type = "flannel"; + delegate = { + isDefaultGateway = true; + bridge = "docker0"; + }; + }]; + tlsKeyFile = worker_key; + tlsCertFile = worker_cert; + hostname = "${config.networking.hostName}.nixos.xyz"; + extraOpts = "--node-ip ${config.networking.primaryIPAddress}"; + clusterDns = config.networking.primaryIPAddress; + }; + etcd = { + servers = ["https://etcd.kubernetes.nixos.xyz:2379"]; + keyFile = etcd_client_key; + certFile = etcd_client_cert; + caFile = ca_pem; + }; + kubeconfig = { + server = "https://kubernetes.nixos.xyz:4443"; + caFile = rootCaFile; + certFile = worker_cert; + keyFile = worker_key; + }; + + # make sure you cover kubernetes.apiserver.portalNet and flannel networks + clusterCidr = "10.0.0.0/8"; + + dns.enable = true; + dns.port = 4453; + }; + + services.dnsmasq.enable = true; + services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; + + virtualisation.docker.enable = true; + virtualisation.docker.storageDriver = "overlay"; +} diff --git a/nixos/tests/kubernetes/kubernetes-master.nix b/nixos/tests/kubernetes/kubernetes-master.nix new file mode 100644 index 00000000000..b9577fa0964 --- /dev/null +++ b/nixos/tests/kubernetes/kubernetes-master.nix @@ -0,0 +1,148 @@ +{ config, pkgs, certs }: +let + etcd_key = "${certs}/etcd-key.pem"; + etcd_cert = "${certs}/etcd.pem"; + ca_pem = "${certs}/ca.pem"; + etcd_client_cert = "${certs}/etcd-client.crt"; + etcd_client_key = "${certs}/etcd-client-key.pem"; + + apiserver_key = "${certs}/apiserver-key.pem"; + apiserver_cert = "${certs}/apiserver.pem"; + worker_key = "${certs}/worker-key.pem"; + worker_cert = "${certs}/worker.pem"; + + + rootCaFile = pkgs.writeScript "rootCaFile.pem" '' + ${pkgs.lib.readFile "${certs}/ca.pem"} + + ${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")} + ''; +in +{ + networking = { + firewall = { + enable = true; + allowPing = true; + allowedTCPPorts = [ + 2379 2380 # etcd + 4443 # kubernetes + ]; + }; + }; + + services.etcd = { + enable = pkgs.lib.mkForce true; + keyFile = etcd_key; + certFile = etcd_cert; + trustedCaFile = rootCaFile; + peerClientCertAuth = true; + listenClientUrls = ["https://0.0.0.0:2379"]; + listenPeerUrls = ["https://0.0.0.0:2380"]; + + advertiseClientUrls = ["https://etcd.kubernetes.nixos.xyz:2379"]; + initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"]; + initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"]; + }; + services.kubernetes = { + roles = ["master"]; + scheduler.leaderElect = true; + controllerManager.leaderElect = true; + controllerManager.rootCaFile = rootCaFile; + controllerManager.serviceAccountKeyFile = apiserver_key; + apiserver = { + securePort = 4443; + publicAddress = "192.168.1.1"; + advertiseAddress = "192.168.1.1"; + tlsKeyFile = apiserver_key; + tlsCertFile = apiserver_cert; + clientCaFile = rootCaFile; + kubeletClientCaFile = rootCaFile; + kubeletClientKeyFile = worker_key; + kubeletClientCertFile = worker_cert; + portalNet = "10.1.10.0/24"; # --service-cluster-ip-range + runtimeConfig = ""; + /*extraOpts = "--v=2";*/ + authorizationMode = ["ABAC"]; + authorizationPolicy = [ + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "kubecfg"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "kubelet"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "kube-worker"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "kube_proxy"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "client"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + group = "system:serviceaccounts"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + group = "system:authenticated"; + readonly = true; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + ]; + }; + }; +} diff --git a/nixos/tests/kubernetes/multinode-kubectl.nix b/nixos/tests/kubernetes/multinode-kubectl.nix new file mode 100644 index 00000000000..97108163d2c --- /dev/null +++ b/nixos/tests/kubernetes/multinode-kubectl.nix @@ -0,0 +1,147 @@ +{ system ? builtins.currentSystem }: + +with import ../../lib/testing.nix { inherit system; }; +with import ../../lib/qemu-flags.nix; +with pkgs.lib; + +let + servers.master = "192.168.1.1"; + servers.one = "192.168.1.10"; + servers.two = "192.168.1.20"; + servers.three = "192.168.1.30"; + + certs = import ./certs.nix { inherit servers; }; + + kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON { + kind = "Pod"; + apiVersion = "v1"; + metadata.name = "kubectl"; + metadata.labels.name = "kubectl"; + spec.containers = [{ + name = "kubectl"; + image = "kubectl:latest"; + command = ["${pkgs.busybox}/bin/tail" "-f"]; + imagePullPolicy = "Never"; + tty = true; + }]; + }); + + kubectlImage = pkgs.dockerTools.buildImage { + name = "kubectl"; + tag = "latest"; + contents = [ pkgs.kubernetes pkgs.busybox certs kubeconfig ]; + config.Entrypoint = "${pkgs.busybox}/bin/sh"; + }; + + kubeconfig = pkgs.writeTextDir "kubeconfig.json" (builtins.toJSON { + apiVersion = "v1"; + kind = "Config"; + clusters = [{ + name = "local"; + cluster.certificate-authority = "/ca.pem"; + cluster.server = "https://${servers.master}:4443/"; + }]; + users = [{ + name = "kubelet"; + user = { + client-certificate = "/admin.crt"; + client-key = "/admin-key.pem"; + }; + }]; + contexts = [{ + context = { + cluster = "local"; + user = "kubelet"; + }; + current-context = "kubelet-context"; + }]; + }); + + test = '' + $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready"); + $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready"); + $master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready"); + $master->waitUntilSucceeds("kubectl get node three.nixos.xyz | grep Ready"); + + $one->execute("docker load < ${kubectlImage}"); + $two->execute("docker load < ${kubectlImage}"); + $three->execute("docker load < ${kubectlImage}"); + + $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}"); + + $master->waitUntilSucceeds("kubectl get pod kubectl | grep Running"); + + $master->succeed("kubectl exec -ti kubectl -- kubectl --kubeconfig=/kubeconfig.json version"); + ''; + +in makeTest { + name = "kubernetes-multinode-kubectl"; + + nodes = { + master = + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 4096; + # networking.hostName = mkForce "master"; + networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}]; + # networking.nat.externalIP = "192.168.1.1"; + networking.primaryIPAddress = mkForce servers.master; + } + (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) + (import ./kubernetes-master.nix { inherit pkgs config certs; }) + ]; + + one = + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 4096; + # networking.hostName = mkForce "one"; + networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}]; + # networking.nat.externalIP = "192.168.1.2"; + networking.primaryIPAddress = mkForce servers.one; + services.kubernetes.roles = ["node"]; + } + (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) + ]; + + two = + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 4096; + # networking.hostName = mkForce "two"; + networking.interfaces.eth1.ip4 = mkForce [{address = servers.two; prefixLength = 24;}]; + # networking.nat.externalIP = "192.168.1.3"; + networking.primaryIPAddress = mkForce servers.two; + services.kubernetes.roles = ["node"]; + } + (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) + ]; + + three = + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 4096; + # networking.hostName = mkForce "three"; + networking.interfaces.eth1.ip4 = mkForce [{address = servers.three; prefixLength = 24;}]; + # networking.nat.externalIP = "192.168.1.4"; + networking.primaryIPAddress = mkForce servers.three; + services.kubernetes.roles = ["node"]; + } + (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) + ]; + }; + + testScript = '' + startAll; + + ${test} + ''; +} diff --git a/nixos/tests/kubernetes/rbac.nix b/nixos/tests/kubernetes/rbac.nix new file mode 100644 index 00000000000..6388fe7ceb9 --- /dev/null +++ b/nixos/tests/kubernetes/rbac.nix @@ -0,0 +1,206 @@ +{ system ? builtins.currentSystem }: + +with import ../../lib/testing.nix { inherit system; }; +with import ../../lib/qemu-flags.nix; +with pkgs.lib; + +let + servers.master = "192.168.1.1"; + servers.one = "192.168.1.10"; + + certs = import ./certs.nix { inherit servers; }; + + roServiceAccount = pkgs.writeText "ro-service-account.json" (builtins.toJSON { + kind = "ServiceAccount"; + apiVersion = "v1"; + metadata = { + name = "read-only"; + namespace = "default"; + }; + }); + + roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON { + "apiVersion" = "rbac.authorization.k8s.io/v1beta1"; + "kind" = "RoleBinding"; + "metadata" = { + "name" = "read-pods"; + "namespace" = "default"; + }; + "roleRef" = { + "apiGroup" = "rbac.authorization.k8s.io"; + "kind" = "Role"; + "name" = "pod-reader"; + }; + "subjects" = [{ + "kind" = "ServiceAccount"; + "name" = "read-only"; + "namespace" = "default"; + }]; + }); + + roRole = pkgs.writeText "ro-role.json" (builtins.toJSON { + "apiVersion" = "rbac.authorization.k8s.io/v1beta1"; + "kind" = "Role"; + "metadata" = { + "name" = "pod-reader"; + "namespace" = "default"; + }; + "rules" = [{ + "apiGroups" = [""]; + "resources" = ["pods"]; + "verbs" = ["get" "list" "watch"]; + }]; + }); + + kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON { + kind = "Pod"; + apiVersion = "v1"; + metadata.name = "kubectl"; + metadata.namespace = "default"; + metadata.labels.name = "kubectl"; + spec.serviceAccountName = "read-only"; + spec.containers = [{ + name = "kubectl"; + image = "kubectl:latest"; + command = ["${pkgs.busybox}/bin/tail" "-f"]; + imagePullPolicy = "Never"; + tty = true; + }]; + }); + + kubectlPod2 = pkgs.writeTextDir "kubectl-pod-2.json" (builtins.toJSON { + kind = "Pod"; + apiVersion = "v1"; + metadata.name = "kubectl-2"; + metadata.namespace = "default"; + metadata.labels.name = "kubectl-2"; + spec.serviceAccountName = "read-only"; + spec.containers = [{ + name = "kubectl-2"; + image = "kubectl:latest"; + command = ["${pkgs.busybox}/bin/tail" "-f"]; + imagePullPolicy = "Never"; + tty = true; + }]; + }); + + kubectlImage = pkgs.dockerTools.buildImage { + name = "kubectl"; + tag = "latest"; + contents = [ pkgs.kubernetes pkgs.busybox kubectlPod2 ]; # certs kubeconfig + config.Entrypoint = "${pkgs.busybox}/bin/sh"; + }; + + test = '' + $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready"); + $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready"); + + $one->execute("docker load < ${kubectlImage}"); + + $master->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}"); + $master->waitUntilSucceeds("kubectl apply -f ${roRole}"); + $master->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}"); + $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}"); + + $master->waitUntilSucceeds("kubectl get pod kubectl | grep Running"); + + $master->succeed("kubectl exec -ti kubectl -- kubectl get pods"); + $master->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json"); + $master->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl"); + ''; + +in makeTest { + name = "kubernetes-multinode-rbac"; + + nodes = { + master = + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 4096; + networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}]; + networking.primaryIPAddress = mkForce servers.master; + services.kubernetes.apiserver.authorizationMode = mkForce ["ABAC" "RBAC"]; + services.kubernetes.apiserver.authorizationPolicy = mkForce [ + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "kubecfg"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "kubelet"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "kube-worker"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "kube_proxy"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + { + apiVersion = "abac.authorization.kubernetes.io/v1beta1"; + kind = "Policy"; + spec = { + user = "client"; + namespace = "*"; + resource = "*"; + apiGroup = "*"; + nonResourcePath = "*"; + }; + } + ]; + } + (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) + (import ./kubernetes-master.nix { inherit pkgs config certs; }) + ]; + + one = + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 4096; + networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}]; + networking.primaryIPAddress = mkForce servers.one; + services.kubernetes.roles = ["node"]; + } + (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) + ]; + }; + + testScript = '' + startAll; + + ${test} + ''; +} diff --git a/nixos/tests/kubernetes/singlenode-kubectl.nix b/nixos/tests/kubernetes/singlenode-kubectl.nix new file mode 100644 index 00000000000..d3a78a06e43 --- /dev/null +++ b/nixos/tests/kubernetes/singlenode-kubectl.nix @@ -0,0 +1,97 @@ +{ system ? builtins.currentSystem }: + +with import ../../lib/testing.nix { inherit system; }; +with import ../../lib/qemu-flags.nix; +with pkgs.lib; + +let + certs = import ./certs.nix { servers = {}; }; + + kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON { + kind = "Pod"; + apiVersion = "v1"; + metadata.name = "kubectl"; + metadata.labels.name = "kubectl"; + spec.containers = [{ + name = "kubectl"; + image = "kubectl:latest"; + command = ["${pkgs.busybox}/bin/tail" "-f"]; + imagePullPolicy = "Never"; + tty = true; + }]; + }); + + kubectlImage = pkgs.dockerTools.buildImage { + name = "kubectl"; + tag = "latest"; + contents = [ pkgs.kubernetes pkgs.busybox certs kubeconfig ]; + config.Entrypoint = "${pkgs.busybox}/bin/sh"; + }; + + kubeconfig = pkgs.writeTextDir "kubeconfig.json" (builtins.toJSON { + apiVersion = "v1"; + kind = "Config"; + clusters = [{ + name = "local"; + cluster.certificate-authority = "/ca.pem"; + cluster.server = "https://192.168.1.1:4443/"; + }]; + users = [{ + name = "kubelet"; + user = { + client-certificate = "/admin.crt"; + client-key = "/admin-key.pem"; + }; + }]; + contexts = [{ + context = { + cluster = "local"; + user = "kubelet"; + }; + current-context = "kubelet-context"; + }]; + }); + + test = '' + $kubernetes->execute("docker load < ${kubectlImage}"); + $kubernetes->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}"); + $kubernetes->waitUntilSucceeds("kubectl get pod kubectl | grep Running"); + + # FIXME: this test fails, for some reason it can not reach host ip address + $kubernetes->succeed("kubectl exec -ti kubectl -- kubectl --kubeconfig=/kubeconfig.json version"); + ''; +in makeTest { + name = "kubernetes-singlenode-kubectl"; + + nodes = { + kubernetes = + { config, pkgs, lib, nodes, ... }: + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 4096; + + programs.bash.enableCompletion = true; + environment.systemPackages = with pkgs; [ netcat bind ]; + + services.kubernetes.roles = ["master" "node"]; + services.kubernetes.apiserver.securePort = 4443; + services.kubernetes.dns.port = 4453; + services.kubernetes.clusterCidr = "10.0.0.0/8"; + virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0"; + + networking.interfaces.eth1.ip4 = mkForce [{address = "192.168.1.1"; prefixLength = 24;}]; + networking.primaryIPAddress = mkForce "192.168.1.1"; + networking.bridges.cbr0.interfaces = []; + networking.interfaces.cbr0 = {}; + + services.dnsmasq.enable = true; + services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; + }; + }; + + testScript = '' + startAll; + + ${test} + ''; +} diff --git a/nixos/tests/kubernetes/singlenode.nix b/nixos/tests/kubernetes/singlenode.nix new file mode 100644 index 00000000000..9bab51c4982 --- /dev/null +++ b/nixos/tests/kubernetes/singlenode.nix @@ -0,0 +1,82 @@ +{ system ? builtins.currentSystem }: + +with import ../../lib/testing.nix { inherit system; }; +with import ../../lib/qemu-flags.nix; +with pkgs.lib; + +let + redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON { + kind = "Pod"; + apiVersion = "v1"; + metadata.name = "redis"; + metadata.labels.name = "redis"; + spec.containers = [{ + name = "redis"; + image = "redis"; + args = ["--bind" "0.0.0.0"]; + imagePullPolicy = "Never"; + ports = [{ + name = "redis-server"; + containerPort = 6379; + }]; + }]; + }); + + redisService = pkgs.writeText "redis-service.json" (builtins.toJSON { + kind = "Service"; + apiVersion = "v1"; + metadata.name = "redis"; + spec = { + ports = [{port = 6379; targetPort = 6379;}]; + selector = {name = "redis";}; + }; + }); + + redisImage = pkgs.dockerTools.buildImage { + name = "redis"; + tag = "latest"; + contents = pkgs.redis; + config.Entrypoint = "/bin/redis-server"; + }; + + testSimplePod = '' + $kubernetes->execute("docker load < ${redisImage}"); + $kubernetes->waitUntilSucceeds("kubectl create -f ${redisPod}"); + $kubernetes->succeed("kubectl create -f ${redisService}"); + $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running"); + $kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379"); + ''; +in { + # This test runs kubernetes on a single node + singlenode = makeTest { + name = "kubernetes-singlenode"; + + nodes = { + kubernetes = + { config, pkgs, lib, nodes, ... }: + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 2048; + + programs.bash.enableCompletion = true; + environment.systemPackages = with pkgs; [ netcat bind ]; + + services.kubernetes.roles = ["master" "node"]; + services.kubernetes.dns.port = 4453; + virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0"; + + networking.bridges.cbr0.interfaces = []; + networking.interfaces.cbr0 = {}; + + services.dnsmasq.enable = true; + services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; + }; + }; + + testScript = '' + startAll; + + ${testSimplePod} + ''; + }; +} |