blob: b6cd811c5aefd8d508c3bfc5cefbebecddf87d9a (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
with import ./base.nix { inherit system; };
let
domain = "my.zyx";
redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON {
kind = "Pod";
apiVersion = "v1";
metadata.name = "redis";
metadata.labels.name = "redis";
spec.containers = [{
name = "redis";
image = "redis";
args = ["--bind" "0.0.0.0"];
imagePullPolicy = "Never";
ports = [{
name = "redis-server";
containerPort = 6379;
}];
}];
});
redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {
kind = "Service";
apiVersion = "v1";
metadata.name = "redis";
spec = {
ports = [{port = 6379; targetPort = 6379;}];
selector = {name = "redis";};
};
});
redisImage = pkgs.dockerTools.buildImage {
name = "redis";
tag = "latest";
contents = [ pkgs.redis pkgs.bind.host ];
config.Entrypoint = ["/bin/redis-server"];
};
probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {
kind = "Pod";
apiVersion = "v1";
metadata.name = "probe";
metadata.labels.name = "probe";
spec.containers = [{
name = "probe";
image = "probe";
args = [ "-f" ];
tty = true;
imagePullPolicy = "Never";
}];
});
probeImage = pkgs.dockerTools.buildImage {
name = "probe";
tag = "latest";
contents = [ pkgs.bind.host pkgs.busybox ];
config.Entrypoint = ["/bin/tail"];
};
extraConfiguration = { config, pkgs, lib, ... }: {
environment.systemPackages = [ pkgs.bind.host ];
services.dnsmasq.enable = true;
services.dnsmasq.servers = [
"/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
];
};
base = {
name = "dns";
inherit domain extraConfiguration;
};
singleNodeTest = {
test = ''
# prepare machine1 for test
machine1.wait_until_succeeds("kubectl get node machine1.${domain} | grep -w Ready")
machine1.wait_until_succeeds(
"${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(
"kubectl create -f ${redisPod}"
)
machine1.wait_until_succeeds(
"kubectl create -f ${redisService}"
)
machine1.wait_until_succeeds(
"${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(
"kubectl create -f ${probePod}"
)
# check if pods are running
machine1.wait_until_succeeds("kubectl get pod redis | grep Running")
machine1.wait_until_succeeds("kubectl get pod probe | grep Running")
machine1.wait_until_succeeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'")
# check dns on host (dnsmasq)
machine1.succeed("host redis.default.svc.cluster.local")
# check dns inside the container
machine1.succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local")
'';
};
multiNodeTest = {
test = ''
# Node token exchange
machine1.wait_until_succeeds(
"cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret"
)
machine2.wait_until_succeeds(
"cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join"
)
# prepare machines for test
machine1.wait_until_succeeds("kubectl get node machine2.${domain} | grep -w Ready")
machine2.wait_until_succeeds(
"${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(
"kubectl create -f ${redisPod}"
)
machine1.wait_until_succeeds(
"kubectl create -f ${redisService}"
)
machine2.wait_until_succeeds(
"${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(
"kubectl create -f ${probePod}"
)
# check if pods are running
machine1.wait_until_succeeds("kubectl get pod redis | grep Running")
machine1.wait_until_succeeds("kubectl get pod probe | grep Running")
machine1.wait_until_succeeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'")
# check dns on hosts (dnsmasq)
machine1.succeed("host redis.default.svc.cluster.local")
machine2.succeed("host redis.default.svc.cluster.local")
# check dns inside the container
machine1.succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local")
'';
};
in {
singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest);
multinode = mkKubernetesMultiNodeTest (base // multiNodeTest);
}
|