1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
import ./make-test.nix ({ pkgs, lib, ... }:
let
# Change this to test a different version of Cassandra:
testPackage = pkgs.cassandra;
clusterName = "NixOS Automated-Test Cluster";
testRemoteAuth = lib.versionAtLeast testPackage.version "3.11";
jmxRoles = [{ username = "me"; password = "password"; }];
jmxRolesFile = ./cassandra-jmx-roles;
jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}";
jmxPort = 7200; # Non-standard port so it doesn't accidentally work
# Would usually be assigned to 512M
numMaxHeapSize = "400";
getHeapLimitCommand = ''
nodetool info -p ${toString jmxPort} | grep "^Heap Memory" | awk \'{print $NF}\'
'';
checkHeapLimitCommand = ''
[ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ]
'';
cassandraCfg = ipAddress:
{ enable = true;
inherit clusterName;
listenAddress = ipAddress;
rpcAddress = ipAddress;
seedAddresses = [ "192.168.1.1" ];
package = testPackage;
maxHeapSize = "${numMaxHeapSize}M";
heapNewSize = "100M";
inherit jmxPort;
};
nodeCfg = ipAddress: extra: {pkgs, config, ...}: rec {
environment.systemPackages = [ testPackage ];
networking = {
firewall.allowedTCPPorts = [ 7000 9042 services.cassandra.jmxPort ];
useDHCP = false;
interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
{ address = ipAddress; prefixLength = 24; }
];
};
services.cassandra = cassandraCfg ipAddress // extra;
virtualisation.memorySize = 1024;
};
in
{
name = "cassandra-ci";
nodes = {
cass0 = nodeCfg "192.168.1.1" {};
cass1 = nodeCfg "192.168.1.2" (lib.optionalAttrs testRemoteAuth { inherit jmxRoles; remoteJmx = true; });
cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
};
testScript = let
jmxPortS = toString jmxPort;
in ''
# Check configuration
subtest "Timers exist", sub {
$cass0->succeed("systemctl list-timers | grep cassandra-full-repair.timer");
$cass0->succeed("systemctl list-timers | grep cassandra-incremental-repair.timer");
};
subtest "Can connect via cqlsh", sub {
$cass0->waitForUnit("cassandra.service");
$cass0->waitUntilSucceeds("nc -z cass0 9042");
$cass0->succeed("echo 'show version;' | cqlsh cass0");
};
subtest "Nodetool is operational", sub {
$cass0->waitForUnit("cassandra.service");
$cass0->waitUntilSucceeds("nc -z localhost ${jmxPortS}");
$cass0->succeed("nodetool status -p ${jmxPortS} --resolve-ip | egrep '^UN[[:space:]]+cass0'");
};
subtest "Cluster name was set", sub {
$cass0->waitForUnit("cassandra.service");
$cass0->waitUntilSucceeds("nc -z localhost ${jmxPortS}");
$cass0->waitUntilSucceeds("nodetool describecluster -p ${jmxPortS} | grep 'Name: ${clusterName}'");
};
subtest "Heap limit set correctly", sub {
# Nodetool takes a while until it can display info
$cass0->waitUntilSucceeds('nodetool info -p ${jmxPortS}');
$cass0->succeed('${checkHeapLimitCommand}');
};
# Check cluster interaction
subtest "Bring up cluster", sub {
$cass1->waitForUnit("cassandra.service");
$cass1->waitUntilSucceeds("nodetool -p ${jmxPortS} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2");
$cass0->succeed("nodetool status -p ${jmxPortS} --resolve-ip | egrep '^UN[[:space:]]+cass1'");
};
'' + lib.optionalString testRemoteAuth ''
subtest "Remote authenticated jmx", sub {
# Doesn't work if not enabled
$cass0->waitUntilSucceeds("nc -z localhost ${jmxPortS}");
$cass1->fail("nc -z 192.168.1.1 ${toString jmxPort}");
$cass1->fail("nodetool -p ${jmxPortS} -h 192.168.1.1 status");
# Works if enabled
$cass1->waitUntilSucceeds("nc -z localhost ${toString jmxPort}");
$cass0->succeed("nodetool -p ${jmxPortS} -h 192.168.1.2 ${jmxAuthArgs} status");
};
'' + ''
subtest "Break and fix node", sub {
$cass1->block;
$cass0->waitUntilSucceeds("nodetool status -p ${jmxPortS} --resolve-ip | egrep -c '^DN[[:space:]]+cass1'");
$cass0->succeed("nodetool status -p ${jmxPortS} | egrep -c '^UN' | grep 1");
$cass1->unblock;
$cass1->waitUntilSucceeds("nodetool -p ${jmxPortS} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2");
$cass0->succeed("nodetool status -p ${jmxPortS} | egrep -c '^UN' | grep 2");
};
subtest "Replace crashed node", sub {
$cass1->crash;
$cass2->waitForUnit("cassandra.service");
$cass0->waitUntilFails("nodetool status -p ${jmxPortS} --resolve-ip | egrep '^UN[[:space:]]+cass1'");
$cass0->waitUntilSucceeds("nodetool status -p ${jmxPortS} --resolve-ip | egrep '^UN[[:space:]]+cass2'");
};
'';
})
|