import ./make-test-python.nix (
{ pkgs, lib, ... }:
let
initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
in
{
name = "iscsi";
meta = {
maintainers = pkgs.lib.teams.deshaw.members;
};
nodes = {
target = { config, pkgs, lib, ... }: {
virtualisation.vlans = [ 1 2 ];
services.target = {
enable = true;
config = {
fabric_modules = [ ];
storage_objects = [
{
dev = "/dev/vdb";
name = "test";
plugin = "block";
write_back = true;
wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
}
];
targets = [
{
fabric = "iscsi";
tpgs = [
{
enable = true;
attributes = {
authentication = 0;
generate_node_acls = 1;
};
luns = [
{
alias = "94dfe06967";
alua_tg_pt_gp_name = "default_tg_pt_gp";
index = 0;
storage_object = "/backstores/block/test";
}
];
node_acls = [
{
mapped_luns = [
{
alias = "d42f5bdf8a";
index = 0;
tpg_lun = 0;
write_protect = false;
}
];
node_wwn = initiatorName;
}
];
portals = [
{
ip_address = "0.0.0.0";
iser = false;
offload = false;
port = 3260;
}
];
tag = 1;
}
];
wwn = targetName;
}
];
};
};
networking.firewall.allowedTCPPorts = [ 3260 ];
networking.firewall.allowedUDPPorts = [ 3260 ];
virtualisation.memorySize = 2048;
virtualisation.emptyDiskImages = [ 2048 ];
};
initiatorAuto = { nodes, config, pkgs, ... }: {
virtualisation.vlans = [ 1 2 ];
services.multipath = {
enable = true;
defaults = ''
find_multipaths yes
user_friendly_names yes
'';
pathGroups = [
{
alias = 123456;
wwid = "3600140592b17c3f6b404168b082ceeb7";
}
];
};
services.openiscsi = {
enable = true;
enableAutoLoginOut = true;
discoverPortal = "target";
name = initiatorName;
};
environment.systemPackages = with pkgs; [
xfsprogs
];
environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
nix.settings = {
substituters = lib.mkForce [ ];
hashed-mirrors = null;
connect-timeout = 1;
};
};
initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
boot.initrd.network.enable = true;
boot.loader.grub.enable = false;
boot.kernelParams = lib.mkOverride 5 (
[
"boot.shell_on_fail"
"console=tty1"
"ip=192.168.1.1:::255.255.255.0::ens9:none"
"ip=192.168.2.1:::255.255.255.0::ens10:none"
]
);
# defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
virtualisation.writableStore = false;
virtualisation.vlans = [ 1 2 ];
services.multipath = {
enable = true;
defaults = ''
find_multipaths yes
user_friendly_names yes
'';
pathGroups = [
{
alias = 123456;
wwid = "3600140592b17c3f6b404168b082ceeb7";
}
];
};
fileSystems = lib.mkOverride 5 {
"/" = {
fsType = "xfs";
device = "/dev/mapper/123456";
options = [ "_netdev" ];
};
};
boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
boot.iscsi-initiator = {
discoverPortal = "target";
name = initiatorName;
target = targetName;
extraIscsiCommands = ''
iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
'';
};
};
};
testScript = { nodes, ... }: ''
target.start()
target.wait_for_unit("iscsi-target.service")
initiatorAuto.start()
initiatorAuto.wait_for_unit("iscsid.service")
initiatorAuto.wait_for_unit("iscsi.service")
initiatorAuto.get_unit_info("iscsi")
# Expecting this to fail since we should already know about 192.168.1.3
initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
# Expecting this to succeed since we don't yet know about 192.168.2.3
initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
# /dev/sda is provided by iscsi on target
initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
initiatorAuto.succeed("mkfs.xfs /dev/sda")
initiatorAuto.succeed("mkdir /mnt")
# Start by verifying /dev/sda and /dev/sdb are both the same disk
initiatorAuto.succeed("mount /dev/sda /mnt")
initiatorAuto.succeed("touch /mnt/hi")
initiatorAuto.succeed("umount /mnt")
initiatorAuto.succeed("mount /dev/sdb /mnt")
initiatorAuto.succeed("test -e /mnt/hi")
initiatorAuto.succeed("umount /mnt")
initiatorAuto.succeed("systemctl restart multipathd")
initiatorAuto.succeed("multipath -ll | systemd-cat")
# Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
initiatorAuto.succeed(
"nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
)
initiatorAuto.succeed("umount /mnt")
initiatorAuto.shutdown()
initiatorRootDisk.start()
initiatorRootDisk.wait_for_unit("multi-user.target")
initiatorRootDisk.wait_for_unit("iscsid")
# Log in over both nodes
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
initiatorRootDisk.succeed("systemctl restart multipathd")
initiatorRootDisk.succeed("multipath -ll | systemd-cat")
# Verify we can write and sync the root disk
initiatorRootDisk.succeed("mkdir /scratch")
initiatorRootDisk.succeed("touch /scratch/both-up")
initiatorRootDisk.succeed("sync /scratch")
# Verify we can write to the root with ens9 (sda, 192.168.1.3) down
initiatorRootDisk.succeed("ip link set ens9 down")
initiatorRootDisk.succeed("touch /scratch/ens9-down")
initiatorRootDisk.succeed("sync /scratch")
initiatorRootDisk.succeed("ip link set ens9 up")
# todo: better way to wait until multipath notices the link is back
initiatorRootDisk.succeed("sleep 5")
initiatorRootDisk.succeed("touch /scratch/both-down")
initiatorRootDisk.succeed("sync /scratch")
# Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
initiatorRootDisk.succeed("ip link set ens10 down")
initiatorRootDisk.succeed("touch /scratch/ens10-down")
initiatorRootDisk.succeed("sync /scratch")
initiatorRootDisk.succeed("ip link set ens10 up")
initiatorRootDisk.succeed("touch /scratch/ens10-down")
initiatorRootDisk.succeed("sync /scratch")
initiatorRootDisk.succeed("ip link set ens9 up")
initiatorRootDisk.succeed("ip link set ens10 up")
initiatorRootDisk.shutdown()
# Verify we can boot with the target's eth1 down, forcing
# it to multipath via the second link
target.succeed("ip link set eth1 down")
initiatorRootDisk.start()
initiatorRootDisk.wait_for_unit("multi-user.target")
initiatorRootDisk.wait_for_unit("iscsid")
initiatorRootDisk.succeed("test -e /scratch/both-up")
'';
}
)