diff options
| -rw-r--r-- | pkgs/os-specific/linux/spectrum/default.nix | 2 | ||||
| -rw-r--r-- | pkgs/os-specific/linux/spectrum/testhost/default.nix | 205 |
2 files changed, 207 insertions, 0 deletions
diff --git a/pkgs/os-specific/linux/spectrum/default.nix b/pkgs/os-specific/linux/spectrum/default.nix index 7e07ee60f43..c4cccab3787 100644 --- a/pkgs/os-specific/linux/spectrum/default.nix +++ b/pkgs/os-specific/linux/spectrum/default.nix @@ -8,6 +8,8 @@ let spectrum-vm = callPackage ./spectrum-vm { linux = linux_vm; }; + spectrum-testhost = callPackage ./testhost { }; + linux_vm = callPackage ./linux/vm.nix { linux = linux_cros; }; makeRootfs = callPackage ./rootfs { }; diff --git a/pkgs/os-specific/linux/spectrum/testhost/default.nix b/pkgs/os-specific/linux/spectrum/testhost/default.nix new file mode 100644 index 00000000000..7e1a973e8c6 --- /dev/null +++ b/pkgs/os-specific/linux/spectrum/testhost/default.nix @@ -0,0 +1,205 @@ +{ lib, runCommandNoCC, writeScript, writeScriptBin, writeShellScript, writeText +, coreutils, cloud-hypervisor, crosvm, curl, execline, gnutar, gnused, iproute +, iptables, jq, kmod, mktuntap, rsync, s6, s6-rc, sys-vms, utillinux +}: + +let + inherit (lib) concatStrings escapeShellArg makeBinPath mapAttrsToList + optionalString; + + compose2 = f: g: a: b: f (g a b); + + concatMapAttrs = compose2 concatStrings mapAttrsToList; + + makeServicesDir = { services }: + runCommandNoCC "services" {} '' + mkdir $out + ${concatMapAttrs (name: attrs: '' + mkdir $out/${name} + ${concatMapAttrs (key: value: '' + cp -r ${value} $out/${name}/${key} + '') attrs} + '') services} + ''; + + s6RcCompile = { fdhuser ? null }: source: + runCommandNoCC "s6-rc-compile" {} '' + ${s6-rc}/bin/s6-rc-compile \ + ${optionalString (fdhuser != null) "-h ${escapeShellArg fdhuser}"} \ + dest ${source} + tar -C dest -cf $out . + ''; + + compiledRcServicesDir = s6RcCompile {} (makeServicesDir { + services = { + vm-app = { + run = writeScript "app-run" '' + #! ${execline}/bin/execlineb -S0 + # fdclose 0 + + # Checking the return value of the bridge creation is + # important, because if it fails due to the bridge already + # existing that means something else could already be using + # this bridge. + if { ip link add name br0 type bridge } + if { ip link set br0 up } + + # Calculate the MACs for our TAP and the router's TAP. + backtick -in router_nic_dec { + expr ${toString sys-vms.app.vmID} * 2 + 64 * 256 * 256 + } + backtick -in client_nic_dec { + expr ${toString sys-vms.app.vmID} * 2 + 64 * 256 * 256 + 1 + } + multisubstitute { + importas -iu router_nic_dec router_nic_dec + importas -iu client_nic_dec client_nic_dec + } + backtick -i router_mac { + pipeline { printf %x $router_nic_dec } + sed s/^\\(..\\)\\(..\\)\\(..\\)$/0A:B3:EC:\\1:\\2:\\3/ + } + backtick -i client_mac { + pipeline { printf %x $client_nic_dec } + sed s/^\\(..\\)\\(..\\)\\(..\\)$/0A:B3:EC:\\1:\\2:\\3/ + } + multisubstitute { + importas -iu router_mac router_mac + importas -iu client_mac client_mac + } + + # Create the net VM end, and attach it to the net VM. + # + # Use a hardcoded name for now because if we use a dynamic + # one iproute2 has no way of telling us the name that was + # chosen: + # https://lore.kernel.org/netdev/20210406134240.wwumpnrzfjbttnmd@eve.qyliss.net/ + define other_tap_name vmtapnet + # Try to delete the device in case the VM was powered off + # (as the finish script wouldn't have been run in that + # case.) Since we check the return value of ip tuntap add, + # in the case of a race condition between deleting the + # device and creating it again, we'll just fail and try + # again. + foreground { ip link delete $other_tap_name } + if { ip tuntap add name $other_tap_name mode tap } + if { ip link set $other_tap_name master br0 } + if { ip link set $other_tap_name up } + if { + pipeline { + jq -n "$ARGS.named" + --arg tap $other_tap_name + --arg mac $router_mac + } + curl -iX PUT + -H "Accept: application/json" + -H "Content-Type: application/json" + --data-binary @- + --unix-socket ../vm-net/env/cloud-hypervisor.sock + http://localhost/api/v1/vm.add-net + } + + mktuntap -pvBi vmtap%d 6 + importas -iu tap_name TUNTAP_NAME + if { ip link set $tap_name master br0 } + if { ip link set $tap_name up } + if { iptables -t nat -A POSTROUTING -o $tap_name -j MASQUERADE } + + ${crosvm}/bin/crosvm run -p init=/sbin/init -p notifyport=''${port} + # --serial type=file,path=/tmp/app.log + --cid 4 + --tap-fd 6,mac=''${client_mac} + --root ${sys-vms.app.rootfs.squashfs} ${sys-vms.app.linux}/bzImage + ''; + finish = writeScript "app-finish" '' + #! ${execline}/bin/execlineb -S0 + # TODO: remove from vm-net + foreground { ip link delete vmtapnet } + ip link delete br0 + ''; + type = writeText "app-type" '' + longrun + ''; + dependencies = writeText "app-dependencies" '' + vm-net + ''; + }; + + vm-net = { + run = writeScript "net-run" '' + #! ${execline}/bin/execlineb -S0 + # This is only necessary for when running s6 from a tty. + # (i.e. when debugging or running the demo). + redirfd -w 0 /dev/null + + define PCI_LOCATION 0000:00:19.0 + define PCI_PATH /sys/bus/pci/devices/''${PCI_LOCATION} + + # Unbind the network device from the driver it's already + # attached to, if any. + foreground { + redirfd -w 1 ''${PCI_PATH}/driver/unbind + printf "%s" $PCI_LOCATION + } + + # (Re)bind the device to the VFIO PCI driver. + if { modprobe vfio-pci } + backtick -in device_id { + if { dd bs=2 skip=1 count=2 status=none if=''${PCI_PATH}/vendor } + if { printf " " } + dd bs=2 skip=1 count=2 status=none if=''${PCI_PATH}/device + } + importas -iu device_id device_id + foreground { + redirfd -w 1 /sys/bus/pci/drivers/vfio-pci/new_id + printf "%s" $device_id + } + + foreground { mkdir env } + + ${cloud-hypervisor}/bin/cloud-hypervisor + --api-socket env/cloud-hypervisor.sock + --console off + # --serial tty + --cmdline "console=ttyS0 panic=30 root=/dev/vda" + --device path=''${PCI_PATH} + --disk path=${sys-vms.net.rootfs.squashfs},readonly=on + --kernel ${sys-vms.net.linux.dev}/vmlinux + ''; + type = writeText "net-type" '' + longrun + ''; + }; + }; + }); + + servicesDir = makeServicesDir { + services = { + ".s6-svscan" = { + finish = writeShellScript ".s6-svscan-finish" ""; + }; + }; + }; +in + +writeScriptBin "spectrum-testhost" '' + #! ${execline}/bin/execlineb -S0 + export PATH ${makeBinPath [ + coreutils curl execline gnused gnutar iproute iptables jq kmod mktuntap rsync + s6 s6-rc + ]} + + if { redirfd -w 1 /proc/sys/net/ipv4/ip_forward echo 1 } + + importas -iu runtime_dir XDG_RUNTIME_DIR + backtick -in TOP { mktemp -dp $runtime_dir spectrum.XXXXXXXXXX } + importas -iu top TOP + if { echo $top } + if { rsync -r --chmod=Du+w ${servicesDir}/ ''${top}/service } + background { + if { mkdir -p ''${top}/s6-rc/compiled } + if { tar -C ''${top}/s6-rc/compiled -xf ${compiledRcServicesDir} } + s6-rc-init -c ''${top}/s6-rc/compiled -l ''${top}/s6-rc/live ''${top}/service + } + s6-svscan ''${top}/service +'' |