diff options
Diffstat (limited to 'nixos/lib/testing-python.nix')
-rw-r--r-- | nixos/lib/testing-python.nix | 251 |
1 files changed, 251 insertions, 0 deletions
diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix new file mode 100644 index 00000000000..0d3c3a89e78 --- /dev/null +++ b/nixos/lib/testing-python.nix @@ -0,0 +1,251 @@ +{ system +, pkgs ? import ../.. { inherit system config; } + # Use a minimal kernel? +, minimal ? false + # Ignored +, config ? { } + # !!! See comment about args in lib/modules.nix +, specialArgs ? { } + # Modules to add to each VM +, extraConfigurations ? [ ] +}: + +with pkgs; + +rec { + + inherit pkgs; + + # Run an automated test suite in the given virtual network. + runTests = { driver, driverInteractive, pos }: + stdenv.mkDerivation { + name = "vm-test-run-${driver.testName}"; + + requiredSystemFeatures = [ "kvm" "nixos-test" ]; + + buildCommand = + '' + mkdir -p $out + + # effectively mute the XMLLogger + export LOGFILE=/dev/null + + ${driver}/bin/nixos-test-driver -o $out + ''; + + passthru = driver.passthru // { + inherit driver driverInteractive; + }; + + inherit pos; # for better debugging + }; + + # Generate convenience wrappers for running the test driver + # has vlans, vms and test script defaulted through env variables + # also instantiates test script with nodes, if it's a function (contract) + setupDriverForTest = { + testScript + , testName + , nodes + , qemu_pkg ? pkgs.qemu_test + , enableOCR ? false + , skipLint ? false + , passthru ? {} + , interactive ? false + }: + let + # Reifies and correctly wraps the python test driver for + # the respective qemu version and with or without ocr support + testDriver = pkgs.callPackage ./test-driver { + inherit enableOCR; + qemu_pkg = qemu_test; + imagemagick_light = imagemagick_light.override { inherit libtiff; }; + tesseract4 = tesseract4.override { enableLanguages = [ "eng" ]; }; + }; + + + testDriverName = + let + # A standard store path to the vm monitor is built like this: + # /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor + # The max filename length of a unix domain socket is 108 bytes. + # This means $name can at most be 50 bytes long. + maxTestNameLen = 50; + testNameLen = builtins.stringLength testName; + in with builtins; + if testNameLen > maxTestNameLen then + abort + ("The name of the test '${testName}' must not be longer than ${toString maxTestNameLen} " + + "it's currently ${toString testNameLen} characters long.") + else + "nixos-test-driver-${testName}"; + + vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes); + vms = map (m: m.config.system.build.vm) (lib.attrValues nodes); + + nodeHostNames = let + nodesList = map (c: c.config.system.name) (lib.attrValues nodes); + in nodesList ++ lib.optional (lib.length nodesList == 1) "machine"; + + # TODO: This is an implementation error and needs fixing + # the testing famework cannot legitimately restrict hostnames further + # beyond RFC1035 + invalidNodeNames = lib.filter + (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) + nodeHostNames; + + testScript' = + # Call the test script with the computed nodes. + if lib.isFunction testScript + then testScript { inherit nodes; } + else testScript; + + in + if lib.length invalidNodeNames > 0 then + throw '' + Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})! + All machines are referenced as python variables in the testing framework which will break the + script when special characters are used. + + This is an IMPLEMENTATION ERROR and needs to be fixed. Meanwhile, + please stick to alphanumeric chars and underscores as separation. + '' + else lib.warnIf skipLint "Linting is disabled" (runCommand testDriverName + { + inherit testName; + nativeBuildInputs = [ makeWrapper ]; + testScript = testScript'; + preferLocalBuild = true; + passthru = passthru // { + inherit nodes; + }; + } + '' + mkdir -p $out/bin + + vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) + echo -n "$testScript" > $out/test-script + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver + + ${testDriver}/bin/generate-driver-symbols + ${lib.optionalString (!skipLint) '' + PYFLAKES_BUILTINS="$( + echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)}, + < ${lib.escapeShellArg "driver-symbols"} + )" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script + ''} + + # set defaults through environment + # see: ./test-driver/test-driver.py argparse implementation + wrapProgram $out/bin/nixos-test-driver \ + --set startScripts "''${vmStartScripts[*]}" \ + --set testScript "$out/test-script" \ + --set vlans '${toString vlans}' \ + ${lib.optionalString (interactive) "--add-flags --interactive"} + ''); + + # Make a full-blown test + makeTest = + { testScript + , enableOCR ? false + , name ? "unnamed" + # Skip linting (mainly intended for faster dev cycles) + , skipLint ? false + , passthru ? {} + , # For meta.position + pos ? # position used in error messages and for meta.position + (if t.meta.description or null != null + then builtins.unsafeGetAttrPos "description" t.meta + else builtins.unsafeGetAttrPos "testScript" t) + , ... + } @ t: + let + nodes = qemu_pkg: + let + testScript' = + # Call the test script with the computed nodes. + if lib.isFunction testScript + then testScript { nodes = nodes qemu_pkg; } + else testScript; + + build-vms = import ./build-vms.nix { + inherit system lib pkgs minimal specialArgs; + extraConfigurations = extraConfigurations ++ [( + { config, ... }: + { + virtualisation.qemu.package = qemu_pkg; + + # Make sure all derivations referenced by the test + # script are available on the nodes. When the store is + # accessed through 9p, this isn't important, since + # everything in the store is available to the guest, + # but when building a root image it is, as all paths + # that should be available to the guest has to be + # copied to the image. + virtualisation.additionalPaths = + lib.optional + # A testScript may evaluate nodes, which has caused + # infinite recursions. The demand cycle involves: + # testScript --> + # nodes --> + # toplevel --> + # additionalPaths --> + # hasContext testScript' --> + # testScript (ad infinitum) + # If we don't need to build an image, we can break this + # cycle by short-circuiting when useNixStoreImage is false. + (config.virtualisation.useNixStoreImage && builtins.hasContext testScript') + (pkgs.writeStringReferencesToFile testScript'); + + # Ensure we do not use aliases. Ideally this is only set + # when the test framework is used by Nixpkgs NixOS tests. + nixpkgs.config.allowAliases = false; + } + )]; + }; + in + build-vms.buildVirtualNetwork ( + t.nodes or (if t ? machine then { machine = t.machine; } else { }) + ); + + driver = setupDriverForTest { + inherit testScript enableOCR skipLint passthru; + testName = name; + qemu_pkg = pkgs.qemu_test; + nodes = nodes pkgs.qemu_test; + }; + driverInteractive = setupDriverForTest { + inherit testScript enableOCR skipLint passthru; + testName = name; + qemu_pkg = pkgs.qemu; + nodes = nodes pkgs.qemu; + interactive = true; + }; + + test = + let + passMeta = drv: drv // lib.optionalAttrs (t ? meta) { + meta = (drv.meta or { }) // t.meta; + }; + in passMeta (runTests { inherit driver pos driverInteractive; }); + + in + test // { + inherit test driver driverInteractive nodes; + }; + + abortForFunction = functionName: abort ''The ${functionName} function was + removed because it is not an essential part of the NixOS testing + infrastructure. It had no usage in NixOS or Nixpkgs and it had no designated + maintainer. You are free to reintroduce it by documenting it in the manual + and adding yourself as maintainer. It was removed in + https://github.com/NixOS/nixpkgs/pull/137013 + ''; + + runInMachine = abortForFunction "runInMachine"; + + runInMachineWithX = abortForFunction "runInMachineWithX"; + + simpleTest = as: (makeTest as).test; + +} |