Merge pull request #95924 from helsinki-systems/port/hardened-test

nixos/hardened: Port test to Python and fix it

2 files changed, 64 insertions, 57 deletions

diff --git a/nixos/lib/test-driver/test-driver.py b/nixos/lib/test-driver/test-driver.py
index 7b8d5803aa5..f4e2bb6100f 100644
--- a/nixos/lib/test-driver/test-driver.py
+++ b/nixos/lib/test-driver/test-driver.py
@@ -424,15 +424,18 @@ class Machine:
                 output += out
         return output
 
-    def fail(self, *commands: str) -> None:
+    def fail(self, *commands: str) -> str:
         """Execute each command and check that it fails."""
+        output = ""
         for command in commands:
             with self.nested("must fail: {}".format(command)):
-                status, output = self.execute(command)
+                (status, out) = self.execute(command)
                 if status == 0:
                     raise Exception(
                         "command `{}` unexpectedly succeeded".format(command)
                     )
+                output += out
+        return output
 
     def wait_until_succeeds(self, command: str) -> str:
         """Wait until a command returns success and return its output.
diff --git a/nixos/tests/hardened.nix b/nixos/tests/hardened.nix
index 5ed0dfcf9ab..8d845de70e2 100644
--- a/nixos/tests/hardened.nix
+++ b/nixos/tests/hardened.nix
@@ -1,4 +1,4 @@
-import ./make-test.nix ({ pkgs, latestKernel ? false, ... } : {
+import ./make-test-python.nix ({ pkgs, latestKernel ? false, ... } : {
   name = "hardened";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ joachifm ];
@@ -47,84 +47,88 @@ import ./make-test.nix ({ pkgs, latestKernel ? false, ... } : {
       };
     in
     ''
-      $machine->waitForUnit("multi-user.target");
+      machine.wait_for_unit("multi-user.target")
+
+
+      with subtest("AppArmor profiles are loaded"):
+          machine.succeed("systemctl status apparmor.service")
 
-      subtest "apparmor-loaded", sub {
-          $machine->succeed("systemctl status apparmor.service");
-      };
 
       # AppArmor securityfs
-      subtest "apparmor-securityfs", sub {
-          $machine->succeed("mountpoint -q /sys/kernel/security");
-          $machine->succeed("cat /sys/kernel/security/apparmor/profiles");
-      };
+      with subtest("AppArmor securityfs is mounted"):
+          machine.succeed("mountpoint -q /sys/kernel/security")
+          machine.succeed("cat /sys/kernel/security/apparmor/profiles")
+
 
       # Test loading out-of-tree modules
-      subtest "extra-module-packages", sub {
-          $machine->succeed("grep -Fq wireguard /proc/modules");
-      };
+      with subtest("Out-of-tree modules can be loaded"):
+          machine.succeed("grep -Fq wireguard /proc/modules")
+
 
       # Test hidepid
-      subtest "hidepid", sub {
-          $machine->succeed("grep -Fq hidepid=2 /proc/mounts");
+      with subtest("hidepid=2 option is applied and works"):
+          machine.succeed("grep -Fq hidepid=2 /proc/mounts")
           # cannot use pgrep -u here, it segfaults when access to process info is denied
-          $machine->succeed("[ `su - sybil -c 'ps --no-headers --user root | wc -l'` = 0 ]");
-          $machine->succeed("[ `su - alice -c 'ps --no-headers --user root | wc -l'` != 0 ]");
-      };
+          machine.succeed("[ `su - sybil -c 'ps --no-headers --user root | wc -l'` = 0 ]")
+          machine.succeed("[ `su - alice -c 'ps --no-headers --user root | wc -l'` != 0 ]")
+
 
       # Test kernel module hardening
-      subtest "lock-modules", sub {
+      with subtest("No more kernel modules can be loaded"):
           # note: this better a be module we normally wouldn't load ...
-          $machine->fail("modprobe dccp");
-      };
+          machine.fail("modprobe dccp")
+
 
       # Test userns
-      subtest "userns", sub {
-          $machine->succeed("unshare --user true");
-          $machine->fail("su -l alice -c 'unshare --user true'");
-      };
+      with subtest("User namespaces are restricted"):
+          machine.succeed("unshare --user true")
+          machine.fail("su -l alice -c 'unshare --user true'")
+
 
       # Test dmesg restriction
-      subtest "dmesg", sub {
-          $machine->fail("su -l alice -c dmesg");
-      };
+      with subtest("Regular users cannot access dmesg"):
+          machine.fail("su -l alice -c dmesg")
+
 
       # Test access to kcore
-      subtest "kcore", sub {
-          $machine->fail("cat /proc/kcore");
-      };
+      with subtest("Kcore is inaccessible as root"):
+          machine.fail("cat /proc/kcore")
+
 
       # Test deferred mount
-      subtest "mount", sub {
-        $machine->fail("mountpoint -q /efi"); # was deferred
-        $machine->execute("mkdir -p /efi");
-        $machine->succeed("mount /dev/disk/by-label/EFISYS /efi");
-        $machine->succeed("mountpoint -q /efi"); # now mounted
-      };
+      with subtest("Deferred mounts work"):
+          machine.fail("mountpoint -q /efi")  # was deferred
+          machine.execute("mkdir -p /efi")
+          machine.succeed("mount /dev/disk/by-label/EFISYS /efi")
+          machine.succeed("mountpoint -q /efi")  # now mounted
+
 
       # Test Nix dæmon usage
-      subtest "nix-daemon", sub {
-        $machine->fail("su -l nobody -s /bin/sh -c 'nix ping-store'");
-        $machine->succeed("su -l alice -c 'nix ping-store'") =~ "OK";
-      };
+      with subtest("nix-daemon cannot be used by all users"):
+          machine.fail("su -l nobody -s /bin/sh -c 'nix ping-store'")
+          machine.succeed("su -l alice -c 'nix ping-store'")
+
 
       # Test kernel image protection
-      subtest "kernelimage", sub {
-        $machine->fail("systemctl hibernate");
-        $machine->fail("systemctl kexec");
-      };
+      with subtest("The kernel image is protected"):
+          machine.fail("systemctl hibernate")
+          machine.fail("systemctl kexec")
 
-      # Test hardened memory allocator
-      sub runMallocTestProg {
-          my ($progName, $errorText) = @_;
-          my $text = "fatal allocator error: " . $errorText;
-          $machine->fail("${hardened-malloc-tests}/bin/" . $progName) =~ $text;
-      };
 
-      subtest "hardenedmalloc", sub {
-        runMallocTestProg("double_free_large", "invalid free");
-        runMallocTestProg("unaligned_free_small", "invalid unaligned free");
-        runMallocTestProg("write_after_free_small", "detected write after free");
-      };
+      # Test hardened memory allocator
+      def runMallocTestProg(prog_name, error_text):
+          text = "fatal allocator error: " + error_text
+          if not text in machine.fail(
+              "${hardened-malloc-tests}/bin/"
+              + prog_name
+              + " 2>&1"
+          ):
+              raise Exception("Hardened malloc does not work for {}".format(error_text))
+
+
+      with subtest("The hardened memory allocator works"):
+          runMallocTestProg("double_free_large", "invalid free")
+          runMallocTestProg("unaligned_free_small", "invalid unaligned free")
+          runMallocTestProg("write_after_free_small", "detected write after free")
     '';
 })