darwin.stdenv: use CoreFoundation instead of CF

This patch switches the CoreFoundation on x86_64-darwin from the open
source swift-corelibs-foundation (CF) to the system CoreFoundation.

This change was motivated by failures building packages for the current
staging-next cycle #263535 due to an apparent incompatibility with the
rpath-based approach to choosing CF or CoreFoundation and macOS 14. This
error often manifests as a crash with an Illegal Instruction.

For example, building aws-sdk-cpp for building Nix will fail this way.

https://hydra.nixos.org/build/239459417/nixlog/1

    Application Specific Information:
    CF objects must have a non-zero isa

    Error Formulating Crash Report:
    PC register does not match crashing frame (0x0 vs 0x7FF8094DD640)

    Thread 0 Crashed::  Dispatch queue: com.apple.main-thread
    0   CoreFoundation                	    0x7ff8094dd640 CF_IS_OBJC.cold.1 + 14
    1   CoreFoundation                	    0x7ff8094501d0 CF_IS_OBJC + 60
    2   CoreFoundation                	    0x7ff8093155e8 CFRelease + 40
    3   ???                           	       0x10c7a2c61 s_aws_secure_transport_ctx_destroy + 65
    4   ???                           	       0x10c87ba32 aws_ref_count_release + 34
    5   ???                           	       0x10c7b7adb aws_tls_connection_options_clean_up + 27
    6   ???                           	       0x10c596db4 Aws::Crt::Io::TlsConnectionOptions::~TlsConnectionOptions() + 20
    7   ???                           	       0x10c2d249c Aws::CleanupCrt() + 92
    8   ???                           	       0x10c2d1ff0 Aws::ShutdownAPI(Aws::SDKOptions const&) + 64
    9   ???                           	       0x102d9bc6f main + 335
    10  dyld                          	       0x202f333a6 start + 1942

According to a [post][1] on the Apple developer forums, hardening was
added to CoreFoundation, and this particular message occurs when you
attempt to release an object it does not recognize as a valid CF object.
(Thank you to @lilyinstarlight for finding this post).

When I switched aws-sdk-cpp to link against CoreFoundation instead of
CF, the error went away. Somehow both libraries were being used. To
prevent dependent packages from linking the wrong CoreFoundation, it
would need to be added as a propagated build input.

Note that there are other issues related to mixing CF and CoreFoundation
frameworks. #264503 fixes an issue with abseil-cpp where it propagates
CF, causing issues when using a different SDK version. Mixing versions
can also cause crashes with Python when a shared object is loaded that
is linked to the “wrong” CoreFoundation.

`NIX_COREFOUNDATION_RPATH` is supposed to make sure the right
CoreFoundation is being used, but it does not appear to be enough on
macOS 14 (presumably due to the hardening). While it is possible to
propagate CoreFoundation manually, the cleaner solution is to make it
the default. CF remains available as `darwin.swift-corelibs-foundation`.

[1]: https://developer.apple.com/forums/thread/739355

1 files changed, 42 insertions, 23 deletions

diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix
index 9ee6dfb1080..c94c56daae1 100644
--- a/pkgs/stdenv/darwin/default.nix
+++ b/pkgs/stdenv/darwin/default.nix
@@ -245,7 +245,8 @@ in
       coreutils = bootstrapTools;
       gnugrep = bootstrapTools;
 
-      pbzx = bootstrapTools;
+      # Either pbzx or Libsystem is required from bootstrap tools (one is used building the other).
+      pbzx = if localSystem.isAarch64 then bootstrapTools else super.pbzx;
       cpio = self.stdenv.mkDerivation {
         name = "bootstrap-stage0-cpio";
         buildCommand = ''
@@ -255,7 +256,11 @@ in
         passthru.isFromBootstrapFiles = true;
       };
 
-      darwin = super.darwin.overrideScope (selfDarwin: _: {
+      darwin = super.darwin.overrideScope (selfDarwin: superDarwin: {
+        # Prevent CF from being propagated to the initial stdenv. Packages that require it
+        # will have to manually add it to their build inputs.
+        CF = null;
+
         binutils-unwrapped = bootstrapTools // {
           version = "boot";
         };
@@ -296,15 +301,6 @@ in
 
         sigtool = bootstrapTools;
       } // lib.optionalAttrs (! useAppleSDKLibs) {
-        CF = self.stdenv.mkDerivation {
-          name = "bootstrap-stage0-CF";
-          buildCommand = ''
-            mkdir -p $out/Library/Frameworks
-            ln -s ${bootstrapTools}/Library/Frameworks/CoreFoundation.framework $out/Library/Frameworks
-          '';
-          passthru.isFromBootstrapFiles = true;
-        };
-
         Libsystem = self.stdenv.mkDerivation {
           name = "bootstrap-stage0-Libsystem";
           buildCommand = ''
@@ -424,15 +420,18 @@ in
   # making sure both packages are present on x86_64-darwin and aarch64-darwin.
   (prevStage:
     # previous stage0 stdenv:
-    assert lib.all isFromBootstrapFiles (with prevStage; [ bash coreutils cpio gnugrep pbzx ]);
+    assert lib.all isFromBootstrapFiles (
+      with prevStage; [ bash coreutils cpio gnugrep ] ++ lib.optionals useAppleSDKLibs [ pbzx ]
+    );
 
     assert lib.all isFromBootstrapFiles (with prevStage.darwin; [
       binutils-unwrapped cctools print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isFromBootstrapFiles (with prevStage.darwin; [ CF Libsystem ]);
-    assert    useAppleSDKLibs  -> lib.all        isFromNixpkgs (with prevStage.darwin; [ CF Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all isFromBootstrapFiles (with prevStage.darwin; [ Libsystem ]);
+    assert    useAppleSDKLibs  -> lib.all        isFromNixpkgs (with prevStage.darwin; [ Libsystem ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd xnu ]);
+    assert (with prevStage.darwin; (! useAppleSDKLibs) -> CF == null);
 
     assert lib.all isFromBootstrapFiles (with prevStage.llvmPackages; [
       clang-unwrapped libclang libllvm llvm compiler-rt libcxx libcxxabi
@@ -445,7 +444,11 @@ in
       inherit (prevStage) ccWrapperStdenv
         coreutils gnugrep;
 
-      cmake = super.cmakeMinimal;
+      # Use this stage’s CF to build CMake. It’s required but can’t be included in the stdenv.
+      cmake = self.cmakeMinimal;
+      cmakeMinimal = super.cmakeMinimal.overrideAttrs (old: {
+        buildInputs = old.buildInputs ++ [ self.darwin.CF ];
+      });
 
       curl = super.curlMinimal;
 
@@ -457,9 +460,18 @@ in
 
       ninja = super.ninja.override { buildDocs = false; };
 
-      python3 = super.python3Minimal;
+      # Use this stage’s CF to build Python. It’s required but can’t be included in the stdenv.
+      python3 = self.python3Minimal;
+      python3Minimal = super.python3Minimal.overrideAttrs (old: {
+        buildInputs = old.buildInputs ++ [ self.darwin.CF ];
+      });
 
       darwin = super.darwin.overrideScope (selfDarwin: superDarwin: {
+        # Use this stage’s CF to build configd. It’s required but can’t be included in the stdenv.
+        configd = superDarwin.configd.overrideAttrs (old: {
+          buildInputs = old.buildInputs or [ ] ++ [ self.darwin.CF ];
+        });
+
         signingUtils = prevStage.darwin.signingUtils.override {
           inherit (selfDarwin) sigtool;
         };
@@ -529,7 +541,8 @@ in
     assert lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [
       binutils-unwrapped cctools locale libtapi print-reexports rewrite-tbd sigtool
     ]);
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ CF Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd xnu ]);
 
@@ -628,7 +641,8 @@ in
       binutils-unwrapped cctools locale libtapi print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ CF Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd xnu ]);
 
@@ -725,7 +739,8 @@ in
       binutils-unwrapped cctools locale libtapi print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ CF Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);
 
@@ -824,8 +839,9 @@ in
       binutils-unwrapped cctools locale libtapi print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ CF configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ configd ]);
     assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);
 
@@ -951,7 +967,8 @@ in
     ]);
 
     assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ configd ]);
-    assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ CF Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);
 
@@ -1031,7 +1048,8 @@ in
     ]);
 
     assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ configd ]);
-    assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ CF Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);
 
@@ -1197,7 +1215,8 @@ in
       binutils-unwrapped cctools libtapi locale print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByNixpkgsCompiler (with prevStage.darwin; [ CF Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByNixpkgsCompiler (with prevStage.darwin; [ Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all            isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all            isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);