wine: improve Darwin support

- Add a setup hook to allow Darwin to build PE DLLs using MinGW;
- Add a postConfigure script to fix preloader brekage on Apple Silicon
  Macs running under Rosetta 2;
- Disable linking against X11 libraries (pulled in by ffmpeg); and
- Allow building Wine 7.0 using the 10.12 SDK on x86_64-darwin.

3 files changed, 101 insertions, 4 deletions

diff --git a/pkgs/applications/emulators/wine/base.nix b/pkgs/applications/emulators/wine/base.nix
index 89aa4b3e68c..131f573288b 100644
--- a/pkgs/applications/emulators/wine/base.nix
+++ b/pkgs/applications/emulators/wine/base.nix
@@ -1,4 +1,4 @@
-{ stdenv, lib, pkgArches, callPackage,
+{ stdenv, lib, pkgArches, callPackage, makeSetupHook,
   name, version, src, mingwGccs, monos, geckos, platforms,
   bison, flex, fontforge, makeWrapper, pkg-config,
   autoconf, hexdump, perl, nixosTests,
@@ -15,9 +15,31 @@ let
   prevName = name;
   prevPlatforms = platforms;
   prevConfigFlags = configureFlags;
+  setupHookDarwin = makeSetupHook {
+    name = "darwin-mingw-hook";
+    substitutions = {
+      darwinSuffixSalt = stdenv.cc.suffixSalt;
+      mingwGccsSuffixSalts = map (gcc: gcc.suffixSalt) mingwGccs;
+    };
+  } ./setup-hook-darwin.sh;
 in
 stdenv.mkDerivation ((lib.optionalAttrs (buildScript != null) {
   builder = buildScript;
+}) // (lib.optionalAttrs stdenv.isDarwin {
+  postConfigure = ''
+    # dynamic fallback, so this shouldn’t cause problems for older versions of macOS and will
+    # provide additional functionality on newer ones. This can be removed once the x86_64-darwin
+    # SDK is updated.
+    sed 's|/\* #undef HAVE_MTLDEVICE_REGISTRYID \*/|#define HAVE_MTLDEVICE_REGISTRYID 1|' \
+      -i include/config.h
+  '';
+  postBuild = ''
+    # The Wine preloader must _not_ be linked to any system libraries, but `NIX_LDFLAGS` will link
+    # to libintl, libiconv, and CoreFoundation no matter what. Delete the one that was built and
+    # rebuild it with empty NIX_LDFLAGS.
+    rm loader/wine64-preloader
+    make loader/wine64-preloader NIX_LDFLAGS="" NIX_LDFLAGS_${stdenv.cc.suffixSalt}=""
+  '';
 }) // rec {
   inherit src;
 
@@ -38,11 +60,13 @@ stdenv.mkDerivation ((lib.optionalAttrs (buildScript != null) {
     hexdump
     perl
   ]
-  ++ lib.optionals supportFlags.mingwSupport mingwGccs;
+  ++ lib.optionals supportFlags.mingwSupport (mingwGccs
+    ++ lib.optional stdenv.isDarwin setupHookDarwin);
 
   buildInputs = toBuildInputs pkgArches (with supportFlags; (pkgs:
   [ pkgs.freetype pkgs.perl pkgs.libunwind ]
   ++ lib.optional stdenv.isLinux         pkgs.libcap
+  ++ lib.optional stdenv.isDarwin        pkgs.libinotify-kqueue
   ++ lib.optional cupsSupport            pkgs.cups
   ++ lib.optional gettextSupport         pkgs.gettext
   ++ lib.optional dbusSupport            pkgs.dbus
@@ -85,12 +109,17 @@ stdenv.mkDerivation ((lib.optionalAttrs (buildScript != null) {
      wayland libxkbcommon wayland-protocols wayland.dev libxkbcommon.dev
   ])));
 
-  patches = [ ] ++ patches';
+  patches = [ ]
+    # Wine requires `MTLDevice.registryID` for `winemac.drv`, but that property is not available
+    # in the 10.12 SDK (current SDK on x86_64-darwin). Work around that by using selector syntax.
+    ++ lib.optional stdenv.isDarwin ./darwin-metal-compat.patch
+    ++ patches';
 
   configureFlags = prevConfigFlags
     ++ lib.optionals supportFlags.waylandSupport [ "--with-wayland" ]
     ++ lib.optionals supportFlags.vulkanSupport [ "--with-vulkan" ]
-    ++ lib.optionals supportFlags.vkd3dSupport [ "--with-vkd3d" ];
+    ++ lib.optionals supportFlags.vkd3dSupport [ "--with-vkd3d" ]
+    ++ lib.optionals (stdenv.isDarwin && !supportFlags.xineramaSupport) [ "--without-x" ];
 
   # Wine locates a lot of libraries dynamically through dlopen().  Add
   # them to the RPATH so that the user doesn't have to set them in
diff --git a/pkgs/applications/emulators/wine/darwin-metal-compat.patch b/pkgs/applications/emulators/wine/darwin-metal-compat.patch
new file mode 100644
index 00000000000..aeee7533bbd
--- /dev/null
+++ b/pkgs/applications/emulators/wine/darwin-metal-compat.patch
@@ -0,0 +1,31 @@
+diff --git a/dlls/winemac.drv/cocoa_display.m b/dlls/winemac.drv/cocoa_display.m
+index f64a6c0f6ad..6da0391e3fa 100644
+--- a/dlls/winemac.drv/cocoa_display.m
++++ b/dlls/winemac.drv/cocoa_display.m
+@@ -289,7 +289,7 @@ static int macdrv_get_gpus_from_metal(struct macdrv_gpu** new_gpus, int* count)
+      * the primary GPU because we need to hide the integrated GPU for an automatic graphic switching pair to avoid apps
+      * using the integrated GPU. This is the behavior of Windows on a Mac. */
+     primary_device = [MTLCreateSystemDefaultDevice() autorelease];
+-    if (macdrv_get_gpu_info_from_registry_id(&primary_gpu, primary_device.registryID))
++    if (macdrv_get_gpu_info_from_registry_id(&primary_gpu, (uint64_t)[primary_device registryID]))
+         goto done;
+ 
+     /* Hide the integrated GPU if the system default device is a dedicated GPU */
+@@ -301,7 +301,7 @@ static int macdrv_get_gpus_from_metal(struct macdrv_gpu** new_gpus, int* count)
+ 
+     for (i = 0; i < devices.count; i++)
+     {
+-        if (macdrv_get_gpu_info_from_registry_id(&gpus[gpu_count], devices[i].registryID))
++        if (macdrv_get_gpu_info_from_registry_id(&gpus[gpu_count], (uint64_t)[devices[i] registryID]))
+             goto done;
+ 
+         if (hide_integrated && devices[i].isLowPower)
+@@ -354,7 +354,7 @@ static int macdrv_get_gpu_info_from_display_id_using_metal(struct macdrv_gpu* gp
+ 
+     device = [CGDirectDisplayCopyCurrentMetalDevice(display_id) autorelease];
+     if (device && [device respondsToSelector:@selector(registryID)])
+-        ret = macdrv_get_gpu_info_from_registry_id(gpu, device.registryID);
++        ret = macdrv_get_gpu_info_from_registry_id(gpu, (uint64_t)[device registryID]);
+ 
+ done:
+     [pool release];
diff --git a/pkgs/applications/emulators/wine/setup-hook-darwin.sh b/pkgs/applications/emulators/wine/setup-hook-darwin.sh
new file mode 100644
index 00000000000..39eee193ded
--- /dev/null
+++ b/pkgs/applications/emulators/wine/setup-hook-darwin.sh
@@ -0,0 +1,37 @@
+
+fixupCFlagsForDarwin() {
+    # Because it’s getting called from a Darwin stdenv, MinGW will pick up on Darwin-specific
+    # flags, and the ./configure tests will fail to consider it a working cross-compiler.
+    # Strip them out, so Wine can use MinGW to build its DLLs instead of trying to use Clang.
+    # Ideally, it would be possible to build the DLLs on Windows (i.e., as part of `pkgsCross``),
+    # but that is not the case currently with Wine’s build system.
+    cflagsFilter='s|-F[^ ]*||g;s|-iframework [^ ]*||g;s|-isystem [^ ]*||g;s|  *| |g'
+
+    # libiconv and libintl aren’t needed by Wine, and having them causes linking to fail.
+    # The `CoreFoundation` reference is added by `linkSystemCoreFoundationFramework` in the
+    # Apple SDK’s setup hook. Remove that because MingW will fail due to file not found.
+    ldFlagsFilter='s|-lintl||g;s|-liconv||g;s|/nix/store/[^-]*-apple-framework-CoreFoundation[^ ]*||g'
+
+    # `cc-wrapper.sh`` supports getting flags from a system-specific salt. While it is currently a
+    # tuple, that’s not considered a stable interface, so the Wine derivation will provide them:
+    # - for Darwin: The source is `stdenv.cc.suffixSalt`; and
+    # - for MinGW: The source is the `suffixSalt`` attribute of each of the `mingwGccs`.
+    export NIX_CFLAGS_COMPILE_@darwinSuffixSalt@=${NIX_CFLAGS_COMPILE-}
+    export NIX_LDFLAGS_@darwinSuffixSalt@=${NIX_LDFLAGS-}
+    for mingwSalt in @mingwGccsSuffixSalts@; do
+        echo removing @darwinSuffixSalt@-specific flags from NIX_CFLAGS_COMPILE for $mingwSalt
+        export NIX_CFLAGS_COMPILE_$mingwSalt+="$(sed "$cflagsFilter" <<< "$NIX_CFLAGS_COMPILE")"
+        echo removing @darwinSuffixSalt@-specific flags from NIX_LDFLAGS for $mingwSalt
+        export NIX_LDFLAGS_$mingwSalt+="$(sed "$ldFlagsFilter;$cflagsFilter" <<< "$NIX_LDFLAGS")"
+    done
+
+    # Make sure the global flags aren’t accidentally influencing the platform-specific flags.
+    export NIX_CFLAGS_COMPILE=""
+    export NIX_LDFLAGS=""
+}
+
+# This is pretty hacky, but this hook _must_ run after `linkSystemCoreFoundationFramework`.
+function runFixupCFlagsForDarwinLast() {
+    preConfigureHooks+=(fixupCFlagsForDarwin)
+}
+postUnpackHooks+=(runFixupCFlagsForDarwinLast)