openssl: stop static binaries referencing libs

Previously, the "out" output of openssl would contain would contain a
couple of tiny libraries in etc/, and the big OpenSSL libraries in
lib/.  This bloated closures when building things against OpenSSL with
pkgsStatic.  To fix this, introduce a lib output, so only the config
files are left in out.

Additionally, we have to disable support for dynamic engines in static
builds to avoid a reference to the engines directory in $lib.  I don't
think it's likely that this would ever have worked anyway.

1 files changed, 16 insertions, 2 deletions

diff --git a/pkgs/development/libraries/openssl/default.nix b/pkgs/development/libraries/openssl/default.nix
index 33ddbf7018c..3e190d4b450 100644
--- a/pkgs/development/libraries/openssl/default.nix
+++ b/pkgs/development/libraries/openssl/default.nix
@@ -45,7 +45,7 @@ let
                   '!defined(__ANDROID__) && !defined(__OpenBSD__) && 0'
     '';
 
-    outputs = [ "bin" "dev" "out" "man" ] ++ lib.optional withDocs "doc";
+    outputs = [ "bin" "dev" "out" "lib" "man" ] ++ lib.optional withDocs "doc";
     setOutputFlags = false;
     separateDebugInfo =
       !stdenv.hostPlatform.isDarwin &&
@@ -94,7 +94,7 @@ let
     dontAddStaticConfigureFlags = true;
     configureFlags = [
       "shared" # "shared" builds both shared and static libraries
-      "--libdir=lib"
+      "--libdir=${placeholder "lib"}/lib"
       "--openssldir=etc/ssl"
     ] ++ lib.optionals withCryptodev [
       "-DHAVE_CRYPTODEV"
@@ -103,6 +103,7 @@ let
       ++ lib.optional enableSSL3 "enable-ssl3"
       ++ lib.optional (lib.versionAtLeast version "3.0.0") "enable-ktls"
       ++ lib.optional (lib.versionAtLeast version "1.1.0" && stdenv.hostPlatform.isAarch64) "no-afalgeng"
+      ++ lib.optional static "disable-dynamic-engine"
       # OpenSSL needs a specific `no-shared` configure flag.
       # See https://wiki.openssl.org/index.php/Compilation_and_Installation#Configure_Options
       # for a comprehensive list of configuration options.
@@ -117,6 +118,19 @@ let
       "MANSUFFIX=ssl"
     ];
 
+    buildFlags = lib.optionals static [
+      # Even though engines are disabled in static builds, we have to
+      # override ENGINESDIR so the bin output doesn't end up with an
+      # reference to the lib output.
+      "ENGINESDIR=/"
+    ];
+
+    installFlags = lib.optionals static [
+      # Build system wants to be able to create the engines directory
+      # even though nothing will get installed to it.
+      "ENGINESDIR=/build/engines"
+    ];
+
     enableParallelBuilding = true;
 
     postInstall =