mailman: refactor package structure

* Removed unused `.package`-option.
* Added explicit postgresql support.
* Create a new meta-package for mailman to make sure each component has
  the **same** python and packages can be downgraded if needed (e.g.
  psycopg2 or sqlalchemy) without interfering with `pythonPackages` in any way.
* Document why certain python overrides are needed.

Closes #170035
Closes #158424

8 files changed, 183 insertions, 137 deletions

diff --git a/pkgs/servers/mail/mailman/default.nix b/pkgs/servers/mail/mailman/default.nix
index f5632df7949..7345f78dec5 100644
--- a/pkgs/servers/mail/mailman/default.nix
+++ b/pkgs/servers/mail/mailman/default.nix
@@ -1,95 +1,30 @@
-{ lib, buildPythonPackage, fetchPypi, fetchpatch, pythonOlder, python3, postfix, lynx
-}:
+{ newScope, lib, python3 }:
 
 let
-  # Mailman does not support sqlalchemy >= 1.4 https://gitlab.com/mailman/mailman/-/issues/845
-  pythonOverride = python3.override {
-    packageOverrides = self: super: {
-      alembic = super.alembic.overridePythonAttrs (oldAttrs:  {
-        # does not find tests
-        doCheck = false;
-      });
-      sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec {
-        version = "1.3.24";
-        src = oldAttrs.src.override {
-          inherit version;
-          hash = "sha256-67t3fL+TEjWbiXv4G6ANrg9ctp+6KhgmXcwYpvXvdRk=";
-        };
-        # does not find tests
-        doCheck = false;
-      });
-    };
-  };
-in
+  callPackage = newScope self;
 
-buildPythonPackage rec {
-  pname = "mailman";
-  version = "3.3.5";
-  disabled = pythonOlder "3.6";
+  self = lib.makeExtensible (self: {
+    python3 = callPackage ./python.nix { inherit python3; };
 
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "12mgxs1ndhdjjkydx48b95na9k9h0disfqgrr6wxx7vda6dqvcwz";
-  };
+    hyperkitty = callPackage ./hyperkitty.nix { };
 
-  propagatedBuildInputs = with pythonOverride.pkgs; [
-    aiosmtpd
-    alembic
-    authheaders
-    click
-    dnspython
-    falcon
-    flufl_bounce
-    flufl_i18n
-    flufl_lock
-    gunicorn
-    importlib-resources
-    lazr_config
-    passlib
-    requests
-    sqlalchemy
-    zope_component
-    zope_configuration
-  ];
+    mailman = callPackage ./package.nix { };
 
-  patches = [
-    (fetchpatch {
-      url = "https://gitlab.com/mailman/mailman/-/commit/4b206e2a5267a0e17f345fd7b2d957122ba57566.patch";
-      sha256 = "06axmrn74p81wvcki36c7gfj5fp5q15zxz2yl3lrvijic7hbs4n2";
-    })
-    (fetchpatch {
-      url = "https://gitlab.com/mailman/mailman/-/commit/9613154f3c04fa2383fbf017031ef263c291418d.patch";
-      sha256 = "0vyw87s857vfxbf7kihwb6w094xyxmxbi1bpdqi3ybjamjycp55r";
-    })
-    ./log-stderr.patch
-  ];
+    mailman-hyperkitty = callPackage ./mailman-hyperkitty.nix { };
 
-  postPatch = ''
-    substituteInPlace setup.py \
-      --replace "alembic>=1.6.2,<1.7" "alembic>=1.6.2"
+    postorius = callPackage ./postorius.nix { };
 
-    substituteInPlace src/mailman/config/postfix.cfg \
-      --replace /usr/sbin/postmap ${postfix}/bin/postmap
-    substituteInPlace src/mailman/config/schema.cfg \
-      --replace /usr/bin/lynx ${lynx}/bin/lynx
-  '';
+    web = callPackage ./web.nix { };
 
-  # Mailman assumes that those scripts in $out/bin are Python scripts. Wrapping
-  # them in shell code breaks this assumption. Use the wrapped version (see
-  # wrapped.nix) if you need the CLI (rather than the Python library).
-  #
-  # This gives a properly wrapped 'mailman' command plus an interpreter that
-  # has all the necessary search paths to execute unwrapped 'master' and
-  # 'runner' scripts.
-  dontWrapPythonPrograms = true;
+    buildEnv = { web ? self.web
+               , mailman ? self.mailman
+               , mailman-hyperkitty ? self.mailman-hyperkitty
+               , withHyperkitty ? false
+               }:
+      self.python3.withPackages
+        (ps:
+          [ web mailman ps.psycopg2 ]
+          ++ lib.optional withHyperkitty mailman-hyperkitty);
+  });
 
-  # requires flufl.testing, which the upstream has archived
-  doCheck = false;
-
-  meta = {
-    homepage = "https://www.gnu.org/software/mailman/";
-    description = "Free software for managing electronic mail discussion and newsletter lists";
-    license = lib.licenses.gpl3Plus;
-    maintainers = with lib.maintainers; [ qyliss ];
-  };
-}
+in self
diff --git a/pkgs/servers/mail/mailman/hyperkitty.nix b/pkgs/servers/mail/mailman/hyperkitty.nix
index b84e78e323d..c10f2c1ba75 100644
--- a/pkgs/servers/mail/mailman/hyperkitty.nix
+++ b/pkgs/servers/mail/mailman/hyperkitty.nix
@@ -1,34 +1,10 @@
 { lib
-, buildPythonPackage
+, python3
 , fetchpatch
-, fetchPypi
-, pythonOlder
-
-# dependencies
-, defusedxml
-, django
-, django-gravatar2
-, django-haystack
-, django-mailman3
-, django-paintstore
-, django-q
-, django_compressor
-, django-extensions
-, djangorestframework
-, flufl_lock
-, mistune_2_0
-, networkx
-, psycopg2
-, python-dateutil
-, robot-detection
-
-# tests
-, beautifulsoup4
-, elasticsearch
-, mock
-, whoosh
 }:
 
+with python3.pkgs;
+
 buildPythonPackage rec {
   pname = "HyperKitty";
   # Note: Mailman core must be on the latest version before upgrading HyperKitty.
diff --git a/pkgs/servers/mail/mailman/mailman-hyperkitty.nix b/pkgs/servers/mail/mailman/mailman-hyperkitty.nix
new file mode 100644
index 00000000000..39f9b5ec37a
--- /dev/null
+++ b/pkgs/servers/mail/mailman/mailman-hyperkitty.nix
@@ -0,0 +1,47 @@
+{ lib
+, python3
+, mailman
+}:
+
+with python3.pkgs;
+buildPythonPackage rec {
+  pname = "mailman-hyperkitty";
+  version = "1.2.0";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.9";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-EQBx1KX3z/Wv3QAHOi+s/ihLOjpiupIQBYyE6IPbJto=";
+  };
+
+  propagatedBuildInputs = [
+    mailman
+    requests
+    zope_interface
+  ];
+
+  checkInputs = [
+    mock
+    nose2
+  ];
+
+  checkPhase = ''
+    ${python.interpreter} -m nose2 -v
+  '';
+
+  # There is an AssertionError
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "mailman_hyperkitty"
+  ];
+
+  meta = with lib; {
+    description = "Mailman archiver plugin for HyperKitty";
+    homepage = "https://gitlab.com/mailman/mailman-hyperkitty";
+    license = licenses.gpl3Plus;
+    maintainers = with maintainers; [ globin qyliss ];
+  };
+}
diff --git a/pkgs/servers/mail/mailman/package.nix b/pkgs/servers/mail/mailman/package.nix
new file mode 100644
index 00000000000..a990f3cdb05
--- /dev/null
+++ b/pkgs/servers/mail/mailman/package.nix
@@ -0,0 +1,76 @@
+{ lib, fetchpatch, python3, postfix, lynx
+}:
+
+with python3.pkgs;
+
+buildPythonPackage rec {
+  pname = "mailman";
+  version = "3.3.5";
+  disabled = pythonOlder "3.6";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "12mgxs1ndhdjjkydx48b95na9k9h0disfqgrr6wxx7vda6dqvcwz";
+  };
+
+  propagatedBuildInputs = with python3.pkgs; [
+    aiosmtpd
+    alembic
+    authheaders
+    click
+    dnspython
+    falcon
+    flufl_bounce
+    flufl_i18n
+    flufl_lock
+    gunicorn
+    importlib-resources
+    lazr_config
+    passlib
+    requests
+    sqlalchemy
+    zope_component
+    zope_configuration
+  ];
+
+  patches = [
+    (fetchpatch {
+      url = "https://gitlab.com/mailman/mailman/-/commit/4b206e2a5267a0e17f345fd7b2d957122ba57566.patch";
+      sha256 = "06axmrn74p81wvcki36c7gfj5fp5q15zxz2yl3lrvijic7hbs4n2";
+    })
+    (fetchpatch {
+      url = "https://gitlab.com/mailman/mailman/-/commit/9613154f3c04fa2383fbf017031ef263c291418d.patch";
+      sha256 = "0vyw87s857vfxbf7kihwb6w094xyxmxbi1bpdqi3ybjamjycp55r";
+    })
+    ./log-stderr.patch
+  ];
+
+  postPatch = ''
+    substituteInPlace setup.py \
+      --replace "alembic>=1.6.2,<1.7" "alembic>=1.6.2"
+
+    substituteInPlace src/mailman/config/postfix.cfg \
+      --replace /usr/sbin/postmap ${postfix}/bin/postmap
+    substituteInPlace src/mailman/config/schema.cfg \
+      --replace /usr/bin/lynx ${lynx}/bin/lynx
+  '';
+
+  # Mailman assumes that those scripts in $out/bin are Python scripts. Wrapping
+  # them in shell code breaks this assumption. Use the wrapped version (see
+  # wrapped.nix) if you need the CLI (rather than the Python library).
+  #
+  # This gives a properly wrapped 'mailman' command plus an interpreter that
+  # has all the necessary search paths to execute unwrapped 'master' and
+  # 'runner' scripts.
+  dontWrapPythonPrograms = true;
+
+  # requires flufl.testing, which the upstream has archived
+  doCheck = false;
+
+  meta = {
+    homepage = "https://www.gnu.org/software/mailman/";
+    description = "Free software for managing electronic mail discussion and newsletter lists";
+    license = lib.licenses.gpl3Plus;
+    maintainers = with lib.maintainers; [ qyliss ma27 ];
+  };
+}
diff --git a/pkgs/servers/mail/mailman/postorius.nix b/pkgs/servers/mail/mailman/postorius.nix
index 4022b6fe91c..bd4425e5efa 100644
--- a/pkgs/servers/mail/mailman/postorius.nix
+++ b/pkgs/servers/mail/mailman/postorius.nix
@@ -1,6 +1,6 @@
-{ lib, buildPythonPackage, fetchPypi, beautifulsoup4, vcrpy, mock
-, django-mailman3, mailmanclient, readme_renderer
-}:
+{ lib, python3 }:
+
+with python3.pkgs;
 
 buildPythonPackage rec {
   pname = "postorius";
diff --git a/pkgs/servers/mail/mailman/python.nix b/pkgs/servers/mail/mailman/python.nix
new file mode 100644
index 00000000000..359f974f598
--- /dev/null
+++ b/pkgs/servers/mail/mailman/python.nix
@@ -0,0 +1,30 @@
+{ python3 }:
+
+python3.override {
+  packageOverrides = self: super: {
+    # does not find tests
+    alembic = super.alembic.overridePythonAttrs (oldAttrs:  {
+      doCheck = false;
+    });
+    # Needed by mailman, see https://gitlab.com/mailman/mailman/-/issues/964
+    sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec {
+      version = "1.3.24";
+      src = super.fetchPypi {
+        inherit version;
+        inherit (oldAttrs) pname;
+        sha256 = "06bmxzssc66cblk1hamskyv5q3xf1nh1py3vi6dka4lkpxy7gfzb";
+      };
+      # does not find tests
+      doCheck = false;
+    });
+    # Fixes `AssertionError: database connection isn't set to UTC`
+    psycopg2 = super.psycopg2.overridePythonAttrs (a: (rec {
+      version = "2.8.6";
+      src = super.fetchPypi {
+        inherit version;
+        inherit (a) pname;
+        sha256 = "fb23f6c71107c37fd667cb4ea363ddeb936b348bbd6449278eb92c189699f543";
+      };
+    }));
+  };
+}
diff --git a/pkgs/servers/mail/mailman/web.nix b/pkgs/servers/mail/mailman/web.nix
index adad9d21943..64489bc3e41 100644
--- a/pkgs/servers/mail/mailman/web.nix
+++ b/pkgs/servers/mail/mailman/web.nix
@@ -1,7 +1,9 @@
-{ buildPythonPackage, lib, fetchPypi, pythonOlder
-, sassc, hyperkitty, postorius, whoosh, setuptools-scm
+{ lib, python3
+, sassc, hyperkitty, postorius
 }:
 
+with python3.pkgs;
+
 buildPythonPackage rec {
   pname = "mailman-web";
   version = "0.0.5";
diff --git a/pkgs/servers/mail/mailman/wrapped.nix b/pkgs/servers/mail/mailman/wrapped.nix
deleted file mode 100644
index c244af10e78..00000000000
--- a/pkgs/servers/mail/mailman/wrapped.nix
+++ /dev/null
@@ -1,20 +0,0 @@
-{ runCommand, lib, makeWrapper, python3
-, archivers ? [ python3.pkgs.mailman-hyperkitty ]
-}:
-
-let
-  inherit (python3.pkgs) makePythonPath mailman;
-in
-
-runCommand "${mailman.name}-wrapped" {
-  inherit (mailman) meta;
-  nativeBuildInputs = [ makeWrapper ];
-  passthru = mailman.passthru // { unwrapped = mailman; };
-} ''
-  mkdir -p "$out/bin"
-  cd "${mailman}/bin"
-  for exe in *; do
-    makeWrapper "${mailman}/bin/$exe" "$out/bin/$exe" \
-        --set PYTHONPATH ${makePythonPath ([ mailman ] ++ archivers)}
-  done
-''