sage: 8.9 -> 9.2 fixups

12 files changed, 217 insertions, 36 deletions

diff --git a/pkgs/applications/science/math/maxima/default.nix b/pkgs/applications/science/math/maxima/default.nix
index 1bf7d16f290..cdc81a87646 100644
--- a/pkgs/applications/science/math/maxima/default.nix
+++ b/pkgs/applications/science/math/maxima/default.nix
@@ -87,6 +87,10 @@ stdenv.mkDerivation ({
 
   enableParallelBuilding = true;
 
+  passthru = {
+    ecl = ecl;
+  };
+
   meta = {
     description = "Computer algebra system";
     homepage = "http://maxima.sourceforge.net";
diff --git a/pkgs/applications/science/math/sage/default.nix b/pkgs/applications/science/math/sage/default.nix
index 2a95e9a5985..416f2b957b7 100644
--- a/pkgs/applications/science/math/sage/default.nix
+++ b/pkgs/applications/science/math/sage/default.nix
@@ -9,12 +9,13 @@
 let
   inherit (pkgs) symlinkJoin callPackage nodePackages;
 
-  python = pkgs.python3.override {
+  python3 = pkgs.python3.override {
     packageOverrides = self: super: {
       # `sagelib`, i.e. all of sage except some wrappers and runtime dependencies
       sagelib = self.callPackage ./sagelib.nix {
         inherit flint arb;
         inherit sage-src env-locations pynac singular;
+        ecl = maxima-ecl.ecl;
         linbox = pkgs.linbox.override { withSage = true; };
         pkg-config = pkgs.pkgconfig; # not to confuse with pythonPackages.pkgconfig
       };
@@ -42,7 +43,8 @@ let
   env-locations = callPackage ./env-locations.nix {
     inherit pari_data;
     inherit singular maxima-ecl;
-    cysignals = python.pkgs.cysignals;
+    ecl = maxima-ecl.ecl;
+    cysignals = python3.pkgs.cysignals;
     three = nodePackages.three;
     mathjax = nodePackages.mathjax;
   };
@@ -50,21 +52,22 @@ let
   # The shell file that gets sourced on every sage start. Will also source
   # the env-locations file.
   sage-env = callPackage ./sage-env.nix {
-    sagelib = python.pkgs.sagelib;
+    sagelib = python3.pkgs.sagelib;
     inherit env-locations;
-    inherit python singular palp flint pynac pythonEnv maxima-ecl;
+    inherit python3 singular palp flint pynac pythonEnv maxima-ecl;
+    ecl = maxima-ecl.ecl;
     pkg-config = pkgs.pkgconfig; # not to confuse with pythonPackages.pkgconfig
   };
 
   # The documentation for sage, building it takes a lot of ram.
   sagedoc = callPackage ./sagedoc.nix {
     inherit sage-with-env;
-    inherit python maxima-ecl;
+    inherit python3 maxima-ecl;
   };
 
   # sagelib with added wrappers and a dependency on sage-tests to make sure thet tests were run.
   sage-with-env = callPackage ./sage-with-env.nix {
-    inherit pythonEnv;
+    inherit python3 pythonEnv;
     inherit sage-env;
     inherit pynac singular maxima-ecl;
     pkg-config = pkgs.pkgconfig; # not to confuse with pythonPackages.pkgconfig
@@ -81,7 +84,7 @@ let
 
   sage-src = callPackage ./sage-src.nix {};
 
-  pythonRuntimeDeps = with python.pkgs; [
+  pythonRuntimeDeps = with python3.pkgs; [
     sagelib
     cvxopt
     networkx
@@ -95,11 +98,10 @@ let
     ipywidgets
     rpy2
     sphinx
-    typing
     pillow
   ];
 
-  pythonEnv = python.buildEnv.override {
+  pythonEnv = python3.buildEnv.override {
     extraLibs = pythonRuntimeDeps;
     ignoreCollisions = true;
   } // { extraLibs = pythonRuntimeDeps; }; # make the libs accessible
@@ -108,7 +110,21 @@ let
 
   singular = pkgs.singular.override { inherit flint; };
 
-  maxima-ecl = pkgs.maxima-ecl;
+  maxima-ecl = pkgs.maxima-ecl.override {
+    ecl = pkgs.ecl.override {
+      # "echo syntax error | ecl > /dev/full 2>&1" segfaults in
+      # ECL. We apply a patch to fix it (write_error.patch), but it
+      # only works if threads are disabled.  sage 9.2 tests this
+      # (src/sage/interfaces/tests.py) and ships ecl like so.
+      # https://gitlab.com/embeddable-common-lisp/ecl/-/merge_requests/1#note_1657275
+      threadSupport = false;
+
+      # if we don't use the system boehmgc, sending a SIGINT to ecl
+      # can segfault if we it happens during memory allocation.
+      # src/sage/libs/ecl.pyx would intermittently fail in this case.
+      useBoehmgc = true;
+    };
+  };
 
   # *not* to confuse with the python package "pynac"
   pynac = pkgs.pynac.override { inherit singular flint; };
diff --git a/pkgs/applications/science/math/sage/env-locations.nix b/pkgs/applications/science/math/sage/env-locations.nix
index bda2d23b9e5..b1ad0aad9c7 100644
--- a/pkgs/applications/science/math/sage/env-locations.nix
+++ b/pkgs/applications/science/math/sage/env-locations.nix
@@ -44,6 +44,6 @@ writeTextFile rec {
     export JSMOL_DIR="${jmol}/share/jsmol"
     export MATHJAX_DIR="${mathjax}/lib/node_modules/mathjax"
     export THREEJS_DIR="${three}/lib/node_modules/three"
-    export SAGE_INCLUDE_DIRECTORIES="${cysignals}/lib/python2.7/site-packages"
+    export SAGE_INCLUDE_DIRECTORIES="${cysignals}/${cysignals.pythonModule.sitePackages}"
   '';
 }
diff --git a/pkgs/applications/science/math/sage/patches/pillow-update.patch b/pkgs/applications/science/math/sage/patches/pillow-update.patch
new file mode 100644
index 00000000000..19d61552262
--- /dev/null
+++ b/pkgs/applications/science/math/sage/patches/pillow-update.patch
@@ -0,0 +1,39 @@
+diff --git a/src/sage/repl/image.py b/src/sage/repl/image.py
+index d7d00b0..cd1607a 100644
+--- a/src/sage/repl/image.py
++++ b/src/sage/repl/image.py
+@@ -77,7 +77,7 @@ class Image(SageObject):
+ 
+         - ``size`` -- 2-tuple, containing (width, height) in pixels.
+ 
+-        - ``color`` -- string or tuple of numeric. What colour to use
++        - ``color`` -- string, numeric or tuple of numeric. What colour to use
+           for the image. Default is black.  If given, this should be a
+           a tuple with one value per band. When creating RGB images,
+           you can also use colour strings as supported by the
+@@ -91,9 +91,15 @@ class Image(SageObject):
+         EXAMPLES::
+ 
+             sage: from sage.repl.image import Image
+-            sage: Image('P', (16, 16), (13,))
++            sage: Image('P', (16, 16), 13)
+             16x16px 8-bit Color image
+         """
++        # pillow does not support Sage integers as color
++        from sage.rings.integer import Integer
++        if isinstance(color, Integer):
++            color = int(color)
++        elif isinstance(color, tuple):
++            color = tuple(int(i) if isinstance(i, Integer) else i for i in color)
+         self._pil = PIL.Image.new(mode, size, color)
+ 
+     @property
+@@ -233,7 +239,7 @@ class Image(SageObject):
+         EXAMPLES::
+ 
+             sage: from sage.repl.image import Image
+-            sage: img = Image('P', (12, 34), (13,))
++            sage: img = Image('P', (12, 34), 13)
+             sage: filename = tmp_filename(ext='.png')
+             sage: img.save(filename)
+             sage: with open(filename, 'rb') as f:
diff --git a/pkgs/applications/science/math/sage/patches/register-pretty-printer-earlier.patch b/pkgs/applications/science/math/sage/patches/register-pretty-printer-earlier.patch
new file mode 100644
index 00000000000..83bd83a6d33
--- /dev/null
+++ b/pkgs/applications/science/math/sage/patches/register-pretty-printer-earlier.patch
@@ -0,0 +1,36 @@
+diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py
+index cb3667659e..867f547d71 100644
+--- a/src/sage/doctest/forker.py
++++ b/src/sage/doctest/forker.py
+@@ -200,6 +200,15 @@ def init_sage(controller=None):
+     from sage.cpython._py2_random import Random
+     sage.misc.randstate.DEFAULT_PYTHON_RANDOM = Random
+ 
++    # IPython's pretty printer sorts the repr of dicts by their keys by default
++    # (or their keys' str() if they are not otherwise orderable).  However, it
++    # disables this for CPython 3.6+ opting to instead display dicts' "natural"
++    # insertion order, which is preserved in those versions).
++    # However, this order is random in some instances.
++    # Also modifications of code may affect the order.
++    # So here we fore sorted dict printing.
++    IPython.lib.pretty.for_type(dict, _sorted_dict_pprinter_factory('{', '}'))
++
+     if controller is None:
+         import sage.repl.ipython_kernel.all_jupyter
+     else:
+@@ -222,15 +231,6 @@ def init_sage(controller=None):
+     from sage.repl.rich_output.backend_doctest import BackendDoctest
+     dm.switch_backend(BackendDoctest())
+ 
+-    # IPython's pretty printer sorts the repr of dicts by their keys by default
+-    # (or their keys' str() if they are not otherwise orderable).  However, it
+-    # disables this for CPython 3.6+ opting to instead display dicts' "natural"
+-    # insertion order, which is preserved in those versions).
+-    # However, this order is random in some instances.
+-    # Also modifications of code may affect the order.
+-    # So here we fore sorted dict printing.
+-    IPython.lib.pretty.for_type(dict, _sorted_dict_pprinter_factory('{', '}'))
+-
+     # Switch on extra debugging
+     from sage.structure.debug_options import debug
+     debug.refine_category_hash_check = True
diff --git a/pkgs/applications/science/math/sage/patches/sagespawn-implicit-casting.patch b/pkgs/applications/science/math/sage/patches/sagespawn-implicit-casting.patch
new file mode 100644
index 00000000000..2ee5db3e29c
--- /dev/null
+++ b/pkgs/applications/science/math/sage/patches/sagespawn-implicit-casting.patch
@@ -0,0 +1,13 @@
+diff --git a/src/sage/interfaces/sagespawn.pyx b/src/sage/interfaces/sagespawn.pyx
+index 9041238f1d..469befbc66 100644
+--- a/src/sage/interfaces/sagespawn.pyx
++++ b/src/sage/interfaces/sagespawn.pyx
+@@ -228,7 +228,7 @@ class SagePtyProcess(PtyProcess):
+         Check that the process eventually dies after calling
+         ``terminate_async``::
+ 
+-            sage: s.ptyproc.terminate_async(interval=0.2)
++            sage: s.ptyproc.terminate_async(interval=float(0.2))
+             sage: while True:
+             ....:     try:
+             ....:         os.kill(s.pid, 0)
diff --git a/pkgs/applications/science/math/sage/patches/sphinx-docbuild-subprocesses.patch b/pkgs/applications/science/math/sage/patches/sphinx-docbuild-subprocesses.patch
index 56f09279622..661077cfa8c 100644
--- a/pkgs/applications/science/math/sage/patches/sphinx-docbuild-subprocesses.patch
+++ b/pkgs/applications/science/math/sage/patches/sphinx-docbuild-subprocesses.patch
@@ -36,7 +36,7 @@ index 73a078e619..059125c59f 100644
          # Run Sphinx with Sage's special logger
 -        sys.argv = ["sphinx-build"] + build_command.split()
 -        from .sphinxbuild import runsphinx
-+        args = "python -um sage_setup.docbuild.sphinxbuild -N".split() + build_command.split()
++        args = "python3 -um sage_setup.docbuild.sphinxbuild -N".split() + build_command.split()
          try:
 -            runsphinx()
 +            subprocess.check_call(args)
diff --git a/pkgs/applications/science/math/sage/sage-env.nix b/pkgs/applications/science/math/sage/sage-env.nix
index ba7f6ccd18b..568a4a39f39 100644
--- a/pkgs/applications/science/math/sage/sage-env.nix
+++ b/pkgs/applications/science/math/sage/sage-env.nix
@@ -1,7 +1,6 @@
 { stdenv
 , lib
 , writeTextFile
-, python
 , sagelib
 , env-locations
 , gfortran
@@ -67,11 +66,6 @@ let
     "@sage-local@"
     "@sage-local@/build"
     pythonEnv
-    # empty python env to add python wrapper that clears PYTHONHOME (see
-    # wrapper.nix). This is necessary because sage will call the python3 binary
-    # (from python2 code). The python2 PYTHONHOME (again set in wrapper.nix)
-    # will then confuse python3, if it is not overwritten.
-    python3.buildEnv
     gfortran # for inline fortran
     stdenv.cc # for cython
     bash
@@ -129,8 +123,21 @@ writeTextFile rec {
       ]
     }'
     export SAGE_ROOT='${sagelib.src}'
-    export SAGE_LOCAL='@sage-local@'
+  '' +
+    # TODO: is using pythonEnv instead of @sage-local@ here a good
+    # idea? there is a test in src/sage/env.py that checks if the values
+    # SAGE_ROOT and SAGE_LOCAL set here match the ones set in env.py.
+    # we fix up env.py's SAGE_ROOT in sage-src.nix (which does not
+    # have access to sage-with-env), but env.py autodetects
+    # SAGE_LOCAL to be pythonEnv.
+    # setting SAGE_LOCAL to pythonEnv also avoids having to create
+    # python3, ipython, ipython3 and jupyter symlinks in
+    # sage-with-env.nix.
+  ''
+    export SAGE_LOCAL='${pythonEnv}'
+
     export SAGE_SHARE='${sagelib}/share'
+    export SAGE_ENV_CONFIG_SOURCED=1 # sage-env complains if sage-env-config is not sourced beforehand
     orig_path="$PATH"
     export PATH='${runtimepath}'
 
@@ -177,9 +184,9 @@ writeTextFile rec {
       ])
     }'
 
-    export SAGE_LIB='${sagelib}/${python.sitePackages}'
+    export SAGE_LIB='${sagelib}/${python3.sitePackages}'
 
-    export SAGE_EXTCODE='${sagelib.src}/src/ext'
+    export SAGE_EXTCODE='${sagelib.src}/src/sage/ext_data'
 
   # for find_library
     export DYLD_LIBRARY_PATH="${lib.makeLibraryPath [stdenv.cc.libc singular]}''${DYLD_LIBRARY_PATH:+:}$DYLD_LIBRARY_PATH"
diff --git a/pkgs/applications/science/math/sage/sage-src.nix b/pkgs/applications/science/math/sage/sage-src.nix
index b0182e9693f..8948621a025 100644
--- a/pkgs/applications/science/math/sage/sage-src.nix
+++ b/pkgs/applications/science/math/sage/sage-src.nix
@@ -39,6 +39,18 @@ stdenv.mkDerivation rec {
     # Parallelize docubuild using subprocesses, fixing an isolation issue. See
     # https://groups.google.com/forum/#!topic/sage-packaging/YGOm8tkADrE
     ./patches/sphinx-docbuild-subprocesses.patch
+
+    # Sage's workaround to pretty print dicts (in
+    # src/sage/doctest/forker.py:init_sage) runs too late (after
+    # controller.load_environment(), which imports sage.all.*) to to
+    # affect sage.sandpiles.Sandpile{Config,Divisor}'s pretty printer.
+    # Due to the sandpiles module being lazily loaded, this only
+    # affects the first run (subsequent runs read from an import cache
+    # at ~/.sage/cache and are not affected), which is probably why
+    # other distributions don't hit this bug. This breaks two sandpile
+    # tests, so do the workaround a little bit earlier.
+    # https://trac.sagemath.org/ticket/31053
+    ./patches/register-pretty-printer-earlier.patch
   ];
 
   # Since sage unfortunately does not release bugfix releases, packagers must
@@ -48,6 +60,17 @@ stdenv.mkDerivation rec {
     # To help debug the transient error in
     # https://trac.sagemath.org/ticket/23087 when it next occurs.
     ./patches/configurationpy-error-verbose.patch
+
+    # fix intermittent errors in Sage 9.2's psage.py (this patch is
+    # already included in Sage 9.3): https://trac.sagemath.org/ticket/30730
+    (fetchpatch {
+      name = "fix-psage-is-locked.patch";
+      url = "https://git.sagemath.org/sage.git/patch/?id=75df605f216ddc7b6ca719be942d666b241520e9";
+      sha256 = "0g9pl1wbb3sgs26d3bvv70cpa77sfskylv4kd255y1794f1fgk4q";
+    })
+
+    # fix intermittent errors in sagespawn.pyx: https://trac.sagemath.org/ticket/31052
+    ./patches/sagespawn-implicit-casting.patch
   ];
 
   # Patches needed because of package updates. We could just pin the versions of
@@ -77,6 +100,9 @@ stdenv.mkDerivation rec {
 
     # ignore a deprecation warning for usage of `cmp` in the attrs library in the doctests
     ./patches/ignore-cmp-deprecation.patch
+
+    # adapt sage's Image class to pillow 8.0.1 (https://trac.sagemath.org/ticket/30971)
+    ./patches/pillow-update.patch
   ];
 
   patches = nixPatches ++ bugfixPatches ++ packageUpgradePatches;
@@ -84,16 +110,16 @@ stdenv.mkDerivation rec {
   postPatch = ''
     # make sure shebangs etc are fixed, but sage-python23 still works
     find . -type f -exec sed \
-      -e 's/sage-python23/python/g' \
+      -e 's/sage-python23/python3/g' \
       -i {} \;
 
     echo '#!${runtimeShell}
-    python "$@"' > build/bin/sage-python23
+    python3 "$@"' > build/bin/sage-python23
 
     # Make sure sage can at least be imported without setting any environment
     # variables. It won't be close to feature complete though.
     sed -i \
-      "s|var('SAGE_LOCAL',.*|var('SAGE_LOCAL', '$out/src')|" \
+      "s|var('SAGE_ROOT'.*|var('SAGE_ROOT', '$out')|" \
       src/sage/env.py
 
     # Do not use sage-env-config (generated by ./configure).
@@ -101,7 +127,39 @@ stdenv.mkDerivation rec {
     echo '# do nothing' >  src/bin/sage-env-config
   '';
 
-  configurePhase = "# do nothing";
+  # Test src/doc/en/reference/spkg/conf.py will fail if
+  # src/doc/en/reference/spkg/index.rst is not generated.  It is
+  # generated by src/doc/bootstrap, so I've copied the relevant part
+  # here. An alternative would be to create an empty
+  # src/doc/en/reference/spkg/index.rst file.
+  configurePhase = ''
+    OUTPUT_DIR="src/doc/en/reference/spkg"
+    mkdir -p "$OUTPUT_DIR"
+    OUTPUT_INDEX="$OUTPUT_DIR"/index.rst
+    cat > "$OUTPUT_INDEX" <<EOF
+
+    External Packages
+    =================
+
+    .. toctree::
+       :maxdepth: 1
+
+    EOF
+    for PKG_SCRIPTS in build/pkgs/*; do
+        if [ -d "$PKG_SCRIPTS" ]; then
+            PKG_BASE=$(basename "$PKG_SCRIPTS")
+            if [ -f "$PKG_SCRIPTS"/SPKG.rst ]; then
+                # Instead of just copying, we may want to call
+                # a version of sage-spkg-info to format extra information.
+                cp "$PKG_SCRIPTS"/SPKG.rst "$OUTPUT_DIR"/$PKG_BASE.rst
+                echo >> "$OUTPUT_INDEX" "   $PKG_BASE"
+            fi
+        fi
+    done
+    cat >> "$OUTPUT_INDEX" <<EOF
+    .. include:: ../footer.txt
+    EOF
+  '';
 
   buildPhase = "# do nothing";
 
diff --git a/pkgs/applications/science/math/sage/sage-with-env.nix b/pkgs/applications/science/math/sage/sage-with-env.nix
index 88436e1e0f5..f204d97961b 100644
--- a/pkgs/applications/science/math/sage/sage-with-env.nix
+++ b/pkgs/applications/science/math/sage/sage-with-env.nix
@@ -50,10 +50,10 @@ let
   ];
 
   # remove python prefix, replace "-" in the name by "_", apply patch_names
-  # python2.7-some-pkg-1.0 -> some_pkg-1.0
+  # python3.8-some-pkg-1.0 -> some_pkg-1.0
   pkg_to_spkg_name = pkg: patch_names: let
     parts = lib.splitString "-" pkg.name;
-    # remove python2.7-
+    # remove python3.8-
     stripped_parts = if (builtins.head parts) == python3.libPrefix then builtins.tail parts else parts;
     version = lib.last stripped_parts;
     orig_pkgname = lib.init stripped_parts;
@@ -126,8 +126,6 @@ stdenv.mkDerivation rec {
     cp -r src/bin "$out/bin"
     cp -r build/bin "$out/build/bin"
 
-    ln -s "${python3}/bin/python3" "$out/bin/python3"
-
     cp -f '${sage-env}/sage-env' "$out/bin/sage-env"
     substituteInPlace "$out/bin/sage-env" \
       --subst-var-by sage-local "$out"
diff --git a/pkgs/applications/science/math/sage/sagedoc.nix b/pkgs/applications/science/math/sage/sagedoc.nix
index 619a69b7aac..a10672f3a09 100644
--- a/pkgs/applications/science/math/sage/sagedoc.nix
+++ b/pkgs/applications/science/math/sage/sagedoc.nix
@@ -1,6 +1,6 @@
 { stdenv
 , sage-with-env
-, python
+, python3
 , maxima-ecl
 , tachyon
 , jmol
@@ -17,12 +17,12 @@ stdenv.mkDerivation rec {
   # modules are imported and because matplotlib is used to produce plots.
   buildInputs = [
     sage-with-env.env.lib
-    python
+    python3
     maxima-ecl
     tachyon
     jmol
     cddlib
-  ] ++ (with python.pkgs; [
+  ] ++ (with python3.pkgs; [
     psutil
     future
     sphinx
@@ -34,7 +34,6 @@ stdenv.mkDerivation rec {
     ipykernel
     ipywidgets
     jupyter_client
-    typing
   ]);
 
   unpackPhase = ''
@@ -45,13 +44,26 @@ stdenv.mkDerivation rec {
     chmod -R 755 "$SAGE_DOC_SRC_OVERRIDE"
   '';
 
+  postPatch = ''
+    # src/doc/bootstrap generates installation instructions for
+    # arch, debian, fedora, cygwin and homebrew. as a hack, disable
+    # including the generated files.
+    sed -i "/literalinclude/d" $SAGE_DOC_SRC_OVERRIDE/en/installation/source.rst
+  '';
+
   buildPhase = ''
     export SAGE_NUM_THREADS="$NIX_BUILD_CORES"
     export HOME="$TMPDIR/sage_home"
     mkdir -p "$HOME"
 
     # needed to link them in the sage docs using intersphinx
-    export PPLPY_DOCS=${python.pkgs.pplpy.doc}/share/doc/pplpy
+    export PPLPY_DOCS=${python3.pkgs.pplpy.doc}/share/doc/pplpy
+
+    # adapted from src/doc/bootstrap
+    OUTPUT_DIR="$SAGE_DOC_SRC_OVERRIDE/en/reference/repl"
+    mkdir -p "$OUTPUT_DIR"
+    OUTPUT="$OUTPUT_DIR/options.txt"
+    ${sage-with-env}/bin/sage -advanced > "$OUTPUT"
 
     ${sage-with-env}/bin/sage -python -m sage_setup.docbuild \
       --mathjax \
diff --git a/pkgs/applications/science/math/sage/sagelib.nix b/pkgs/applications/science/math/sage/sagelib.nix
index a60680139f9..1c2235bf71b 100644
--- a/pkgs/applications/science/math/sage/sagelib.nix
+++ b/pkgs/applications/science/math/sage/sagelib.nix
@@ -53,7 +53,6 @@
 , gmpy2
 , pplpy
 , sqlite
-, boehmgc # FIXME
 }:
 
 assert (!blas.isILP64) && (!lapack.isILP64);
@@ -129,7 +128,6 @@ buildPythonPackage rec {
     gmpy2
     pplpy
     sqlite
-    boehmgc # FIXME
   ];
 
   buildPhase = ''