ceph: 16.2.10 -> 17.2.5

Updates Ceph from the Pacific to the Quincy release tree.

Upgrade to boost179 and python310, cleans up the python environment,
builds with more system libraries.

Disables parquet support for the radosgw, because I couldn't
get it to build.

Also properly enable the testsuite on ceph-common and generally unroll
and sort lists for a better overview and history.

Co-Authored-By: name_snrl <Demogorgon-74@ya.ru>

2 files changed, 209 insertions, 109 deletions

diff --git a/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch b/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch
deleted file mode 100644
index a117408b000..00000000000
--- a/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- a/cmake/modules/BuildSPDK.cmake
-+++ b/cmake/modules/BuildSPDK.cmake
-@@ -35,7 +35,7 @@ macro(build_spdk)
-     # unset $CFLAGS, otherwise it will interfere with how SPDK sets
-     # its include directory.
-     # unset $LDFLAGS, otherwise SPDK will fail to mock some functions.
--    BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} ${make_cmd} EXTRA_CFLAGS="${spdk_CFLAGS}"
-+    BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} ${make_cmd} EXTRA_CFLAGS="${spdk_CFLAGS}" C_OPT="-mssse3"
-     BUILD_IN_SOURCE 1
-     INSTALL_COMMAND "true")
-   unset(make_cmd)
diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix
index 15e5071f963..34befedf18b 100644
--- a/pkgs/tools/filesystems/ceph/default.nix
+++ b/pkgs/tools/filesystems/ceph/default.nix
@@ -1,44 +1,82 @@
-{ lib, stdenv, runCommand, fetchurl, fetchpatch
+{ lib
+, stdenv
+, runCommand
+, fetchurl
+, fetchFromGitHub
+
+# Build time
+, cmake
 , ensureNewerSourcesHook
-, cmake, pkg-config
-, which, git
-, boost175, xz
-, libxml2, zlib, lz4
-, openldap, lttng-ust
-, babeltrace, gperf
-, gtest
-, cunit, snappy
+, fmt
+, git
 , makeWrapper
-, leveldb, oath-toolkit
-, libnl, libcap_ng
-, rdkafka
+, pkg-config
+, which
+
+# Tests
 , nixosTests
-, cryptsetup
-, sqlite
-, lua
-, icu
+
+# Runtime dependencies
+, arrow-cpp
+, babeltrace
+, boost179
 , bzip2
+, cryptsetup
+, cimg
+, cunit
 , doxygen
+, gperf
 , graphviz
-, fmt
-, python39
+, gtest
+, icu
+, jsoncpp
+, libcap_ng
+, libnl
+, libxml2
+, lttng-ust
+, lua
+, lz4
+, oath-toolkit
+, openldap
+, python310
+, rdkafka
+, rocksdb
+, snappy
+, sqlite
+, utf8proc
+, zlib
+, zstd
 
 # Optional Dependencies
-, yasm ? null, fcgi ? null, expat ? null
-, curl ? null, fuse ? null
-, libedit ? null, libatomic_ops ? null
+, curl ? null
+, expat ? null
+, fuse ? null
+, libatomic_ops ? null
+, libedit ? null
 , libs3 ? null
+, yasm ? null
 
 # Mallocs
-, jemalloc ? null, gperftools ? null
+, gperftools ? null
+, jemalloc ? null
 
 # Crypto Dependencies
 , cryptopp ? null
-, nss ? null, nspr ? null
+, nspr ? null
+, nss ? null
 
 # Linux Only Dependencies
-, linuxHeaders, util-linux, libuuid, udev, keyutils, rdma-core, rabbitmq-c
-, libaio ? null, libxfs ? null, zfs ? null, liburing ? null
+, linuxHeaders
+, util-linux
+, libuuid
+, udev
+, keyutils
+, rdma-core
+, rabbitmq-c
+, libaio ? null
+, libxfs ? null
+, liburing ? null
+, zfs ? null
 , ...
 }:
 
@@ -49,7 +87,6 @@ let
   shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null;
 
   optYasm = shouldUsePkg yasm;
-  optFcgi = shouldUsePkg fcgi;
   optExpat = shouldUsePkg expat;
   optCurl = shouldUsePkg curl;
   optFuse = shouldUsePkg fuse;
@@ -68,8 +105,18 @@ let
   optLibxfs = shouldUsePkg libxfs;
   optZfs = shouldUsePkg zfs;
 
-  hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null;
+  # Downgrade rocksdb, 7.10 breaks ceph
+  rocksdb' = rocksdb.overrideAttrs (oldAttrs: {
+    version = "7.9.2";
+    src = fetchFromGitHub {
+      owner = "facebook";
+      repo = "rocksdb";
+      rev = "refs/tags/v7.9.2";
+      hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
+    };
+  });
 
+  hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
 
   # Malloc implementation (can be jemalloc, tcmalloc or null)
   malloc = if optJemalloc != null then optJemalloc else optGperftools;
@@ -92,20 +139,30 @@ let
      platforms = [ "x86_64-linux" "aarch64-linux" ];
    };
 
-  ceph-common = python.pkgs.buildPythonPackage rec{
+  ceph-common = with python.pkgs; buildPythonPackage {
     pname = "ceph-common";
     inherit src version;
 
     sourceRoot = "ceph-${version}/src/python-common";
 
-    nativeCheckInputs = [ python.pkgs.pytest ];
-    propagatedBuildInputs = with python.pkgs; [ pyyaml six ];
+    propagatedBuildInputs = [
+      pyyaml
+    ];
+
+    nativeCheckInputs = [
+      pytestCheckHook
+    ];
+
+    disabledTests = [
+      # requires network access
+      "test_valid_addr"
+    ];
 
     meta = getMeta "Ceph common module for code shared by manager modules";
   };
 
-  # Boost 1.75 is not compatible with Python 3.10
-  python = python39.override {
+  # Watch out for python <> boost compatibility
+  python = python310.override {
     packageOverrides = self: super: {
       sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec {
         version = "1.4.46";
@@ -125,91 +182,128 @@ let
     };
   };
 
-  boost = boost175.override {
+  boost = boost179.override {
     enablePython = true;
     inherit python;
   };
 
-  ceph-python-env = python.withPackages (ps: [
-    # Check .requires files below https://github.com/ceph/ceph/tree/main/debian for dependencies
-    ps.sphinx
-    ps.flask
-    ps.routes
-    ps.cython
-    ps.setuptools
-    ps.virtualenv
-    # Libraries needed by the python tools
-    ps.mako
+  # TODO: split this off in build and runtime environment
+  ceph-python-env = python.withPackages (ps: with ps; [
     ceph-common
-    ps.cherrypy
-    ps.cmd2
-    ps.colorama
-    ps.python-dateutil
-    ps.jsonpatch
-    ps.pecan
-    ps.prettytable
-    ps.pyopenssl
-    ps.pyjwt
-    ps.webob
-    ps.bcrypt
-    ps.scipy
-    ps.six
-    ps.pyyaml
+
+    # build time
+    cython
+
+    # debian/control
+    bcrypt
+    cherrypy
+    influxdb
+    jinja2
+    kubernetes
+    natsort
+    numpy
+    pecan
+    prettytable
+    pyjwt
+    pyopenssl
+    python-dateutil
+    pyyaml
+    requests
+    routes
+    scikit-learn
+    scipy
+    setuptools
+    sphinx
+    virtualenv
+    werkzeug
+
+    # src/pybind/mgr/requirements-required.txt
+    cryptography
+    jsonpatch
+
+    # src/tools/cephfs/shell/setup.py
+    cmd2
+    colorama
   ]);
   sitePackages = ceph-python-env.python.sitePackages;
 
-  version = "16.2.10";
+  version = "17.2.5";
   src = fetchurl {
     url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz";
-    sha256 = "sha256-342+nUV3mCX7QJfZSnKEfnQFCJwJmVQeYnefJwW/AtU=";
+    hash = "sha256-NiJpwUeROvh0siSaRoRrDm+C0s61CvRiIrbd7JmRspo=";
   };
 in rec {
   ceph = stdenv.mkDerivation {
     pname = "ceph";
     inherit src version;
 
-    patches = [
-      ./0000-fix-SPDK-build-env.patch
-      # pacific: include/buffer: include <memory>
-      # fixes build with gcc 12
-      # https://github.com/ceph/ceph/pull/47295
-      (fetchpatch {
-        url = "https://github.com/ceph/ceph/pull/47295/commits/df88789a38c053513d3b2a9b7d12a952fc0c9042.patch";
-        hash = "sha256-je65kBfa5hR0ZKo6ZI10XmD5ZUbKj5rxlGxxI9ZJVfo=";
-      })
-      (fetchpatch {
-        url = "https://github.com/ceph/ceph/pull/47295/commits/2abcbe4e47705e6e0fcc7d9d9b75625f563199af.patch";
-        hash = "sha256-8sWQKoZNHuGuhzX/F+3fY4+kjsrwsfoMdVpfVSj2x5w=";
-      })
-      (fetchpatch {
-        url = "https://github.com/ceph/ceph/pull/47295/commits/13dc077cf6c65a3b8c4f13d896847b9964b3fcbb.patch";
-        hash = "sha256-byfiZh9OJrux/y5m3QCPg0LET6q33ZDXmp/CN+yOSQQ=";
-      })
-    ];
-
     nativeBuildInputs = [
       cmake
-      pkg-config which git python.pkgs.wrapPython makeWrapper
+      fmt
+      git
+      makeWrapper
+      pkg-config
+      python
       python.pkgs.python # for the toPythonPath function
+      python.pkgs.wrapPython
+      which
       (ensureNewerSourcesHook { year = "1980"; })
-      python
-      fmt
       # for building docs/man-pages presumably
       doxygen
       graphviz
     ];
 
+    enableParallelBuilding = true;
+
     buildInputs = cryptoLibsMap.${cryptoStr} ++ [
-      boost xz ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3
-      malloc zlib openldap lttng-ust babeltrace gperf gtest cunit
-      snappy lz4 oath-toolkit leveldb libnl libcap_ng rdkafka
-      cryptsetup sqlite lua icu bzip2
+      arrow-cpp
+      babeltrace
+      boost
+      bzip2
+      ceph-python-env
+      cimg
+      cryptsetup
+      cunit
+      gperf
+      gtest
+      jsoncpp
+      icu
+      libcap_ng
+      libnl
+      libxml2
+      lttng-ust
+      lua
+      lz4
+      malloc
+      oath-toolkit
+      openldap
+      optLibatomic_ops
+      optLibs3
+      optYasm
+      rdkafka
+      rocksdb'
+      snappy
+      sqlite
+      utf8proc
+      zlib
+      zstd
     ] ++ lib.optionals stdenv.isLinux [
-      linuxHeaders util-linux libuuid udev keyutils liburing optLibaio optLibxfs optZfs
-      # ceph 14
-      rdma-core rabbitmq-c
+      keyutils
+      liburing
+      libuuid
+      linuxHeaders
+      optLibaio
+      optLibxfs
+      optZfs
+      rabbitmq-c
+      rdma-core
+      udev
+      util-linux
     ] ++ lib.optionals hasRadosgw [
-      optFcgi optExpat optCurl optFuse optLibedit
+      optCurl
+      optExpat
+      optFuse
+      optLibedit
     ];
 
     pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ];
@@ -226,17 +320,27 @@ in rec {
     '';
 
     cmakeFlags = [
-      "-DWITH_SYSTEM_ROCKSDB=OFF"  # breaks Bluestore
       "-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
 
-      "-DWITH_SYSTEM_BOOST=ON"
-      "-DWITH_SYSTEM_GTEST=ON"
       "-DMGR_PYTHON_VERSION=${ceph-python-env.python.pythonVersion}"
-      "-DWITH_SYSTEMD=OFF"
-      "-DWITH_TESTS=OFF"
-      "-DWITH_CEPHFS_SHELL=ON"
+      "-DWITH_CEPHFS_SHELL:BOOL=ON"
+      "-DWITH_SYSTEMD:BOOL=OFF"
+      "-DWITH_TESTS:BOOL=OFF"
+
+      # Use our own libraries, where possible
+      "-DWITH_SYSTEM_ARROW:BOOL=ON"
+      "-DWITH_SYSTEM_BOOST:BOOL=ON"
+      "-DWITH_SYSTEM_CIMG:BOOL=ON"
+      "-DWITH_SYSTEM_JSONCPP:BOOL=ON"
+      "-DWITH_SYSTEM_GTEST:BOOL=ON"
+      "-DWITH_SYSTEM_ROCKSDB:BOOL=ON"
+      "-DWITH_SYSTEM_UTF8PROC:BOOL=ON"
+      "-DWITH_SYSTEM_ZSTD:BOOL=ON"
+
       # TODO breaks with sandbox, tries to download stuff with npm
-      "-DWITH_MGR_DASHBOARD_FRONTEND=OFF"
+      "-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF"
+      # no matching function for call to 'parquet::PageReader::Open(std::shared_ptr<arrow::io::InputStream>&, int64_t, arrow::Compression::type, parquet::MemoryPool*, parquet::CryptoContext*)'
+      "-DWITH_RADOSGW_SELECT_PARQUET:BOOL=OFF"
       # WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now
       ''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}''
     ] ++ lib.optional stdenv.isLinux "-DWITH_SYSTEM_LIBURING=ON";
@@ -259,8 +363,15 @@ in rec {
 
     meta = getMeta "Distributed storage system";
 
-    passthru.version = version;
-    passthru.tests = { inherit (nixosTests) ceph-single-node ceph-multi-node ceph-single-node-bluestore; };
+    passthru = {
+      inherit version;
+      tests = {
+        inherit (nixosTests)
+          ceph-multi-node
+          ceph-single-node
+          ceph-single-node-bluestore;
+      };
+    };
   };
 
   ceph-client = runCommand "ceph-client-${version}" {