python3Packages.pyarrow: 9.0.0 -> 10.0.1

1 files changed, 29 insertions, 15 deletions

diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix
index a862fa69a9a..ed7297bf20d 100644
--- a/pkgs/development/python-modules/pyarrow/default.nix
+++ b/pkgs/development/python-modules/pyarrow/default.nix
@@ -17,18 +17,15 @@
 , pkg-config
 , scipy
 , setuptools-scm
-, six
 }:
 
 let
   zero_or_one = cond: if cond then 1 else 0;
-
-  _arrow-cpp = arrow-cpp.override { python3 = python; };
 in
 
 buildPythonPackage rec {
   pname = "pyarrow";
-  inherit (_arrow-cpp) version src;
+  inherit (arrow-cpp) version src;
 
   disabled = pythonOlder "3.7";
 
@@ -41,13 +38,14 @@ buildPythonPackage rec {
     setuptools-scm
   ];
 
+  buildInputs = [ arrow-cpp ];
+
   propagatedBuildInputs = [
     cffi
     cloudpickle
     fsspec
     numpy
     scipy
-    six
   ];
 
   nativeCheckInputs = [
@@ -60,20 +58,24 @@ buildPythonPackage rec {
   PYARROW_BUILD_TYPE = "release";
 
   PYARROW_WITH_DATASET = zero_or_one true;
-  PYARROW_WITH_FLIGHT = zero_or_one _arrow-cpp.enableFlight;
+  PYARROW_WITH_FLIGHT = zero_or_one arrow-cpp.enableFlight;
   PYARROW_WITH_HDFS = zero_or_one true;
   PYARROW_WITH_PARQUET = zero_or_one true;
-  PYARROW_WITH_PLASMA = zero_or_one (!stdenv.isDarwin);
-  PYARROW_WITH_S3 = zero_or_one _arrow-cpp.enableS3;
+  PYARROW_WITH_PARQUET_ENCRYPTION = zero_or_one true;
+  # Plasma is deprecated since arrow 10.0.0
+  PYARROW_WITH_PLASMA = zero_or_one false;
+  PYARROW_WITH_S3 = zero_or_one arrow-cpp.enableS3;
+  PYARROW_WITH_GCS = zero_or_one arrow-cpp.enableGcs;
+  PYARROW_BUNDLE_ARROW_CPP_HEADERS = zero_or_one false;
 
   PYARROW_CMAKE_OPTIONS = [
     "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib"
   ];
 
-  ARROW_HOME = _arrow-cpp;
-  PARQUET_HOME = _arrow-cpp;
+  ARROW_HOME = arrow-cpp;
+  PARQUET_HOME = arrow-cpp;
 
-  ARROW_TEST_DATA = lib.optionalString doCheck _arrow-cpp.ARROW_TEST_DATA;
+  ARROW_TEST_DATA = lib.optionalString doCheck arrow-cpp.ARROW_TEST_DATA;
 
   doCheck = true;
 
@@ -85,6 +87,13 @@ buildPythonPackage rec {
     export PYARROW_PARALLEL=$NIX_BUILD_CORES
   '';
 
+  postInstall = ''
+    # copy the pyarrow C++ header files to the appropriate location
+    pyarrow_include="$out/${python.sitePackages}/pyarrow/include"
+    mkdir -p "$pyarrow_include/arrow/python"
+    find "$PWD/pyarrow/src/arrow" -type f -name '*.h' -exec cp {} "$pyarrow_include/arrow/python" \;
+  '';
+
   pytestFlagsArray = [
     # Deselect a single test because pyarrow prints a 2-line error message where
     # only a single line is expected. The additional line of output comes from
@@ -102,6 +111,8 @@ buildPythonPackage rec {
     "--deselect=pyarrow/tests/test_pandas.py::test_threaded_pandas_import"
     # Flaky test, works locally but not on Hydra
     "--deselect=pyarrow/tests/test_csv.py::TestThreadedCSVTableRead::test_cancellation"
+    # expects arrow-cpp headers to be bundled
+    "--deselect=pyarrow/tests/test_misc.py::test_get_include"
   ] ++ lib.optionals stdenv.isDarwin [
     # Requires loopback networking
     "--deselect=pyarrow/tests/test_ipc.py::test_socket_"
@@ -109,8 +120,13 @@ buildPythonPackage rec {
     "--deselect=pyarrow/tests/test_flight.py::test_large_descriptor"
     "--deselect=pyarrow/tests/test_flight.py::test_large_metadata_client"
     "--deselect=pyarrow/tests/test_flight.py::test_none_action_side_effect"
+  ] ++ lib.optionals stdenv.isLinux [
+    # this test requires local networking
+    "--deselect=pyarrow/tests/test_fs.py::test_filesystem_from_uri_gcs"
   ];
 
+  disabledTests = [ "GcsFileSystem" ];
+
   dontUseSetuptoolsCheck = true;
 
   preCheck = ''
@@ -125,7 +141,7 @@ buildPythonPackage rec {
 
   pythonImportsCheck = [
     "pyarrow"
-  ] ++ map (module: "pyarrow.${module}") ([
+  ] ++ map (module: "pyarrow.${module}") [
     "compute"
     "csv"
     "dataset"
@@ -135,9 +151,7 @@ buildPythonPackage rec {
     "hdfs"
     "json"
     "parquet"
-  ] ++ lib.optionals (!stdenv.isDarwin) [
-    "plasma"
-  ]);
+  ];
 
   meta = with lib; {
     description = "A cross-language development platform for in-memory data";