diff options
| -rw-r--r-- | pkgs/development/libraries/arrow-cpp/default.nix | 49 | ||||
| -rw-r--r-- | pkgs/development/libraries/arrow-cpp/double-conversion_cmake.patch | 43 | ||||
| -rw-r--r-- | pkgs/development/libraries/parquet-cpp/default.nix | 39 | ||||
| -rw-r--r-- | pkgs/development/python-modules/pyarrow/default.nix | 18 | ||||
| -rw-r--r-- | pkgs/top-level/aliases.nix | 1 | ||||
| -rw-r--r-- | pkgs/top-level/all-packages.nix | 2 |
6 files changed, 90 insertions, 62 deletions
diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix index 16fc7e0c960..77ce77dee5a 100644 --- a/pkgs/development/libraries/arrow-cpp/default.nix +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -1,42 +1,75 @@ -{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, python, rapidjson, snappy, zlib, zstd }: +{ stdenv, symlinkJoin, fetchurl, fetchFromGitHub, boost, brotli, cmake, double-conversion, flatbuffers, gflags, glog, gtest, lz4, perl, python, rapidjson, snappy, thrift, which, zlib, zstd }: + +let + parquet-testing = fetchFromGitHub { + owner = "apache"; + repo = "parquet-testing"; + rev = "46ae2605c2de306f5740587107dcf333a527f2d1"; + sha256 = "07ps745gas2zcfmg56m3vwl63yyzmalnxwb5dc40vd004cx5hdik"; + }; +in stdenv.mkDerivation rec { name = "arrow-cpp-${version}"; - version = "0.10.0"; + version = "0.11.0"; src = fetchurl { url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz"; - sha256 = "0bc4krapz1kzdm16npzmgdz7zvg9lip6rnqbwph8vfn7zji0fcll"; + sha256 = "0pc5pqr0dbnx8s1ji102dhw9bbrsq3ml4ac3mmi2022yfyizlf0q"; }; sourceRoot = "apache-arrow-${version}/cpp"; - # patch to fix python-test - patches = [ ./darwin.patch ]; + patches = [ + # fix ARROW-3467 + ./double-conversion_cmake.patch + + # patch to fix python-test + ./darwin.patch + ]; nativeBuildInputs = [ cmake ]; - buildInputs = [ boost python.pkgs.python python.pkgs.numpy ]; + buildInputs = [ boost double-conversion glog python.pkgs.python python.pkgs.numpy ]; preConfigure = '' + substituteInPlace cmake_modules/FindThrift.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY substituteInPlace cmake_modules/FindBrotli.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + substituteInPlace cmake_modules/FindGLOG.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY substituteInPlace cmake_modules/FindLz4.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + + patchShebangs build-support/ ''; BROTLI_HOME = symlinkJoin { name="brotli-wrap"; paths = [ brotli.lib brotli.dev ]; }; + DOUBLE_CONVERSION_HOME = double-conversion; FLATBUFFERS_HOME = flatbuffers; - GTEST_HOME = gtest; GFLAGS_HOME = gflags; + GLOG_HOME = glog; + GTEST_HOME = gtest; LZ4_HOME = symlinkJoin { name="lz4-wrap"; paths = [ lz4 lz4.dev ]; }; RAPIDJSON_HOME = rapidjson; SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; }; - ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; }; + THRIFT_HOME = thrift; + ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib zlib.dev ]; }; ZSTD_HOME = zstd; cmakeFlags = [ "-DARROW_PYTHON=ON" + "-DARROW_PARQUET=ON" ]; + doInstallCheck = true; + PARQUET_TEST_DATA = if doInstallCheck then "${parquet-testing}/data" else null; + installCheckInputs = [ perl which ]; + installCheckPhase = (stdenv.lib.optionalString stdenv.isDarwin '' + for f in release/*-test; do + install_name_tool -add_rpath "$out"/lib "$f" + done + '') + '' + ctest -L unittest -V + ''; + meta = { description = "A cross-language development platform for in-memory data"; homepage = https://arrow.apache.org/; diff --git a/pkgs/development/libraries/arrow-cpp/double-conversion_cmake.patch b/pkgs/development/libraries/arrow-cpp/double-conversion_cmake.patch new file mode 100644 index 00000000000..336fdde9e8f --- /dev/null +++ b/pkgs/development/libraries/arrow-cpp/double-conversion_cmake.patch @@ -0,0 +1,43 @@ +diff --git a/CMakeLists.txt b/cpp/CMakeLists.txt +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -615,7 +615,7 @@ endif(UNIX) + set(ARROW_LINK_LIBS) + + # Libraries to link statically with libarrow.so +-set(ARROW_STATIC_LINK_LIBS double-conversion) ++set(ARROW_STATIC_LINK_LIBS ${DOUBLE_CONVERSION_TARGET}) + + if (ARROW_WITH_BROTLI) + SET(ARROW_STATIC_LINK_LIBS +@@ -694,7 +694,7 @@ else () + set(ARROW_MIN_TEST_LIBS + arrow_shared + ${ARROW_LINK_LIBS} +- double-conversion ++ ${DOUBLE_CONVERSION_TARGET} + ${BOOST_SYSTEM_LIBRARY} + ${BOOST_FILESYSTEM_LIBRARY} + ${BOOST_REGEX_LIBRARY} +diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +--- a/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cmake_modules/ThirdpartyToolchain.cmake +@@ -469,14 +469,16 @@ if("${DOUBLE_CONVERSION_HOME}" STREQUAL "") + set(DOUBLE_CONVERSION_VENDORED 1) + else() + find_package(double-conversion REQUIRED) ++ set(DOUBLE_CONVERSION_TARGET double-conversion::double-conversion) + set(DOUBLE_CONVERSION_VENDORED 0) + endif() + + include_directories(SYSTEM ${DOUBLE_CONVERSION_INCLUDE_DIR}) +-ADD_THIRDPARTY_LIB(double-conversion +- STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB}) + + if (DOUBLE_CONVERSION_VENDORED) ++ ADD_THIRDPARTY_LIB(double-conversion ++ STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB}) ++ set(DOUBLE_CONVERSION_TARGET double-conversion) + add_dependencies(arrow_dependencies double-conversion_ep) + endif() + diff --git a/pkgs/development/libraries/parquet-cpp/default.nix b/pkgs/development/libraries/parquet-cpp/default.nix deleted file mode 100644 index 804ddb136f0..00000000000 --- a/pkgs/development/libraries/parquet-cpp/default.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ stdenv, symlinkJoin, fetchurl, arrow-cpp, boost, cmake, gtest, snappy, thrift, zlib }: - -stdenv.mkDerivation rec { - name = "parquet-cpp-${version}"; - version = "1.5.0"; - - src = fetchurl { - url = "https://github.com/apache/parquet-cpp/archive/apache-${name}.tar.gz"; - sha256 = "19nwqahc0igr0jfprbf2m86rmzz6zicw4z7b8z832wbsyc904wli"; - }; - - patches = [ ./api.patch ]; - - nativeBuildInputs = [ cmake ]; - buildInputs = [ boost ]; - - preConfigure = '' - substituteInPlace cmake_modules/FindThrift.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY - substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY - ''; - - ARROW_HOME = arrow-cpp; - THRIFT_HOME = thrift; - GTEST_HOME = gtest; - SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; }; - ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; }; - - cmakeFlags = [ - "-DPARQUET_BUILD_BENCHMARKS=OFF" - ]; - - meta = { - description = "A C++ library to read and write the Apache Parquet columnar data format"; - homepage = http://parquet.apache.org; - license = stdenv.lib.licenses.asl20; - platforms = stdenv.lib.platforms.unix; - maintainers = with stdenv.lib.maintainers; [ veprbl ]; - }; -} diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index e73b1717331..8ebb88fa6a7 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -1,8 +1,7 @@ -{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, JPype1, numpy, pandas, pytest, pytestrunner, parquet-cpp, pkgconfig, setuptools_scm, six }: +{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, JPype1, numpy, pandas, pytest, pytestrunner, pkgconfig, setuptools_scm, six }: let - _arrow-cpp = arrow-cpp.override { inherit python;}; - _parquet-cpp = parquet-cpp.override { arrow-cpp = _arrow-cpp; }; + _arrow-cpp = arrow-cpp.override { inherit python; }; in buildPythonPackage rec { @@ -17,18 +16,11 @@ buildPythonPackage rec { checkInputs = [ pandas pytest pytestrunner JPype1 ]; PYARROW_BUILD_TYPE = "release"; - PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib;${PARQUET_HOME}/lib"; - - preBuild = '' - substituteInPlace CMakeLists.txt --replace "\''${ARROW_ABI_VERSION}" '"0.0.0"' - substituteInPlace CMakeLists.txt --replace "\''${ARROW_SO_VERSION}" '"0"' - - # fix the hardcoded value - substituteInPlace cmake_modules/FindParquet.cmake --replace 'set(PARQUET_ABI_VERSION "1.0.0")' 'set(PARQUET_ABI_VERSION "${_parquet-cpp.version}")' - ''; + PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib"; preCheck = '' rm pyarrow/tests/test_hdfs.py + rm pyarrow/tests/test_cuda.py # fails: "ArrowNotImplementedError: Unsupported numpy type 22" substituteInPlace pyarrow/tests/test_feather.py --replace "test_timedelta_with_nulls" "_disabled" @@ -47,7 +39,7 @@ buildPythonPackage rec { ''; ARROW_HOME = _arrow-cpp; - PARQUET_HOME = _parquet-cpp; + PARQUET_HOME = _arrow-cpp; setupPyBuildFlags = ["--with-parquet" ]; diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index 0abf0e60846..8fcf0baba02 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -207,6 +207,7 @@ mapAliases ({ openssh_with_kerberos = openssh; # added 2018-01-28 owncloudclient = owncloud-client; # added 2016-08 p11_kit = p11-kit; # added 2018-02-25 + parquet-cpp = arrow-cpp; # added 2018-09-08 pass-otp = pass.withExtensions (ext: [ext.pass-otp]); # added 2018-05-04 pgp-tools = signing-party; # added 2017-03-26 pidgin-with-plugins = pidgin; # added 2016-06 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 8f4bc519285..e05d180457e 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -11600,8 +11600,6 @@ with pkgs; paperkey = callPackage ../tools/security/paperkey { }; - parquet-cpp = callPackage ../development/libraries/parquet-cpp {}; - pangoxsl = callPackage ../development/libraries/pangoxsl { }; pcaudiolib = callPackage ../development/libraries/pcaudiolib { |