Merge pull request #3316 from aherrmann/pr_mpi

MPI support for HDF5 and SGE support for OpenMPI

6 files changed, 142 insertions, 2 deletions

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 678a439fe0a..15160d3afc4 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -1,4 +1,13 @@
-{stdenv, fetchurl, gfortran}:
+{stdenv, fetchurl, gfortran
+
+# Enable the Sun Grid Engine bindings
+, enableSGE ? false
+
+# Pass PATH/LD_LIBRARY_PATH to point to current mpirun by default
+, enablePrefix ? false
+}:
+
+with stdenv.lib;
 
 stdenv.mkDerivation {
   name = "openmpi-1.6.5";
@@ -7,6 +16,10 @@ stdenv.mkDerivation {
     sha256 = "11gws4d3z7934zna2r7m1f80iay2ha17kp42mkh39wjykfwbldzy";
   };
   buildInputs = [ gfortran ];
+  configureFlags = []
+    ++ optional enableSGE "--with-sge"
+    ++ optional enablePrefix "--enable-mpirun-prefix-by-default"
+    ;
   meta = {
     homePage = http://www.open-mpi.org/;
     description = "Open source MPI-2 implementation";
diff --git a/pkgs/development/python-modules/h5py/default.nix b/pkgs/development/python-modules/h5py/default.nix
new file mode 100644
index 00000000000..9ab68ac4cd2
--- /dev/null
+++ b/pkgs/development/python-modules/h5py/default.nix
@@ -0,0 +1,43 @@
+{ stdenv, fetchurl, python, buildPythonPackage
+, numpy, hdf5, cython
+, mpiSupport ? false, mpi4py ? null, mpi ? null }:
+
+assert mpiSupport == hdf5.mpiSupport;
+assert mpiSupport -> mpi != null
+  && mpi4py != null
+  && mpi == mpi4py.mpi
+  && mpi == hdf5.mpi
+  ;
+
+with stdenv.lib;
+
+buildPythonPackage rec {
+  name = "h5py-2.3.1";
+
+  src = fetchurl {
+    url = "https://pypi.python.org/packages/source/h/h5py/${name}.tar.gz";
+    md5 = "8f32f96d653e904d20f9f910c6d9dd91";
+  };
+
+  setupPyBuildFlags = [ "--hdf5=${hdf5}" ]
+    ++ optional mpiSupport "--mpi"
+    ;
+  setupPyInstallFlags = setupPyBuildFlags;
+
+  preBuild = if mpiSupport then "export CC=${mpi}/bin/mpicc" else "";
+
+  buildInputs = [ hdf5 cython ]
+    ++ optional mpiSupport mpi
+    ;
+  propagatedBuildInputs = [ numpy ]
+    ++ optional mpiSupport mpi4py
+    ;
+
+  meta = {
+    description = "
+      The h5py package is a Pythonic interface to the HDF5 binary data format.
+    ";
+    homepage = "http://www.h5py.org/";
+    license = stdenv.lib.licenses.bsd2;
+  };
+}
diff --git a/pkgs/development/python-modules/mpi4py/default.nix b/pkgs/development/python-modules/mpi4py/default.nix
new file mode 100644
index 00000000000..74d46def907
--- /dev/null
+++ b/pkgs/development/python-modules/mpi4py/default.nix
@@ -0,0 +1,45 @@
+{ stdenv, fetchurl, python, buildPythonPackage, mpi, openssh }:
+
+buildPythonPackage rec {
+  name = "mpi4py-1.3.1";
+
+  src = fetchurl {
+    url = "https://bitbucket.org/mpi4py/mpi4py/downloads/${name}.tar.gz";
+    sha256 = "e7bd2044aaac5a6ea87a87b2ecc73b310bb6efe5026031e33067ea3c2efc3507";
+  };
+
+  passthru = {
+    inherit mpi;
+  };
+
+  configurePhase = "";
+
+  installPhase = ''
+    mkdir -p "$out/lib/${python.libPrefix}/site-packages"
+    export PYTHONPATH="$out/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
+
+    ${python}/bin/${python.executable} setup.py install \
+      --install-lib=$out/lib/${python.libPrefix}/site-packages \
+      --prefix="$out"
+
+    # --install-lib:
+    # sometimes packages specify where files should be installed outside the usual
+    # python lib prefix, we override that back so all infrastructure (setup hooks)
+    # work as expected
+  '';
+
+  setupPyBuildFlags = ["--mpicc=${mpi}/bin/mpicc"];
+
+  buildInputs = [ mpi ];
+  # Requires openssh for tests. Tests of dependent packages will also fail,
+  # if openssh is not present. E.g. h5py with mpi support.
+  propagatedBuildInputs = [ openssh ];
+
+  meta = {
+    description = "
+      Provides Python bindings for the Message Passing Interface standard.
+    ";
+    homepage = "http://code.google.com/p/mpi4py/";
+    license = stdenv.lib.licenses.bsd3;
+  };
+}
diff --git a/pkgs/tools/misc/hdf5/default.nix b/pkgs/tools/misc/hdf5/default.nix
index a7d03d1335f..5cdc468c2a5 100644
--- a/pkgs/tools/misc/hdf5/default.nix
+++ b/pkgs/tools/misc/hdf5/default.nix
@@ -3,6 +3,8 @@
 , fetchurl
 , zlib ? null
 , szip ? null
+, mpi ? null
+, enableShared ? true
 }:
 stdenv.mkDerivation rec {
   version = "1.8.13";
@@ -12,11 +14,23 @@ stdenv.mkDerivation rec {
     sha256 = "1h9qdl321gzm3ihdhlijbl9sh9qcdrw94j7izg64yfqhxj7b7xl2";  			
   };
 
+  passthru = {
+    mpiSupport = (mpi != null);
+    inherit mpi;
+  };
+
   buildInputs = []
     ++ stdenv.lib.optional (zlib != null) zlib
     ++ stdenv.lib.optional (szip != null) szip;
 
-  configureFlags = if szip != null then "--with-szlib=${szip}" else "";
+  propagatedBuildInputs = []
+    ++ stdenv.lib.optional (mpi != null) mpi;
+
+  configureFlags = "
+    ${if szip != null then "--with-szlib=${szip}" else ""}
+    ${if mpi != null then "--enable-parallel" else ""}
+    ${if enableShared then "--enable-shared" else ""}
+  ";
   
   patches = [./bin-mv.patch];
   
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 851bf546dcd..396b528519f 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -1286,6 +1286,12 @@ let
 
   hdf5 = callPackage ../tools/misc/hdf5 {
     szip = null;
+    mpi = null;
+  };
+
+  hdf5-mpi = hdf5.override {
+    szip = null;
+    mpi = pkgs.openmpi;
   };
 
   heimdall = callPackage ../tools/misc/heimdall { };
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index d276413f3d3..533f59b78d8 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -90,6 +90,19 @@ rec {
     '';
   };
 
+  h5py = callPackage ../development/python-modules/h5py {
+    inherit (pkgs) stdenv fetchurl;
+    inherit python buildPythonPackage cython numpy;
+    hdf5 = pkgs.hdf5.override { mpi = null; };
+  };
+
+  h5py-mpi = h5py.override {
+    mpiSupport = true;
+    mpi = pkgs.openmpi;
+    hdf5 = pkgs.hdf5.override { mpi = pkgs.openmpi; enableShared = true; };
+    inherit mpi4py;
+  };
+
   ipython = import ../shells/ipython {
     inherit (pkgs) stdenv fetchurl sip pyqt4;
     inherit buildPythonPackage pythonPackages;
@@ -106,6 +119,12 @@ rec {
     pylabQtSupport = false;
   });
 
+  mpi4py = callPackage ../development/python-modules/mpi4py {
+    inherit (pkgs) stdenv fetchurl openssh;
+    inherit python buildPythonPackage;
+    mpi = pkgs.openmpi;
+  };
+
   nixpart = callPackage ../tools/filesystems/nixpart { };
 
   # This is used for NixOps to make sure we won't break it with the next major