blob: 61f2d0c6e202940585ee9d0fac4c6f6541c052c3 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
{ stdenv, fetchurl, jre, bash, simpleBuildTool, python27Packages }:
stdenv.mkDerivation rec {
name = "spark-${version}";
version = "0.9.0";
src = fetchurl {
url = "http://d3kbcqa49mib13.cloudfront.net/${name}-incubating-bin-cdh4.tgz";
sha256 = "0dgirq2ws25accijijanqij6d1mwxkrcqkmq1xsslfpz26svs1w1";
};
unpackPhase = ''tar zxf $src'';
untarDir = "spark-${version}-incubating-bin-cdh4";
installPhase = ''
set -x
mkdir -p $out/lib $out/bin
mv ${untarDir} $out/lib
cat > $out/bin/spark-class <<EOF
#!${bash}/bin/bash
export JAVA_HOME=${jre}
export SPARK_HOME=$out/lib/${untarDir}
if [ -z "\$1" ]; then
echo "Usage: spark-class <class> [<args>]" >&2
exit 1
fi
export SPARK_MEM=\''${SPARK_MEM:-1024m}
JAVA_OPTS=""
JAVA_OPTS="\$JAVA_OPTS -Djava.library.path=\"\$SPARK_LIBRARY_PATH\""
JAVA_OPTS="\$JAVA_OPTS -Xms\$SPARK_MEM -Xmx\$SPARK_MEM"
export JAVA_OPTS
CLASSPATH=\`$out/lib/${untarDir}/bin/compute-classpath.sh\`
export CLASSPATH
exec ${jre}/bin/java -cp "\$CLASSPATH" \$JAVA_OPTS "\$@"
EOF
chmod +x $out/bin/spark-class
cat > $out/bin/spark-shell <<EOF
#!${bash}/bin/bash
set -o posix
export JAVA_HOME=${jre}
export SPARK_HOME=$out/lib/${untarDir}
for o in "\$@"; do
if [ "\$1" = "-c" -o "\$1" = "--cores" ]; then
shift
if [ -n "\$1" ]; then
OPTIONS="-Dspark.cores.max=\$1"
shift
fi
fi
done
exit_status=127
saved_stty=""
function restoreSttySettings() {
stty \$saved_stty
saved_stty=""
}
function onExit() {
if [[ "\$saved_stty" != "" ]]; then
restoreSttySettings
fi
exit \$exit_status
}
trap onExit INT
saved_stty=\$(stty -g 2>/dev/null)
if [[ ! \$? ]]; then
saved_stty=""
fi
$out/bin/spark-class \$OPTIONS org.apache.spark.repl.Main "\$@"
exit_status=\$?
onExit
EOF
chmod +x $out/bin/spark-shell
cat > $out/bin/pyspark <<EOF
#!${bash}/bin/bash
export JAVA_HOME=${jre}
export SPARK_HOME=$out/lib/${untarDir}
export PYTHONPATH=$out/lib/${untarDir}/python:\$PYTHONPATH
export OLD_PYTHONSTARTUP=\$PYTHONSTARTUP
export PYTHONSTARTUP=$out/lib/${untarDir}/python/pyspark/shell.py
export SPARK_MEM=\''${SPARK_MEM:-1024m}
exec ${python27Packages.ipythonLight}/bin/ipython \$@
EOF
chmod +x $out/bin/pyspark
cat > $out/bin/spark-upload-scala <<EOF
#!${bash}/bin/bash
export JAVA_HOME=${jre}
export SPARK_HOME=$out/lib/${untarDir}
export SPARK_MEM=\''${SPARK_MEM:-1024m}
CLASS=\$1; shift
exec ${simpleBuildTool}/bin/sbt package "run-main \$CLASS \$@"
EOF
chmod +x $out/bin/spark-upload-scala
cat > $out/bin/spark-upload-python <<EOF
#!${bash}/bin/bash
exec $out/bin/pyspark \$@
EOF
chmod +x $out/bin/spark-upload-python
'';
phases = "unpackPhase installPhase";
meta = {
description = "Spark cluster computing";
homepage = "http://spark.incubator.apache.org";
license = stdenv.lib.licenses.asl20;
platforms = stdenv.lib.platforms.all;
maintainers = [ stdenv.lib.maintainers.thoughtpolice ];
};
}
|