1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
|
{ lib
, stdenv
, python
, buildPythonPackage
, fetchFromGitHub
, writeText
, alembic
, argcomplete
, attrs
, blinker
, cached-property
, cattrs
, clickclick
, colorlog
, croniter
, cryptography
, dill
, flask
, flask-appbuilder
, flask-caching
, flask_login
, flask_wtf
, GitPython
, graphviz
, gunicorn
, httpx
, iso8601
, importlib-resources
, importlib-metadata
, inflection
, itsdangerous
, jinja2
, jsonschema
, lazy-object-proxy
, lockfile
, markdown
, markupsafe
, marshmallow-oneofschema
, numpy
, openapi-spec-validator
, pandas
, pendulum
, psutil
, pygments
, pyjwt
, python-daemon
, python-dateutil
, python-nvd3
, python-slugify
, python3-openid
, pyyaml
, rich
, setproctitle
, sqlalchemy
, sqlalchemy-jsonfield
, swagger-ui-bundle
, tabulate
, tenacity
, termcolor
, unicodecsv
, werkzeug
, pytest
, freezegun
, mkYarnPackage
}:
let
version = "2.1.1rc1";
airflow-src = fetchFromGitHub rec {
owner = "apache";
repo = "airflow";
rev = version;
sha256 = "1vzzmcfgqni9rkf7ggh8mswnm3ffwaishcz1ysrwx0a96ilhm9q2";
};
# airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
# This replicates this shell script, fixing bugs in yarn.lock and package.json
airflow-frontend = mkYarnPackage {
name = "airflow-frontend";
src = "${airflow-src}/airflow/www";
packageJSON = ./package.json;
yarnLock = ./yarn.lock;
yarnNix = ./yarn.nix;
distPhase = "true";
configurePhase = ''
cp -r $node_modules node_modules
'';
buildPhase = ''
yarn --offline build
find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
'';
installPhase = ''
mkdir -p $out/static/
cp -r static/dist $out/static
'';
};
in
buildPythonPackage rec {
pname = "apache-airflow";
inherit version;
src = airflow-src;
propagatedBuildInputs = [
alembic
argcomplete
attrs
blinker
cached-property
cattrs
clickclick
colorlog
croniter
cryptography
dill
flask
flask-appbuilder
flask-caching
flask_login
flask_wtf
GitPython
graphviz
gunicorn
httpx
iso8601
importlib-resources
importlib-metadata
inflection
itsdangerous
jinja2
jsonschema
lazy-object-proxy
lockfile
markdown
markupsafe
marshmallow-oneofschema
numpy
openapi-spec-validator
pandas
pendulum
psutil
pygments
pyjwt
python-daemon
python-dateutil
python-nvd3
python-slugify
python3-openid
pyyaml
rich
setproctitle
sqlalchemy
sqlalchemy-jsonfield
swagger-ui-bundle
tabulate
tenacity
termcolor
unicodecsv
werkzeug
];
buildInputs = [
airflow-frontend
];
checkInputs = [
freezegun
pytest
];
INSTALL_PROVIDERS_FROM_SOURCES = "true";
postPatch = ''
substituteInPlace setup.cfg \
--replace "importlib_resources~=1.4" "importlib_resources" \
--replace "importlib_metadata~=1.7" "importlib_metadata" \
--replace "tenacity~=6.2.0" "tenacity" \
--replace "pyjwt<2" "pyjwt" \
--replace "flask>=1.1.0, <2.0" "flask" \
--replace "flask-login>=0.3, <0.5" "flask-login" \
--replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \
--replace "jinja2>=2.10.1, <2.12.0" "jinja2" \
--replace "attrs>=20.0, <21.0" "attrs" \
--replace "cattrs~=1.1, <1.7.0" "cattrs" \
--replace "markupsafe>=1.1.1, <2.0" "markupsafe" \
--replace "docutils<0.17" "docutils" \
--replace "sqlalchemy>=1.3.18, <1.4" "sqlalchemy" \
--replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \
--replace "werkzeug~=1.0, >=1.0.1" "werkzeug" \
--replace "itsdangerous>=1.1.0, <2.0" "itsdangerous"
substituteInPlace tests/core/test_core.py \
--replace "/bin/bash" "${stdenv.shell}"
'';
# allow for gunicorn processes to have access to python packages
makeWrapperArgs = [ "--prefix PYTHONPATH : $PYTHONPATH" ];
checkPhase = ''
export HOME=$(mktemp -d)
export AIRFLOW_HOME=$HOME
export AIRFLOW__CORE__UNIT_TEST_MODE=True
export AIRFLOW_DB="$HOME/airflow.db"
export PATH=$PATH:$out/bin
airflow version
airflow db init
airflow db reset -y
pytest tests/core/test_core.py
'';
postInstall = ''
cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
'';
meta = with lib; {
description = "Programmatically author, schedule and monitor data pipelines";
homepage = "http://airflow.apache.org/";
license = licenses.asl20;
maintainers = with maintainers; [ bhipple costrouc ingenieroariel ];
};
}
|