From f41a3e7d7d327ea66459d17bfbe4a751b2496cb1 Mon Sep 17 00:00:00 2001 From: Joachim Schiele Date: Sun, 18 Mar 2018 13:34:46 +0100 Subject: Emscripten 1.37.16 to 1.37.36 (#37291) * trying to build emscriptenPackages not all fail * reading the console.log it turns out python executable is not in place and that is why emconfigure didnt work * backup commit * much more targets are compiling now * added common revisioning * revision bump to 1.37.36 (not tested) * fixed xmllint * forcing unit testing, will implement the tests after i get home * json_c test working * added tests * tiny fixes * added documentation --- doc/default.nix | 4 + doc/languages-frameworks/emscripten.md | 185 +++++++++++++++++++++ doc/languages-frameworks/index.xml | 1 + .../emscripten-fastcomp/emscripten-fastcomp.nix | 17 +- pkgs/development/compilers/emscripten/default.nix | 24 ++- pkgs/development/em-modules/generic/default.nix | 15 +- pkgs/top-level/all-packages.nix | 2 + pkgs/top-level/emscripten-packages.nix | 130 ++++++++++++--- 8 files changed, 340 insertions(+), 38 deletions(-) create mode 100644 doc/languages-frameworks/emscripten.md diff --git a/doc/default.nix b/doc/default.nix index 60c613878c7..ec458634a42 100644 --- a/doc/default.nix +++ b/doc/default.nix @@ -81,6 +81,10 @@ pkgs.stdenv.mkDerivation { inputFile = ./languages-frameworks/vim.md; outputFile = "./languages-frameworks/vim.xml"; } + + toDocbook { + inputFile = ./languages-frameworks/emscripten.md; + outputFile = "./languages-frameworks/emscripten.xml"; + } + '' echo ${lib.nixpkgsVersion} > .version diff --git a/doc/languages-frameworks/emscripten.md b/doc/languages-frameworks/emscripten.md new file mode 100644 index 00000000000..24c49ec1409 --- /dev/null +++ b/doc/languages-frameworks/emscripten.md @@ -0,0 +1,185 @@ +# User's Guide to Emscripten in Nixpkgs + +[Emscripten](https://github.com/kripken/emscripten): An LLVM-to-JavaScript Compiler + +This section of the manual covers how to use `emscripten` in nixpkgs. + +Minimal requirements: + +* nix +* nixpkgs + +Modes of use of `emscripten`: + +* **Imperative usage** (on the command line): + + If you want to work with `emcc`, `emconfigure` and `emmake` as you are used to from Ubuntu and similar distributions you can use these commands: + + * `nix-env -i emscripten` + * `nix-shell -p emscripten` + +* **Declarative usage**: + + This mode is far more power full since this makes use of `nix` for dependency management of emscripten libraries and targets by using the `mkDerivation` which is implemented by `pkgs.emscriptenStdenv` and `pkgs.buildEmscriptenPackage`. The source for the packages is in `pkgs/top-level/emscripten-packages.nix` and the abstraction behind it in `pkgs/development/em-modules/generic/default.nix`. + * build and install all packages: + * `nix-env -iA emscriptenPackages` + + * dev-shell for zlib implementation hacking: + * `nix-shell -A emscriptenPackages.zlib` + + +## Imperative usage + +A few things to note: + +* `export EMCC_DEBUG=2` is nice for debugging +* `~/.emscripten`, the build artifact cache sometimes creates issues and needs to be removed from time to time + + +## Declarative usage + +Let's see two different examples from `pkgs/top-level/emscripten-packages.nix`: + +* `pkgs.zlib.override` +* `pkgs.buildEmscriptenPackage` + +Both are interesting concepts. + +A special requirement of the `pkgs.buildEmscriptenPackage` is the `doCheck = true` is a default meaning that each emscriptenPackage requires a `checkPhase` implemented. + +* Use `export EMCC_DEBUG=2` from within a emscriptenPackage's `phase` to get more detailed debug output what is going wrong. +* ~/.emscripten cache is requiring us to set `HOME=$TMPDIR` in individual phases. This makes compilation slower but also makes it more deterministic. + +### Usage 1: pkgs.zlib.override + +This example uses `zlib` from nixpkgs but instead of compiling **C** to **ELF** it compiles **C** to **JS** since we were using `pkgs.zlib.override` and changed stdenv to `pkgs.emscriptenStdenv`. A few adaptions and hacks were set in place to make it working. One advantage is that when `pkgs.zlib` is updated, it will automatically update this package as well. However, this can also be the downside... + +See the `zlib` example: + + zlib = (pkgs.zlib.override { + stdenv = pkgs.emscriptenStdenv; + }).overrideDerivation + (old: rec { + buildInputs = old.buildInputs ++ [ pkgconfig ]; + # we need to reset this setting! + NIX_CFLAGS_COMPILE=""; + configurePhase = '' + # FIXME: Some tests require writing at $HOME + HOME=$TMPDIR + runHook preConfigure + + #export EMCC_DEBUG=2 + emconfigure ./configure --prefix=$out --shared + + runHook postConfigure + ''; + dontStrip = true; + outputs = [ "out" ]; + buildPhase = '' + emmake make + ''; + installPhase = '' + emmake make install + ''; + checkPhase = '' + echo "================= testing zlib using node =================" + + echo "Compiling a custom test" + set -x + emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 test/example.c -DZ_SOLO \ + libz.so.${old.version} -I . -o example.js + + echo "Using node to execute the test" + ${pkgs.nodejs}/bin/node ./example.js + + set +x + if [ $? -ne 0 ]; then + echo "test failed for some reason" + exit 1; + else + echo "it seems to work! very good." + fi + echo "================= /testing zlib using node =================" + ''; + + postPatch = pkgs.stdenv.lib.optionalString pkgs.stdenv.isDarwin '' + substituteInPlace configure \ + --replace '/usr/bin/libtool' 'ar' \ + --replace 'AR="libtool"' 'AR="ar"' \ + --replace 'ARFLAGS="-o"' 'ARFLAGS="-r"' + ''; + }); + +### Usage 2: pkgs.buildEmscriptenPackage + +This `xmlmirror` example features a emscriptenPackage which is defined completely from this context and no `pkgs.zlib.override` is used. + + xmlmirror = pkgs.buildEmscriptenPackage rec { + name = "xmlmirror"; + + buildInputs = [ pkgconfig autoconf automake libtool gnumake libxml2 nodejs openjdk json_c ]; + nativeBuildInputs = [ pkgconfig zlib ]; + + src = pkgs.fetchgit { + url = "https://gitlab.com/odfplugfest/xmlmirror.git"; + rev = "4fd7e86f7c9526b8f4c1733e5c8b45175860a8fd"; + sha256 = "1jasdqnbdnb83wbcnyrp32f36w3xwhwp0wq8lwwmhqagxrij1r4b"; + }; + + configurePhase = '' + rm -f fastXmlLint.js* + # a fix for ERROR:root:For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was 234217728 + # https://gitlab.com/odfplugfest/xmlmirror/issues/8 + sed -e "s/TOTAL_MEMORY=234217728/TOTAL_MEMORY=268435456/g" -i Makefile.emEnv + # https://github.com/kripken/emscripten/issues/6344 + # https://gitlab.com/odfplugfest/xmlmirror/issues/9 + sed -e "s/\$(JSONC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(LIBXML20_LDFLAGS)/\$(JSONC_LDFLAGS) \$(LIBXML20_LDFLAGS) \$(ZLIB_LDFLAGS) /g" -i Makefile.emEnv + # https://gitlab.com/odfplugfest/xmlmirror/issues/11 + sed -e "s/-o fastXmlLint.js/-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -o fastXmlLint.js/g" -i Makefile.emEnv + ''; + + buildPhase = '' + HOME=$TMPDIR + make -f Makefile.emEnv + ''; + + outputs = [ "out" "doc" ]; + + installPhase = '' + mkdir -p $out/share + mkdir -p $doc/share/${name} + + cp Demo* $out/share + cp -R codemirror-5.12 $out/share + cp fastXmlLint.js* $out/share + cp *.xsd $out/share + cp *.js $out/share + cp *.xhtml $out/share + cp *.html $out/share + cp *.json $out/share + cp *.rng $out/share + cp README.md $doc/share/${name} + ''; + checkPhase = '' + + ''; + }; + +### Declarative debugging + +Use `nix-shell -I nixpkgs=/some/dir/nixpkgs -A emscriptenPackages.libz` and from there you can go trough the individual steps. This makes it easy to build a good `unit test` or list the files of the project. + +1. `nix-shell -I nixpkgs=/some/dir/nixpkgs -A emscriptenPackages.libz` +2. `cd /tmp/` +3. `unpackPhase` +4. cd libz-1.2.3 +5. `configurePhase` +6. `buildPhase` +7. ... happy hacking... + +## Summary + +Using this toolchain makes it easy to leverage `nix` from NixOS, MacOSX or even Windows (WSL+ubuntu+nix). This toolchain is reproducible, behaves like the rest of the packages from nixpkgs and contains a set of well working examples to learn and adapt from. + +If in trouble, ask the maintainers. + diff --git a/doc/languages-frameworks/index.xml b/doc/languages-frameworks/index.xml index fc15d847d15..6743c131201 100644 --- a/doc/languages-frameworks/index.xml +++ b/doc/languages-frameworks/index.xml @@ -30,6 +30,7 @@ such as Perl or Haskell. These are described in this chapter. + diff --git a/pkgs/development/compilers/emscripten-fastcomp/emscripten-fastcomp.nix b/pkgs/development/compilers/emscripten-fastcomp/emscripten-fastcomp.nix index f35ca26584f..5317cf25d41 100644 --- a/pkgs/development/compilers/emscripten-fastcomp/emscripten-fastcomp.nix +++ b/pkgs/development/compilers/emscripten-fastcomp/emscripten-fastcomp.nix @@ -1,7 +1,7 @@ -{ stdenv, fetchFromGitHub, cmake, python, ... }: +{ emscriptenVersion, stdenv, llvm, fetchFromGitHub, cmake, python, gtest, ... }: let - rev = "1.37.16"; + rev = emscriptenVersion; gcc = if stdenv.cc.isGNU then stdenv.cc.cc else stdenv.cc.cc.gcc; in stdenv.mkDerivation rec { @@ -10,18 +10,18 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "kripken"; repo = "emscripten-fastcomp"; - sha256 = "0wj9sc0gciaiidcjv6wb0qn6ks06xds7q34351masc7qpvd217by"; + sha256 = "04j698gmp686b5lricjakm5hyh2z2kh28m1ffkghmkyz4zkzmx98"; inherit rev; }; srcFL = fetchFromGitHub { owner = "kripken"; repo = "emscripten-fastcomp-clang"; - sha256 = "1akdgxzxhzjbhp4d14ajcrp9jrf39x004a726ly2gynqc185l4j7"; + sha256 = "1ici51mmpgg80xk3y8f376nbbfak6rz27qdy98l8lxkrymklp5g5"; inherit rev; }; - nativeBuildInputs = [ cmake python ]; + nativeBuildInputs = [ cmake python gtest ]; preConfigure = '' cp -Lr ${srcFL} tools/clang chmod +w -R tools/clang @@ -30,9 +30,10 @@ stdenv.mkDerivation rec { "-DCMAKE_BUILD_TYPE=Release" "-DLLVM_TARGETS_TO_BUILD='X86;JSBackend'" "-DLLVM_INCLUDE_EXAMPLES=OFF" - "-DLLVM_INCLUDE_TESTS=OFF" - # "-DCLANG_INCLUDE_EXAMPLES=OFF" - "-DCLANG_INCLUDE_TESTS=OFF" + "-DLLVM_INCLUDE_TESTS=ON" + #"-DLLVM_CONFIG=${llvm}/bin/llvm-config" + "-DLLVM_BUILD_TESTS=ON" + "-DCLANG_INCLUDE_TESTS=ON" ] ++ (stdenv.lib.optional stdenv.isLinux # necessary for clang to find crtend.o "-DGCC_INSTALL_PREFIX=${gcc}" diff --git a/pkgs/development/compilers/emscripten/default.nix b/pkgs/development/compilers/emscripten/default.nix index ae80ad6fc80..ac972c99a95 100644 --- a/pkgs/development/compilers/emscripten/default.nix +++ b/pkgs/development/compilers/emscripten/default.nix @@ -1,9 +1,9 @@ -{ stdenv, fetchFromGitHub, emscriptenfastcomp, python, nodejs, closurecompiler -, jre, binaryen, enableWasm ? true +{ emscriptenVersion, stdenv, fetchFromGitHub, emscriptenfastcomp, python, nodejs, closurecompiler, pkgs +, jre, binaryen, enableWasm ? true , python2Packages, cmake }: let - rev = "1.37.16"; + rev = emscriptenVersion; appdir = "share/emscripten"; in @@ -13,10 +13,12 @@ stdenv.mkDerivation { src = fetchFromGitHub { owner = "kripken"; repo = "emscripten"; - sha256 = "1qyhjx5zza01vnwmj6qzxbkagxknn4kzb6gw12fqw5q8pa8fy4zy"; + sha256 = "02p0cp86vd1mydlpq544xbydggpnrq9dhbxx7h08j235frjm5cdc"; inherit rev; }; + buildInputs = [ nodejs cmake python ]; + buildCommand = '' mkdir -p $out/${appdir} cp -r $src/* $out/${appdir} @@ -38,9 +40,23 @@ stdenv.mkDerivation { echo "COMPILER_ENGINE = NODE_JS" >> $out/${appdir}/config echo "CLOSURE_COMPILER = '${closurecompiler}/share/java/closure-compiler-v${closurecompiler.version}.jar'" >> $out/${appdir}/config echo "JAVA = '${jre}/bin/java'" >> $out/${appdir}/config + # to make the test(s) below work + echo "SPIDERMONKEY_ENGINE = []" >> $out/${appdir}/config '' + stdenv.lib.optionalString enableWasm '' echo "BINARYEN_ROOT = '${binaryen}'" >> $out/share/emscripten/config + '' + + + '' + echo "--------------- running test -----------------" + # quick hack to get the test working + HOME=$TMPDIR + cp $out/${appdir}/config $HOME/.emscripten + export PATH=$PATH:$out/bin + + #export EMCC_DEBUG=2 + ${python}/bin/python $src/tests/runner.py test_hello_world + echo "--------------- /running test -----------------" ''; meta = with stdenv.lib; { diff --git a/pkgs/development/em-modules/generic/default.nix b/pkgs/development/em-modules/generic/default.nix index 16b8f1df595..332fab8e14a 100644 --- a/pkgs/development/em-modules/generic/default.nix +++ b/pkgs/development/em-modules/generic/default.nix @@ -1,4 +1,4 @@ -{ pkgs, lib, emscripten }: +{ pkgs, lib, emscripten, python }: { buildInputs ? [], nativeBuildInputs ? [] @@ -11,8 +11,8 @@ pkgs.stdenv.mkDerivation ( { name = "emscripten-${args.name}"; - buildInputs = [ emscripten ] ++ buildInputs; - nativeBuildInputs = [ emscripten ] ++ nativeBuildInputs; + buildInputs = [ emscripten python ] ++ buildInputs; + nativeBuildInputs = [ emscripten python ] ++ nativeBuildInputs; # fake conftest results with emscripten's python magic EMCONFIGURE_JS=2; @@ -38,9 +38,18 @@ pkgs.stdenv.mkDerivation ( runHook postBuild ''; + doCheck = true; + checkPhase = args.checkPhase or '' runHook preCheck + echo "Please provide a test for your emscripten based library/tool, see libxml2 as an exmple on how to use emcc/node to verify your build" + echo "" + echo "In normal C 'unresolved symbols' would yield an error and a breake of execution. In contrast, in emscripten they are only a warning which is ok given that emscripten assumptions about shared libraries." + echo " -> https://github.com/kripken/emscripten/wiki/Linking" + echo "So just assume the dependencies were built using hydra, then YOU WILL NEVER see the warning and your code depending on a library will always fail!" + exit 1 + runHook postCheck ''; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 0676d59d25a..054febb6223 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2079,6 +2079,8 @@ with pkgs; buildEmscriptenPackage = callPackage ../development/em-modules/generic { }; + emscriptenVersion = "1.37.36"; + emscripten = callPackage ../development/compilers/emscripten { }; emscriptenfastcompPackages = callPackage ../development/compilers/emscripten-fastcomp { }; diff --git a/pkgs/top-level/emscripten-packages.nix b/pkgs/top-level/emscripten-packages.nix index 7e63041867d..5b5b32b89da 100644 --- a/pkgs/top-level/emscripten-packages.nix +++ b/pkgs/top-level/emscripten-packages.nix @@ -1,49 +1,92 @@ { pkgs }: +with pkgs; + # emscripten toolchain abstraction for nix # https://github.com/NixOS/nixpkgs/pull/16208 -with pkgs; rec { - json_c = pkgs.json_c.override { - stdenv = emscriptenStdenv; - }; +rec { + json_c = (pkgs.json_c.override { + stdenv = pkgs.emscriptenStdenv; + }).overrideDerivation + (old: { + nativeBuildInputs = [ autoreconfHook pkgconfig ]; + propagatedBuildInputs = [ zlib ]; + buildInputs = old.buildInputs ++ [ automake autoconf ]; + configurePhase = '' + HOME=$TMPDIR + emconfigure ./configure --prefix=$out + ''; + checkPhase = '' + echo "================= testing json_c using node =================" + + echo "Compiling a custom test" + set -x + emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 tests/test1.c \ + `pkg-config zlib --cflags` \ + `pkg-config zlib --libs` \ + -I . \ + .libs/libjson-c.so \ + -o ./test1.js + + echo "Using node to execute the test which basically outputs an error on stderr which we grep for" + ${pkgs.nodejs}/bin/node ./test1.js + + set +x + if [ $? -ne 0 ]; then + echo "test1.js execution failed -> unit test failed, please fix" + exit 1; + else + echo "test1.js execution seems to work! very good." + fi + echo "================= /testing json_c using node =================" + ''; + }); libxml2 = (pkgs.libxml2.override { stdenv = emscriptenStdenv; pythonSupport = false; }).overrideDerivation (old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ autoreconfHook pkgconfig ]; - buildInputs = old.buildInputs ++ [ zlib nodejs ]; + propagatedBuildInputs = [ zlib ]; + buildInputs = old.buildInputs ++ [ pkgconfig ]; + # just override it with nothing so it does not fail autoreconfPhase = "echo autoreconfPhase not used..."; + configurePhase = '' + HOME=$TMPDIR + emconfigure ./configure --prefix=$out --without-python + ''; checkPhase = '' - echo "================= testing xmllint using node =================" + echo "================= testing libxml2 using node =================" + + echo "Compiling a custom test" + set -x emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 xmllint.o \ ./.libs/libxml2.a `pkg-config zlib --cflags` `pkg-config zlib --libs` -o ./xmllint.test.js \ --embed-file ./test/xmlid/id_err1.xml - # test/xmlid/id_err2.xml:3: validity error : xml:id : attribute type should be ID - # - # ^ - node ./xmllint.test.js --noout test/xmlid/id_err1.xml 2>&1 | grep 0bar + + echo "Using node to execute the test which basically outputs an error on stderr which we grep for" + ${pkgs.nodejs}/bin/node ./xmllint.test.js --noout test/xmlid/id_err1.xml 2>&1 | grep 0bar + + set +x if [ $? -ne 0 ]; then echo "xmllint unit test failed, please fix this package" exit 1; else echo "since there is no stupid text containing 'foo xml:id' it seems to work! very good." fi - echo "================= /testing xmllint using node =================" + echo "================= /testing libxml2 using node =================" ''; }); - xmlmirror = buildEmscriptenPackage rec { + xmlmirror = pkgs.buildEmscriptenPackage rec { name = "xmlmirror"; - nativeBuildInputs = [ pkgconfig ]; - buildInputs = [ autoconf automake libtool gnumake libxml2 nodejs - python openjdk json_c zlib ]; + buildInputs = [ pkgconfig autoconf automake libtool gnumake libxml2 nodejs openjdk json_c ]; + nativeBuildInputs = [ pkgconfig zlib ]; - src = fetchgit { + src = pkgs.fetchgit { url = "https://gitlab.com/odfplugfest/xmlmirror.git"; rev = "4fd7e86f7c9526b8f4c1733e5c8b45175860a8fd"; sha256 = "1jasdqnbdnb83wbcnyrp32f36w3xwhwp0wq8lwwmhqagxrij1r4b"; @@ -51,6 +94,14 @@ with pkgs; rec { configurePhase = '' rm -f fastXmlLint.js* + # a fix for ERROR:root:For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was 234217728 + # https://gitlab.com/odfplugfest/xmlmirror/issues/8 + sed -e "s/TOTAL_MEMORY=234217728/TOTAL_MEMORY=268435456/g" -i Makefile.emEnv + # https://github.com/kripken/emscripten/issues/6344 + # https://gitlab.com/odfplugfest/xmlmirror/issues/9 + sed -e "s/\$(JSONC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(LIBXML20_LDFLAGS)/\$(JSONC_LDFLAGS) \$(LIBXML20_LDFLAGS) \$(ZLIB_LDFLAGS) /g" -i Makefile.emEnv + # https://gitlab.com/odfplugfest/xmlmirror/issues/11 + sed -e "s/-o fastXmlLint.js/-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -o fastXmlLint.js/g" -i Makefile.emEnv ''; buildPhase = '' @@ -75,25 +126,58 @@ with pkgs; rec { cp *.rng $out/share cp README.md $doc/share/${name} ''; - - postInstall = '' + checkPhase = '' + ''; }; zlib = (pkgs.zlib.override { - stdenv = emscriptenStdenv; + stdenv = pkgs.emscriptenStdenv; }).overrideDerivation - (old: { + (old: rec { + buildInputs = old.buildInputs ++ [ pkgconfig ]; + # we need to reset this setting! + NIX_CFLAGS_COMPILE=""; configurePhase = '' # FIXME: Some tests require writing at $HOME HOME=$TMPDIR runHook preConfigure - emconfigure ./configure --prefix=$out + #export EMCC_DEBUG=2 + emconfigure ./configure --prefix=$out --shared runHook postConfigure ''; - postPatch = stdenv.lib.optionalString stdenv.isDarwin '' + dontStrip = true; + outputs = [ "out" ]; + buildPhase = '' + emmake make + ''; + installPhase = '' + emmake make install + ''; + checkPhase = '' + echo "================= testing zlib using node =================" + + echo "Compiling a custom test" + set -x + emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 test/example.c -DZ_SOLO \ + libz.so.${old.version} -I . -o example.js + + echo "Using node to execute the test" + ${pkgs.nodejs}/bin/node ./example.js + + set +x + if [ $? -ne 0 ]; then + echo "test failed for some reason" + exit 1; + else + echo "it seems to work! very good." + fi + echo "================= /testing zlib using node =================" + ''; + + postPatch = pkgs.stdenv.lib.optionalString pkgs.stdenv.isDarwin '' substituteInPlace configure \ --replace '/usr/bin/libtool' 'ar' \ --replace 'AR="libtool"' 'AR="ar"' \ -- cgit 1.4.1