diff options
6 files changed, 141 insertions, 64 deletions
diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix index 533751734fa..244aae44596 100644 --- a/nixos/modules/services/continuous-integration/buildbot/master.nix +++ b/nixos/modules/services/continuous-integration/buildbot/master.nix @@ -12,7 +12,7 @@ let factory = util.BuildFactory() c = BuildmasterConfig = dict( workers = [${concatStringsSep "," cfg.workers}], - protocols = { 'pb': {'port': ${cfg.bpPort} } }, + protocols = { 'pb': {'port': ${toString cfg.bpPort} } }, title = '${escapeStr cfg.title}', titleURL = '${escapeStr cfg.titleUrl}', buildbotURL = '${escapeStr cfg.buildbotUrl}', @@ -62,7 +62,7 @@ in { extraConfig = mkOption { type = types.str; description = "Extra configuration to append to master.cfg"; - default = ""; + default = "c['buildbotNetUsageData'] = None"; }; masterCfg = mkOption { @@ -92,10 +92,7 @@ in { workers = mkOption { type = types.listOf types.str; description = "List of Workers."; - default = [ - "worker.Worker('example-worker', 'pass')" - ]; - example = [ "worker.LocalWorker('example-worker')" ]; + default = [ "worker.Worker('example-worker', 'pass')" ]; }; status = mkOption { @@ -135,9 +132,8 @@ in { }; bpPort = mkOption { - default = "9989"; - type = types.string; - example = "tcp:10000:interface=127.0.0.1"; + default = 9989; + type = types.int; description = "Port where the master will listen to Buildbot Worker."; }; @@ -179,14 +175,14 @@ in { package = mkOption { type = types.package; - default = pkgs.buildbot-ui; - defaultText = "pkgs.buildbot-ui"; + default = pkgs.buildbot-full; + defaultText = "pkgs.buildbot-full"; description = "Package to use for buildbot."; example = literalExample "pkgs.buildbot-full"; }; packages = mkOption { - default = [ ]; + default = with pkgs; [ python27Packages.twisted git ]; example = literalExample "[ pkgs.git ]"; type = types.listOf types.package; description = "Packages to add to PATH for the buildbot process."; @@ -212,29 +208,30 @@ in { systemd.services.buildbot-master = { description = "Buildbot Continuous Integration Server."; - after = [ "network.target" ]; + after = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; path = cfg.packages; + preStart = '' + env > envvars + mkdir -vp ${cfg.buildbotDir} + ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg + rm -fv $cfg.buildbotDir}/buildbot.tac + ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir} + ''; + serviceConfig = { Type = "simple"; User = cfg.user; Group = cfg.group; WorkingDirectory = cfg.home; - ExecStart = "${cfg.package}/bin/buildbot start --nodaemon ${cfg.buildbotDir}"; - }; + Environment = "PYTHONPATH=${cfg.package}/lib/python2.7/site-packages:${pkgs.buildbot-plugins.www}/lib/python2.7/site-packages:${pkgs.buildbot-plugins.waterfall-view}/lib/python2.7/site-packages:${pkgs.buildbot-plugins.console-view}/lib/python2.7/site-packages:${pkgs.python27Packages.future}/lib/python2.7/site-packages:${pkgs.python27Packages.dateutil}/lib/python2.7/site-packages:${pkgs.python27Packages.six}/lib/python2.7/site-packages:${pkgs.python27Packages.sqlalchemy}/lib/python2.7/site-packages:${pkgs.python27Packages.jinja2}/lib/python2.7/site-packages:${pkgs.python27Packages.markupsafe}/lib/python2.7/site-packages:${pkgs.python27Packages.sqlalchemy_migrate}/lib/python2.7/site-packages:${pkgs.python27Packages.tempita}/lib/python2.7/site-packages:${pkgs.python27Packages.decorator}/lib/python2.7/site-packages:${pkgs.python27Packages.sqlparse}/lib/python2.7/site-packages:${pkgs.python27Packages.txaio}/lib/python2.7/site-packages:${pkgs.python27Packages.autobahn}/lib/python2.7/site-packages:${pkgs.python27Packages.pyjwt}/lib/python2.7/site-packages:${pkgs.python27Packages.distro}/lib/python2.7/site-packages:${pkgs.python27Packages.pbr}/lib/python2.7/site-packages:${pkgs.python27Packages.urllib3}/lib/python2.7/site-packages"; - preStart = '' - ${pkgs.coreutils}/bin/mkdir -vp ${cfg.buildbotDir} - ${pkgs.coreutils}/bin/ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg - ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir} - ''; + # NOTE: call twistd directly with stdout logging for systemd + #ExecStart = "${cfg.package}/bin/buildbot start --nodaemon ${cfg.buildbotDir}"; + ExecStart = "${pkgs.python27Packages.twisted}/bin/twistd -n -l - -y ${cfg.buildbotDir}/buildbot.tac"; + }; - postStart = '' - until [[ $(${pkgs.curl}/bin/curl -s --head -w '\n%{http_code}' http://localhost:${toString cfg.port} | tail -n1) =~ ^(200|403)$ ]]; do - sleep 1 - done - ''; }; }; diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix index e4ee4dd861e..a97f571e89d 100644 --- a/nixos/modules/services/continuous-integration/buildbot/worker.nix +++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix @@ -74,7 +74,7 @@ in { }; packages = mkOption { - default = [ ]; + default = with pkgs; [ python27Packages.twisted git ]; example = literalExample "[ pkgs.git ]"; type = types.listOf types.package; description = "Packages to add to PATH for the buildbot process."; @@ -106,7 +106,8 @@ in { path = cfg.packages; preStart = '' - ${pkgs.coreutils}/bin/mkdir -vp ${cfg.buildbotDir} + mkdir -vp ${cfg.buildbotDir} + rm -fv $cfg.buildbotDir}/buildbot.tac ${cfg.package}/bin/buildbot-worker create-worker ${cfg.buildbotDir} ${cfg.masterUrl} ${cfg.workerUser} ${cfg.workerPass} ''; @@ -115,7 +116,11 @@ in { User = cfg.user; Group = cfg.group; WorkingDirectory = cfg.home; - ExecStart = "${cfg.package}/bin/buildbot-worker start --nodaemon ${cfg.buildbotDir}"; + Environment = "PYTHONPATH=${cfg.package}/lib/python2.7/site-packages:${pkgs.python27Packages.future}/lib/python2.7/site-packages"; + + # NOTE: call twistd directly with stdout logging for systemd + #ExecStart = "${cfg.package}/bin/buildbot-worker start --nodaemon ${cfg.buildbotDir}"; + ExecStart = "${pkgs.python27Packages.twisted}/bin/twistd -n -l - -y ${cfg.buildbotDir}/buildbot.tac"; }; }; diff --git a/nixos/tests/buildbot.nix b/nixos/tests/buildbot.nix index 13a162e6c6e..828846f17c8 100644 --- a/nixos/tests/buildbot.nix +++ b/nixos/tests/buildbot.nix @@ -4,18 +4,22 @@ import ./make-test.nix ({ pkgs, ... } : { name = "buildbot"; nodes = { - bbmaster = { config, pkgs, nodes, ... }: { + bbmaster = { config, pkgs, ... }: { services.buildbot-master = { enable = true; + package = pkgs.buildbot-full; + + # NOTE: use fake repo due to no internet in hydra ci factorySteps = [ - "steps.Git(repourl='git://github.com/buildbot/pyflakes.git', mode='incremental')" - "steps.ShellCommand(command=['trial', 'pyflakes'])" + "steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')" + "steps.ShellCommand(command=['bash', 'fakerepo.sh'])" ]; changeSource = [ - "changes.GitPoller('git://github.com/buildbot/pyflakes.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)" + "changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)" ]; }; - networking.firewall.allowedTCPPorts = [ 8010 9989 ]; + networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ]; + environment.systemPackages = with pkgs; [ git buildbot-full ]; }; bbworker = { config, pkgs, ... }: { @@ -23,24 +27,85 @@ import ./make-test.nix ({ pkgs, ... } : { enable = true; masterUrl = "bbmaster:9989"; }; + environment.systemPackages = with pkgs; [ git buildbot-worker ]; + }; + + gitrepo = { config, pkgs, ... }: { + services.openssh.enable = true; + networking.firewall.allowedTCPPorts = [ 22 9418 ]; + environment.systemPackages = with pkgs; [ git ]; }; }; testScript = '' + #Start up and populate fake repo + $gitrepo->waitForUnit("multi-user.target"); + print($gitrepo->execute(" \ + git config --global user.name 'Nobody Fakeuser' && \ + git config --global user.email 'nobody\@fakerepo.com' && \ + rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \ + mkdir -pv /srv/repos/fakerepo ~/.ssh && \ + ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \ + cat ~/.ssh/known_hosts && \ + cd /srv/repos/fakerepo && \ + git init && \ + echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \ + cat fakerepo.sh && \ + touch .git/git-daemon-export-ok && \ + git add fakerepo.sh .git/git-daemon-export-ok && \ + git commit -m fakerepo && \ + git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \ + ")); + + # Test gitrepo + $bbmaster->waitForUnit("network-online.target"); + #$bbmaster->execute("nc -z gitrepo 9418"); + print($bbmaster->execute(" \ + rm -rfv /tmp/fakerepo && \ + git clone git://gitrepo/fakerepo /tmp/fakerepo && \ + pwd && \ + ls -la && \ + ls -la /tmp/fakerepo \ + ")); - $bbmaster->waitForUnit("network.target"); - $bbworker->waitForUnit("network.target"); + # Test start master and connect worker + $bbmaster->waitForUnit("buildbot-master.service"); + $bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/; + $bbworker->waitForUnit("network-online.target"); + $bbworker->execute("nc -z bbmaster 8010"); + $bbworker->execute("nc -z bbmaster 9989"); + $bbworker->waitForUnit("buildbot-worker.service"); + print($bbworker->execute("ls -la /home/bbworker/worker")); - # Additional tests to be added - #$bbmaster->waitForUnit("buildbot-master.service"); - #$bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/ or die; - #$bbworker->waitForUnit("buildbot-worker.service"); - #$bbworker->waitUntilSucceeds("tail -10 /home/bbworker/worker/twistd.log") =~ /success/ or die; + + # Test stop buildbot master and worker + print($bbmaster->execute(" \ + systemctl -l --no-pager status buildbot-master && \ + systemctl stop buildbot-master \ + ")); + $bbworker->fail("nc -z bbmaster 8010"); + $bbworker->fail("nc -z bbmaster 9989"); + print($bbworker->execute(" \ + systemctl -l --no-pager status buildbot-worker && \ + systemctl stop buildbot-worker && \ + ls -la /home/bbworker/worker \ + ")); + + + # Test buildbot daemon mode + # NOTE: daemon mode tests disabled due to broken PYTHONPATH child inheritence + # + #$bbmaster->execute("buildbot create-master /tmp"); + #$bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg"); + #$bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg"); + #$bbmaster->execute("buildbot start /tmp"); + #$bbworker->execute("nc -z bbmaster 8011"); + #$bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/; + #$bbmaster->execute("buildbot stop /tmp"); + #$bbworker->fail("nc -z bbmaster 8011"); ''; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ nand0p ]; - }; + meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ]; }) diff --git a/pkgs/development/tools/build-managers/buildbot/default.nix b/pkgs/development/tools/build-managers/buildbot/default.nix index ed24e917a1c..8c8caf06303 100644 --- a/pkgs/development/tools/build-managers/buildbot/default.nix +++ b/pkgs/development/tools/build-managers/buildbot/default.nix @@ -5,17 +5,19 @@ let buildInputs = [ makeWrapper ] ++ plugins; passthru.withPlugins = moarPlugins: withPlugins (moarPlugins ++ plugins); } '' - makeWrapper ${package}/bin/buildbot $out/bin/buildbot --prefix PYTHONPATH : $PYTHONPATH + makeWrapper ${package}/bin/buildbot $out/bin/buildbot \ + --prefix PYTHONPATH : "${package}/lib/python2.7/site-packages:$PYTHONPATH" + ln -sfv ${package}/lib $out/lib ''; package = pythonPackages.buildPythonApplication (rec { name = "${pname}-${version}"; pname = "buildbot"; - version = "0.9.4"; + version = "0.9.5"; src = pythonPackages.fetchPypi { inherit pname version; - sha256 = "0wklrn4fszac9wi8zw3vbsznwyff6y57cz0i81zvh46skb6n3086"; + sha256 = "11r553nmh87a9pm58wycimapk2pw9hnlc7hffn97xwbqprd8qh66"; }; buildInputs = with pythonPackages; [ @@ -49,6 +51,7 @@ let txaio autobahn pyjwt + distro # tls pyopenssl @@ -68,6 +71,14 @@ let postPatch = '' substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)" + + # NOTE: secrets management tests currently broken + rm -fv buildbot/test/integration/test_integration_secrets.py + rm -fv buildbot/test/integration/test_integration_secrets_with_vault.py + rm -fv buildbot/test/unit/test_fake_secrets_manager.py + rm -fv buildbot/test/unit/test_interpolate_secrets.py + rm -fv buildbot/test/unit/test_secret_in_file.py + rm -fv buildbot/test/unit/test_secret_in_vault.py ''; passthru = { inherit withPlugins; }; diff --git a/pkgs/development/tools/build-managers/buildbot/plugins.nix b/pkgs/development/tools/build-managers/buildbot/plugins.nix index b22759af8fe..77e97ed7ac3 100644 --- a/pkgs/development/tools/build-managers/buildbot/plugins.nix +++ b/pkgs/development/tools/build-managers/buildbot/plugins.nix @@ -1,14 +1,14 @@ -{ stdenv, fetchurl, pythonPackages }: +{ stdenv, pythonPackages }: let buildbot-pkg = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot-pkg"; - version = "0.9.4"; + version = "0.9.5"; - src = fetchurl { - url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "09a3yvs5hhf8syrkyydznmymgg86dpvgrwy9rb3bryq00wpjb3wn"; + src = pythonPackages.fetchPypi { + inherit pname version; + sha256 = "1xpi4w0lc6z97pmmms85dvdspacbzlvs8zi3kv1r4rypk3znwmi1"; }; propagatedBuildInputs = with pythonPackages; [ setuptools ]; @@ -25,15 +25,14 @@ in { www = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot_www"; - version = "0.9.4"; + version = "0.9.5"; # NOTE: wheel is used due to buildbot circular dependency format = "wheel"; src = pythonPackages.fetchPypi { inherit pname version format; - python = "py2"; - sha256 = "08m4h2pf6hgi8igh2j0qzfq49izc2z0qqj6ddxk0di5l306jx4im"; + sha256 = "1d7yjxka6slflm3wbdpq4sr1kagmgbqdv2zgx9bq77jvjh7ga0py"; }; meta = with stdenv.lib; { @@ -47,11 +46,11 @@ in { console-view = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot-console-view"; - version = "0.9.4"; + version = "0.9.5"; - src = fetchurl { - url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "1w2vv8iyzl7ak4161avp9n6mhh08adav2fl82bbm17a3064apl8n"; + src = pythonPackages.fetchPypi { + inherit pname version; + sha256 = "1s6mvw955dsgk7hvb1xa32bbd7w2yma62py5s0vmi5shv8nwq3hb"; }; propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ]; @@ -67,11 +66,11 @@ in { waterfall-view = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot-waterfall-view"; - version = "0.9.4"; + version = "0.9.5"; - src = fetchurl { - url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "17xn6vrr0k2xabw6hr9sdyy0ry3llyjfmc79qrpgp5bsly2qv3jf"; + src = pythonPackages.fetchPypi { + inherit pname version; + sha256 = "116846d987wp1bz78f0h4lypqcns5073vzhb4vsqbf08sppgr67k"; }; propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ]; diff --git a/pkgs/development/tools/build-managers/buildbot/worker.nix b/pkgs/development/tools/build-managers/buildbot/worker.nix index b9b4b86aa44..145221c6353 100644 --- a/pkgs/development/tools/build-managers/buildbot/worker.nix +++ b/pkgs/development/tools/build-managers/buildbot/worker.nix @@ -3,11 +3,11 @@ pythonPackages.buildPythonApplication (rec { name = "${pname}-${version}"; pname = "buildbot-worker"; - version = "0.9.4"; + version = "0.9.5"; src = pythonPackages.fetchPypi { inherit pname version; - sha256 = "0rdrr8x7sn2nxl51p6h9ad42s3c28lb6sys84zrg0d7fm4zhv7hj"; + sha256 = "1al7jam351sf781axj4kfhj70cc0g21zv81ynk410kdccjyxp2dy"; }; buildInputs = with pythonPackages; [ setuptoolsTrial mock ]; |