diff options
Diffstat (limited to 'nixos')
235 files changed, 5632 insertions, 2704 deletions
diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index f9de2db1a08..6ca75f869f4 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -62,14 +62,13 @@ let "--stringparam html.stylesheet 'style.css overrides.css highlightjs/mono-blue.css'" "--stringparam html.script './highlightjs/highlight.pack.js ./highlightjs/loader.js'" "--param xref.with.number.and.title 1" - "--param toc.section.depth 3" + "--param toc.section.depth 0" "--stringparam admon.style ''" "--stringparam callout.graphics.extension .svg" "--stringparam current.docid manual" "--param chunk.section.depth 0" "--param chunk.first.sections 1" "--param use.id.as.filename 1" - "--stringparam generate.toc 'book toc appendix toc'" "--stringparam chunk.toc ${toc}" ]; diff --git a/nixos/doc/manual/development/running-nixos-tests-interactively.xml b/nixos/doc/manual/development/running-nixos-tests-interactively.xml index e390d62fde2..ea3ba0e4bf7 100644 --- a/nixos/doc/manual/development/running-nixos-tests-interactively.xml +++ b/nixos/doc/manual/development/running-nixos-tests-interactively.xml @@ -14,14 +14,14 @@ starting VDE switch for network 1 <prompt>></prompt> </screen> - You can then take any Perl statement, e.g. + You can then take any Python statement, e.g. <screen> -<prompt>></prompt> startAll -<prompt>></prompt> testScript -<prompt>></prompt> $machine->succeed("touch /tmp/foo") -<prompt>></prompt> print($machine->succeed("pwd")) # Show stdout of command +<prompt>></prompt> start_all() +<prompt>></prompt> test_script() +<prompt>></prompt> machine.succeed("touch /tmp/foo") +<prompt>></prompt> print(machine.succeed("pwd")) # Show stdout of command </screen> - The function <command>testScript</command> executes the entire test script + The function <command>test_script</command> executes the entire test script and drops you back into the test driver command line upon its completion. This allows you to inspect the state of the VMs after the test (e.g. to debug the test script). diff --git a/nixos/doc/manual/development/writing-nixos-tests.xml b/nixos/doc/manual/development/writing-nixos-tests.xml index 6be2d0a4d23..24efd2e3273 100644 --- a/nixos/doc/manual/development/writing-nixos-tests.xml +++ b/nixos/doc/manual/development/writing-nixos-tests.xml @@ -8,7 +8,7 @@ <para> A NixOS test is a Nix expression that has the following structure: <programlisting> -import ./make-test.nix { +import ./make-test-python.nix { # Either the configuration of a single machine: machine = @@ -27,11 +27,11 @@ import ./make-test.nix { testScript = '' - <replaceable>Perl code…</replaceable> + <replaceable>Python code…</replaceable> ''; } </programlisting> - The attribute <literal>testScript</literal> is a bit of Perl code that + The attribute <literal>testScript</literal> is a bit of Python code that executes the test (described below). During the test, it will start one or more virtual machines, the configuration of which is described by the attribute <literal>machine</literal> (if you need only one machine in your @@ -96,26 +96,27 @@ xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualis </para> <para> - The test script is a sequence of Perl statements that perform various + The test script is a sequence of Python statements that perform various actions, such as starting VMs, executing commands in the VMs, and so on. Each virtual machine is represented as an object stored in the variable - <literal>$<replaceable>name</replaceable></literal>, where - <replaceable>name</replaceable> is the identifier of the machine (which is - just <literal>machine</literal> if you didn’t specify multiple machines - using the <literal>nodes</literal> attribute). For instance, the following - starts the machine, waits until it has finished booting, then executes a - command and checks that the output is more-or-less correct: + <literal><replaceable>name</replaceable></literal> if this is also the + identifier of the machine in the declarative config. + If you didn't specify multiple machines using the <literal>nodes</literal> + attribute, it is just <literal>machine</literal>. + The following example starts the machine, waits until it has finished booting, + then executes a command and checks that the output is more-or-less correct: <programlisting> -$machine->start; -$machine->waitForUnit("default.target"); -$machine->succeed("uname") =~ /Linux/ or die; +machine.start() +machine.wait_for_unit("default.target") +if not "Linux" in machine.succeed("uname"): + raise Exception("Wrong OS") </programlisting> The first line is actually unnecessary; machines are implicitly started when - you first execute an action on them (such as <literal>waitForUnit</literal> + you first execute an action on them (such as <literal>wait_for_unit</literal> or <literal>succeed</literal>). If you have multiple machines, you can speed up the test by starting them in parallel: <programlisting> -startAll; +start_all() </programlisting> </para> @@ -187,7 +188,7 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>getScreenText</methodname> + <methodname>get_screen_text</methodname> </term> <listitem> <para> @@ -204,7 +205,7 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>sendMonitorCommand</methodname> + <methodname>send_monitor_command</methodname> </term> <listitem> <para> @@ -215,23 +216,23 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>sendKeys</methodname> + <methodname>send_keys</methodname> </term> <listitem> <para> Simulate pressing keys on the virtual keyboard, e.g., - <literal>sendKeys("ctrl-alt-delete")</literal>. + <literal>send_keys("ctrl-alt-delete")</literal>. </para> </listitem> </varlistentry> <varlistentry> <term> - <methodname>sendChars</methodname> + <methodname>send_chars</methodname> </term> <listitem> <para> Simulate typing a sequence of characters on the virtual keyboard, e.g., - <literal>sendKeys("foobar\n")</literal> will type the string + <literal>send_keys("foobar\n")</literal> will type the string <literal>foobar</literal> followed by the Enter key. </para> </listitem> @@ -272,7 +273,7 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>waitUntilSucceeds</methodname> + <methodname>wait_until_succeeds</methodname> </term> <listitem> <para> @@ -282,7 +283,7 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>waitUntilFails</methodname> + <methodname>wait_until_fails</methodname> </term> <listitem> <para> @@ -292,7 +293,7 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>waitForUnit</methodname> + <methodname>wait_for_unit</methodname> </term> <listitem> <para> @@ -302,7 +303,7 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>waitForFile</methodname> + <methodname>wait_for_file</methodname> </term> <listitem> <para> @@ -312,7 +313,7 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>waitForOpenPort</methodname> + <methodname>wait_for_open_port</methodname> </term> <listitem> <para> @@ -323,7 +324,7 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>waitForClosedPort</methodname> + <methodname>wait_for_closed_port</methodname> </term> <listitem> <para> @@ -333,7 +334,7 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>waitForX</methodname> + <methodname>wait_for_x</methodname> </term> <listitem> <para> @@ -343,13 +344,13 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>waitForText</methodname> + <methodname>wait_for_text</methodname> </term> <listitem> <para> Wait until the supplied regular expressions matches the textual contents of the screen by using optical character recognition (see - <methodname>getScreenText</methodname>). + <methodname>get_screen_text</methodname>). </para> <note> <para> @@ -361,23 +362,23 @@ startAll; </varlistentry> <varlistentry> <term> - <methodname>waitForWindow</methodname> + <methodname>wait_for_window</methodname> </term> <listitem> <para> Wait until an X11 window has appeared whose name matches the given - regular expression, e.g., <literal>waitForWindow(qr/Terminal/)</literal>. + regular expression, e.g., <literal>wait_for_window("Terminal")</literal>. </para> </listitem> </varlistentry> <varlistentry> <term> - <methodname>copyFileFromHost</methodname> + <methodname>copy_file_from_host</methodname> </term> <listitem> <para> Copies a file from host to machine, e.g., - <literal>copyFileFromHost("myfile", "/etc/my/important/file")</literal>. + <literal>copy_file_from_host("myfile", "/etc/my/important/file")</literal>. </para> <para> The first argument is the file on the host. The file needs to be @@ -397,8 +398,8 @@ startAll; </para> <para> <programlisting> -$machine->systemctl("list-jobs --no-pager"); // runs `systemctl list-jobs --no-pager` -$machine->systemctl("list-jobs --no-pager", "any-user"); // spawns a shell for `any-user` and runs `systemctl --user list-jobs --no-pager` +machine.systemctl("list-jobs --no-pager") # runs `systemctl list-jobs --no-pager` +machine.systemctl("list-jobs --no-pager", "any-user") # spawns a shell for `any-user` and runs `systemctl --user list-jobs --no-pager` </programlisting> </para> </listitem> @@ -408,14 +409,14 @@ $machine->systemctl("list-jobs --no-pager", "any-user"); // spawns a shell for ` <para> To test user units declared by <literal>systemd.user.services</literal> the - optional <literal>$user</literal> argument can be used: + optional <literal>user</literal> argument can be used: <programlisting> -$machine->start; -$machine->waitForX; -$machine->waitForUnit("xautolock.service", "x-session-user"); +machine.start() +machine.wait_for_x() +machine.wait_for_unit("xautolock.service", "x-session-user") </programlisting> - This applies to <literal>systemctl</literal>, <literal>getUnitInfo</literal>, - <literal>waitForUnit</literal>, <literal>startJob</literal> and - <literal>stopJob</literal>. + This applies to <literal>systemctl</literal>, <literal>get_unit_info</literal>, + <literal>wait_for_unit</literal>, <literal>start_job</literal> and + <literal>stop_job</literal>. </para> </section> diff --git a/nixos/doc/manual/installation/installing-virtualbox-guest.xml b/nixos/doc/manual/installation/installing-virtualbox-guest.xml index 5c86eacfbf4..0ba909fa953 100644 --- a/nixos/doc/manual/installation/installing-virtualbox-guest.xml +++ b/nixos/doc/manual/installation/installing-virtualbox-guest.xml @@ -49,6 +49,11 @@ </listitem> <listitem> <para> + Click on Settings / Display / Screen and select VBoxVGA as Graphics Controller + </para> + </listitem> + <listitem> + <para> Save the settings, start the virtual machine, and continue installation like normal </para> diff --git a/nixos/doc/manual/man-nixos-option.xml b/nixos/doc/manual/man-nixos-option.xml index 81e3739b3be..beabf020c92 100644 --- a/nixos/doc/manual/man-nixos-option.xml +++ b/nixos/doc/manual/man-nixos-option.xml @@ -19,14 +19,10 @@ </arg> <arg> - <option>--verbose</option> + <option>--all</option> </arg> <arg> - <option>--xml</option> - </arg> - - <arg choice="plain"> <replaceable>option.name</replaceable> </arg> </cmdsynopsis> @@ -62,22 +58,11 @@ </varlistentry> <varlistentry> <term> - <option>--verbose</option> - </term> - <listitem> - <para> - This option enables verbose mode, which currently is just the Bash - <command>set</command> <option>-x</option> debug mode. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term> - <option>--xml</option> + <option>--all</option> </term> <listitem> <para> - This option causes the output to be rendered as XML. + Print the values of all options. </para> </listitem> </varlistentry> diff --git a/nixos/doc/manual/man-nixos-rebuild.xml b/nixos/doc/manual/man-nixos-rebuild.xml index c697b7ee047..495dbc8859b 100644 --- a/nixos/doc/manual/man-nixos-rebuild.xml +++ b/nixos/doc/manual/man-nixos-rebuild.xml @@ -494,6 +494,20 @@ </para> </listitem> </varlistentry> + + <varlistentry> + <term> + <option>--use-remote-sudo</option> + </term> + <listitem> + <para> + When set, nixos-rebuild prefixes remote commands that run on + the <option>--build-host</option> and <option>--target-host</option> + systems with <command>sudo</command>. Setting this option allows + deploying as a non-root user. + </para> + </listitem> + </varlistentry> </variablelist> <para> diff --git a/nixos/doc/manual/manual.xml b/nixos/doc/manual/manual.xml index 12f52e1997c..18a67a2dd94 100644 --- a/nixos/doc/manual/manual.xml +++ b/nixos/doc/manual/manual.xml @@ -8,32 +8,7 @@ <subtitle>Version <xi:include href="./generated/version" parse="text" /> </subtitle> </info> - <preface xml:id="preface"> - <title>Preface</title> - <para> - This manual describes how to install, use and extend NixOS, a Linux - distribution based on the purely functional package management system Nix. - </para> - <para> - If you encounter problems, please report them on the - <literal - xlink:href="https://discourse.nixos.org">Discourse</literal> or - on the <link - xlink:href="irc://irc.freenode.net/#nixos"> - <literal>#nixos</literal> channel on Freenode</link>. Bugs should be - reported in - <link - xlink:href="https://github.com/NixOS/nixpkgs/issues">NixOS’ - GitHub issue tracker</link>. - </para> - <note> - <para> - Commands prefixed with <literal>#</literal> have to be run as root, either - requiring to login as root user or temporarily switching to it using - <literal>sudo</literal> for example. - </para> - </note> - </preface> + <xi:include href="preface.xml" /> <xi:include href="installation/installation.xml" /> <xi:include href="configuration/configuration.xml" /> <xi:include href="administration/running.xml" /> diff --git a/nixos/doc/manual/preface.xml b/nixos/doc/manual/preface.xml new file mode 100644 index 00000000000..6ac9ae7e786 --- /dev/null +++ b/nixos/doc/manual/preface.xml @@ -0,0 +1,37 @@ +<preface xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xml:id="preface"> + <title>Preface</title> + <para> + This manual describes how to install, use and extend NixOS, a Linux + distribution based on the purely functional package management system + <link xlink:href="https://nixos.org/nix">Nix</link>, that is composed + using modules and packages defined in the + <link xlink:href="https://nixos.org/nixpkgs">Nixpkgs</link> project. + </para> + <para> + Additional information regarding the Nix package manager and the Nixpkgs + project can be found in respectively the + <link xlink:href="https://nixos.org/nix/manual">Nix manual</link> and the + <link xlink:href="https://nixos.org/nixpkgs/manual">Nixpkgs manual</link>. + </para> + <para> + If you encounter problems, please report them on the + <literal + xlink:href="https://discourse.nixos.org">Discourse</literal> or + on the <link + xlink:href="irc://irc.freenode.net/#nixos"> + <literal>#nixos</literal> channel on Freenode</link>. Bugs should be + reported in + <link + xlink:href="https://github.com/NixOS/nixpkgs/issues">NixOS’ + GitHub issue tracker</link>. + </para> + <note> + <para> + Commands prefixed with <literal>#</literal> have to be run as root, either + requiring to login as root user or temporarily switching to it using + <literal>sudo</literal> for example. + </para> + </note> +</preface> diff --git a/nixos/doc/manual/release-notes/rl-2003.xml b/nixos/doc/manual/release-notes/rl-2003.xml index f001a18b1c1..72766f16eb9 100644 --- a/nixos/doc/manual/release-notes/rl-2003.xml +++ b/nixos/doc/manual/release-notes/rl-2003.xml @@ -49,6 +49,12 @@ zfs as soon as any zfs mountpoint is configured in <varname>fileSystems</varname>. </para> </listitem> + <listitem> + <para> + <command>nixos-option</command> has been rewritten in C++, speeding it up, improving correctness, + and adding a <option>--all</option> option which prints all options and their values. + </para> + </listitem> </itemizedlist> </section> diff --git a/nixos/lib/test-driver/test-driver.py b/nixos/lib/test-driver/test-driver.py new file mode 100644 index 00000000000..c8940d78af4 --- /dev/null +++ b/nixos/lib/test-driver/test-driver.py @@ -0,0 +1,809 @@ +#! /somewhere/python3 + +from contextlib import contextmanager +from xml.sax.saxutils import XMLGenerator +import _thread +import atexit +import json +import os +import ptpython.repl +import pty +import queue +import re +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import unicodedata + +CHAR_TO_KEY = { + "A": "shift-a", + "N": "shift-n", + "-": "0x0C", + "_": "shift-0x0C", + "B": "shift-b", + "O": "shift-o", + "=": "0x0D", + "+": "shift-0x0D", + "C": "shift-c", + "P": "shift-p", + "[": "0x1A", + "{": "shift-0x1A", + "D": "shift-d", + "Q": "shift-q", + "]": "0x1B", + "}": "shift-0x1B", + "E": "shift-e", + "R": "shift-r", + ";": "0x27", + ":": "shift-0x27", + "F": "shift-f", + "S": "shift-s", + "'": "0x28", + '"': "shift-0x28", + "G": "shift-g", + "T": "shift-t", + "`": "0x29", + "~": "shift-0x29", + "H": "shift-h", + "U": "shift-u", + "\\": "0x2B", + "|": "shift-0x2B", + "I": "shift-i", + "V": "shift-v", + ",": "0x33", + "<": "shift-0x33", + "J": "shift-j", + "W": "shift-w", + ".": "0x34", + ">": "shift-0x34", + "K": "shift-k", + "X": "shift-x", + "/": "0x35", + "?": "shift-0x35", + "L": "shift-l", + "Y": "shift-y", + " ": "spc", + "M": "shift-m", + "Z": "shift-z", + "\n": "ret", + "!": "shift-0x02", + "@": "shift-0x03", + "#": "shift-0x04", + "$": "shift-0x05", + "%": "shift-0x06", + "^": "shift-0x07", + "&": "shift-0x08", + "*": "shift-0x09", + "(": "shift-0x0A", + ")": "shift-0x0B", +} + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def create_vlan(vlan_nr): + global log + log.log("starting VDE switch for network {}".format(vlan_nr)) + vde_socket = os.path.abspath("./vde{}.ctl".format(vlan_nr)) + pty_master, pty_slave = pty.openpty() + vde_process = subprocess.Popen( + ["vde_switch", "-s", vde_socket, "--dirmode", "0777"], + bufsize=1, + stdin=pty_slave, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) + fd = os.fdopen(pty_master, "w") + fd.write("version\n") + # TODO: perl version checks if this can be read from + # an if not, dies. we could hang here forever. Fix it. + vde_process.stdout.readline() + if not os.path.exists(os.path.join(vde_socket, "ctl")): + raise Exception("cannot start vde_switch") + + return (vlan_nr, vde_socket, vde_process, fd) + + +def retry(fn): + """Call the given function repeatedly, with 1 second intervals, + until it returns True or a timeout is reached. + """ + + for _ in range(900): + if fn(False): + return + time.sleep(1) + + if not fn(True): + raise Exception("action timed out") + + +class Logger: + def __init__(self): + self.logfile = os.environ.get("LOGFILE", "/dev/null") + self.logfile_handle = open(self.logfile, "wb") + self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8") + self.queue = queue.Queue(1000) + + self.xml.startDocument() + self.xml.startElement("logfile", attrs={}) + + def close(self): + self.xml.endElement("logfile") + self.xml.endDocument() + self.logfile_handle.close() + + def sanitise(self, message): + return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C") + + def maybe_prefix(self, message, attributes): + if "machine" in attributes: + return "{}: {}".format(attributes["machine"], message) + return message + + def log_line(self, message, attributes): + self.xml.startElement("line", attributes) + self.xml.characters(message) + self.xml.endElement("line") + + def log(self, message, attributes={}): + eprint(self.maybe_prefix(message, attributes)) + self.drain_log_queue() + self.log_line(message, attributes) + + def enqueue(self, message): + self.queue.put(message) + + def drain_log_queue(self): + try: + while True: + item = self.queue.get_nowait() + attributes = {"machine": item["machine"], "type": "serial"} + self.log_line(self.sanitise(item["msg"]), attributes) + except queue.Empty: + pass + + @contextmanager + def nested(self, message, attributes={}): + eprint(self.maybe_prefix(message, attributes)) + + self.xml.startElement("nest", attrs={}) + self.xml.startElement("head", attributes) + self.xml.characters(message) + self.xml.endElement("head") + + tic = time.time() + self.drain_log_queue() + yield + self.drain_log_queue() + toc = time.time() + self.log("({:.2f} seconds)".format(toc - tic)) + + self.xml.endElement("nest") + + +class Machine: + def __init__(self, args): + if "name" in args: + self.name = args["name"] + else: + self.name = "machine" + try: + cmd = args["startCommand"] + self.name = re.search("run-(.+)-vm$", cmd).group(1) + except KeyError: + pass + except AttributeError: + pass + + self.script = args.get("startCommand", self.create_startcommand(args)) + + tmp_dir = os.environ.get("TMPDIR", tempfile.gettempdir()) + + def create_dir(name): + path = os.path.join(tmp_dir, name) + os.makedirs(path, mode=0o700, exist_ok=True) + return path + + self.state_dir = create_dir("vm-state-{}".format(self.name)) + self.shared_dir = create_dir("xchg-shared") + + self.booted = False + self.connected = False + self.pid = None + self.socket = None + self.monitor = None + self.logger = args["log"] + self.allow_reboot = args.get("allowReboot", False) + + @staticmethod + def create_startcommand(args): + net_backend = "-netdev user,id=net0" + net_frontend = "-device virtio-net-pci,netdev=net0" + + if "netBackendArgs" in args: + net_backend += "," + args["netBackendArgs"] + + if "netFrontendArgs" in args: + net_frontend += "," + args["netFrontendArgs"] + + start_command = ( + "qemu-kvm -m 384 " + net_backend + " " + net_frontend + " $QEMU_OPTS " + ) + + if "hda" in args: + hda_path = os.path.abspath(args["hda"]) + if args.get("hdaInterface", "") == "scsi": + start_command += ( + "-drive id=hda,file=" + + hda_path + + ",werror=report,if=none " + + "-device scsi-hd,drive=hda " + ) + else: + start_command += ( + "-drive file=" + + hda_path + + ",if=" + + args["hdaInterface"] + + ",werror=report " + ) + + if "cdrom" in args: + start_command += "-cdrom " + args["cdrom"] + " " + + if "usb" in args: + start_command += ( + "-device piix3-usb-uhci -drive " + + "id=usbdisk,file=" + + args["usb"] + + ",if=none,readonly " + + "-device usb-storage,drive=usbdisk " + ) + if "bios" in args: + start_command += "-bios " + args["bios"] + " " + + start_command += args.get("qemuFlags", "") + + return start_command + + def is_up(self): + return self.booted and self.connected + + def log(self, msg): + self.logger.log(msg, {"machine": self.name}) + + def nested(self, msg, attrs={}): + my_attrs = {"machine": self.name} + my_attrs.update(attrs) + return self.logger.nested(msg, my_attrs) + + def wait_for_monitor_prompt(self): + while True: + answer = self.monitor.recv(1024).decode() + if answer.endswith("(qemu) "): + return answer + + def send_monitor_command(self, command): + message = ("{}\n".format(command)).encode() + self.log("sending monitor command: {}".format(command)) + self.monitor.send(message) + return self.wait_for_monitor_prompt() + + def wait_for_unit(self, unit, user=None): + while True: + info = self.get_unit_info(unit, user) + state = info["ActiveState"] + if state == "failed": + raise Exception('unit "{}" reached state "{}"'.format(unit, state)) + + if state == "inactive": + status, jobs = self.systemctl("list-jobs --full 2>&1", user) + if "No jobs" in jobs: + info = self.get_unit_info(unit, user) + if info["ActiveState"] == state: + raise Exception( + ( + 'unit "{}" is inactive and there ' "are no pending jobs" + ).format(unit) + ) + if state == "active": + return True + + def get_unit_info(self, unit, user=None): + status, lines = self.systemctl('--no-pager show "{}"'.format(unit), user) + if status != 0: + raise Exception( + 'retrieving systemctl info for unit "{}" {} failed with exit code {}'.format( + unit, "" if user is None else 'under user "{}"'.format(user), status + ) + ) + + line_pattern = re.compile(r"^([^=]+)=(.*)$") + + def tuple_from_line(line): + match = line_pattern.match(line) + return match[1], match[2] + + return dict( + tuple_from_line(line) + for line in lines.split("\n") + if line_pattern.match(line) + ) + + def systemctl(self, q, user=None): + if user is not None: + q = q.replace("'", "\\'") + return self.execute( + ( + "su -l {} -c " + "$'XDG_RUNTIME_DIR=/run/user/`id -u` " + "systemctl --user {}'" + ).format(user, q) + ) + return self.execute("systemctl {}".format(q)) + + def require_unit_state(self, unit, require_state="active"): + with self.nested( + "checking if unit ‘{}’ has reached state '{}'".format(unit, require_state) + ): + info = self.get_unit_info(unit) + state = info["ActiveState"] + if state != require_state: + raise Exception( + "Expected unit ‘{}’ to to be in state ".format(unit) + + "'active' but it is in state ‘{}’".format(state) + ) + + def execute(self, command): + self.connect() + + out_command = "( {} ); echo '|!EOF' $?\n".format(command) + self.shell.send(out_command.encode()) + + output = "" + status_code_pattern = re.compile(r"(.*)\|\!EOF\s+(\d+)") + + while True: + chunk = self.shell.recv(4096).decode() + match = status_code_pattern.match(chunk) + if match: + output += match[1] + status_code = int(match[2]) + return (status_code, output) + output += chunk + + def succeed(self, *commands): + """Execute each command and check that it succeeds.""" + for command in commands: + with self.nested("must succeed: {}".format(command)): + status, output = self.execute(command) + if status != 0: + self.log("output: {}".format(output)) + raise Exception( + "command `{}` failed (exit code {})".format(command, status) + ) + return output + + def fail(self, *commands): + """Execute each command and check that it fails.""" + for command in commands: + with self.nested("must fail: {}".format(command)): + status, output = self.execute(command) + if status == 0: + raise Exception( + "command `{}` unexpectedly succeeded".format(command) + ) + + def wait_until_succeeds(self, command): + with self.nested("waiting for success: {}".format(command)): + while True: + status, output = self.execute(command) + if status == 0: + return output + + def wait_until_fails(self, command): + with self.nested("waiting for failure: {}".format(command)): + while True: + status, output = self.execute(command) + if status != 0: + return output + + def wait_for_shutdown(self): + if not self.booted: + return + + with self.nested("waiting for the VM to power off"): + sys.stdout.flush() + self.process.wait() + + self.pid = None + self.booted = False + self.connected = False + + def get_tty_text(self, tty): + status, output = self.execute( + "fold -w$(stty -F /dev/tty{0} size | " + "awk '{{print $2}}') /dev/vcs{0}".format(tty) + ) + return output + + def wait_until_tty_matches(self, tty, regexp): + matcher = re.compile(regexp) + with self.nested("waiting for {} to appear on tty {}".format(regexp, tty)): + while True: + text = self.get_tty_text(tty) + if len(matcher.findall(text)) > 0: + return True + + def send_chars(self, chars): + with self.nested("sending keys ‘{}‘".format(chars)): + for char in chars: + self.send_key(char) + + def wait_for_file(self, filename): + with self.nested("waiting for file ‘{}‘".format(filename)): + while True: + status, _ = self.execute("test -e {}".format(filename)) + if status == 0: + return True + + def wait_for_open_port(self, port): + def port_is_open(_): + status, _ = self.execute("nc -z localhost {}".format(port)) + return status == 0 + + with self.nested("waiting for TCP port {}".format(port)): + retry(port_is_open) + + def wait_for_closed_port(self, port): + def port_is_closed(_): + status, _ = self.execute("nc -z localhost {}".format(port)) + return status != 0 + + retry(port_is_closed) + + def start_job(self, jobname, user=None): + return self.systemctl("start {}".format(jobname), user) + + def stop_job(self, jobname, user=None): + return self.systemctl("stop {}".format(jobname), user) + + def wait_for_job(self, jobname): + return self.wait_for_unit(jobname) + + def connect(self): + if self.connected: + return + + with self.nested("waiting for the VM to finish booting"): + self.start() + + tic = time.time() + self.shell.recv(1024) + # TODO: Timeout + toc = time.time() + + self.log("connected to guest root shell") + self.log("(connecting took {:.2f} seconds)".format(toc - tic)) + self.connected = True + + def screenshot(self, filename): + out_dir = os.environ.get("out", os.getcwd()) + word_pattern = re.compile(r"^\w+$") + if word_pattern.match(filename): + filename = os.path.join(out_dir, "{}.png".format(filename)) + tmp = "{}.ppm".format(filename) + + with self.nested( + "making screenshot {}".format(filename), + {"image": os.path.basename(filename)}, + ): + self.send_monitor_command("screendump {}".format(tmp)) + ret = subprocess.run("pnmtopng {} > {}".format(tmp, filename), shell=True) + os.unlink(tmp) + if ret.returncode != 0: + raise Exception("Cannot convert screenshot") + + def dump_tty_contents(self, tty): + """Debugging: Dump the contents of the TTY<n> + """ + self.execute("fold -w 80 /dev/vcs{} | systemd-cat".format(tty)) + + def get_screen_text(self): + if shutil.which("tesseract") is None: + raise Exception("get_screen_text used but enableOCR is false") + + magick_args = ( + "-filter Catrom -density 72 -resample 300 " + + "-contrast -normalize -despeckle -type grayscale " + + "-sharpen 1 -posterize 3 -negate -gamma 100 " + + "-blur 1x65535" + ) + + tess_args = "-c debug_file=/dev/null --psm 11 --oem 2" + + with self.nested("performing optical character recognition"): + with tempfile.NamedTemporaryFile() as tmpin: + self.send_monitor_command("screendump {}".format(tmpin.name)) + + cmd = "convert {} {} tiff:- | tesseract - - {}".format( + magick_args, tmpin.name, tess_args + ) + ret = subprocess.run(cmd, shell=True, capture_output=True) + if ret.returncode != 0: + raise Exception( + "OCR failed with exit code {}".format(ret.returncode) + ) + + return ret.stdout.decode("utf-8") + + def wait_for_text(self, regex): + def screen_matches(last): + text = self.get_screen_text() + m = re.search(regex, text) + + if last and not m: + self.log("Last OCR attempt failed. Text was: {}".format(text)) + + return m + + with self.nested("waiting for {} to appear on screen".format(regex)): + retry(screen_matches) + + def send_key(self, key): + key = CHAR_TO_KEY.get(key, key) + self.send_monitor_command("sendkey {}".format(key)) + + def start(self): + if self.booted: + return + + self.log("starting vm") + + def create_socket(path): + if os.path.exists(path): + os.unlink(path) + s = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM) + s.bind(path) + s.listen(1) + return s + + monitor_path = os.path.join(self.state_dir, "monitor") + self.monitor_socket = create_socket(monitor_path) + + shell_path = os.path.join(self.state_dir, "shell") + self.shell_socket = create_socket(shell_path) + + qemu_options = ( + " ".join( + [ + "" if self.allow_reboot else "-no-reboot", + "-monitor unix:{}".format(monitor_path), + "-chardev socket,id=shell,path={}".format(shell_path), + "-device virtio-serial", + "-device virtconsole,chardev=shell", + "-device virtio-rng-pci", + "-serial stdio" if "DISPLAY" in os.environ else "-nographic", + ] + ) + + " " + + os.environ.get("QEMU_OPTS", "") + ) + + environment = { + "QEMU_OPTS": qemu_options, + "SHARED_DIR": self.shared_dir, + "USE_TMPDIR": "1", + } + environment.update(dict(os.environ)) + + self.process = subprocess.Popen( + self.script, + bufsize=1, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + cwd=self.state_dir, + env=environment, + ) + self.monitor, _ = self.monitor_socket.accept() + self.shell, _ = self.shell_socket.accept() + + def process_serial_output(): + for line in self.process.stdout: + line = line.decode("unicode_escape").replace("\r", "").rstrip() + eprint("{} # {}".format(self.name, line)) + self.logger.enqueue({"msg": line, "machine": self.name}) + + _thread.start_new_thread(process_serial_output, ()) + + self.wait_for_monitor_prompt() + + self.pid = self.process.pid + self.booted = True + + self.log("QEMU running (pid {})".format(self.pid)) + + def shutdown(self): + if not self.booted: + return + + self.shell.send("poweroff\n".encode()) + self.wait_for_shutdown() + + def crash(self): + if not self.booted: + return + + self.log("forced crash") + self.send_monitor_command("quit") + self.wait_for_shutdown() + + def wait_for_x(self): + """Wait until it is possible to connect to the X server. Note that + testing the existence of /tmp/.X11-unix/X0 is insufficient. + """ + with self.nested("waiting for the X11 server"): + while True: + cmd = ( + "journalctl -b SYSLOG_IDENTIFIER=systemd | " + + 'grep "Reached target Current graphical"' + ) + status, _ = self.execute(cmd) + if status != 0: + continue + status, _ = self.execute("[ -e /tmp/.X11-unix/X0 ]") + if status == 0: + return + + def get_window_names(self): + return self.succeed( + r"xwininfo -root -tree | sed 's/.*0x[0-9a-f]* \"\([^\"]*\)\".*/\1/; t; d'" + ).splitlines() + + def wait_for_window(self, regexp): + pattern = re.compile(regexp) + + def window_is_visible(last_try): + names = self.get_window_names() + if last_try: + self.log( + "Last chance to match {} on the window list,".format(regexp) + + " which currently contains: " + + ", ".join(names) + ) + return any(pattern.search(name) for name in names) + + with self.nested("Waiting for a window to appear"): + retry(window_is_visible) + + def sleep(self, secs): + time.sleep(secs) + + def forward_port(self, host_port=8080, guest_port=80): + """Forward a TCP port on the host to a TCP port on the guest. + Useful during interactive testing. + """ + self.send_monitor_command( + "hostfwd_add tcp::{}-:{}".format(host_port, guest_port) + ) + + def block(self): + """Make the machine unreachable by shutting down eth1 (the multicast + interface used to talk to the other VMs). We keep eth0 up so that + the test driver can continue to talk to the machine. + """ + self.send_monitor_command("set_link virtio-net-pci.1 off") + + def unblock(self): + """Make the machine reachable. + """ + self.send_monitor_command("set_link virtio-net-pci.1 on") + + +def create_machine(args): + global log + args["log"] = log + args["redirectSerial"] = os.environ.get("USE_SERIAL", "0") == "1" + return Machine(args) + + +def start_all(): + with log.nested("starting all VMs"): + for machine in machines: + machine.start() + + +def join_all(): + with log.nested("waiting for all VMs to finish"): + for machine in machines: + machine.wait_for_shutdown() + + +def test_script(): + exec(os.environ["testScript"]) + + +def run_tests(): + tests = os.environ.get("tests", None) + if tests is not None: + with log.nested("running the VM test script"): + try: + exec(tests) + except Exception as e: + eprint("error: {}".format(str(e))) + sys.exit(1) + else: + ptpython.repl.embed(locals(), globals()) + + # TODO: Collect coverage data + + for machine in machines: + if machine.is_up(): + machine.execute("sync") + + if nr_tests != 0: + log.log("{} out of {} tests succeeded".format(nr_succeeded, nr_tests)) + + +@contextmanager +def subtest(name): + global nr_tests + global nr_succeeded + + with log.nested(name): + nr_tests += 1 + try: + yield + nr_succeeded += 1 + return True + except Exception as e: + log.log("error: {}".format(str(e))) + + return False + + +if __name__ == "__main__": + global log + log = Logger() + + vlan_nrs = list(dict.fromkeys(os.environ["VLANS"].split())) + vde_sockets = [create_vlan(v) for v in vlan_nrs] + for nr, vde_socket, _, _ in vde_sockets: + os.environ["QEMU_VDE_SOCKET_{}".format(nr)] = vde_socket + + vm_scripts = sys.argv[1:] + machines = [create_machine({"startCommand": s}) for s in vm_scripts] + machine_eval = [ + "{0} = machines[{1}]".format(m.name, idx) for idx, m in enumerate(machines) + ] + exec("\n".join(machine_eval)) + + nr_tests = 0 + nr_succeeded = 0 + + @atexit.register + def clean_up(): + with log.nested("cleaning up"): + for machine in machines: + if machine.pid is None: + continue + log.log("killing {} (pid {})".format(machine.name, machine.pid)) + machine.process.kill() + + for _, _, process, _ in vde_sockets: + process.kill() + log.close() + + tic = time.time() + run_tests() + toc = time.time() + print("test script finished in {:.2f}s".format(toc - tic)) diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix new file mode 100644 index 00000000000..21f6172e967 --- /dev/null +++ b/nixos/lib/testing-python.nix @@ -0,0 +1,279 @@ +{ system +, pkgs ? import ../.. { inherit system config; } + # Use a minimal kernel? +, minimal ? false + # Ignored +, config ? {} + # Modules to add to each VM +, extraConfigurations ? [] }: + +with import ./build-vms.nix { inherit system pkgs minimal extraConfigurations; }; +with pkgs; + +let + jquery-ui = callPackage ./testing/jquery-ui.nix { }; + jquery = callPackage ./testing/jquery.nix { }; + +in rec { + + inherit pkgs; + + + testDriver = let + testDriverScript = ./test-driver/test-driver.py; + in stdenv.mkDerivation { + name = "nixos-test-driver"; + + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ]; + checkInputs = with python3Packages; [ pylint black ]; + + dontUnpack = true; + + preferLocalBuild = true; + + doCheck = true; + checkPhase = '' + pylint --errors-only ${testDriverScript} + black --check --diff ${testDriverScript} + ''; + + installPhase = + '' + mkdir -p $out/bin + cp ${testDriverScript} $out/bin/nixos-test-driver + chmod u+x $out/bin/nixos-test-driver + # TODO: copy user script part into this file (append) + + wrapProgram $out/bin/nixos-test-driver \ + --prefix PATH : "${lib.makeBinPath [ qemu_test vde2 netpbm coreutils ]}" \ + ''; + }; + + + # Run an automated test suite in the given virtual network. + # `driver' is the script that runs the network. + runTests = driver: + stdenv.mkDerivation { + name = "vm-test-run-${driver.testName}"; + + requiredSystemFeatures = [ "kvm" "nixos-test" ]; + + buildInputs = [ libxslt ]; + + buildCommand = + '' + mkdir -p $out/nix-support + + LOGFILE=$out/log.xml tests='exec(os.environ["testScript"])' ${driver}/bin/nixos-test-driver + + # Generate a pretty-printed log. + xsltproc --output $out/log.html ${./test-driver/log2html.xsl} $out/log.xml + ln -s ${./test-driver/logfile.css} $out/logfile.css + ln -s ${./test-driver/treebits.js} $out/treebits.js + ln -s ${jquery}/js/jquery.min.js $out/ + ln -s ${jquery}/js/jquery.js $out/ + ln -s ${jquery-ui}/js/jquery-ui.min.js $out/ + ln -s ${jquery-ui}/js/jquery-ui.js $out/ + + touch $out/nix-support/hydra-build-products + echo "report testlog $out log.html" >> $out/nix-support/hydra-build-products + + for i in */xchg/coverage-data; do + mkdir -p $out/coverage-data + mv $i $out/coverage-data/$(dirname $(dirname $i)) + done + ''; + }; + + + makeTest = + { testScript + , makeCoverageReport ? false + , enableOCR ? false + , name ? "unnamed" + , ... + } @ t: + + let + # A standard store path to the vm monitor is built like this: + # /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor + # The max filename length of a unix domain socket is 108 bytes. + # This means $name can at most be 50 bytes long. + maxTestNameLen = 50; + testNameLen = builtins.stringLength name; + + testDriverName = with builtins; + if testNameLen > maxTestNameLen then + abort ("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " + + "it's currently ${toString testNameLen} characters long.") + else + "nixos-test-driver-${name}"; + + nodes = buildVirtualNetwork ( + t.nodes or (if t ? machine then { machine = t.machine; } else { })); + + testScript' = + # Call the test script with the computed nodes. + if lib.isFunction testScript + then testScript { inherit nodes; } + else testScript; + + vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes); + + vms = map (m: m.config.system.build.vm) (lib.attrValues nodes); + + ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; }; + + imagemagick_tiff = imagemagick_light.override { inherit libtiff; }; + + # Generate onvenience wrappers for running the test driver + # interactively with the specified network, and for starting the + # VMs from the command line. + driver = runCommand testDriverName + { buildInputs = [ makeWrapper]; + testScript = testScript'; + preferLocalBuild = true; + testName = name; + } + '' + mkdir -p $out/bin + + echo -n "$testScript" > $out/test-script + ${python3Packages.black}/bin/black --check --diff $out/test-script + + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/ + vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) + wrapProgram $out/bin/nixos-test-driver \ + --add-flags "''${vms[*]}" \ + ${lib.optionalString enableOCR + "--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \ + --run "export testScript=\"\$(cat $out/test-script)\"" \ + --set VLANS '${toString vlans}' + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms + wrapProgram $out/bin/nixos-run-vms \ + --add-flags "''${vms[*]}" \ + ${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \ + --set tests 'start_all(); join_all();' \ + --set VLANS '${toString vlans}' \ + ${lib.optionalString (builtins.length vms == 1) "--set USE_SERIAL 1"} + ''; # " + + passMeta = drv: drv // lib.optionalAttrs (t ? meta) { + meta = (drv.meta or {}) // t.meta; + }; + + test = passMeta (runTests driver); + report = passMeta (releaseTools.gcovReport { coverageRuns = [ test ]; }); + + nodeNames = builtins.attrNames nodes; + invalidNodeNames = lib.filter + (node: builtins.match "^[A-z_][A-z0-9_]+$" node == null) nodeNames; + + in + if lib.length invalidNodeNames > 0 then + throw '' + Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})! + All machines are referenced as perl variables in the testing framework which will break the + script when special characters are used. + + Please stick to alphanumeric chars and underscores as separation. + '' + else + (if makeCoverageReport then report else test) // { + inherit nodes driver test; + }; + + runInMachine = + { drv + , machine + , preBuild ? "" + , postBuild ? "" + , ... # ??? + }: + let + vm = buildVM { } + [ machine + { key = "run-in-machine"; + networking.hostName = "client"; + nix.readOnlyStore = false; + virtualisation.writableStore = false; + } + ]; + + buildrunner = writeText "vm-build" '' + source $1 + + ${coreutils}/bin/mkdir -p $TMPDIR + cd $TMPDIR + + exec $origBuilder $origArgs + ''; + + testScript = '' + startAll; + $client->waitForUnit("multi-user.target"); + ${preBuild} + $client->succeed("env -i ${bash}/bin/bash ${buildrunner} /tmp/xchg/saved-env >&2"); + ${postBuild} + $client->succeed("sync"); # flush all data before pulling the plug + ''; + + vmRunCommand = writeText "vm-run" '' + xchg=vm-state-client/xchg + ${coreutils}/bin/mkdir $out + ${coreutils}/bin/mkdir -p $xchg + + for i in $passAsFile; do + i2=''${i}Path + _basename=$(${coreutils}/bin/basename ''${!i2}) + ${coreutils}/bin/cp ''${!i2} $xchg/$_basename + eval $i2=/tmp/xchg/$_basename + ${coreutils}/bin/ls -la $xchg + done + + unset i i2 _basename + export | ${gnugrep}/bin/grep -v '^xchg=' > $xchg/saved-env + unset xchg + + export tests='${testScript}' + ${testDriver}/bin/nixos-test-driver ${vm.config.system.build.vm}/bin/run-*-vm + ''; # */ + + in + lib.overrideDerivation drv (attrs: { + requiredSystemFeatures = [ "kvm" ]; + builder = "${bash}/bin/sh"; + args = ["-e" vmRunCommand]; + origArgs = attrs.args; + origBuilder = attrs.builder; + }); + + + runInMachineWithX = { require ? [], ... } @ args: + let + client = + { ... }: + { + inherit require; + virtualisation.memorySize = 1024; + services.xserver.enable = true; + services.xserver.displayManager.slim.enable = false; + services.xserver.displayManager.auto.enable = true; + services.xserver.windowManager.default = "icewm"; + services.xserver.windowManager.icewm.enable = true; + services.xserver.desktopManager.default = "none"; + }; + in + runInMachine ({ + machine = client; + preBuild = + '' + $client->waitForX; + ''; + } // args); + + + simpleTest = as: (makeTest as).test; + +} diff --git a/nixos/maintainers/scripts/ec2/create-amis.sh b/nixos/maintainers/scripts/ec2/create-amis.sh index c4149e3e8ff..f08e500e079 100755 --- a/nixos/maintainers/scripts/ec2/create-amis.sh +++ b/nixos/maintainers/scripts/ec2/create-amis.sh @@ -14,7 +14,7 @@ set -euo pipefail # configuration -state_dir=/home/deploy/amis/ec2-images +state_dir=$HOME/amis/ec2-images home_region=eu-west-1 bucket=nixos-amis diff --git a/nixos/modules/config/fonts/corefonts.nix b/nixos/modules/config/fonts/corefonts.nix deleted file mode 100644 index b9f69879a10..00000000000 --- a/nixos/modules/config/fonts/corefonts.nix +++ /dev/null @@ -1,36 +0,0 @@ -# This module is deprecated, since you can just say ‘fonts.fonts = [ -# pkgs.corefonts ];’ instead. - -{ config, lib, pkgs, ... }: - -with lib; - -{ - - options = { - - fonts = { - - enableCoreFonts = mkOption { - visible = false; - default = false; - description = '' - Whether to include Microsoft's proprietary Core Fonts. These fonts - are redistributable, but only verbatim, among other restrictions. - See <link xlink:href="http://corefonts.sourceforge.net/eula.htm"/> - for details. - ''; - }; - - }; - - }; - - - config = mkIf config.fonts.enableCoreFonts { - - fonts.fonts = [ pkgs.corefonts ]; - - }; - -} diff --git a/nixos/modules/config/fonts/fontconfig-ultimate.nix b/nixos/modules/config/fonts/fontconfig-ultimate.nix deleted file mode 100644 index 84d90899dff..00000000000 --- a/nixos/modules/config/fonts/fontconfig-ultimate.nix +++ /dev/null @@ -1,86 +0,0 @@ -{ config, pkgs, lib, ... }: - -with lib; - -let cfg = config.fonts.fontconfig.ultimate; - - latestVersion = pkgs.fontconfig.configVersion; - - # The configuration to be included in /etc/font/ - confPkg = pkgs.runCommand "font-ultimate-conf" { preferLocalBuild = true; } '' - support_folder=$out/etc/fonts/conf.d - latest_folder=$out/etc/fonts/${latestVersion}/conf.d - - mkdir -p $support_folder - mkdir -p $latest_folder - - # fontconfig ultimate substitutions - ${optionalString (cfg.substitutions != "none") '' - ln -s ${pkgs.fontconfig-ultimate}/etc/fonts/presets/${cfg.substitutions}/*.conf \ - $support_folder - ln -s ${pkgs.fontconfig-ultimate}/etc/fonts/presets/${cfg.substitutions}/*.conf \ - $latest_folder - ''} - - # fontconfig ultimate various configuration files - ln -s ${pkgs.fontconfig-ultimate}/etc/fonts/conf.d/*.conf \ - $support_folder - ln -s ${pkgs.fontconfig-ultimate}/etc/fonts/conf.d/*.conf \ - $latest_folder - ''; - -in -{ - - options = { - - fonts = { - - fontconfig = { - - ultimate = { - enable = mkOption { - type = types.bool; - default = false; - description = '' - Enable fontconfig-ultimate settings (formerly known as - Infinality). Besides the customizable settings in this NixOS - module, fontconfig-ultimate also provides many font-specific - rendering tweaks. - ''; - }; - - substitutions = mkOption { - type = types.enum ["free" "combi" "ms" "none"]; - default = "free"; - description = '' - Font substitutions to replace common Type 1 fonts with nicer - TrueType fonts. <literal>free</literal> uses free fonts, - <literal>ms</literal> uses Microsoft fonts, - <literal>combi</literal> uses a combination, and - <literal>none</literal> disables the substitutions. - ''; - }; - - preset = mkOption { - type = types.enum ["ultimate1" "ultimate2" "ultimate3" "ultimate4" "ultimate5" "osx" "windowsxp"]; - default = "ultimate3"; - description = '' - FreeType rendering settings preset. Any of the presets may be - customized by setting environment variables. - ''; - }; - }; - }; - }; - - }; - - config = mkIf (config.fonts.fontconfig.enable && cfg.enable) { - - fonts.fontconfig.confPackages = [ confPkg ]; - environment.variables.INFINALITY_FT = cfg.preset; - - }; - -} diff --git a/nixos/modules/config/pulseaudio.nix b/nixos/modules/config/pulseaudio.nix index b3bc4a451aa..9baad9b5854 100644 --- a/nixos/modules/config/pulseaudio.nix +++ b/nixos/modules/config/pulseaudio.nix @@ -98,11 +98,12 @@ in { description = '' If false, a PulseAudio server is launched automatically for each user that tries to use the sound system. The server runs - with user privileges. This is the recommended and most secure - way to use PulseAudio. If true, one system-wide PulseAudio + with user privileges. If true, one system-wide PulseAudio server is launched on boot, running as the user "pulse", and only users in the "audio" group will have access to the server. Please read the PulseAudio documentation for more details. + + Don't enable this option unless you know what you are doing. ''; }; diff --git a/nixos/modules/config/shells-environment.nix b/nixos/modules/config/shells-environment.nix index d939cbb393e..b79e16cd797 100644 --- a/nixos/modules/config/shells-environment.nix +++ b/nixos/modules/config/shells-environment.nix @@ -118,6 +118,14 @@ in type = with types; attrsOf (nullOr (either str path)); }; + environment.homeBinInPath = mkOption { + description = '' + Include ~/bin/ in $PATH. + ''; + default = true; + type = types.bool; + }; + environment.binsh = mkOption { default = "${config.system.build.binsh}/bin/sh"; defaultText = "\${config.system.build.binsh}/bin/sh"; @@ -186,8 +194,10 @@ in ${cfg.extraInit} - # ~/bin if it exists overrides other bin directories. - export PATH="$HOME/bin:$PATH" + ${optionalString cfg.homeBinInPath '' + # ~/bin if it exists overrides other bin directories. + export PATH="$HOME/bin:$PATH" + ''} ''; system.activationScripts.binsh = stringAfter [ "stdio" ] diff --git a/nixos/modules/config/xdg/sounds.nix b/nixos/modules/config/xdg/sounds.nix index 148240d631c..14d6340fc33 100644 --- a/nixos/modules/config/xdg/sounds.nix +++ b/nixos/modules/config/xdg/sounds.nix @@ -1,4 +1,4 @@ -{ config, lib, ... }: +{ config, lib, pkgs, ... }: with lib; { @@ -14,6 +14,10 @@ with lib; }; config = mkIf config.xdg.sounds.enable { + environment.systemPackages = [ + pkgs.sound-theme-freedesktop + ]; + environment.pathsToLink = [ "/share/sounds" ]; diff --git a/nixos/modules/hardware/brillo.nix b/nixos/modules/hardware/brillo.nix new file mode 100644 index 00000000000..e970c948099 --- /dev/null +++ b/nixos/modules/hardware/brillo.nix @@ -0,0 +1,22 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.hardware.brillo; +in +{ + options = { + hardware.brillo = { + enable = mkEnableOption '' + Enable brillo in userspace. + This will allow brightness control from users in the video group. + ''; + }; + }; + + + config = mkIf cfg.enable { + services.udev.packages = [ pkgs.brillo ]; + environment.systemPackages = [ pkgs.brillo ]; + }; +} diff --git a/nixos/modules/hardware/video/displaylink.nix b/nixos/modules/hardware/video/displaylink.nix index 669ac849cba..912f53da836 100644 --- a/nixos/modules/hardware/video/displaylink.nix +++ b/nixos/modules/hardware/video/displaylink.nix @@ -19,6 +19,21 @@ in config = mkIf enabled { boot.extraModulePackages = [ evdi ]; + boot.kernelModules = [ "evdi" ]; + + environment.etc."X11/xorg.conf.d/40-displaylink.conf".text = '' + Section "OutputClass" + Identifier "DisplayLink" + MatchDriver "evdi" + Driver "modesetting" + Option "AccelMethod" "none" + EndSection + ''; + + # make the device available + services.xserver.displayManager.sessionCommands = '' + ${lib.getBin pkgs.xorg.xrandr}/bin/xrandr --setprovideroutputsource 1 0 + ''; # Those are taken from displaylink-installer.sh and from Arch Linux AUR package. @@ -47,18 +62,13 @@ in description = "DisplayLink Manager Service"; after = [ "display-manager.service" ]; conflicts = [ "getty@tty7.service" ]; - path = [ pkgs.kmod ]; serviceConfig = { ExecStart = "${displaylink}/bin/DisplayLinkManager"; Restart = "always"; RestartSec = 5; + LogsDirectory = "displaylink"; }; - - preStart = '' - mkdir -p /var/log/displaylink - modprobe evdi - ''; }; }; diff --git a/nixos/modules/installer/tools/nixos-option.sh b/nixos/modules/installer/tools/nixos-option.sh deleted file mode 100644 index 4560e9c7403..00000000000 --- a/nixos/modules/installer/tools/nixos-option.sh +++ /dev/null @@ -1,327 +0,0 @@ -#! @shell@ -e - -# FIXME: rewrite this in a more suitable language. - -usage () { - exec man nixos-option - exit 1 -} - -##################### -# Process Arguments # -##################### - -xml=false -verbose=false -nixPath="" - -option="" -exit_code=0 - -argfun="" -for arg; do - if test -z "$argfun"; then - case $arg in - -*) - sarg="$arg" - longarg="" - while test "$sarg" != "-"; do - case $sarg in - --*) longarg=$arg; sarg="--";; - -I) argfun="include_nixpath";; - -*) usage;; - esac - # remove the first letter option - sarg="-${sarg#??}" - done - ;; - *) longarg=$arg;; - esac - for larg in $longarg; do - case $larg in - --xml) xml=true;; - --verbose) verbose=true;; - --help) usage;; - -*) usage;; - *) if test -z "$option"; then - option="$larg" - else - usage - fi;; - esac - done - else - case $argfun in - set_*) - var=$(echo $argfun | sed 's,^set_,,') - eval $var=$arg - ;; - include_nixpath) - nixPath="-I $arg $nixPath" - ;; - esac - argfun="" - fi -done - -if $verbose; then - set -x -else - set +x -fi - -############################# -# Process the configuration # -############################# - -evalNix(){ - # disable `-e` flag, it's possible that the evaluation of `nix-instantiate` fails (e.g. due to broken pkgs) - set +e - result=$(nix-instantiate ${nixPath:+$nixPath} - --eval-only "$@" 2>&1) - exit_code=$? - set -e - - if test $exit_code -eq 0; then - sed '/^warning: Nix search path/d' <<EOF -$result -EOF - return 0; - else - sed -n ' - /^error/ { s/, at (string):[0-9]*:[0-9]*//; p; }; - /^warning: Nix search path/ { p; }; -' >&2 <<EOF -$result -EOF - exit_code=1 - fi -} - -header="let - nixos = import <nixpkgs/nixos> {}; - nixpkgs = import <nixpkgs> {}; -in with nixpkgs.lib; -" - -# This function is used for converting the option definition path given by -# the user into accessors for reaching the definition and the declaration -# corresponding to this option. -generateAccessors(){ - if result=$(evalNix --strict --show-trace <<EOF -$header - -let - path = "${option:+$option}"; - pathList = splitString "." path; - - walkOptions = attrsNames: result: - if attrsNames == [] then - result - else - let name = head attrsNames; rest = tail attrsNames; in - if isOption result.options then - walkOptions rest { - options = result.options.type.getSubOptions ""; - opt = ''(\${result.opt}.type.getSubOptions "")''; - cfg = ''\${result.cfg}."\${name}"''; - } - else - walkOptions rest { - options = result.options.\${name}; - opt = ''\${result.opt}."\${name}"''; - cfg = ''\${result.cfg}."\${name}"''; - } - ; - - walkResult = (if path == "" then x: x else walkOptions pathList) { - options = nixos.options; - opt = ''nixos.options''; - cfg = ''nixos.config''; - }; - -in - ''let option = \${walkResult.opt}; config = \${walkResult.cfg}; in'' -EOF -) - then - echo $result - else - # In case of error we want to ignore the error message roduced by the - # script above, as it is iterating over each attribute, which does not - # produce a nice error message. The following code is a fallback - # solution which is cause a nicer error message in the next - # evaluation. - echo "\"let option = nixos.options${option:+.$option}; config = nixos.config${option:+.$option}; in\"" - fi -} - -header="$header -$(eval echo $(generateAccessors)) -" - -evalAttr(){ - local prefix="$1" - local strict="$2" - local suffix="$3" - - # If strict is set, then set it to "true". - test -n "$strict" && strict=true - - evalNix ${strict:+--strict} <<EOF -$header - -let - value = $prefix${suffix:+.$suffix}; - strict = ${strict:-false}; - cleanOutput = x: with nixpkgs.lib; - if isDerivation x then x.outPath - else if isFunction x then "<CODE>" - else if strict then - if isAttrs x then mapAttrs (n: cleanOutput) x - else if isList x then map cleanOutput x - else x - else x; -in - cleanOutput value -EOF -} - -evalOpt(){ - evalAttr "option" "" "$@" -} - -evalCfg(){ - local strict="$1" - evalAttr "config" "$strict" -} - -findSources(){ - local suffix=$1 - evalNix --strict <<EOF -$header - -option.$suffix -EOF -} - -# Given a result from nix-instantiate, recover the list of attributes it -# contains. -attrNames() { - local attributeset=$1 - # sed is used to replace un-printable subset by 0s, and to remove most of - # the inner-attribute set, which reduce the likelyhood to encounter badly - # pre-processed input. - echo "builtins.attrNames $attributeset" | \ - sed 's,<[A-Z]*>,0,g; :inner; s/{[^\{\}]*};/0;/g; t inner;' | \ - evalNix --strict -} - -# map a simple list which contains strings or paths. -nixMap() { - local fun="$1" - local list="$2" - local elem - for elem in $list; do - test $elem = '[' -o $elem = ']' && continue; - $fun $elem - done -} - -# This duplicates the work made below, but it is useful for processing -# the output of nixos-option with other tools such as nixos-gui. -if $xml; then - evalNix --xml --no-location <<EOF -$header - -let - sources = builtins.map (f: f.source); - opt = option; - cfg = config; -in - -with nixpkgs.lib; - -let - optStrict = v: - let - traverse = x : - if isAttrs x then - if x ? outPath then true - else all id (mapAttrsFlatten (n: traverseNoAttrs) x) - else traverseNoAttrs x; - traverseNoAttrs = x: - # do not continue in attribute sets - if isAttrs x then true - else if isList x then all id (map traverse x) - else true; - in assert traverse v; v; -in - -if isOption opt then - optStrict ({} - // optionalAttrs (opt ? default) { inherit (opt) default; } - // optionalAttrs (opt ? example) { inherit (opt) example; } - // optionalAttrs (opt ? description) { inherit (opt) description; } - // optionalAttrs (opt ? type) { typename = opt.type.description; } - // optionalAttrs (opt ? options) { inherit (opt) options; } - // { - # to disambiguate the xml output. - _isOption = true; - declarations = sources opt.declarations; - definitions = sources opt.definitions; - value = cfg; - }) -else - opt -EOF - exit $? -fi - -if test "$(evalOpt "_type" 2> /dev/null)" = '"option"'; then - echo "Value:" - evalCfg 1 - - echo - - echo "Default:" - if default=$(evalOpt "default" - 2> /dev/null); then - echo "$default" - else - echo "<None>" - fi - echo - if example=$(evalOpt "example" - 2> /dev/null); then - echo "Example:" - echo "$example" - echo - fi - echo "Description:" - echo - echo $(evalOpt "description") - - echo $desc; - - printPath () { echo " $1"; } - - echo "Declared by:" - nixMap printPath "$(findSources "declarations")" - echo - echo "Defined by:" - nixMap printPath "$(findSources "files")" - echo - -else - # echo 1>&2 "Warning: This value is not an option." - - result=$(evalCfg "") - if [ ! -z "$result" ]; then - names=$(attrNames "$result" 2> /dev/null) - echo 1>&2 "This attribute set contains:" - escapeQuotes () { eval echo "$1"; } - nixMap escapeQuotes "$names" - else - echo 1>&2 "An error occurred while looking for attribute names. Are you sure that '$option' exists?" - fi -fi - -exit $exit_code diff --git a/nixos/modules/installer/tools/nixos-option/CMakeLists.txt b/nixos/modules/installer/tools/nixos-option/CMakeLists.txt new file mode 100644 index 00000000000..e5834598c4f --- /dev/null +++ b/nixos/modules/installer/tools/nixos-option/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required (VERSION 2.6) +project (nixos-option) + +add_executable(nixos-option nixos-option.cc libnix-copy-paste.cc) +target_link_libraries(nixos-option PRIVATE -lnixmain -lnixexpr -lnixstore -lnixutil) +target_compile_features(nixos-option PRIVATE cxx_std_17) + +install (TARGETS nixos-option DESTINATION bin) diff --git a/nixos/modules/installer/tools/nixos-option/default.nix b/nixos/modules/installer/tools/nixos-option/default.nix new file mode 100644 index 00000000000..753fd92c7bb --- /dev/null +++ b/nixos/modules/installer/tools/nixos-option/default.nix @@ -0,0 +1,11 @@ +{lib, stdenv, boost, cmake, pkgconfig, nix, ... }: +stdenv.mkDerivation rec { + name = "nixos-option"; + src = ./.; + nativeBuildInputs = [ cmake pkgconfig ]; + buildInputs = [ boost nix ]; + meta = { + license = stdenv.lib.licenses.lgpl2Plus; + maintainers = with lib.maintainers; [ chkno ]; + }; +} diff --git a/nixos/modules/installer/tools/nixos-option/libnix-copy-paste.cc b/nixos/modules/installer/tools/nixos-option/libnix-copy-paste.cc new file mode 100644 index 00000000000..875c07da639 --- /dev/null +++ b/nixos/modules/installer/tools/nixos-option/libnix-copy-paste.cc @@ -0,0 +1,83 @@ +// These are useful methods inside the nix library that ought to be exported. +// Since they are not, copy/paste them here. +// TODO: Delete these and use the ones in the library as they become available. + +#include <nix/config.h> // for nix/globals.hh's reference to SYSTEM + +#include "libnix-copy-paste.hh" +#include <boost/format/alt_sstream.hpp> // for basic_altstringbuf... +#include <boost/format/alt_sstream_impl.hpp> // for basic_altstringbuf... +#include <boost/format/format_class.hpp> // for basic_format +#include <boost/format/format_fwd.hpp> // for format +#include <boost/format/format_implementation.hpp> // for basic_format::basi... +#include <boost/optional/optional.hpp> // for get_pointer +#include <iostream> // for operator<<, basic_... +#include <nix/types.hh> // for Strings, Error +#include <string> // for string, basic_string + +using boost::format; +using nix::Error; +using nix::Strings; +using std::string; + +// From nix/src/libexpr/attr-path.cc +Strings parseAttrPath(const string & s) +{ + Strings res; + string cur; + string::const_iterator i = s.begin(); + while (i != s.end()) { + if (*i == '.') { + res.push_back(cur); + cur.clear(); + } else if (*i == '"') { + ++i; + while (1) { + if (i == s.end()) + throw Error(format("missing closing quote in selection path '%1%'") % s); + if (*i == '"') + break; + cur.push_back(*i++); + } + } else + cur.push_back(*i); + ++i; + } + if (!cur.empty()) + res.push_back(cur); + return res; +} + +// From nix/src/nix/repl.cc +bool isVarName(const string & s) +{ + if (s.size() == 0) + return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') + return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || (i >= 'A' && i <= 'Z') || (i >= '0' && i <= '9') || i == '_' || i == '-' || + i == '\'')) + return false; + return true; +} + +// From nix/src/nix/repl.cc +std::ostream & printStringValue(std::ostream & str, const char * string) +{ + str << "\""; + for (const char * i = string; *i; i++) + if (*i == '\"' || *i == '\\') + str << "\\" << *i; + else if (*i == '\n') + str << "\\n"; + else if (*i == '\r') + str << "\\r"; + else if (*i == '\t') + str << "\\t"; + else + str << *i; + str << "\""; + return str; +} diff --git a/nixos/modules/installer/tools/nixos-option/libnix-copy-paste.hh b/nixos/modules/installer/tools/nixos-option/libnix-copy-paste.hh new file mode 100644 index 00000000000..2274e9a0f85 --- /dev/null +++ b/nixos/modules/installer/tools/nixos-option/libnix-copy-paste.hh @@ -0,0 +1,9 @@ +#pragma once + +#include <iostream> +#include <nix/types.hh> +#include <string> + +nix::Strings parseAttrPath(const std::string & s); +bool isVarName(const std::string & s); +std::ostream & printStringValue(std::ostream & str, const char * string); diff --git a/nixos/modules/installer/tools/nixos-option/nixos-option.cc b/nixos/modules/installer/tools/nixos-option/nixos-option.cc new file mode 100644 index 00000000000..9b92dc829cd --- /dev/null +++ b/nixos/modules/installer/tools/nixos-option/nixos-option.cc @@ -0,0 +1,618 @@ +#include <nix/config.h> // for nix/globals.hh's reference to SYSTEM + +#include <exception> // for exception_ptr, current_exception +#include <functional> // for function +#include <iostream> // for operator<<, basic_ostream, ostrin... +#include <iterator> // for next +#include <list> // for _List_iterator +#include <memory> // for allocator, unique_ptr, make_unique +#include <new> // for operator new +#include <nix/args.hh> // for argvToStrings, UsageError +#include <nix/attr-path.hh> // for findAlongAttrPath +#include <nix/attr-set.hh> // for Attr, Bindings, Bindings::iterator +#include <nix/common-eval-args.hh> // for MixEvalArgs +#include <nix/eval-inline.hh> // for EvalState::forceValue +#include <nix/eval.hh> // for EvalState, initGC, operator<< +#include <nix/globals.hh> // for initPlugins, Settings, settings +#include <nix/nixexpr.hh> // for Pos +#include <nix/shared.hh> // for getArg, LegacyArgs, printVersion +#include <nix/store-api.hh> // for openStore +#include <nix/symbol-table.hh> // for Symbol, SymbolTable +#include <nix/types.hh> // for Error, Path, Strings, PathSet +#include <nix/util.hh> // for absPath, baseNameOf +#include <nix/value.hh> // for Value, Value::(anonymous), Value:... +#include <string> // for string, operator+, operator== +#include <utility> // for move +#include <variant> // for get, holds_alternative, variant +#include <vector> // for vector<>::iterator, vector + +#include "libnix-copy-paste.hh" + +using nix::absPath; +using nix::Bindings; +using nix::Error; +using nix::EvalError; +using nix::EvalState; +using nix::Path; +using nix::PathSet; +using nix::Strings; +using nix::Symbol; +using nix::tAttrs; +using nix::ThrownError; +using nix::tLambda; +using nix::tString; +using nix::UsageError; +using nix::Value; + +// An ostream wrapper to handle nested indentation +class Out +{ + public: + class Separator + {}; + const static Separator sep; + enum LinePolicy + { + ONE_LINE, + MULTI_LINE + }; + explicit Out(std::ostream & ostream) : ostream(ostream), policy(ONE_LINE), writeSinceSep(true) {} + Out(Out & o, const std::string & start, const std::string & end, LinePolicy policy); + Out(Out & o, const std::string & start, const std::string & end, int count) + : Out(o, start, end, count < 2 ? ONE_LINE : MULTI_LINE) + {} + Out(const Out &) = delete; + Out(Out &&) = default; + Out & operator=(const Out &) = delete; + Out & operator=(Out &&) = delete; + ~Out() { ostream << end; } + + private: + std::ostream & ostream; + std::string indentation; + std::string end; + LinePolicy policy; + bool writeSinceSep; + template <typename T> friend Out & operator<<(Out & o, T thing); +}; + +template <typename T> Out & operator<<(Out & o, T thing) +{ + if (!o.writeSinceSep && o.policy == Out::MULTI_LINE) { + o.ostream << o.indentation; + } + o.writeSinceSep = true; + o.ostream << thing; + return o; +} + +template <> Out & operator<<<Out::Separator>(Out & o, Out::Separator /* thing */) +{ + o.ostream << (o.policy == Out::ONE_LINE ? " " : "\n"); + o.writeSinceSep = false; + return o; +} + +Out::Out(Out & o, const std::string & start, const std::string & end, LinePolicy policy) + : ostream(o.ostream), indentation(policy == ONE_LINE ? o.indentation : o.indentation + " "), + end(policy == ONE_LINE ? end : o.indentation + end), policy(policy), writeSinceSep(true) +{ + o << start; + *this << Out::sep; +} + +// Stuff needed for evaluation +struct Context +{ + Context(EvalState & state, Bindings & autoArgs, Value optionsRoot, Value configRoot) + : state(state), autoArgs(autoArgs), optionsRoot(optionsRoot), configRoot(configRoot), + underscoreType(state.symbols.create("_type")) + {} + EvalState & state; + Bindings & autoArgs; + Value optionsRoot; + Value configRoot; + Symbol underscoreType; +}; + +Value evaluateValue(Context & ctx, Value & v) +{ + ctx.state.forceValue(v); + if (ctx.autoArgs.empty()) { + return v; + } + Value called{}; + ctx.state.autoCallFunction(ctx.autoArgs, v, called); + return called; +} + +bool isOption(Context & ctx, const Value & v) +{ + if (v.type != tAttrs) { + return false; + } + const auto & atualType = v.attrs->find(ctx.underscoreType); + if (atualType == v.attrs->end()) { + return false; + } + try { + Value evaluatedType = evaluateValue(ctx, *atualType->value); + if (evaluatedType.type != tString) { + return false; + } + return static_cast<std::string>(evaluatedType.string.s) == "option"; + } catch (Error &) { + return false; + } +} + +// Add quotes to a component of a path. +// These are needed for paths like: +// fileSystems."/".fsType +// systemd.units."dbus.service".text +std::string quoteAttribute(const std::string & attribute) +{ + if (isVarName(attribute)) { + return attribute; + } + std::ostringstream buf; + printStringValue(buf, attribute.c_str()); + return buf.str(); +} + +const std::string appendPath(const std::string & prefix, const std::string & suffix) +{ + if (prefix.empty()) { + return quoteAttribute(suffix); + } + return prefix + "." + quoteAttribute(suffix); +} + +bool forbiddenRecursionName(std::string name) { return (!name.empty() && name[0] == '_') || name == "haskellPackages"; } + +void recurse(const std::function<bool(const std::string & path, std::variant<Value, std::exception_ptr>)> & f, + Context & ctx, Value v, const std::string & path) +{ + std::variant<Value, std::exception_ptr> evaluated; + try { + evaluated = evaluateValue(ctx, v); + } catch (Error &) { + evaluated = std::current_exception(); + } + if (!f(path, evaluated)) { + return; + } + if (std::holds_alternative<std::exception_ptr>(evaluated)) { + return; + } + const Value & evaluated_value = std::get<Value>(evaluated); + if (evaluated_value.type != tAttrs) { + return; + } + for (const auto & child : evaluated_value.attrs->lexicographicOrder()) { + if (forbiddenRecursionName(child->name)) { + continue; + } + recurse(f, ctx, *child->value, appendPath(path, child->name)); + } +} + +// Calls f on all the option names +void mapOptions(const std::function<void(const std::string & path)> & f, Context & ctx, Value root) +{ + recurse( + [f, &ctx](const std::string & path, std::variant<Value, std::exception_ptr> v) { + bool isOpt = std::holds_alternative<std::exception_ptr>(v) || isOption(ctx, std::get<Value>(v)); + if (isOpt) { + f(path); + } + return !isOpt; + }, + ctx, root, ""); +} + +// Calls f on all the config values inside one option. +// Simple options have one config value inside, like sound.enable = true. +// Compound options have multiple config values. For example, the option +// "users.users" has about 1000 config values inside it: +// users.users.avahi.createHome = false; +// users.users.avahi.cryptHomeLuks = null; +// users.users.avahi.description = "`avahi-daemon' privilege separation user"; +// ... +// users.users.avahi.openssh.authorizedKeys.keyFiles = [ ]; +// users.users.avahi.openssh.authorizedKeys.keys = [ ]; +// ... +// users.users.avahi.uid = 10; +// users.users.avahi.useDefaultShell = false; +// users.users.cups.createHome = false; +// ... +// users.users.cups.useDefaultShell = false; +// users.users.gdm = ... ... ... +// users.users.messagebus = ... .. ... +// users.users.nixbld1 = ... .. ... +// ... +// users.users.systemd-timesync = ... .. ... +void mapConfigValuesInOption( + const std::function<void(const std::string & path, std::variant<Value, std::exception_ptr> v)> & f, + const std::string & path, Context & ctx) +{ + Value * option; + try { + option = findAlongAttrPath(ctx.state, path, ctx.autoArgs, ctx.configRoot); + } catch (Error &) { + f(path, std::current_exception()); + return; + } + recurse( + [f, ctx](const std::string & path, std::variant<Value, std::exception_ptr> v) { + bool leaf = std::holds_alternative<std::exception_ptr>(v) || std::get<Value>(v).type != tAttrs || + ctx.state.isDerivation(std::get<Value>(v)); + if (!leaf) { + return true; // Keep digging + } + f(path, v); + return false; + }, + ctx, *option, path); +} + +std::string describeError(const Error & e) { return "«error: " + e.msg() + "»"; } + +void describeDerivation(Context & ctx, Out & out, Value v) +{ + // Copy-pasted from nix/src/nix/repl.cc :( + Bindings::iterator i = v.attrs->find(ctx.state.sDrvPath); + PathSet pathset; + try { + Path drvPath = i != v.attrs->end() ? ctx.state.coerceToPath(*i->pos, *i->value, pathset) : "???"; + out << "«derivation " << drvPath << "»"; + } catch (Error & e) { + out << describeError(e); + } +} + +Value parseAndEval(EvalState & state, const std::string & expression, const std::string & path) +{ + Value v{}; + state.eval(state.parseExprFromString(expression, absPath(path)), v); + return v; +} + +void printValue(Context & ctx, Out & out, std::variant<Value, std::exception_ptr> maybeValue, const std::string & path); + +void printList(Context & ctx, Out & out, Value & v) +{ + Out listOut(out, "[", "]", v.listSize()); + for (unsigned int n = 0; n < v.listSize(); ++n) { + printValue(ctx, listOut, *v.listElems()[n], ""); + listOut << Out::sep; + } +} + +void printAttrs(Context & ctx, Out & out, Value & v, const std::string & path) +{ + Out attrsOut(out, "{", "}", v.attrs->size()); + for (const auto & a : v.attrs->lexicographicOrder()) { + std::string name = a->name; + attrsOut << name << " = "; + printValue(ctx, attrsOut, *a->value, appendPath(path, name)); + attrsOut << ";" << Out::sep; + } +} + +void multiLineStringEscape(Out & out, const std::string & s) +{ + int i; + for (i = 1; i < s.size(); i++) { + if (s[i - 1] == '$' && s[i] == '{') { + out << "''${"; + i++; + } else if (s[i - 1] == '\'' && s[i] == '\'') { + out << "'''"; + i++; + } else { + out << s[i - 1]; + } + } + if (i == s.size()) { + out << s[i - 1]; + } +} + +void printMultiLineString(Out & out, const Value & v) +{ + std::string s = v.string.s; + Out strOut(out, "''", "''", Out::MULTI_LINE); + std::string::size_type begin = 0; + while (begin < s.size()) { + std::string::size_type end = s.find('\n', begin); + if (end == std::string::npos) { + multiLineStringEscape(strOut, s.substr(begin, s.size() - begin)); + break; + } + multiLineStringEscape(strOut, s.substr(begin, end - begin)); + strOut << Out::sep; + begin = end + 1; + } +} + +void printValue(Context & ctx, Out & out, std::variant<Value, std::exception_ptr> maybeValue, const std::string & path) +{ + try { + if (auto ex = std::get_if<std::exception_ptr>(&maybeValue)) { + std::rethrow_exception(*ex); + } + Value v = evaluateValue(ctx, std::get<Value>(maybeValue)); + if (ctx.state.isDerivation(v)) { + describeDerivation(ctx, out, v); + } else if (v.isList()) { + printList(ctx, out, v); + } else if (v.type == tAttrs) { + printAttrs(ctx, out, v, path); + } else if (v.type == tString && std::string(v.string.s).find('\n') != std::string::npos) { + printMultiLineString(out, v); + } else { + ctx.state.forceValueDeep(v); + out << v; + } + } catch (ThrownError & e) { + if (e.msg() == "The option `" + path + "' is used but not defined.") { + // 93% of errors are this, and just letting this message through would be + // misleading. These values may or may not actually be "used" in the + // config. The thing throwing the error message assumes that if anything + // ever looks at this value, it is a "use" of this value. But here in + // nixos-option, we are looking at this value only to print it. + // In order to avoid implying that this undefined value is actually + // referenced, eat the underlying error message and emit "«not defined»". + out << "«not defined»"; + } else { + out << describeError(e); + } + } catch (Error & e) { + out << describeError(e); + } +} + +void printConfigValue(Context & ctx, Out & out, const std::string & path, std::variant<Value, std::exception_ptr> v) +{ + out << path << " = "; + printValue(ctx, out, std::move(v), path); + out << ";\n"; +} + +void printAll(Context & ctx, Out & out) +{ + mapOptions( + [&ctx, &out](const std::string & optionPath) { + mapConfigValuesInOption( + [&ctx, &out](const std::string & configPath, std::variant<Value, std::exception_ptr> v) { + printConfigValue(ctx, out, configPath, v); + }, + optionPath, ctx); + }, + ctx, ctx.optionsRoot); +} + +void printAttr(Context & ctx, Out & out, const std::string & path, Value & root) +{ + try { + printValue(ctx, out, *findAlongAttrPath(ctx.state, path, ctx.autoArgs, root), path); + } catch (Error & e) { + out << describeError(e); + } +} + +bool hasExample(Context & ctx, Value & option) +{ + try { + findAlongAttrPath(ctx.state, "example", ctx.autoArgs, option); + return true; + } catch (Error &) { + return false; + } +} + +void printOption(Context & ctx, Out & out, const std::string & path, Value & option) +{ + out << "Value:\n"; + printAttr(ctx, out, path, ctx.configRoot); + + out << "\n\nDefault:\n"; + printAttr(ctx, out, "default", option); + + out << "\n\nType:\n"; + printAttr(ctx, out, "type.description", option); + + if (hasExample(ctx, option)) { + out << "\n\nExample:\n"; + printAttr(ctx, out, "example", option); + } + + out << "\n\nDescription:\n"; + printAttr(ctx, out, "description", option); + + out << "\n\nDeclared by:\n"; + printAttr(ctx, out, "declarations", option); + + out << "\n\nDefined by:\n"; + printAttr(ctx, out, "files", option); + out << "\n"; +} + +void printListing(Out & out, Value & v) +{ + out << "This attribute set contains:\n"; + for (const auto & a : v.attrs->lexicographicOrder()) { + std::string name = a->name; + if (!name.empty() && name[0] != '_') { + out << name << "\n"; + } + } +} + +bool optionTypeIs(Context & ctx, Value & v, const std::string & soughtType) +{ + try { + const auto & typeLookup = v.attrs->find(ctx.state.sType); + if (typeLookup == v.attrs->end()) { + return false; + } + Value type = evaluateValue(ctx, *typeLookup->value); + if (type.type != tAttrs) { + return false; + } + const auto & nameLookup = type.attrs->find(ctx.state.sName); + if (nameLookup == type.attrs->end()) { + return false; + } + Value name = evaluateValue(ctx, *nameLookup->value); + if (name.type != tString) { + return false; + } + return name.string.s == soughtType; + } catch (Error &) { + return false; + } +} + +bool isAggregateOptionType(Context & ctx, Value & v) +{ + return optionTypeIs(ctx, v, "attrsOf") || optionTypeIs(ctx, v, "listOf") || optionTypeIs(ctx, v, "loaOf"); +} + +MakeError(OptionPathError, EvalError); + +Value getSubOptions(Context & ctx, Value & option) +{ + Value getSubOptions = evaluateValue(ctx, *findAlongAttrPath(ctx.state, "type.getSubOptions", ctx.autoArgs, option)); + if (getSubOptions.type != tLambda) { + throw OptionPathError("Option's type.getSubOptions isn't a function"); + } + Value emptyString{}; + nix::mkString(emptyString, ""); + Value v; + ctx.state.callFunction(getSubOptions, emptyString, v, nix::Pos{}); + return v; +} + +// Carefully walk an option path, looking for sub-options when a path walks past +// an option value. +Value findAlongOptionPath(Context & ctx, const std::string & path) +{ + Strings tokens = parseAttrPath(path); + Value v = ctx.optionsRoot; + for (auto i = tokens.begin(); i != tokens.end(); i++) { + const auto & attr = *i; + try { + bool lastAttribute = std::next(i) == tokens.end(); + v = evaluateValue(ctx, v); + if (attr.empty()) { + throw OptionPathError("empty attribute name"); + } + if (isOption(ctx, v) && optionTypeIs(ctx, v, "submodule")) { + v = getSubOptions(ctx, v); + } + if (isOption(ctx, v) && isAggregateOptionType(ctx, v) && !lastAttribute) { + v = getSubOptions(ctx, v); + // Note that we've consumed attr, but didn't actually use it. This is the path component that's looked + // up in the list or attribute set that doesn't name an option -- the "root" in "users.users.root.name". + } else if (v.type != tAttrs) { + throw OptionPathError("Value is %s while a set was expected", showType(v)); + } else { + const auto & next = v.attrs->find(ctx.state.symbols.create(attr)); + if (next == v.attrs->end()) { + throw OptionPathError("Attribute not found", attr, path); + } + v = *next->value; + } + } catch (OptionPathError & e) { + throw OptionPathError("At '%s' in path '%s': %s", attr, path, e.msg()); + } + } + return v; +} + +void printOne(Context & ctx, Out & out, const std::string & path) +{ + try { + Value option = findAlongOptionPath(ctx, path); + option = evaluateValue(ctx, option); + if (isOption(ctx, option)) { + printOption(ctx, out, path, option); + } else { + printListing(out, option); + } + } catch (Error & e) { + std::cerr << "error: " << e.msg() + << "\nAn error occurred while looking for attribute names. Are " + "you sure that '" + << path << "' exists?\n"; + } +} + +int main(int argc, char ** argv) +{ + bool all = false; + std::string path = "."; + std::string optionsExpr = "(import <nixpkgs/nixos> {}).options"; + std::string configExpr = "(import <nixpkgs/nixos> {}).config"; + std::vector<std::string> args; + + struct MyArgs : nix::LegacyArgs, nix::MixEvalArgs + { + using nix::LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(nix::baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") { + nix::showManPage("nixos-option"); + } else if (*arg == "--version") { + nix::printVersion("nixos-option"); + } else if (*arg == "--all") { + all = true; + } else if (*arg == "--path") { + path = nix::getArg(*arg, arg, end); + } else if (*arg == "--options_expr") { + optionsExpr = nix::getArg(*arg, arg, end); + } else if (*arg == "--config_expr") { + configExpr = nix::getArg(*arg, arg, end); + } else if (!arg->empty() && arg->at(0) == '-') { + return false; + } else { + args.push_back(*arg); + } + return true; + }); + + myArgs.parseCmdline(nix::argvToStrings(argc, argv)); + + nix::initPlugins(); + nix::initGC(); + nix::settings.readOnlyMode = true; + auto store = nix::openStore(); + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + + Value optionsRoot = parseAndEval(*state, optionsExpr, path); + Value configRoot = parseAndEval(*state, configExpr, path); + + Context ctx{*state, *myArgs.getAutoArgs(*state), optionsRoot, configRoot}; + Out out(std::cout); + + if (all) { + if (!args.empty()) { + throw UsageError("--all cannot be used with arguments"); + } + printAll(ctx, out); + } else { + if (args.empty()) { + printOne(ctx, out, ""); + } + for (const auto & arg : args) { + printOne(ctx, out, arg); + } + } + + ctx.state.printStats(); + + return 0; +} diff --git a/nixos/modules/installer/tools/nixos-rebuild.sh b/nixos/modules/installer/tools/nixos-rebuild.sh index 891f374df53..c53dc1000c4 100644 --- a/nixos/modules/installer/tools/nixos-rebuild.sh +++ b/nixos/modules/installer/tools/nixos-rebuild.sh @@ -90,6 +90,11 @@ while [ "$#" -gt 0 ]; do targetHost="$1" shift 1 ;; + --use-remote-sudo) + # note the trailing space + maybeSudo="sudo " + shift 1 + ;; *) echo "$0: unknown option \`$i'" exit 1 @@ -97,10 +102,6 @@ while [ "$#" -gt 0 ]; do esac done -if [ -n "$SUDO_USER" ]; then - maybeSudo="sudo " -fi - if [ -z "$buildHost" -a -n "$targetHost" ]; then buildHost="$targetHost" fi diff --git a/nixos/modules/installer/tools/tools.nix b/nixos/modules/installer/tools/tools.nix index 9e6eead3c4d..e4db39b5c81 100644 --- a/nixos/modules/installer/tools/tools.nix +++ b/nixos/modules/installer/tools/tools.nix @@ -41,10 +41,7 @@ let inherit (config.system.nixos-generate-config) configuration; }; - nixos-option = makeProg { - name = "nixos-option"; - src = ./nixos-option.sh; - }; + nixos-option = pkgs.callPackage ./nixos-option { }; nixos-version = makeProg { name = "nixos-version"; @@ -123,7 +120,7 @@ in # programs.gnupg.agent = { # enable = true; # enableSSHSupport = true; - # flavour = "gnome3"; + # pinentryFlavor = "gnome3"; # }; # List services that you want to enable: diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix index 3e8a5b07a5e..a4db2c9d1d8 100644 --- a/nixos/modules/misc/ids.nix +++ b/nixos/modules/misc/ids.nix @@ -128,7 +128,7 @@ tcpcryptd = 93; # tcpcryptd uses a hard-coded uid. We patch it in Nixpkgs to match this choice. firebird = 95; #keys = 96; # unused - haproxy = 97; + #haproxy = 97; # DynamicUser as of 2019-11-08 mongodb = 98; openldap = 99; #users = 100; # unused @@ -443,7 +443,7 @@ #tcpcryptd = 93; # unused firebird = 95; keys = 96; - haproxy = 97; + #haproxy = 97; # DynamicUser as of 2019-11-08 #mongodb = 98; # unused openldap = 99; munin = 102; diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index d3739ae0960..f7c66166c5c 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1,9 +1,7 @@ [ ./config/debug-info.nix - ./config/fonts/corefonts.nix ./config/fonts/fontconfig.nix ./config/fonts/fontconfig-penultimate.nix - ./config/fonts/fontconfig-ultimate.nix ./config/fonts/fontdir.nix ./config/fonts/fonts.nix ./config/fonts/ghostscript.nix @@ -44,6 +42,7 @@ ./hardware/all-firmware.nix ./hardware/bladeRF.nix ./hardware/brightnessctl.nix + ./hardware/brillo.nix ./hardware/ckb-next.nix ./hardware/cpu/amd-microcode.nix ./hardware/cpu/intel-microcode.nix @@ -227,6 +226,7 @@ ./services/backup/rsnapshot.nix ./services/backup/tarsnap.nix ./services/backup/tsm.nix + ./services/backup/zfs-replication.nix ./services/backup/znapzend.nix ./services/cluster/hadoop/default.nix ./services/cluster/kubernetes/addons/dns.nix @@ -549,6 +549,8 @@ ./services/network-filesystems/nfsd.nix ./services/network-filesystems/openafs/client.nix ./services/network-filesystems/openafs/server.nix + ./services/network-filesystems/orangefs/server.nix + ./services/network-filesystems/orangefs/client.nix ./services/network-filesystems/rsyncd.nix ./services/network-filesystems/samba.nix ./services/network-filesystems/tahoe.nix @@ -602,6 +604,7 @@ ./services/networking/gdomap.nix ./services/networking/git-daemon.nix ./services/networking/gnunet.nix + ./services/networking/go-shadowsocks2.nix ./services/networking/gogoclient.nix ./services/networking/gvpe.nix ./services/networking/hans.nix @@ -615,7 +618,6 @@ ./services/networking/iodine.nix ./services/networking/iperf3.nix ./services/networking/ircd-hybrid/default.nix - ./services/networking/jormungandr.nix ./services/networking/iwd.nix ./services/networking/keepalived/default.nix ./services/networking/keybase.nix @@ -714,6 +716,7 @@ ./services/networking/tinc.nix ./services/networking/tinydns.nix ./services/networking/tftpd.nix + ./services/networking/trickster.nix ./services/networking/tox-bootstrapd.nix ./services/networking/tox-node.nix ./services/networking/toxvpn.nix @@ -807,8 +810,10 @@ ./services/web-apps/nexus.nix ./services/web-apps/pgpkeyserver-lite.nix ./services/web-apps/matomo.nix + ./services/web-apps/moinmoin.nix ./services/web-apps/restya-board.nix ./services/web-apps/tt-rss.nix + ./services/web-apps/trac.nix ./services/web-apps/selfoss.nix ./services/web-apps/shiori.nix ./services/web-apps/virtlyst.nix @@ -859,6 +864,7 @@ ./services/x11/hardware/multitouch.nix ./services/x11/hardware/synaptics.nix ./services/x11/hardware/wacom.nix + ./services/x11/hardware/digimend.nix ./services/x11/hardware/cmt.nix ./services/x11/gdk-pixbuf.nix ./services/x11/redshift.nix diff --git a/nixos/modules/programs/gnupg.nix b/nixos/modules/programs/gnupg.nix index 4fb7c43c8b2..2d262d90657 100644 --- a/nixos/modules/programs/gnupg.nix +++ b/nixos/modules/programs/gnupg.nix @@ -14,7 +14,7 @@ let "qt" else if xserverCfg.desktopManager.xfce.enable then "gtk2" - else if xserverCfg.enable then + else if xserverCfg.enable || config.programs.sway.enable then "gnome3" else null; @@ -121,6 +121,8 @@ in wantedBy = [ "sockets.target" ]; }; + services.dbus.packages = mkIf (cfg.agent.pinentryFlavor == "gnome3") [ pkgs.gcr ]; + environment.systemPackages = with pkgs; [ cfg.package ]; systemd.packages = [ cfg.package ]; diff --git a/nixos/modules/programs/ssh.nix b/nixos/modules/programs/ssh.nix index 733b8f7636f..80198990ed1 100644 --- a/nixos/modules/programs/ssh.nix +++ b/nixos/modules/programs/ssh.nix @@ -115,6 +115,16 @@ in ''; }; + agentPKCS11Whitelist = mkOption { + type = types.nullOr types.str; + default = null; + example = "\${pkgs.opensc}/lib/opensc-pkcs11.so"; + description = '' + A pattern-list of acceptable paths for PKCS#11 shared libraries + that may be used with the -s option to ssh-add. + ''; + }; + package = mkOption { type = types.package; default = pkgs.openssh; @@ -241,6 +251,7 @@ in ExecStart = "${cfg.package}/bin/ssh-agent " + optionalString (cfg.agentTimeout != null) ("-t ${cfg.agentTimeout} ") + + optionalString (cfg.agentPKCS11Whitelist != null) ("-P ${cfg.agentPKCS11Whitelist} ") + "-a %t/ssh-agent"; StandardOutput = "null"; Type = "forking"; diff --git a/nixos/modules/programs/x2goserver.nix b/nixos/modules/programs/x2goserver.nix index 77a1a0da799..7d74231e956 100644 --- a/nixos/modules/programs/x2goserver.nix +++ b/nixos/modules/programs/x2goserver.nix @@ -69,6 +69,7 @@ in { users.users.x2go = { home = "/var/lib/x2go/db"; group = "x2go"; + isSystemUser = true; }; security.wrappers.x2gosqliteWrapper = { diff --git a/nixos/modules/programs/zsh/zsh-syntax-highlighting.nix b/nixos/modules/programs/zsh/zsh-syntax-highlighting.nix index 7184e5d9b9a..c84d26a7921 100644 --- a/nixos/modules/programs/zsh/zsh-syntax-highlighting.nix +++ b/nixos/modules/programs/zsh/zsh-syntax-highlighting.nix @@ -81,7 +81,7 @@ in ]; programs.zsh.interactiveShellInit = with pkgs; - lib.concatStringsSep "\n" ([ + lib.mkAfter (lib.concatStringsSep "\n" ([ "source ${zsh-syntax-highlighting}/share/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh" ] ++ optional (length(cfg.highlighters) > 0) "ZSH_HIGHLIGHT_HIGHLIGHTERS=(${concatStringsSep " " cfg.highlighters})" @@ -95,6 +95,6 @@ in styles: design: "ZSH_HIGHLIGHT_STYLES[${styles}]='${design}'" ) cfg.styles) - ); + )); }; } diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix index df8ebe50584..7d8cf55b827 100644 --- a/nixos/modules/rename.nix +++ b/nixos/modules/rename.nix @@ -135,7 +135,8 @@ with lib; # piwik was renamed to matomo (mkRenamedOptionModule [ "services" "piwik" "enable" ] [ "services" "matomo" "enable" ]) (mkRenamedOptionModule [ "services" "piwik" "webServerUser" ] [ "services" "matomo" "webServerUser" ]) - (mkRenamedOptionModule [ "services" "piwik" "phpfpmProcessManagerConfig" ] [ "services" "matomo" "phpfpmProcessManagerConfig" ]) + (mkRemovedOptionModule [ "services" "piwik" "phpfpmProcessManagerConfig" ] "Use services.phpfpm.pools.<name>.settings") + (mkRemovedOptionModule [ "services" "matomo" "phpfpmProcessManagerConfig" ] "Use services.phpfpm.pools.<name>.settings") (mkRenamedOptionModule [ "services" "piwik" "nginx" ] [ "services" "matomo" "nginx" ]) # tarsnap @@ -233,6 +234,7 @@ with lib; (mkRemovedOptionModule [ "services" "mysql" "rootPassword" ] "Use socket authentication or set the password outside of the nix store.") (mkRemovedOptionModule [ "services" "zabbixServer" "dbPassword" ] "Use services.zabbixServer.database.passwordFile instead.") (mkRemovedOptionModule [ "systemd" "generator-packages" ] "Use systemd.packages instead.") + (mkRemovedOptionModule [ "fonts" "enableCoreFonts" ] "Use fonts.fonts = [ pkgs.corefonts ]; instead.") # ZSH (mkRenamedOptionModule [ "programs" "zsh" "enableSyntaxHighlighting" ] [ "programs" "zsh" "syntaxHighlighting" "enable" ]) @@ -290,5 +292,14 @@ with lib; (opt: mkRemovedOptionModule [ "services" "prometheus" "${opt}" ] '' The prometheus exporters are now configured using `services.prometheus.exporters'. See the 18.03 release notes for more information. + '' )) + + ++ (forEach [ "enable" "substitutions" "preset" ] + (opt: mkRemovedOptionModule [ "fonts" "fontconfig" "ultimate" "${opt}" ] '' + The fonts.fontconfig.ultimate module and configuration is obsolete. + The repository has since been archived and activity has ceased. + https://github.com/bohoomil/fontconfig-ultimate/issues/171. + No action should be needed for font configuration, as the fonts.fontconfig + module is already used by default. '' )); } diff --git a/nixos/modules/security/acme.nix b/nixos/modules/security/acme.nix index cbeb99cfcef..d14613f22b0 100644 --- a/nixos/modules/security/acme.nix +++ b/nixos/modules/security/acme.nix @@ -20,6 +20,16 @@ let ''; }; + server = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + ACME Directory Resource URI. Defaults to let's encrypt + production endpoint, + https://acme-v02.api.letsencrypt.org/directory, if unset. + ''; + }; + domain = mkOption { type = types.str; default = name; @@ -109,7 +119,15 @@ in { ###### interface - + imports = [ + (mkRemovedOptionModule [ "security" "acme" "production" ] '' + Use security.acme.server to define your staging ACME server URL instead. + + To use the let's encrypt staging server, use security.acme.server = + "https://acme-staging-v02.api.letsencrypt.org/directory". + '' + ) + ]; options = { security.acme = { @@ -129,6 +147,16 @@ in ''; }; + server = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + ACME Directory Resource URI. Defaults to let's encrypt + production endpoint, + <literal>https://acme-v02.api.letsencrypt.org/directory</literal>, if unset. + ''; + }; + preliminarySelfsigned = mkOption { type = types.bool; default = true; @@ -142,20 +170,6 @@ in ''; }; - production = mkOption { - type = types.bool; - default = true; - description = '' - If set to true, use Let's Encrypt's production environment - instead of the staging environment. The main benefit of the - staging environment is to get much higher rate limits. - - See - <literal>https://letsencrypt.org/docs/staging-environment</literal> - for more detail. - ''; - }; - certs = mkOption { default = { }; type = with types; attrsOf (submodule certOpts); @@ -198,7 +212,7 @@ in ++ optionals (data.email != null) [ "--email" data.email ] ++ concatMap (p: [ "-f" p ]) data.plugins ++ concatLists (mapAttrsToList (name: root: [ "-d" (if root == null then name else "${name}:${root}")]) data.extraDomains) - ++ optionals (!cfg.production) ["--server" "https://acme-staging-v02.api.letsencrypt.org/directory"]; + ++ optionals (cfg.server != null || data.server != null) ["--server" (if data.server == null then cfg.server else data.server)]; acmeService = { description = "Renew ACME Certificate for ${cert}"; after = [ "network.target" "network-online.target" ]; diff --git a/nixos/modules/security/pam_mount.nix b/nixos/modules/security/pam_mount.nix index 8b131c54a2a..75f58462d13 100644 --- a/nixos/modules/security/pam_mount.nix +++ b/nixos/modules/security/pam_mount.nix @@ -50,9 +50,6 @@ in <pam_mount> <debug enable="0" /> - ${concatStrings (map userVolumeEntry (attrValues extraUserVolumes))} - ${concatStringsSep "\n" cfg.extraVolumes} - <!-- if activated, requires ofl from hxtools to be present --> <logout wait="0" hup="no" term="no" kill="no" /> <!-- set PATH variable for pam_mount module --> @@ -64,6 +61,9 @@ in <cryptmount>${pkgs.pam_mount}/bin/mount.crypt %(VOLUME) %(MNTPT)</cryptmount> <cryptumount>${pkgs.pam_mount}/bin/umount.crypt %(MNTPT)</cryptumount> <pmvarrun>${pkgs.pam_mount}/bin/pmvarrun -u %(USER) -o %(OPERATION)</pmvarrun> + + ${concatStrings (map userVolumeEntry (attrValues extraUserVolumes))} + ${concatStringsSep "\n" cfg.extraVolumes} </pam_mount> ''; }]; diff --git a/nixos/modules/services/admin/oxidized.nix b/nixos/modules/services/admin/oxidized.nix index 39112c3970d..da81be3f23e 100644 --- a/nixos/modules/services/admin/oxidized.nix +++ b/nixos/modules/services/admin/oxidized.nix @@ -89,6 +89,7 @@ in group = cfg.group; home = cfg.dataDir; createHome = true; + isSystemUser = true; }; systemd.services.oxidized = { diff --git a/nixos/modules/services/audio/jack.nix b/nixos/modules/services/audio/jack.nix index aa3351f401a..ceff366d0bb 100644 --- a/nixos/modules/services/audio/jack.nix +++ b/nixos/modules/services/audio/jack.nix @@ -223,6 +223,7 @@ in { group = "jackaudio"; extraGroups = [ "audio" ]; description = "JACK Audio system service user"; + isSystemUser = true; }; # http://jackaudio.org/faq/linux_rt_config.html security.pam.loginLimits = [ diff --git a/nixos/modules/services/audio/mpd.nix b/nixos/modules/services/audio/mpd.nix index 0df8f9688d2..56dc858b640 100644 --- a/nixos/modules/services/audio/mpd.nix +++ b/nixos/modules/services/audio/mpd.nix @@ -181,6 +181,7 @@ in { ProtectKernelModules = true; RestrictAddressFamilies = "AF_INET AF_INET6 AF_UNIX AF_NETLINK"; RestrictNamespaces = true; + Restart = "always"; }; }; diff --git a/nixos/modules/services/backup/automysqlbackup.nix b/nixos/modules/services/backup/automysqlbackup.nix index 1884f3536a9..e3a8d1f7993 100644 --- a/nixos/modules/services/backup/automysqlbackup.nix +++ b/nixos/modules/services/backup/automysqlbackup.nix @@ -99,7 +99,10 @@ in environment.systemPackages = [ pkg ]; - users.users.${user}.group = group; + users.users.${user} = { + group = group; + isSystemUser = true; + }; users.groups.${group} = { }; systemd.tmpfiles.rules = [ diff --git a/nixos/modules/services/continuous-integration/buildkite-agent.nix b/nixos/modules/services/continuous-integration/buildkite-agent.nix index 12cc3d2b1cc..32f361454bc 100644 --- a/nixos/modules/services/continuous-integration/buildkite-agent.nix +++ b/nixos/modules/services/continuous-integration/buildkite-agent.nix @@ -191,6 +191,7 @@ in createHome = true; description = "Buildkite agent user"; extraGroups = [ "keys" ]; + isSystemUser = true; }; environment.systemPackages = [ cfg.package ]; diff --git a/nixos/modules/services/databases/redis.nix b/nixos/modules/services/databases/redis.nix index 5861323e5ea..95128a641d9 100644 --- a/nixos/modules/services/databases/redis.nix +++ b/nixos/modules/services/databases/redis.nix @@ -185,16 +185,19 @@ in ###### implementation config = mkIf config.services.redis.enable { - - boot.kernel.sysctl = { - "vm.nr_hugepages" = "0"; - } // mkIf cfg.vmOverCommit { "vm.overcommit_memory" = "1"; }; + boot.kernel.sysctl = (mkMerge [ + { "vm.nr_hugepages" = "0"; } + ( mkIf cfg.vmOverCommit { "vm.overcommit_memory" = "1"; } ) + ]); networking.firewall = mkIf cfg.openFirewall { allowedTCPPorts = [ cfg.port ]; }; - users.users.redis.description = "Redis database user"; + users.users.redis = { + description = "Redis database user"; + isSystemUser = true; + }; environment.systemPackages = [ cfg.package ]; diff --git a/nixos/modules/services/databases/rethinkdb.nix b/nixos/modules/services/databases/rethinkdb.nix index 4828e594b32..f18fbaf5b06 100644 --- a/nixos/modules/services/databases/rethinkdb.nix +++ b/nixos/modules/services/databases/rethinkdb.nix @@ -99,6 +99,7 @@ in users.users.rethinkdb = mkIf (cfg.user == "rethinkdb") { name = "rethinkdb"; description = "RethinkDB server user"; + isSystemUser = true; }; users.groups = optionalAttrs (cfg.group == "rethinkdb") (singleton diff --git a/nixos/modules/services/desktops/geoclue2.nix b/nixos/modules/services/desktops/geoclue2.nix index 6007dddf50c..542b2ead410 100644 --- a/nixos/modules/services/desktops/geoclue2.nix +++ b/nixos/modules/services/desktops/geoclue2.nix @@ -188,34 +188,41 @@ in systemd.packages = [ package ]; - users.users.geoclue = { - isSystemUser = true; - home = "/var/lib/geoclue"; - group = "geoclue"; - description = "Geoinformation service"; - }; - - users.groups.geoclue = {}; + # we cannot use DynamicUser as we need the the geoclue user to exist for the dbus policy to work + users = { + users.geoclue = { + isSystemUser = true; + home = "/var/lib/geoclue"; + group = "geoclue"; + description = "Geoinformation service"; + }; - systemd.tmpfiles.rules = [ - "d /var/lib/geoclue 0755 geoclue geoclue" - ]; + groups.geoclue = {}; + }; - # restart geoclue service when the configuration changes - systemd.services.geoclue.restartTriggers = [ - config.environment.etc."geoclue/geoclue.conf".source - ]; + systemd.services.geoclue = { + # restart geoclue service when the configuration changes + restartTriggers = [ + config.environment.etc."geoclue/geoclue.conf".source + ]; + serviceConfig.StateDirectory = "geoclue"; + }; # this needs to run as a user service, since it's associated with the # user who is making the requests systemd.user.services = mkIf cfg.enableDemoAgent { geoclue-agent = { description = "Geoclue agent"; - script = "${package}/libexec/geoclue-2.0/demos/agent"; # this should really be `partOf = [ "geoclue.service" ]`, but # we can't be part of a system service, and the agent should # be okay with the main service coming and going wantedBy = [ "default.target" ]; + serviceConfig = { + Type = "exec"; + ExecStart = "${package}/libexec/geoclue-2.0/demos/agent"; + Restart = "on-failure"; + PrivateTmp = true; + }; }; }; @@ -256,4 +263,6 @@ in }; } // mapAttrs' appConfigToINICompatible cfg.appConfig); }; + + meta.maintainers = with lib.maintainers; [ worldofpeace ]; } diff --git a/nixos/modules/services/editors/infinoted.nix b/nixos/modules/services/editors/infinoted.nix index 9cc8d421270..be366761694 100644 --- a/nixos/modules/services/editors/infinoted.nix +++ b/nixos/modules/services/editors/infinoted.nix @@ -115,6 +115,7 @@ in { { name = "infinoted"; description = "Infinoted user"; group = cfg.group; + isSystemUser = true; }; users.groups = optional (cfg.group == "infinoted") { name = "infinoted"; diff --git a/nixos/modules/services/hardware/fancontrol.nix b/nixos/modules/services/hardware/fancontrol.nix index 616e4add31e..bb4541a784d 100644 --- a/nixos/modules/services/hardware/fancontrol.nix +++ b/nixos/modules/services/hardware/fancontrol.nix @@ -4,42 +4,41 @@ with lib; let cfg = config.hardware.fancontrol; - configFile = pkgs.writeText "fan.conf" cfg.config; - -in { + configFile = pkgs.writeText "fancontrol.conf" cfg.config; +in{ options.hardware.fancontrol = { - enable = mkEnableOption "fancontrol (requires fancontrol.config)"; + enable = mkEnableOption "software fan control (requires fancontrol.config)"; config = mkOption { - type = types.lines; default = null; + type = types.lines; + description = "Fancontrol configuration file content. See <citerefentry><refentrytitle>pwmconfig</refentrytitle><manvolnum>8</manvolnum></citerefentry> from the lm_sensors package."; example = '' # Configuration file generated by pwmconfig - INTERVAL=1 - DEVPATH=hwmon0=devices/platform/nct6775.656 hwmon1=devices/pci0000:00/0000:00:18.3 - DEVNAME=hwmon0=nct6779 hwmon1=k10temp - FCTEMPS=hwmon0/pwm2=hwmon1/temp1_input - FCFANS=hwmon0/pwm2=hwmon0/fan2_input - MINTEMP=hwmon0/pwm2=25 - MAXTEMP=hwmon0/pwm2=60 - MINSTART=hwmon0/pwm2=25 - MINSTOP=hwmon0/pwm2=10 - MINPWM=hwmon0/pwm2=0 - MAXPWM=hwmon0/pwm2=255 + INTERVAL=10 + DEVPATH=hwmon3=devices/virtual/thermal/thermal_zone2 hwmon4=devices/platform/f71882fg.656 + DEVNAME=hwmon3=soc_dts1 hwmon4=f71869a + FCTEMPS=hwmon4/device/pwm1=hwmon3/temp1_input + FCFANS= hwmon4/device/pwm1=hwmon4/device/fan1_input + MINTEMP=hwmon4/device/pwm1=35 + MAXTEMP=hwmon4/device/pwm1=65 + MINSTART=hwmon4/device/pwm1=150 + MINSTOP=hwmon4/device/pwm1=0 ''; - description = "Contents for configuration file. See <citerefentry><refentrytitle>pwmconfig</refentrytitle><manvolnum>8</manvolnum></citerefentry>."; }; }; - config = mkIf cfg.enable { systemd.services.fancontrol = { - description = "Fan speed control from lm_sensors"; + unitConfig.Documentation = "man:fancontrol(8)"; + description = "software fan control"; wantedBy = [ "multi-user.target" ]; + after = [ "lm_sensors.service" ]; + serviceConfig = { Type = "simple"; - ExecStart = "${pkgs.lm_sensors}/bin/fancontrol ${configFile}"; + ExecStart = "${pkgs.lm_sensors}/sbin/fancontrol ${configFile}"; }; }; }; diff --git a/nixos/modules/services/hardware/trezord.nix b/nixos/modules/services/hardware/trezord.nix index 62824ed7350..c517e9fbb2b 100644 --- a/nixos/modules/services/hardware/trezord.nix +++ b/nixos/modules/services/hardware/trezord.nix @@ -44,20 +44,7 @@ in { ### implementation config = mkIf cfg.enable { - services.udev.packages = lib.singleton (pkgs.writeTextFile { - name = "trezord-udev-rules"; - destination = "/etc/udev/rules.d/51-trezor.rules"; - text = '' - # TREZOR v1 (One) - SUBSYSTEM=="usb", ATTR{idVendor}=="534c", ATTR{idProduct}=="0001", MODE="0660", GROUP="trezord", TAG+="uaccess", SYMLINK+="trezor%n" - KERNEL=="hidraw*", ATTRS{idVendor}=="534c", ATTRS{idProduct}=="0001", MODE="0660", GROUP="trezord", TAG+="uaccess" - - # TREZOR v2 (T) - SUBSYSTEM=="usb", ATTR{idVendor}=="1209", ATTR{idProduct}=="53c0", MODE="0660", GROUP="trezord", TAG+="uaccess", SYMLINK+="trezor%n" - SUBSYSTEM=="usb", ATTR{idVendor}=="1209", ATTR{idProduct}=="53c1", MODE="0660", GROUP="trezord", TAG+="uaccess", SYMLINK+="trezor%n" - KERNEL=="hidraw*", ATTRS{idVendor}=="1209", ATTRS{idProduct}=="53c1", MODE="0660", GROUP="trezord", TAG+="uaccess" - ''; - }); + services.udev.packages = [ pkgs.trezor-udev-rules ]; systemd.services.trezord = { description = "TREZOR Bridge"; @@ -74,6 +61,7 @@ in { users.users.trezord = { group = "trezord"; description = "Trezor bridge daemon user"; + isSystemUser = true; }; users.groups.trezord = {}; diff --git a/nixos/modules/services/hardware/usbmuxd.nix b/nixos/modules/services/hardware/usbmuxd.nix index 93ced0b9f04..39bbcaf4627 100644 --- a/nixos/modules/services/hardware/usbmuxd.nix +++ b/nixos/modules/services/hardware/usbmuxd.nix @@ -47,6 +47,7 @@ in name = cfg.user; description = "usbmuxd user"; group = cfg.group; + isSystemUser = true; }; users.groups = optional (cfg.group == defaultUserGroup) { diff --git a/nixos/modules/services/hardware/vdr.nix b/nixos/modules/services/hardware/vdr.nix index 6e246f70f51..8a6cde51b06 100644 --- a/nixos/modules/services/hardware/vdr.nix +++ b/nixos/modules/services/hardware/vdr.nix @@ -66,6 +66,7 @@ in { users.users.vdr = { group = "vdr"; home = libDir; + isSystemUser = true; }; users.groups.vdr = {}; diff --git a/nixos/modules/services/mail/mailhog.nix b/nixos/modules/services/mail/mailhog.nix index b78f4c8e0e6..0f998c6d0ea 100644 --- a/nixos/modules/services/mail/mailhog.nix +++ b/nixos/modules/services/mail/mailhog.nix @@ -27,6 +27,7 @@ in { users.users.mailhog = { name = cfg.user; description = "MailHog service user"; + isSystemUser = true; }; systemd.services.mailhog = { diff --git a/nixos/modules/services/misc/airsonic.nix b/nixos/modules/services/misc/airsonic.nix index 919d3b2f6e6..c296e048cea 100644 --- a/nixos/modules/services/misc/airsonic.nix +++ b/nixos/modules/services/misc/airsonic.nix @@ -148,6 +148,7 @@ in { name = cfg.user; home = cfg.home; createHome = true; + isSystemUser = true; }; }; } diff --git a/nixos/modules/services/misc/docker-registry.nix b/nixos/modules/services/misc/docker-registry.nix index c87607d2666..89bac4f47d7 100644 --- a/nixos/modules/services/misc/docker-registry.nix +++ b/nixos/modules/services/misc/docker-registry.nix @@ -145,11 +145,13 @@ in { }; users.users.docker-registry = - if cfg.storagePath != null + (if cfg.storagePath != null then { createHome = true; home = cfg.storagePath; } - else {}; + else {}) // { + isSystemUser = true; + }; }; } diff --git a/nixos/modules/services/misc/errbot.nix b/nixos/modules/services/misc/errbot.nix index 256adce2f02..b447ba5d438 100644 --- a/nixos/modules/services/misc/errbot.nix +++ b/nixos/modules/services/misc/errbot.nix @@ -76,7 +76,10 @@ in { }; config = mkIf (cfg.instances != {}) { - users.users.errbot.group = "errbot"; + users.users.errbot = { + group = "errbot"; + isSystemUser = true; + }; users.groups.errbot = {}; systemd.services = mapAttrs' (name: instanceCfg: nameValuePair "errbot-${name}" ( diff --git a/nixos/modules/services/misc/gitea.nix b/nixos/modules/services/misc/gitea.nix index 4992b13c9d4..c8c59fb256e 100644 --- a/nixos/modules/services/misc/gitea.nix +++ b/nixos/modules/services/misc/gitea.nix @@ -409,6 +409,7 @@ in home = cfg.stateDir; useDefaultShell = true; group = "gitea"; + isSystemUser = true; }; }; diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index 6ee35aaca56..07ea9c45843 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -608,6 +608,8 @@ in { # objects owners and extensions; for now we tack on what's needed # here. systemd.services.postgresql.postStart = mkAfter (optionalString databaseActuallyCreateLocally '' + set -eu + $PSQL -tAc "SELECT 1 FROM pg_database WHERE datname = '${cfg.databaseName}'" | grep -q 1 || $PSQL -tAc 'CREATE DATABASE "${cfg.databaseName}" OWNER "${cfg.databaseUsername}"' current_owner=$($PSQL -tAc "SELECT pg_catalog.pg_get_userbyid(datdba) FROM pg_catalog.pg_database WHERE datname = '${cfg.databaseName}'") if [[ "$current_owner" != "${cfg.databaseUsername}" ]]; then @@ -739,7 +741,6 @@ in { gitlab-workhorse ]; serviceConfig = { - PermissionsStartOnly = true; # preStart must be run as root Type = "simple"; User = cfg.user; Group = cfg.group; @@ -781,13 +782,18 @@ in { ExecStartPre = let preStartFullPrivileges = '' shopt -s dotglob nullglob + set -eu + chown --no-dereference '${cfg.user}':'${cfg.group}' '${cfg.statePath}'/* chown --no-dereference '${cfg.user}':'${cfg.group}' '${cfg.statePath}'/config/* ''; preStart = '' + set -eu + cp -f ${cfg.packages.gitlab}/share/gitlab/VERSION ${cfg.statePath}/VERSION rm -rf ${cfg.statePath}/db/* rm -rf ${cfg.statePath}/config/initializers/* + rm -f ${cfg.statePath}/lib cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/config.dist/* ${cfg.statePath}/config cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/db/* ${cfg.statePath}/db diff --git a/nixos/modules/services/misc/gollum.nix b/nixos/modules/services/misc/gollum.nix index 7653b415bf0..f4a9c72b154 100644 --- a/nixos/modules/services/misc/gollum.nix +++ b/nixos/modules/services/misc/gollum.nix @@ -71,6 +71,7 @@ in group = config.users.users.gollum.name; description = "Gollum user"; createHome = false; + isSystemUser = true; }; users.groups.gollum = { }; diff --git a/nixos/modules/services/misc/jellyfin.nix b/nixos/modules/services/misc/jellyfin.nix index 55559206568..6ecdfb57dc3 100644 --- a/nixos/modules/services/misc/jellyfin.nix +++ b/nixos/modules/services/misc/jellyfin.nix @@ -41,7 +41,10 @@ in }; users.users = mkIf (cfg.user == "jellyfin") { - jellyfin.group = cfg.group; + jellyfin = { + group = cfg.group; + isSystemUser = true; + }; }; users.groups = mkIf (cfg.group == "jellyfin") { diff --git a/nixos/modules/services/misc/matrix-synapse.nix b/nixos/modules/services/misc/matrix-synapse.nix index 0f4eb2ccfca..50661b873f6 100644 --- a/nixos/modules/services/misc/matrix-synapse.nix +++ b/nixos/modules/services/misc/matrix-synapse.nix @@ -407,6 +407,9 @@ in { "192.168.0.0/16" "100.64.0.0/10" "169.254.0.0/16" + "::1/128" + "fe80::/64" + "fc00::/7" ]; description = '' List of IP address CIDR ranges that the URL preview spider is denied diff --git a/nixos/modules/services/misc/osrm.nix b/nixos/modules/services/misc/osrm.nix index f89f37ccd9d..79c347ab7e0 100644 --- a/nixos/modules/services/misc/osrm.nix +++ b/nixos/modules/services/misc/osrm.nix @@ -59,6 +59,7 @@ in group = config.users.users.osrm.name; description = "OSRM user"; createHome = false; + isSystemUser = true; }; users.groups.osrm = { }; diff --git a/nixos/modules/services/misc/redmine.nix b/nixos/modules/services/misc/redmine.nix index 24b9e27ac2d..bf9a6914a48 100644 --- a/nixos/modules/services/misc/redmine.nix +++ b/nixos/modules/services/misc/redmine.nix @@ -62,20 +62,11 @@ in services.redmine = { enable = mkEnableOption "Redmine"; - # default to the 4.x series not forcing major version upgrade of those on the 3.x series package = mkOption { type = types.package; - default = if versionAtLeast config.system.stateVersion "19.03" - then pkgs.redmine_4 - else pkgs.redmine - ; - defaultText = "pkgs.redmine"; - description = '' - Which Redmine package to use. This defaults to version 3.x if - <literal>system.stateVersion < 19.03</literal> and version 4.x - otherwise. - ''; - example = "pkgs.redmine_4.override { ruby = pkgs.ruby_2_4; }"; + default = pkgs.redmine; + description = "Which Redmine package to use."; + example = "pkgs.redmine.override { ruby = pkgs.ruby_2_4; }"; }; user = mkOption { diff --git a/nixos/modules/services/misc/zoneminder.nix b/nixos/modules/services/misc/zoneminder.nix index 3bff04e7127..d7f7324580c 100644 --- a/nixos/modules/services/misc/zoneminder.nix +++ b/nixos/modules/services/misc/zoneminder.nix @@ -265,7 +265,7 @@ in { } location /cache/ { - alias /var/cache/${dirName}; + alias /var/cache/${dirName}/; } location ~ \.php$ { diff --git a/nixos/modules/services/monitoring/collectd.nix b/nixos/modules/services/monitoring/collectd.nix index b2e44a1e366..731ac743b7c 100644 --- a/nixos/modules/services/monitoring/collectd.nix +++ b/nixos/modules/services/monitoring/collectd.nix @@ -131,6 +131,7 @@ in { users.users = optional (cfg.user == "collectd") { name = "collectd"; + isSystemUser = true; }; }; } diff --git a/nixos/modules/services/monitoring/fusion-inventory.nix b/nixos/modules/services/monitoring/fusion-inventory.nix index b90579bb70c..fe19ed56195 100644 --- a/nixos/modules/services/monitoring/fusion-inventory.nix +++ b/nixos/modules/services/monitoring/fusion-inventory.nix @@ -49,6 +49,7 @@ in { users.users = singleton { name = "fusion-inventory"; description = "FusionInventory user"; + isSystemUser = true; }; systemd.services.fusion-inventory = { diff --git a/nixos/modules/services/monitoring/netdata.nix b/nixos/modules/services/monitoring/netdata.nix index 463b1b882ac..3ffde8e9bce 100644 --- a/nixos/modules/services/monitoring/netdata.nix +++ b/nixos/modules/services/monitoring/netdata.nix @@ -138,7 +138,7 @@ in { description = "Real time performance monitoring"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; - path = (with pkgs; [ gawk curl ]) ++ lib.optional cfg.python.enable + path = (with pkgs; [ curl gawk which ]) ++ lib.optional cfg.python.enable (pkgs.python3.withPackages cfg.python.extraPackages); serviceConfig = { Environment="PYTHONPATH=${pkgs.netdata}/libexec/netdata/python.d/python_modules"; @@ -181,6 +181,7 @@ in { users.users = optional (cfg.user == defaultUser) { name = defaultUser; + isSystemUser = true; }; users.groups = optional (cfg.group == defaultUser) { diff --git a/nixos/modules/services/monitoring/zabbix-agent.nix b/nixos/modules/services/monitoring/zabbix-agent.nix index 856b9432892..b3383ed628b 100644 --- a/nixos/modules/services/monitoring/zabbix-agent.nix +++ b/nixos/modules/services/monitoring/zabbix-agent.nix @@ -131,6 +131,7 @@ in users.users.${user} = { description = "Zabbix Agent daemon user"; inherit group; + isSystemUser = true; }; users.groups.${group} = { }; diff --git a/nixos/modules/services/network-filesystems/orangefs/client.nix b/nixos/modules/services/network-filesystems/orangefs/client.nix new file mode 100644 index 00000000000..b69d9e713c3 --- /dev/null +++ b/nixos/modules/services/network-filesystems/orangefs/client.nix @@ -0,0 +1,97 @@ +{ config, lib, pkgs, ...} : + +with lib; + +let + cfg = config.services.orangefs.client; + +in { + ###### interface + + options = { + services.orangefs.client = { + enable = mkEnableOption "OrangeFS client daemon"; + + extraOptions = mkOption { + type = with types; listOf str; + default = []; + description = "Extra command line options for pvfs2-client."; + }; + + fileSystems = mkOption { + description = '' + The orangefs file systems to be mounted. + This option is prefered over using <option>fileSystems</option> directly since + the pvfs client service needs to be running for it to be mounted. + ''; + + example = [{ + mountPoint = "/orangefs"; + target = "tcp://server:3334/orangefs"; + }]; + + type = with types; listOf (submodule ({ ... } : { + options = { + + mountPoint = mkOption { + type = types.str; + default = "/orangefs"; + description = "Mount point."; + }; + + options = mkOption { + type = with types; listOf str; + default = []; + description = "Mount options"; + }; + + target = mkOption { + type = types.str; + default = null; + example = "tcp://server:3334/orangefs"; + description = "Target URL"; + }; + }; + })); + }; + }; + }; + + + ###### implementation + + config = mkIf cfg.enable { + environment.systemPackages = [ pkgs.orangefs ]; + + boot.supportedFilesystems = [ "pvfs2" ]; + boot.kernelModules = [ "orangefs" ]; + + systemd.services.orangefs-client = { + requires = [ "network-online.target" ]; + after = [ "network-online.target" ]; + + serviceConfig = { + Type = "simple"; + + ExecStart = '' + ${pkgs.orangefs}/bin/pvfs2-client-core \ + --logtype=syslog ${concatStringsSep " " cfg.extraOptions} + ''; + + TimeoutStopSec = "120"; + }; + }; + + systemd.mounts = map (fs: { + requires = [ "orangefs-client.service" ]; + after = [ "orangefs-client.service" ]; + bindsTo = [ "orangefs-client.service" ]; + wantedBy = [ "remote-fs.target" ]; + type = "pvfs2"; + options = concatStringsSep "," fs.options; + what = fs.target; + where = fs.mountPoint; + }) cfg.fileSystems; + }; +} + diff --git a/nixos/modules/services/network-filesystems/orangefs/server.nix b/nixos/modules/services/network-filesystems/orangefs/server.nix new file mode 100644 index 00000000000..74ebdc13402 --- /dev/null +++ b/nixos/modules/services/network-filesystems/orangefs/server.nix @@ -0,0 +1,225 @@ +{ config, lib, pkgs, ...} : + +with lib; + +let + cfg = config.services.orangefs.server; + + aliases = mapAttrsToList (alias: url: alias) cfg.servers; + + # Maximum handle number is 2^63 + maxHandle = 9223372036854775806; + + # One range of handles for each meta/data instance + handleStep = maxHandle / (length aliases) / 2; + + fileSystems = mapAttrsToList (name: fs: '' + <FileSystem> + Name ${name} + ID ${toString fs.id} + RootHandle ${toString fs.rootHandle} + + ${fs.extraConfig} + + <MetaHandleRanges> + ${concatStringsSep "\n" ( + imap0 (i: alias: + let + begin = i * handleStep + 3; + end = begin + handleStep - 1; + in "Range ${alias} ${toString begin}-${toString end}") aliases + )} + </MetaHandleRanges> + + <DataHandleRanges> + ${concatStringsSep "\n" ( + imap0 (i: alias: + let + begin = i * handleStep + 3 + (length aliases) * handleStep; + end = begin + handleStep - 1; + in "Range ${alias} ${toString begin}-${toString end}") aliases + )} + </DataHandleRanges> + + <StorageHints> + TroveSyncMeta ${if fs.troveSyncMeta then "yes" else "no"} + TroveSyncData ${if fs.troveSyncData then "yes" else "no"} + ${fs.extraStorageHints} + </StorageHints> + + </FileSystem> + '') cfg.fileSystems; + + configFile = '' + <Defaults> + LogType ${cfg.logType} + DataStorageSpace ${cfg.dataStorageSpace} + MetaDataStorageSpace ${cfg.metadataStorageSpace} + + BMIModules ${concatStringsSep "," cfg.BMIModules} + ${cfg.extraDefaults} + </Defaults> + + ${cfg.extraConfig} + + <Aliases> + ${concatStringsSep "\n" (mapAttrsToList (alias: url: "Alias ${alias} ${url}") cfg.servers)} + </Aliases> + + ${concatStringsSep "\n" fileSystems} + ''; + +in { + ###### interface + + options = { + services.orangefs.server = { + enable = mkEnableOption "OrangeFS server"; + + logType = mkOption { + type = with types; enum [ "file" "syslog" ]; + default = "syslog"; + description = "Destination for log messages."; + }; + + dataStorageSpace = mkOption { + type = types.str; + default = null; + example = "/data/storage"; + description = "Directory for data storage."; + }; + + metadataStorageSpace = mkOption { + type = types.str; + default = null; + example = "/data/meta"; + description = "Directory for meta data storage."; + }; + + BMIModules = mkOption { + type = with types; listOf str; + default = [ "bmi_tcp" ]; + example = [ "bmi_tcp" "bmi_ib"]; + description = "List of BMI modules to load."; + }; + + extraDefaults = mkOption { + type = types.lines; + default = ""; + description = "Extra config for <literal><Defaults></literal> section."; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = "Extra config for the global section."; + }; + + servers = mkOption { + type = with types; attrsOf types.str; + default = {}; + example = '' + { + node1="tcp://node1:3334"; + node2="tcp://node2:3334"; + } + ''; + description = "URLs for storage server including port. The attribute names define the server alias."; + }; + + fileSystems = mkOption { + description = '' + These options will create the <literal><FileSystem></literal> sections of config file. + ''; + default = { orangefs = {}; }; + defaultText = literalExample "{ orangefs = {}; }"; + example = literalExample '' + { + fs1 = { + id = 101; + }; + + fs2 = { + id = 102; + }; + } + ''; + type = with types; attrsOf (submodule ({ ... } : { + options = { + id = mkOption { + type = types.int; + default = 1; + description = "File system ID (must be unique within configuration)."; + }; + + rootHandle = mkOption { + type = types.int; + default = 3; + description = "File system root ID."; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = "Extra config for <literal><FileSystem></literal> section."; + }; + + troveSyncMeta = mkOption { + type = types.bool; + default = true; + description = "Sync meta data."; + }; + + troveSyncData = mkOption { + type = types.bool; + default = false; + description = "Sync data."; + }; + + extraStorageHints = mkOption { + type = types.lines; + default = ""; + description = "Extra config for <literal><StorageHints></literal> section."; + }; + }; + })); + }; + }; + }; + + ###### implementation + + config = mkIf cfg.enable { + environment.systemPackages = [ pkgs.orangefs ]; + + # orangefs daemon will run as user + users.users.orangefs.isSystemUser = true; + users.groups.orangefs = {}; + + # To format the file system the config file is needed. + environment.etc."orangefs/server.conf" = { + text = configFile; + user = "orangefs"; + group = "orangefs"; + }; + + systemd.services.orangefs-server = { + wantedBy = [ "multi-user.target" ]; + requires = [ "network-online.target" ]; + after = [ "network-online.target" ]; + + serviceConfig = { + # Run as "simple" in forground mode. + # This is more reliable + ExecStart = '' + ${pkgs.orangefs}/bin/pvfs2-server -d \ + /etc/orangefs/server.conf + ''; + TimeoutStopSec = "120"; + User = "orangefs"; + Group = "orangefs"; + }; + }; + }; + +} diff --git a/nixos/modules/services/network-filesystems/samba.nix b/nixos/modules/services/network-filesystems/samba.nix index ce565dbaab8..83995d28179 100644 --- a/nixos/modules/services/network-filesystems/samba.nix +++ b/nixos/modules/services/network-filesystems/samba.nix @@ -12,11 +12,6 @@ let samba = cfg.package; - setupScript = - '' - mkdir -p /var/lock/samba /var/log/samba /var/cache/samba /var/lib/samba/private - ''; - shareConfig = name: let share = getAttr name cfg.shares; in "[${name}]\n " + (smbToString ( @@ -62,6 +57,7 @@ let Type = "notify"; NotifyAccess = "all"; #may not do anything... }; + unitConfig.RequiresMountsFor = "/var/lib/samba"; restartTriggers = [ configFile ]; }; @@ -228,8 +224,7 @@ in systemd = { targets.samba = { description = "Samba Server"; - requires = [ "samba-setup.service" ]; - after = [ "samba-setup.service" "network.target" ]; + after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; }; # Refer to https://github.com/samba-team/samba/tree/master/packaging/systemd @@ -238,12 +233,13 @@ in samba-smbd = daemonService "smbd" ""; samba-nmbd = mkIf cfg.enableNmbd (daemonService "nmbd" ""); samba-winbindd = mkIf cfg.enableWinbindd (daemonService "winbindd" ""); - samba-setup = { - description = "Samba Setup Task"; - script = setupScript; - unitConfig.RequiresMountsFor = "/var/lib/samba"; - }; }; + tmpfiles.rules = [ + "d /var/lock/samba - - - - -" + "d /var/log/samba - - - - -" + "d /var/cache/samba - - - - -" + "d /var/lib/samba/private - - - - -" + ]; }; security.pam.services.samba = {}; diff --git a/nixos/modules/services/networking/bitcoind.nix b/nixos/modules/services/networking/bitcoind.nix index 1439d739da9..90f1291c019 100644 --- a/nixos/modules/services/networking/bitcoind.nix +++ b/nixos/modules/services/networking/bitcoind.nix @@ -187,6 +187,7 @@ in { group = cfg.group; description = "Bitcoin daemon user"; home = cfg.dataDir; + isSystemUser = true; }; users.groups.${cfg.group} = { name = cfg.group; diff --git a/nixos/modules/services/networking/dnscache.nix b/nixos/modules/services/networking/dnscache.nix index 5051fc916d9..d123bca9321 100644 --- a/nixos/modules/services/networking/dnscache.nix +++ b/nixos/modules/services/networking/dnscache.nix @@ -84,7 +84,7 @@ in { config = mkIf config.services.dnscache.enable { environment.systemPackages = [ pkgs.djbdns ]; - users.users.dnscache = {}; + users.users.dnscache.isSystemUser = true; systemd.services.dnscache = { description = "djbdns dnscache server"; diff --git a/nixos/modules/services/networking/dnscrypt-wrapper.nix b/nixos/modules/services/networking/dnscrypt-wrapper.nix index bf13d5c6f5f..79f9e1a4308 100644 --- a/nixos/modules/services/networking/dnscrypt-wrapper.nix +++ b/nixos/modules/services/networking/dnscrypt-wrapper.nix @@ -142,6 +142,7 @@ in { description = "dnscrypt-wrapper daemon user"; home = "${dataDir}"; createHome = true; + isSystemUser = true; }; users.groups.dnscrypt-wrapper = { }; diff --git a/nixos/modules/services/networking/dnsdist.nix b/nixos/modules/services/networking/dnsdist.nix index 12eee136e63..8249da69bc1 100644 --- a/nixos/modules/services/networking/dnsdist.nix +++ b/nixos/modules/services/networking/dnsdist.nix @@ -46,11 +46,10 @@ in { RestartSec="1"; DynamicUser = true; StartLimitInterval="0"; - PrivateTmp=true; PrivateDevices=true; - CapabilityBoundingSet="CAP_NET_BIND_SERVICE CAP_SETGID CAP_SETUID"; + AmbientCapabilities="CAP_NET_BIND_SERVICE"; + CapabilityBoundingSet="CAP_NET_BIND_SERVICE"; ExecStart = "${pkgs.dnsdist}/bin/dnsdist --supervised --disable-syslog --config ${configFile}"; - ProtectSystem="full"; ProtectHome=true; RestrictAddressFamilies="AF_UNIX AF_INET AF_INET6"; LimitNOFILE="16384"; diff --git a/nixos/modules/services/networking/go-shadowsocks2.nix b/nixos/modules/services/networking/go-shadowsocks2.nix new file mode 100644 index 00000000000..afbd7ea27c6 --- /dev/null +++ b/nixos/modules/services/networking/go-shadowsocks2.nix @@ -0,0 +1,30 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.go-shadowsocks2.server; +in { + options.services.go-shadowsocks2.server = { + enable = mkEnableOption "go-shadowsocks2 server"; + + listenAddress = mkOption { + type = types.str; + description = "Server listen address or URL"; + example = "ss://AEAD_CHACHA20_POLY1305:your-password@:8488"; + }; + }; + + config = mkIf cfg.enable { + systemd.services.go-shadowsocks2-server = { + description = "go-shadowsocks2 server"; + + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + ExecStart = "${pkgs.go-shadowsocks2}/bin/go-shadowsocks2 -s '${cfg.listenAddress}'"; + DynamicUser = true; + }; + }; + }; +} diff --git a/nixos/modules/services/networking/hans.nix b/nixos/modules/services/networking/hans.nix index 20e57e4626e..4f60300f5ff 100644 --- a/nixos/modules/services/networking/hans.nix +++ b/nixos/modules/services/networking/hans.nix @@ -138,6 +138,7 @@ in users.users = singleton { name = hansUser; description = "Hans daemon user"; + isSystemUser = true; }; }; diff --git a/nixos/modules/services/networking/haproxy.nix b/nixos/modules/services/networking/haproxy.nix index 0438d0bf8d8..aff71e5e97d 100644 --- a/nixos/modules/services/networking/haproxy.nix +++ b/nixos/modules/services/networking/haproxy.nix @@ -1,7 +1,16 @@ { config, lib, pkgs, ... }: + let cfg = config.services.haproxy; - haproxyCfg = pkgs.writeText "haproxy.conf" cfg.config; + + haproxyCfg = pkgs.writeText "haproxy.conf" '' + global + # needed for hot-reload to work without dropping packets in multi-worker mode + stats socket /run/haproxy/haproxy.sock mode 600 expose-fd listeners level user + + ${cfg.config} + ''; + in with lib; { @@ -25,9 +34,7 @@ with lib; <filename>haproxy.conf</filename>. ''; }; - }; - }; config = mkIf cfg.enable { @@ -42,21 +49,16 @@ with lib; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { - Type = "forking"; - PIDFile = "/run/haproxy.pid"; - ExecStartPre = "${pkgs.haproxy}/sbin/haproxy -c -q -f ${haproxyCfg}"; - ExecStart = "${pkgs.haproxy}/sbin/haproxy -D -f ${haproxyCfg} -p /run/haproxy.pid"; - ExecReload = "-${pkgs.bash}/bin/bash -c \"exec ${pkgs.haproxy}/sbin/haproxy -D -f ${haproxyCfg} -p /run/haproxy.pid -sf $MAINPID\""; + DynamicUser = true; + Type = "notify"; + # when running the config test, don't be quiet so we can see what goes wrong + ExecStartPre = "${pkgs.haproxy}/sbin/haproxy -c -f ${haproxyCfg}"; + ExecStart = "${pkgs.haproxy}/sbin/haproxy -Ws -f ${haproxyCfg}"; + Restart = "on-failure"; + RuntimeDirectory = "haproxy"; + # needed in case we bind to port < 1024 + AmbientCapabilities = "CAP_NET_BIND_SERVICE"; }; }; - - environment.systemPackages = [ pkgs.haproxy ]; - - users.users.haproxy = { - group = "haproxy"; - uid = config.ids.uids.haproxy; - }; - - users.groups.haproxy.gid = config.ids.uids.haproxy; }; } diff --git a/nixos/modules/services/networking/jormungandr.nix b/nixos/modules/services/networking/jormungandr.nix deleted file mode 100644 index 152cceb4bf9..00000000000 --- a/nixos/modules/services/networking/jormungandr.nix +++ /dev/null @@ -1,102 +0,0 @@ -{ config, lib, pkgs, ... }: - -let - cfg = config.services.jormungandr; - - inherit (lib) mkEnableOption mkIf mkOption; - inherit (lib) optionalString types; - - dataDir = "/var/lib/jormungandr"; - - # Default settings so far, as the service matures we will - # move these out as separate settings - configSettings = { - storage = dataDir; - p2p = { - public_address = "/ip4/127.0.0.1/tcp/8299"; - topics_of_interest = { - messages = "high"; - blocks = "high"; - }; - }; - rest = { - listen = "127.0.0.1:8607"; - }; - }; - - configFile = if cfg.configFile == null then - pkgs.writeText "jormungandr.yaml" (builtins.toJSON configSettings) - else cfg.configFile; - -in { - - options = { - - services.jormungandr = { - enable = mkEnableOption "jormungandr service"; - - configFile = mkOption { - type = types.nullOr types.path; - default = null; - example = "/var/lib/jormungandr/node.yaml"; - description = '' - The path of the jormungandr blockchain configuration file in YAML format. - If no file is specified, a file is generated using the other options. - ''; - }; - - secretFile = mkOption { - type = types.nullOr types.path; - default = null; - example = "/etc/secret/jormungandr.yaml"; - description = '' - The path of the jormungandr blockchain secret node configuration file in - YAML format. Do not store this in nix store! - ''; - }; - - genesisBlockHash = mkOption { - type = types.nullOr types.str; - default = null; - example = "d70495af81ae8600aca3e642b2427327cb6001ec4d7a0037e96a00dabed163f9"; - description = '' - Set the genesis block hash (the hash of the block0) so we can retrieve - the genesis block (and the blockchain configuration) from the existing - storage or from the network. - ''; - }; - - genesisBlockFile = mkOption { - type = types.nullOr types.path; - default = null; - example = "/var/lib/jormungandr/block-0.bin"; - description = '' - The path of the genesis block file if we are hosting it locally. - ''; - }; - - }; - }; - - config = mkIf cfg.enable { - - systemd.services.jormungandr = { - description = "jormungandr server"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" ]; - environment = { - RUST_BACKTRACE = "full"; - }; - serviceConfig = { - DynamicUser = true; - StateDirectory = baseNameOf dataDir; - ExecStart = '' - ${pkgs.jormungandr}/bin/jormungandr --config ${configFile} \ - ${optionalString (cfg.secretFile != null) " --secret ${cfg.secretFile}"} \ - ${optionalString (cfg.genesisBlockHash != null) " --genesis-block-hash ${cfg.genesisBlockHash}"} \ - ${optionalString (cfg.genesisBlockFile != null) " --genesis-block ${cfg.genesisBlockFile}"} - ''; - }; - }; - }; -} diff --git a/nixos/modules/services/networking/matterbridge.nix b/nixos/modules/services/networking/matterbridge.nix index 1fd63348c16..682eaa6eb29 100644 --- a/nixos/modules/services/networking/matterbridge.nix +++ b/nixos/modules/services/networking/matterbridge.nix @@ -95,6 +95,7 @@ in users.users = optional (cfg.user == "matterbridge") { name = "matterbridge"; group = "matterbridge"; + isSystemUser = true; }; users.groups = optional (cfg.group == "matterbridge") diff --git a/nixos/modules/services/networking/morty.nix b/nixos/modules/services/networking/morty.nix index 1b3084fe9ab..e3a6444c116 100644 --- a/nixos/modules/services/networking/morty.nix +++ b/nixos/modules/services/networking/morty.nix @@ -74,6 +74,7 @@ in { description = "Morty user"; createHome = true; home = "/var/lib/morty"; + isSystemUser = true; }; systemd.services.morty = diff --git a/nixos/modules/services/networking/nat.nix b/nixos/modules/services/networking/nat.nix index 89d8590093d..5681bda51cb 100644 --- a/nixos/modules/services/networking/nat.nix +++ b/nixos/modules/services/networking/nat.nix @@ -29,7 +29,7 @@ let iptables -w -t nat -N nixos-nat-post # We can't match on incoming interface in POSTROUTING, so - # mark packets coming from the external interfaces. + # mark packets coming from the internal interfaces. ${concatMapStrings (iface: '' iptables -w -t nat -A nixos-nat-pre \ -i '${iface}' -j MARK --set-mark 1 diff --git a/nixos/modules/services/networking/networkmanager.nix b/nixos/modules/services/networking/networkmanager.nix index 918bf891b10..90d1032c41b 100644 --- a/nixos/modules/services/networking/networkmanager.nix +++ b/nixos/modules/services/networking/networkmanager.nix @@ -456,15 +456,19 @@ in { }; # Turn off NixOS' network management when networking is managed entirely by NetworkManager - networking = (mkIf (!delegateWireless) { - useDHCP = false; - # Use mkDefault to trigger the assertion about the conflict above - wireless.enable = mkDefault false; - }) // (mkIf cfg.enableStrongSwan { - networkmanager.packages = [ pkgs.networkmanager_strongswan ]; - }) // (mkIf enableIwd { - wireless.iwd.enable = true; - }); + networking = mkMerge [ + (mkIf (!delegateWireless) { + useDHCP = false; + }) + + (mkIf cfg.enableStrongSwan { + networkmanager.packages = [ pkgs.networkmanager_strongswan ]; + }) + + (mkIf enableIwd { + wireless.iwd.enable = true; + }) + ]; security.polkit.extraConfig = polkitConf; diff --git a/nixos/modules/services/networking/nghttpx/default.nix b/nixos/modules/services/networking/nghttpx/default.nix index d6e1906e388..881a2670f5d 100644 --- a/nixos/modules/services/networking/nghttpx/default.nix +++ b/nixos/modules/services/networking/nghttpx/default.nix @@ -96,6 +96,7 @@ in users.groups.nghttpx = { }; users.users.nghttpx = { group = config.users.groups.nghttpx.name; + isSystemUser = true; }; diff --git a/nixos/modules/services/networking/owamp.nix b/nixos/modules/services/networking/owamp.nix index 821a0258f4b..dbb2e3b4c40 100644 --- a/nixos/modules/services/networking/owamp.nix +++ b/nixos/modules/services/networking/owamp.nix @@ -21,6 +21,7 @@ in name = "owamp"; group = "owamp"; description = "Owamp daemon"; + isSystemUser = true; }; users.groups = singleton { diff --git a/nixos/modules/services/networking/thelounge.nix b/nixos/modules/services/networking/thelounge.nix index b1d23372955..875d8f66169 100644 --- a/nixos/modules/services/networking/thelounge.nix +++ b/nixos/modules/services/networking/thelounge.nix @@ -56,6 +56,7 @@ in { users.users.thelounge = { description = "thelounge service user"; group = "thelounge"; + isSystemUser = true; }; users.groups.thelounge = {}; systemd.services.thelounge = { diff --git a/nixos/modules/services/networking/tinydns.nix b/nixos/modules/services/networking/tinydns.nix index 7d5db71601e..79507b2ebcd 100644 --- a/nixos/modules/services/networking/tinydns.nix +++ b/nixos/modules/services/networking/tinydns.nix @@ -32,11 +32,12 @@ with lib; config = mkIf config.services.tinydns.enable { environment.systemPackages = [ pkgs.djbdns ]; - users.users.tinydns = {}; + users.users.tinydns.isSystemUser = true; systemd.services.tinydns = { description = "djbdns tinydns server"; wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; path = with pkgs; [ daemontools djbdns ]; preStart = '' rm -rf /var/lib/tinydns diff --git a/nixos/modules/services/networking/trickster.nix b/nixos/modules/services/networking/trickster.nix new file mode 100644 index 00000000000..8760dd5a938 --- /dev/null +++ b/nixos/modules/services/networking/trickster.nix @@ -0,0 +1,112 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.trickster; +in +{ + + options = { + services.trickster = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Enable Trickster. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.trickster; + defaultText = "pkgs.trickster"; + description = '' + Package that should be used for trickster. + ''; + }; + + configFile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Path to configuration file. + ''; + }; + + instance-id = mkOption { + type = types.nullOr types.int; + default = null; + description = '' + Instance ID for when running multiple processes (default null). + ''; + }; + + log-level = mkOption { + type = types.str; + default = "info"; + description = '' + Level of Logging to use (debug, info, warn, error) (default "info"). + ''; + }; + + metrics-port = mkOption { + type = types.port; + default = 8082; + description = '' + Port that the /metrics endpoint will listen on. + ''; + }; + + origin = mkOption { + type = types.str; + default = "http://prometheus:9090"; + description = '' + URL to the Prometheus Origin. Enter it like you would in grafana, e.g., http://prometheus:9090 (default http://prometheus:9090). + ''; + }; + + profiler-port = mkOption { + type = types.nullOr types.port; + default = null; + description = '' + Port that the /debug/pprof endpoint will listen on. + ''; + }; + + proxy-port = mkOption { + type = types.port; + default = 9090; + description = '' + Port that the Proxy server will listen on. + ''; + }; + + }; + }; + + config = mkIf cfg.enable { + systemd.services.trickster = { + description = "Dashboard Accelerator for Prometheus"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + DynamicUser = true; + ExecStart = '' + ${cfg.package}/bin/trickster \ + -log-level ${cfg.log-level} \ + -metrics-port ${toString cfg.metrics-port} \ + -origin ${cfg.origin} \ + -proxy-port ${toString cfg.proxy-port} \ + ${optionalString (cfg.configFile != null) "-config ${cfg.configFile}"} \ + ${optionalString (cfg.profiler-port != null) "-profiler-port ${cfg.profiler-port}"} \ + ${optionalString (cfg.instance-id != null) "-instance-id ${cfg.instance-id}"} + ''; + ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + Restart = "always"; + }; + }; + + }; +} + diff --git a/nixos/modules/services/printing/cupsd.nix b/nixos/modules/services/printing/cupsd.nix index 3fcae611dc7..1071c05d514 100644 --- a/nixos/modules/services/printing/cupsd.nix +++ b/nixos/modules/services/printing/cupsd.nix @@ -31,7 +31,7 @@ let # part of CUPS itself, e.g. the SMB backend is part of Samba. Since # we can't update ${cups.out}/lib/cups itself, we create a symlink tree # here and add the additional programs. The ServerBin directive in - # cupsd.conf tells cupsd to use this tree. + # cups-files.conf tells cupsd to use this tree. bindir = pkgs.buildEnv { name = "cups-progs"; paths = diff --git a/nixos/modules/services/scheduling/marathon.nix b/nixos/modules/services/scheduling/marathon.nix index 0961a67770e..2e0d20c64b2 100644 --- a/nixos/modules/services/scheduling/marathon.nix +++ b/nixos/modules/services/scheduling/marathon.nix @@ -93,6 +93,6 @@ in { }; }; - users.users.${cfg.user} = { }; + users.users.${cfg.user}.isSystemUser = true; }; } diff --git a/nixos/modules/services/security/bitwarden_rs/default.nix b/nixos/modules/services/security/bitwarden_rs/default.nix index 80fd65891ff..d1817db0755 100644 --- a/nixos/modules/services/security/bitwarden_rs/default.nix +++ b/nixos/modules/services/security/bitwarden_rs/default.nix @@ -74,7 +74,10 @@ in { webVaultEnabled = mkDefault true; }; - users.users.bitwarden_rs = { inherit group; }; + users.users.bitwarden_rs = { + inherit group; + isSystemUser = true; + }; users.groups.bitwarden_rs = { }; systemd.services.bitwarden_rs = { diff --git a/nixos/modules/services/security/oauth2_proxy.nix b/nixos/modules/services/security/oauth2_proxy.nix index bb03f7fc9e4..2abb9ec32ac 100644 --- a/nixos/modules/services/security/oauth2_proxy.nix +++ b/nixos/modules/services/security/oauth2_proxy.nix @@ -546,6 +546,7 @@ in users.users.oauth2_proxy = { description = "OAuth2 Proxy"; + isSystemUser = true; }; systemd.services.oauth2_proxy = { diff --git a/nixos/modules/services/security/vault.nix b/nixos/modules/services/security/vault.nix index d5962ba9af9..b0ab8fadcbe 100644 --- a/nixos/modules/services/security/vault.nix +++ b/nixos/modules/services/security/vault.nix @@ -119,9 +119,8 @@ in }; users.groups.vault.gid = config.ids.gids.vault; - systemd.tmpfiles.rules = optional (cfg.storagePath != null) [ - "d '${cfg.storagePath}' 0700 vault vault - -" - ]; + systemd.tmpfiles.rules = optional (cfg.storagePath != null) + "d '${cfg.storagePath}' 0700 vault vault - -"; systemd.services.vault = { description = "Vault server daemon"; diff --git a/nixos/modules/services/torrent/magnetico.nix b/nixos/modules/services/torrent/magnetico.nix index 02fa2ac0750..a7acdb78b31 100644 --- a/nixos/modules/services/torrent/magnetico.nix +++ b/nixos/modules/services/torrent/magnetico.nix @@ -171,6 +171,7 @@ in { users.users.magnetico = { description = "Magnetico daemons user"; + isSystemUser = true; }; systemd.services.magneticod = { diff --git a/nixos/modules/services/web-apps/codimd.nix b/nixos/modules/services/web-apps/codimd.nix index 7ae7cd9c52d..5f56f8ed5a0 100644 --- a/nixos/modules/services/web-apps/codimd.nix +++ b/nixos/modules/services/web-apps/codimd.nix @@ -893,6 +893,7 @@ in extraGroups = cfg.groups; home = cfg.workDir; createHome = true; + isSystemUser = true; }; systemd.services.codimd = { diff --git a/nixos/modules/services/web-apps/frab.nix b/nixos/modules/services/web-apps/frab.nix index 7914e5cc0ee..a9a30b40922 100644 --- a/nixos/modules/services/web-apps/frab.nix +++ b/nixos/modules/services/web-apps/frab.nix @@ -177,6 +177,7 @@ in { name = cfg.user; group = cfg.group; home = "${cfg.statePath}"; + isSystemUser = true; } ]; diff --git a/nixos/modules/services/web-apps/limesurvey.nix b/nixos/modules/services/web-apps/limesurvey.nix index 68b57a9b90d..bd524524130 100644 --- a/nixos/modules/services/web-apps/limesurvey.nix +++ b/nixos/modules/services/web-apps/limesurvey.nix @@ -277,7 +277,10 @@ in systemd.services.httpd.after = optional mysqlLocal "mysql.service" ++ optional pgsqlLocal "postgresql.service"; - users.users.${user}.group = group; + users.users.${user} = { + group = group; + isSystemUser = true; + }; }; } diff --git a/nixos/modules/services/web-apps/matomo-doc.xml b/nixos/modules/services/web-apps/matomo-doc.xml index 8485492c51c..79cece551d3 100644 --- a/nixos/modules/services/web-apps/matomo-doc.xml +++ b/nixos/modules/services/web-apps/matomo-doc.xml @@ -105,7 +105,7 @@ GRANT ALL PRIVILEGES ON matomo.* TO 'matomo'@'localhost'; <para> You can use other web servers by forwarding calls for <filename>index.php</filename> and <filename>piwik.php</filename> to the - <literal>/run/phpfpm-matomo.sock</literal> fastcgi unix socket. You can use + <literal><link linkend="opt-services.phpfpm.pools._name_.socket">services.phpfpm.pools.<name>.socket</link></literal> fastcgi unix socket. You can use the nginx configuration in the module code as a reference to what else should be configured. </para> diff --git a/nixos/modules/services/web-apps/matomo.nix b/nixos/modules/services/web-apps/matomo.nix index 1e34aff8d17..352cc4c647b 100644 --- a/nixos/modules/services/web-apps/matomo.nix +++ b/nixos/modules/services/web-apps/matomo.nix @@ -2,15 +2,13 @@ with lib; let cfg = config.services.matomo; + fpm = config.services.phpfpm.pools.${pool}; user = "matomo"; dataDir = "/var/lib/${user}"; deprecatedDataDir = "/var/lib/piwik"; pool = user; - # it's not possible to use /run/phpfpm/${pool}.sock because /run/phpfpm/ is root:root 0770, - # and therefore is not accessible by the web server. - phpSocket = "/run/phpfpm-${pool}.sock"; phpExecutionUnit = "phpfpm-${pool}"; databaseService = "mysql.service"; @@ -50,7 +48,7 @@ in { default = null; example = "lighttpd"; description = '' - Name of the web server user that forwards requests to the ${phpSocket} fastcgi socket for Matomo if the nginx + Name of the web server user that forwards requests to <option>services.phpfpm.pools.<name>.socket</option> the fastcgi socket for Matomo if the nginx option is not used. Either this option or the nginx option is mandatory. If you want to use another webserver than nginx, you need to set this to that server's user and pass fastcgi requests to `index.php`, `matomo.php` and `piwik.php` (legacy name) to this socket. @@ -71,25 +69,6 @@ in { ''; }; - phpfpmProcessManagerConfig = mkOption { - type = types.str; - default = '' - ; default phpfpm process manager settings - pm = dynamic - pm.max_children = 75 - pm.start_servers = 10 - pm.min_spare_servers = 5 - pm.max_spare_servers = 20 - pm.max_requests = 500 - - ; log worker's stdout, but this has a performance hit - catch_workers_output = yes - ''; - description = '' - Settings for phpfpm's process manager. You might need to change this depending on the load for Matomo. - ''; - }; - nginx = mkOption { type = types.nullOr (types.submodule ( recursiveUpdate @@ -233,15 +212,24 @@ in { else if (cfg.webServerUser != null) then cfg.webServerUser else ""; in { ${pool} = { - listen = phpSocket; - extraConfig = '' - listen.owner = ${socketOwner} - listen.group = root - listen.mode = 0600 - user = ${user} - env[PIWIK_USER_PATH] = ${dataDir} - ${cfg.phpfpmProcessManagerConfig} + inherit user; + phpOptions = '' + error_log = 'stderr' + log_errors = on ''; + settings = mapAttrs (name: mkDefault) { + "listen.owner" = socketOwner; + "listen.group" = "root"; + "listen.mode" = "0660"; + "pm" = "dynamic"; + "pm.max_children" = 75; + "pm.start_servers" = 10; + "pm.min_spare_servers" = 5; + "pm.max_spare_servers" = 20; + "pm.max_requests" = 500; + "catch_workers_output" = true; + }; + phpEnv.PIWIK_USER_PATH = dataDir; }; }; @@ -264,15 +252,15 @@ in { }; # allow index.php for webinterface locations."= /index.php".extraConfig = '' - fastcgi_pass unix:${phpSocket}; + fastcgi_pass unix:${fpm.socket}; ''; # allow matomo.php for tracking locations."= /matomo.php".extraConfig = '' - fastcgi_pass unix:${phpSocket}; + fastcgi_pass unix:${fpm.socket}; ''; # allow piwik.php for tracking (deprecated name) locations."= /piwik.php".extraConfig = '' - fastcgi_pass unix:${phpSocket}; + fastcgi_pass unix:${fpm.socket}; ''; # Any other attempt to access any php files is forbidden locations."~* ^.+\\.php$".extraConfig = '' diff --git a/nixos/modules/services/web-apps/mediawiki.nix b/nixos/modules/services/web-apps/mediawiki.nix index ec2568bf952..43edc04e1a4 100644 --- a/nixos/modules/services/web-apps/mediawiki.nix +++ b/nixos/modules/services/web-apps/mediawiki.nix @@ -461,7 +461,10 @@ in systemd.services.httpd.after = optional (cfg.database.createLocally && cfg.database.type == "mysql") "mysql.service"; - users.users.${user}.group = group; + users.users.${user} = { + group = group; + isSystemUser = true; + }; environment.systemPackages = [ mediawikiScripts ]; }; diff --git a/nixos/modules/services/web-apps/moinmoin.nix b/nixos/modules/services/web-apps/moinmoin.nix new file mode 100644 index 00000000000..0fee64be0bb --- /dev/null +++ b/nixos/modules/services/web-apps/moinmoin.nix @@ -0,0 +1,303 @@ +{ config, lib, pkgs, ... }: +with lib; + +let + cfg = config.services.moinmoin; + python = pkgs.python27; + pkg = python.pkgs.moinmoin; + dataDir = "/var/lib/moin"; + usingGunicorn = cfg.webServer == "nginx-gunicorn" || cfg.webServer == "gunicorn"; + usingNginx = cfg.webServer == "nginx-gunicorn"; + user = "moin"; + group = "moin"; + + uLit = s: ''u"${s}"''; + indentLines = n: str: concatMapStrings (line: "${fixedWidthString n " " " "}${line}\n") (splitString "\n" str); + + moinCliWrapper = wikiIdent: pkgs.writeShellScriptBin "moin-${wikiIdent}" '' + ${pkgs.su}/bin/su -s ${pkgs.runtimeShell} -c "${pkg}/bin/moin --config-dir=/var/lib/moin/${wikiIdent}/config $*" ${user} + ''; + + wikiConfig = wikiIdent: w: '' + # -*- coding: utf-8 -*- + + from MoinMoin.config import multiconfig, url_prefix_static + + class Config(multiconfig.DefaultConfig): + ${optionalString (w.webLocation != "/") '' + url_prefix_static = '${w.webLocation}' + url_prefix_static + ''} + + sitename = u'${w.siteName}' + page_front_page = u'${w.frontPage}' + + data_dir = '${dataDir}/${wikiIdent}/data' + data_underlay_dir = '${dataDir}/${wikiIdent}/underlay' + + language_default = u'${w.languageDefault}' + ${optionalString (w.superUsers != []) '' + superuser = [${concatMapStringsSep ", " uLit w.superUsers}] + ''} + + ${indentLines 4 w.extraConfig} + ''; + wikiConfigFile = name: wiki: pkgs.writeText "${name}.py" (wikiConfig name wiki); + +in +{ + options.services.moinmoin = with types; { + enable = mkEnableOption "MoinMoin Wiki Engine"; + + webServer = mkOption { + type = enum [ "nginx-gunicorn" "gunicorn" "none" ]; + default = "nginx-gunicorn"; + example = "none"; + description = '' + Which web server to use to serve the wiki. + Use <literal>none</literal> if you want to configure this yourself. + ''; + }; + + gunicorn.workers = mkOption { + type = ints.positive; + default = 3; + example = 10; + description = '' + The number of worker processes for handling requests. + ''; + }; + + wikis = mkOption { + type = attrsOf (submodule ({ name, ... }: { + options = { + siteName = mkOption { + type = str; + default = "Untitled Wiki"; + example = "ExampleWiki"; + description = '' + Short description of your wiki site, displayed below the logo on each page, and + used in RSS documents as the channel title. + ''; + }; + + webHost = mkOption { + type = str; + description = "Host part of the wiki URL. If undefined, the name of the attribute set will be used."; + example = "wiki.example.org"; + }; + + webLocation = mkOption { + type = str; + default = "/"; + example = "/moin"; + description = "Location part of the wiki URL."; + }; + + frontPage = mkOption { + type = str; + default = "LanguageSetup"; + example = "FrontPage"; + description = '' + Front page name. Set this to something like <literal>FrontPage</literal> once languages are + configured. + ''; + }; + + superUsers = mkOption { + type = listOf str; + default = []; + example = [ "elvis" ]; + description = '' + List of trusted user names with wiki system administration super powers. + + Please note that accounts for these users need to be created using the <command>moin</command> command-line utility, e.g.: + <command>moin-<replaceable>WIKINAME</replaceable> account create --name=<replaceable>NAME</replaceable> --email=<replaceable>EMAIL</replaceable> --password=<replaceable>PASSWORD</replaceable></command>. + ''; + }; + + languageDefault = mkOption { + type = str; + default = "en"; + example = "de"; + description = "The ISO-639-1 name of the main wiki language. Languages that MoinMoin does not support are ignored."; + }; + + extraConfig = mkOption { + type = lines; + default = ""; + example = '' + show_hosts = True + search_results_per_page = 100 + acl_rights_default = u"Known:read,write,delete,revert All:read" + logo_string = u"<h2>\U0001f639</h2>" + theme_default = u"modernized" + + user_checkbox_defaults = {'show_page_trail': 0, 'edit_on_doubleclick': 0} + navi_bar = [u'SomePage'] + multiconfig.DefaultConfig.navi_bar + actions_excluded = multiconfig.DefaultConfig.actions_excluded + ['newaccount'] + + mail_smarthost = "mail.example.org" + mail_from = u"Example.Org Wiki <wiki@example.org>" + ''; + description = '' + Additional configuration to be appended verbatim to this wiki's config. + + See <link xlink:href='http://moinmo.in/HelpOnConfiguration' /> for documentation. + ''; + }; + + }; + config = { + webHost = mkDefault name; + }; + })); + example = literalExample '' + { + "mywiki" = { + siteName = "Example Wiki"; + webHost = "wiki.example.org"; + superUsers = [ "admin" ]; + frontPage = "Index"; + extraConfig = "page_category_regex = ur'(?P<all>(Category|Kategorie)(?P<key>(?!Template)\S+))'" + }; + } + ''; + description = '' + Configurations of the individual wikis. Attribute names must be valid Python + identifiers of the form <literal>[A-Za-z_][A-Za-z0-9_]*</literal>. + + For every attribute <replaceable>WIKINAME</replaceable>, a helper script + moin-<replaceable>WIKINAME</replaceable> is created which runs the + <command>moin</command> command under the <literal>moin</literal> user (to avoid + file ownership issues) and with the right configuration directory passed to it. + ''; + }; + }; + + config = mkIf cfg.enable { + assertions = forEach (attrNames cfg.wikis) (wname: + { assertion = builtins.match "[A-Za-z_][A-Za-z0-9_]*" wname != null; + message = "${wname} is not valid Python identifier"; + } + ); + + users.users = { + moin = { + description = "MoinMoin wiki"; + home = dataDir; + group = group; + isSystemUser = true; + }; + }; + + users.groups = { + moin = { + members = mkIf usingNginx [ config.services.nginx.user ]; + }; + }; + + environment.systemPackages = [ pkg ] ++ map moinCliWrapper (attrNames cfg.wikis); + + systemd.services = mkIf usingGunicorn + (flip mapAttrs' cfg.wikis (wikiIdent: wiki: + nameValuePair "moin-${wikiIdent}" + { + description = "MoinMoin wiki ${wikiIdent} - gunicorn process"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + restartIfChanged = true; + restartTriggers = [ (wikiConfigFile wikiIdent wiki) ]; + + environment = let + penv = python.buildEnv.override { + # setuptools: https://github.com/benoitc/gunicorn/issues/1716 + extraLibs = [ python.pkgs.gevent python.pkgs.setuptools pkg ]; + }; + in { + PYTHONPATH = "${dataDir}/${wikiIdent}/config:${penv}/${python.sitePackages}"; + }; + + preStart = '' + umask 0007 + rm -rf ${dataDir}/${wikiIdent}/underlay + cp -r ${pkg}/share/moin/underlay ${dataDir}/${wikiIdent}/ + chmod -R u+w ${dataDir}/${wikiIdent}/underlay + ''; + + serviceConfig = { + User = user; + Group = group; + WorkingDirectory = "${dataDir}/${wikiIdent}"; + ExecStart = ''${python.pkgs.gunicorn}/bin/gunicorn moin_wsgi \ + --name gunicorn-${wikiIdent} \ + --workers ${toString cfg.gunicorn.workers} \ + --worker-class gevent \ + --bind unix:/run/moin/${wikiIdent}/gunicorn.sock + ''; + + Restart = "on-failure"; + RestartSec = "2s"; + StartLimitIntervalSec = "30s"; + + StateDirectory = "moin/${wikiIdent}"; + StateDirectoryMode = "0750"; + RuntimeDirectory = "moin/${wikiIdent}"; + RuntimeDirectoryMode = "0750"; + + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + PrivateNetwork = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ]; + RestrictNamespaces = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + RestrictRealtime = true; + }; + } + )); + + services.nginx = mkIf usingNginx { + enable = true; + virtualHosts = flip mapAttrs' cfg.wikis (name: w: nameValuePair w.webHost { + forceSSL = mkDefault true; + enableACME = mkDefault true; + locations."${w.webLocation}" = { + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $host; + + proxy_pass http://unix:/run/moin/${name}/gunicorn.sock; + ''; + }; + }); + }; + + systemd.tmpfiles.rules = [ + "d /run/moin 0750 ${user} ${group} - -" + "d ${dataDir} 0550 ${user} ${group} - -" + ] + ++ (concatLists (flip mapAttrsToList cfg.wikis (wikiIdent: wiki: [ + "d ${dataDir}/${wikiIdent} 0750 ${user} ${group} - -" + "d ${dataDir}/${wikiIdent}/config 0550 ${user} ${group} - -" + "L+ ${dataDir}/${wikiIdent}/config/wikiconfig.py - - - - ${wikiConfigFile wikiIdent wiki}" + # needed in order to pass module name to gunicorn + "L+ ${dataDir}/${wikiIdent}/config/moin_wsgi.py - - - - ${pkg}/share/moin/server/moin.wsgi" + # seed data files + "C ${dataDir}/${wikiIdent}/data 0770 ${user} ${group} - ${pkg}/share/moin/data" + # fix nix store permissions + "Z ${dataDir}/${wikiIdent}/data 0770 ${user} ${group} - -" + ]))); + }; + + meta.maintainers = with lib.maintainers; [ b42 ]; +} diff --git a/nixos/modules/services/web-apps/moodle.nix b/nixos/modules/services/web-apps/moodle.nix index 211bc17ee19..ac59f9e0012 100644 --- a/nixos/modules/services/web-apps/moodle.nix +++ b/nixos/modules/services/web-apps/moodle.nix @@ -309,7 +309,9 @@ in systemd.services.httpd.after = optional mysqlLocal "mysql.service" ++ optional pgsqlLocal "postgresql.service"; - users.users.${user}.group = group; - + users.users.${user} = { + group = group; + isSystemUser = true; + }; }; } diff --git a/nixos/modules/services/web-apps/nexus.nix b/nixos/modules/services/web-apps/nexus.nix index 3af97e146d0..d4d507362c9 100644 --- a/nixos/modules/services/web-apps/nexus.nix +++ b/nixos/modules/services/web-apps/nexus.nix @@ -68,6 +68,7 @@ in -Dkaraf.data=${cfg.home}/nexus3 -Djava.io.tmpdir=${cfg.home}/nexus3/tmp -Dkaraf.startLocalConsole=false + -Djava.endorsed.dirs=${cfg.package}/lib/endorsed ''; description = '' diff --git a/nixos/modules/services/web-apps/trac.nix b/nixos/modules/services/web-apps/trac.nix new file mode 100644 index 00000000000..207fb857438 --- /dev/null +++ b/nixos/modules/services/web-apps/trac.nix @@ -0,0 +1,79 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.trac; + + inherit (lib) mkEnableOption mkIf mkOption types; + +in { + + options = { + + services.trac = { + enable = mkEnableOption "Trac service"; + + listen = { + ip = mkOption { + type = types.str; + default = "0.0.0.0"; + description = '' + IP address that Trac should listen on. + ''; + }; + + port = mkOption { + type = types.port; + default = 8000; + description = '' + Listen port for Trac. + ''; + }; + }; + + dataDir = mkOption { + default = "/var/lib/trac"; + type = types.path; + description = '' + The directory for storing the Trac data. + ''; + }; + + openFirewall = mkOption { + type = types.bool; + default = false; + description = '' + Open ports in the firewall for Trac. + ''; + }; + }; + + }; + + config = mkIf cfg.enable { + + systemd.services.trac = { + description = "Trac server"; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + DynamicUser = true; + StateDirectory = baseNameOf cfg.dataDir; + ExecStart = '' + ${pkgs.trac}/bin/tracd -s \ + -b ${toString cfg.listen.ip} \ + -p ${toString cfg.listen.port} \ + ${cfg.dataDir} + ''; + }; + preStart = '' + if [ ! -e ${cfg.dataDir}/VERSION ]; then + ${pkgs.trac}/bin/trac-admin ${cfg.dataDir} initenv Trac "sqlite:db/trac.db" + fi + ''; + }; + + networking.firewall = mkIf cfg.openFirewall { + allowedTCPPorts = [ cfg.listen.port ]; + }; + + }; +} diff --git a/nixos/modules/services/web-apps/virtlyst.nix b/nixos/modules/services/web-apps/virtlyst.nix index e5c0bff2168..37bdbb0e3b4 100644 --- a/nixos/modules/services/web-apps/virtlyst.nix +++ b/nixos/modules/services/web-apps/virtlyst.nix @@ -54,6 +54,7 @@ in home = stateDir; createHome = true; group = mkIf config.virtualisation.libvirtd.enable "libvirtd"; + isSystemUser = true; }; systemd.services.virtlyst = { diff --git a/nixos/modules/services/web-apps/wordpress.nix b/nixos/modules/services/web-apps/wordpress.nix index e311dd917dd..f1370c2854b 100644 --- a/nixos/modules/services/web-apps/wordpress.nix +++ b/nixos/modules/services/web-apps/wordpress.nix @@ -367,7 +367,10 @@ in }) ]; - users.users.${user}.group = group; + users.users.${user} = { + group = group; + isSystemUser = true; + }; }; } diff --git a/nixos/modules/services/web-servers/apache-httpd/default.nix b/nixos/modules/services/web-servers/apache-httpd/default.nix index 99304d0e48a..f5a6051b4b5 100644 --- a/nixos/modules/services/web-servers/apache-httpd/default.nix +++ b/nixos/modules/services/web-servers/apache-httpd/default.nix @@ -6,6 +6,8 @@ let mainCfg = config.services.httpd; + runtimeDir = "/run/httpd"; + httpd = mainCfg.package.out; httpdConf = mainCfg.configFile; @@ -27,41 +29,29 @@ let listenToString = l: "${l.ip}:${toString l.port}"; - extraModules = attrByPath ["extraModules"] [] mainCfg; - extraForeignModules = filter isAttrs extraModules; - extraApacheModules = filter isString extraModules; - allHosts = [mainCfg] ++ mainCfg.virtualHosts; enableSSL = any (vhost: vhost.enableSSL) allHosts; + enableUserDir = any (vhost: vhost.enableUserDir) allHosts; - # Names of modules from ${httpd}/modules that we want to load. - apacheModules = - [ # HTTP authentication mechanisms: basic and digest. - "auth_basic" "auth_digest" - - # Authentication: is the user who he claims to be? - "authn_file" "authn_dbm" "authn_anon" "authn_core" - - # Authorization: is the user allowed access? - "authz_user" "authz_groupfile" "authz_host" "authz_core" - - # Other modules. - "ext_filter" "include" "log_config" "env" "mime_magic" - "cern_meta" "expires" "headers" "usertrack" /* "unique_id" */ "setenvif" - "mime" "dav" "status" "autoindex" "asis" "info" "dav_fs" - "vhost_alias" "negotiation" "dir" "imagemap" "actions" "speling" - "userdir" "alias" "rewrite" "proxy" "proxy_http" - "unixd" "cache" "cache_disk" "slotmem_shm" "socache_shmcb" + # NOTE: generally speaking order of modules is very important + modules = + [ # required apache modules our httpd service cannot run without + "authn_core" "authz_core" + "log_config" + "mime" "autoindex" "negotiation" "dir" + "alias" "rewrite" + "unixd" "slotmem_shm" "socache_shmcb" "mpm_${mainCfg.multiProcessingModule}" - - # For compatibility with old configurations, the new module mod_access_compat is provided. - "access_compat" ] ++ (if mainCfg.multiProcessingModule == "prefork" then [ "cgi" ] else [ "cgid" ]) ++ optional enableSSL "ssl" - ++ extraApacheModules; + ++ optional enableUserDir "userdir" + ++ optional mainCfg.enableMellon { name = "auth_mellon"; path = "${pkgs.apacheHttpdPackages.mod_auth_mellon}/modules/mod_auth_mellon.so"; } + ++ optional mainCfg.enablePHP { name = "php${phpMajorVersion}"; path = "${php}/modules/libphp${phpMajorVersion}.so"; } + ++ optional mainCfg.enablePerl { name = "perl"; path = "${mod_perl}/modules/mod_perl.so"; } + ++ mainCfg.extraModules; allDenied = "Require all denied"; @@ -85,20 +75,22 @@ let browserHacks = '' - BrowserMatch "Mozilla/2" nokeepalive - BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0 - BrowserMatch "RealPlayer 4\.0" force-response-1.0 - BrowserMatch "Java/1\.0" force-response-1.0 - BrowserMatch "JDK/1\.0" force-response-1.0 - BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully - BrowserMatch "^WebDrive" redirect-carefully - BrowserMatch "^WebDAVFS/1.[012]" redirect-carefully - BrowserMatch "^gnome-vfs" redirect-carefully + <IfModule mod_setenvif.c> + BrowserMatch "Mozilla/2" nokeepalive + BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0 + BrowserMatch "RealPlayer 4\.0" force-response-1.0 + BrowserMatch "Java/1\.0" force-response-1.0 + BrowserMatch "JDK/1\.0" force-response-1.0 + BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully + BrowserMatch "^WebDrive" redirect-carefully + BrowserMatch "^WebDAVFS/1.[012]" redirect-carefully + BrowserMatch "^gnome-vfs" redirect-carefully + </IfModule> ''; sslConf = '' - SSLSessionCache shmcb:${mainCfg.stateDir}/ssl_scache(512000) + SSLSessionCache shmcb:${runtimeDir}/ssl_scache(512000) Mutex posixsem @@ -239,13 +231,13 @@ let ServerRoot ${httpd} - DefaultRuntimeDir ${mainCfg.stateDir}/runtime + DefaultRuntimeDir ${runtimeDir}/runtime - PidFile ${mainCfg.stateDir}/httpd.pid + PidFile ${runtimeDir}/httpd.pid ${optionalString (mainCfg.multiProcessingModule != "prefork") '' # mod_cgid requires this. - ScriptSock ${mainCfg.stateDir}/cgisock + ScriptSock ${runtimeDir}/cgisock ''} <IfModule prefork.c> @@ -264,13 +256,12 @@ let Group ${mainCfg.group} ${let - load = {name, path}: "LoadModule ${name}_module ${path}\n"; - allModules = map (name: {inherit name; path = "${httpd}/modules/mod_${name}.so";}) apacheModules - ++ optional mainCfg.enableMellon { name = "auth_mellon"; path = "${pkgs.apacheHttpdPackages.mod_auth_mellon}/modules/mod_auth_mellon.so"; } - ++ optional mainCfg.enablePHP { name = "php${phpMajorVersion}"; path = "${php}/modules/libphp${phpMajorVersion}.so"; } - ++ optional mainCfg.enablePerl { name = "perl"; path = "${mod_perl}/modules/mod_perl.so"; } - ++ extraForeignModules; - in concatMapStrings load (unique allModules) + mkModule = module: + if isString module then { name = module; path = "${httpd}/modules/mod_${module}.so"; } + else if isAttrs module then { inherit (module) name path; } + else throw "Expecting either a string or attribute set including a name and path."; + in + concatMapStringsSep "\n" (module: "LoadModule ${module.name}_module ${module.path}") (unique (map mkModule modules)) } AddHandler type-map var @@ -337,6 +328,7 @@ in imports = [ (mkRemovedOptionModule [ "services" "httpd" "extraSubservices" ] "Most existing subservices have been ported to the NixOS module system. Please update your configuration accordingly.") + (mkRemovedOptionModule [ "services" "httpd" "stateDir" ] "The httpd module now uses /run/httpd as a runtime directory.") ]; ###### interface @@ -384,7 +376,12 @@ in extraModules = mkOption { type = types.listOf types.unspecified; default = []; - example = literalExample ''[ "proxy_connect" { name = "php5"; path = "''${pkgs.php}/modules/libphp5.so"; } ]''; + example = literalExample '' + [ + "proxy_connect" + { name = "jk"; path = "''${pkgs.tomcat_connectors}/modules/mod_jk.so"; } + ] + ''; description = '' Additional Apache modules to be used. These can be specified as a string in the case of modules distributed @@ -431,16 +428,6 @@ in ''; }; - stateDir = mkOption { - type = types.path; - default = "/run/httpd"; - description = '' - Directory for Apache's transient runtime state (such as PID - files). It is created automatically. Note that the default, - <filename>/run/httpd</filename>, is deleted at boot time. - ''; - }; - virtualHosts = mkOption { type = types.listOf (types.submodule ( { options = import ./per-server-options.nix { @@ -595,6 +582,28 @@ in date.timezone = "${config.time.timeZone}" ''; + services.httpd.extraModules = mkBefore [ + # HTTP authentication mechanisms: basic and digest. + "auth_basic" "auth_digest" + + # Authentication: is the user who he claims to be? + "authn_file" "authn_dbm" "authn_anon" + + # Authorization: is the user allowed access? + "authz_user" "authz_groupfile" "authz_host" + + # Other modules. + "ext_filter" "include" "env" "mime_magic" + "cern_meta" "expires" "headers" "usertrack" "setenvif" + "dav" "status" "asis" "info" "dav_fs" + "vhost_alias" "imagemap" "actions" "speling" + "proxy" "proxy_http" + "cache" "cache_disk" + + # For compatibility with old configurations, the new module mod_access_compat is provided. + "access_compat" + ]; + systemd.services.httpd = { description = "Apache HTTPD"; @@ -611,12 +620,6 @@ in preStart = '' - mkdir -m 0750 -p ${mainCfg.stateDir} - [ $(id -u) != 0 ] || chown root.${mainCfg.group} ${mainCfg.stateDir} - - mkdir -m 0750 -p "${mainCfg.stateDir}/runtime" - [ $(id -u) != 0 ] || chown root.${mainCfg.group} "${mainCfg.stateDir}/runtime" - mkdir -m 0700 -p ${mainCfg.logDir} # Get rid of old semaphores. These tend to accumulate across @@ -630,10 +633,13 @@ in serviceConfig.ExecStart = "@${httpd}/bin/httpd httpd -f ${httpdConf}"; serviceConfig.ExecStop = "${httpd}/bin/httpd -f ${httpdConf} -k graceful-stop"; serviceConfig.ExecReload = "${httpd}/bin/httpd -f ${httpdConf} -k graceful"; + serviceConfig.Group = mainCfg.group; serviceConfig.Type = "forking"; - serviceConfig.PIDFile = "${mainCfg.stateDir}/httpd.pid"; + serviceConfig.PIDFile = "${runtimeDir}/httpd.pid"; serviceConfig.Restart = "always"; serviceConfig.RestartSec = "5s"; + serviceConfig.RuntimeDirectory = "httpd httpd/runtime"; + serviceConfig.RuntimeDirectoryMode = "0750"; }; }; diff --git a/nixos/modules/services/web-servers/hitch/default.nix b/nixos/modules/services/web-servers/hitch/default.nix index a6c4cbea122..1812f225b74 100644 --- a/nixos/modules/services/web-servers/hitch/default.nix +++ b/nixos/modules/services/web-servers/hitch/default.nix @@ -102,7 +102,10 @@ with lib; environment.systemPackages = [ pkgs.hitch ]; - users.users.hitch.group = "hitch"; + users.users.hitch = { + group = "hitch"; + isSystemUser = true; + }; users.groups.hitch = {}; }; } diff --git a/nixos/modules/services/web-servers/traefik.nix b/nixos/modules/services/web-servers/traefik.nix index 8de7df0d446..5b0fc467ea4 100644 --- a/nixos/modules/services/web-servers/traefik.nix +++ b/nixos/modules/services/web-servers/traefik.nix @@ -117,6 +117,7 @@ in { group = "traefik"; home = cfg.dataDir; createHome = true; + isSystemUser = true; }; users.groups.traefik = {}; diff --git a/nixos/modules/services/web-servers/unit/default.nix b/nixos/modules/services/web-servers/unit/default.nix index a4a9d370d64..32f6d475b34 100644 --- a/nixos/modules/services/web-servers/unit/default.nix +++ b/nixos/modules/services/web-servers/unit/default.nix @@ -116,6 +116,7 @@ in { users.users = optionalAttrs (cfg.user == "unit") (singleton { name = "unit"; group = cfg.group; + isSystemUser = true; }); users.groups = optionalAttrs (cfg.group == "unit") (singleton { diff --git a/nixos/modules/services/x11/desktop-managers/gnome3.nix b/nixos/modules/services/x11/desktop-managers/gnome3.nix index 5ad31e5b9d0..0aae25662c6 100644 --- a/nixos/modules/services/x11/desktop-managers/gnome3.nix +++ b/nixos/modules/services/x11/desktop-managers/gnome3.nix @@ -217,6 +217,12 @@ in services.xserver.updateDbusEnvironment = true; + # gnome has a custom alert theme but it still + # inherits from the freedesktop theme. + environment.systemPackages = with pkgs; [ + sound-theme-freedesktop + ]; + # Needed for themes and backgrounds environment.pathsToLink = [ "/share" # TODO: https://github.com/NixOS/nixpkgs/issues/47173 diff --git a/nixos/modules/services/x11/display-managers/sddm.nix b/nixos/modules/services/x11/display-managers/sddm.nix index 8847acb0c60..899dd8665a2 100644 --- a/nixos/modules/services/x11/display-managers/sddm.nix +++ b/nixos/modules/services/x11/display-managers/sddm.nix @@ -14,7 +14,7 @@ let xserverWrapper = pkgs.writeScript "xserver-wrapper" '' #!/bin/sh ${concatMapStrings (n: "export ${n}=\"${getAttr n xEnv}\"\n") (attrNames xEnv)} - exec systemd-cat ${dmcfg.xserverBin} ${toString dmcfg.xserverArgs} "$@" + exec systemd-cat -t xserver-wrapper ${dmcfg.xserverBin} ${toString dmcfg.xserverArgs} "$@" ''; Xsetup = pkgs.writeScript "Xsetup" '' diff --git a/nixos/modules/services/x11/hardware/digimend.nix b/nixos/modules/services/x11/hardware/digimend.nix new file mode 100644 index 00000000000..a9f5640905a --- /dev/null +++ b/nixos/modules/services/x11/hardware/digimend.nix @@ -0,0 +1,43 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.services.xserver.digimend; + + pkg = config.boot.kernelPackages.digimend; + +in + +{ + + options = { + + services.xserver.digimend = { + + enable = mkOption { + default = false; + description = '' + Whether to enable the digimend drivers for Huion/XP-Pen/etc. tablets. + ''; + }; + + }; + + }; + + + config = mkIf cfg.enable { + + # digimend drivers use xsetwacom and wacom X11 drivers + services.xserver.wacom.enable = true; + + boot.extraModulePackages = [ pkg ]; + + environment.etc."X11/xorg.conf.d/50-digimend.conf".source = + "${pkg}/usr/share/X11/xorg.conf.d/50-digimend.conf"; + + }; + +} diff --git a/nixos/modules/services/x11/hardware/libinput.nix b/nixos/modules/services/x11/hardware/libinput.nix index bd289976532..4a25232383d 100644 --- a/nixos/modules/services/x11/hardware/libinput.nix +++ b/nixos/modules/services/x11/hardware/libinput.nix @@ -122,7 +122,7 @@ in { description = '' Specify the scrolling method: <literal>twofinger</literal>, <literal>edge</literal>, - or <literal>none</literal> + <literal>button</literal>, or <literal>none</literal> ''; }; diff --git a/nixos/modules/services/x11/window-managers/xmonad.nix b/nixos/modules/services/x11/window-managers/xmonad.nix index 0e131412276..30c59b88f82 100644 --- a/nixos/modules/services/x11/window-managers/xmonad.nix +++ b/nixos/modules/services/x11/window-managers/xmonad.nix @@ -86,7 +86,7 @@ in ${xmonadBin} waitPID=$! '' else '' - ${xmonad}/bin/xmonad & + systemd-cat -t xmonad ${xmonad}/bin/xmonad & waitPID=$! ''; }]; diff --git a/nixos/modules/system/boot/networkd.nix b/nixos/modules/system/boot/networkd.nix index f2060e21509..85a106527fe 100644 --- a/nixos/modules/system/boot/networkd.nix +++ b/nixos/modules/system/boot/networkd.nix @@ -187,7 +187,7 @@ let # Note: For DHCP the values both, none, v4, v6 are deprecated (assertValueOneOf "DHCP" ["yes" "no" "ipv4" "ipv6" "both" "none" "v4" "v6"]) (assertValueOneOf "DHCPServer" boolValues) - (assertValueOneOf "LinkLocalAddressing" ["yes" "no" "ipv4" "ipv6"]) + (assertValueOneOf "LinkLocalAddressing" ["yes" "no" "ipv4" "ipv6" "ipv4-fallback" "fallback"]) (assertValueOneOf "IPv4LLRoute" boolValues) (assertValueOneOf "LLMNR" ["yes" "resolve" "no"]) (assertValueOneOf "MulticastDNS" ["yes" "resolve" "no"]) diff --git a/nixos/modules/system/boot/plymouth.nix b/nixos/modules/system/boot/plymouth.nix index adca3c3f66e..23fce22366d 100644 --- a/nixos/modules/system/boot/plymouth.nix +++ b/nixos/modules/system/boot/plymouth.nix @@ -25,6 +25,7 @@ let [Daemon] ShowDelay=0 Theme=${cfg.theme} + ${cfg.extraConfig} ''; in @@ -65,6 +66,15 @@ in ''; }; + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Literal string to append to <literal>configFile</literal> + and the config file generated by the plymouth module. + ''; + }; + }; }; diff --git a/nixos/modules/system/boot/systemd-unit-options.nix b/nixos/modules/system/boot/systemd-unit-options.nix index c1f2c98afcd..bee21f1a8f3 100644 --- a/nixos/modules/system/boot/systemd-unit-options.nix +++ b/nixos/modules/system/boot/systemd-unit-options.nix @@ -24,7 +24,7 @@ in rec { in if isList (head defs'') then concatLists defs'' - else mergeOneOption loc defs'; + else mergeEqualOption loc defs'; }; sharedOptions = { diff --git a/nixos/modules/virtualisation/amazon-image.nix b/nixos/modules/virtualisation/amazon-image.nix index aadfc5add35..20d48add712 100644 --- a/nixos/modules/virtualisation/amazon-image.nix +++ b/nixos/modules/virtualisation/amazon-image.nix @@ -135,6 +135,9 @@ in services.openssh.enable = true; services.openssh.permitRootLogin = "prohibit-password"; + # Creates symlinks for block device names. + services.udev.packages = [ pkgs.ec2-utils ]; + # Force getting the hostname from EC2. networking.hostName = mkDefault ""; diff --git a/nixos/modules/virtualisation/ec2-amis.nix b/nixos/modules/virtualisation/ec2-amis.nix index f640bb21b13..3b4e55d39d7 100644 --- a/nixos/modules/virtualisation/ec2-amis.nix +++ b/nixos/modules/virtualisation/ec2-amis.nix @@ -291,5 +291,21 @@ let self = { "19.03".sa-east-1.hvm-ebs = "ami-0c6a43c6e0ad1f4e2"; "19.03".ap-south-1.hvm-ebs = "ami-0303deb1b5890f878"; - latest = self."19.03"; + # 19.09.981.205691b7cbe + "19.09".eu-west-1.hvm-ebs = "ami-0ebd3156e21e9642f"; + "19.09".eu-west-2.hvm-ebs = "ami-02a2b5480a79084b7"; + "19.09".eu-west-3.hvm-ebs = "ami-09aa175c7588734f7"; + "19.09".eu-central-1.hvm-ebs = "ami-00a7fafd7e237a330"; + "19.09".us-east-1.hvm-ebs = "ami-00a8eeaf232a74f84"; + "19.09".us-east-2.hvm-ebs = "ami-093efd3a57a1e03a8"; + "19.09".us-west-1.hvm-ebs = "ami-0913e9a2b677fac30"; + "19.09".us-west-2.hvm-ebs = "ami-02d9a19f77b47882a"; + "19.09".ca-central-1.hvm-ebs = "ami-0627dd3f7b3627a29"; + "19.09".ap-southeast-1.hvm-ebs = "ami-083614e4d08f2164d"; + "19.09".ap-southeast-2.hvm-ebs = "ami-0048c704185ded6dc"; + "19.09".ap-northeast-1.hvm-ebs = "ami-0329e7fc2d7f60bd0"; + "19.09".ap-northeast-2.hvm-ebs = "ami-03d4ae7d0b5fc364f"; + "19.09".ap-south-1.hvm-ebs = "ami-0b599690b35aeef23"; + + latest = self."19.09"; }; in self diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index ed3431554be..e313d2b411b 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -23,24 +23,56 @@ let cfg = config.virtualisation; - qemuGraphics = lib.optionalString (!cfg.graphics) "-nographic"; - consoles = lib.concatMapStringsSep " " (c: "console=${c}") cfg.qemu.consoles; - # XXX: This is very ugly and in the future we really should use attribute - # sets to build ALL of the QEMU flags instead of this mixed mess of Nix - # expressions and shell script stuff. - mkDiskIfaceDriveFlag = idx: driveArgs: let - inherit (cfg.qemu) diskInterface; - # The drive identifier created by incrementing the index by one using the - # shell. - drvId = "drive$((${idx} + 1))"; - # NOTE: DO NOT shell escape, because this may contain shell variables. - commonArgs = "index=${idx},id=${drvId},${driveArgs}"; - isSCSI = diskInterface == "scsi"; - devArgs = "${diskInterface}-hd,drive=${drvId}"; - args = "-drive ${commonArgs},if=none -device lsi53c895a -device ${devArgs}"; - in if isSCSI then args else "-drive ${commonArgs},if=${diskInterface}"; + driveOpts = { ... }: { + + options = { + + file = mkOption { + type = types.str; + description = "The file image used for this drive."; + }; + + driveExtraOpts = mkOption { + type = types.attrsOf types.str; + default = {}; + description = "Extra options passed to drive flag."; + }; + + deviceExtraOpts = mkOption { + type = types.attrsOf types.str; + default = {}; + description = "Extra options passed to device flag."; + }; + + }; + + }; + + driveCmdline = idx: { file, driveExtraOpts, deviceExtraOpts, ... }: + let + drvId = "drive${toString idx}"; + mkKeyValue = generators.mkKeyValueDefault {} "="; + mkOpts = opts: concatStringsSep "," (mapAttrsToList mkKeyValue opts); + driveOpts = mkOpts (driveExtraOpts // { + index = idx; + id = drvId; + "if" = "none"; + inherit file; + }); + deviceOpts = mkOpts (deviceExtraOpts // { + drive = drvId; + }); + device = + if cfg.qemu.diskInterface == "scsi" then + "-device lsi53c895a -device scsi-hd,${deviceOpts}" + else + "-device virtio-blk-pci,${deviceOpts}"; + in + "-drive ${driveOpts} ${device}"; + + drivesCmdLine = drives: concatStringsSep " " (imap1 driveCmdline drives); # Shell script to start the VM. startVM = @@ -77,13 +109,11 @@ let ''} cd $TMPDIR - idx=2 - extraDisks="" + idx=0 ${flip concatMapStrings cfg.emptyDiskImages (size: '' if ! test -e "empty$idx.qcow2"; then ${qemu}/bin/qemu-img create -f qcow2 "empty$idx.qcow2" "${toString size}M" fi - extraDisks="$extraDisks ${mkDiskIfaceDriveFlag "$idx" "file=$(pwd)/empty$idx.qcow2,werror=report"}" idx=$((idx + 1)) '')} @@ -97,21 +127,7 @@ let -virtfs local,path=/nix/store,security_model=none,mount_tag=store \ -virtfs local,path=$TMPDIR/xchg,security_model=none,mount_tag=xchg \ -virtfs local,path=''${SHARED_DIR:-$TMPDIR/xchg},security_model=none,mount_tag=shared \ - ${if cfg.useBootLoader then '' - ${mkDiskIfaceDriveFlag "0" "file=$NIX_DISK_IMAGE,cache=writeback,werror=report"} \ - ${mkDiskIfaceDriveFlag "1" "file=$TMPDIR/disk.img,media=disk"} \ - ${if cfg.useEFIBoot then '' - -pflash $TMPDIR/bios.bin \ - '' else '' - ''} - '' else '' - ${mkDiskIfaceDriveFlag "0" "file=$NIX_DISK_IMAGE,cache=writeback,werror=report"} \ - -kernel ${config.system.build.toplevel}/kernel \ - -initrd ${config.system.build.toplevel}/initrd \ - -append "$(cat ${config.system.build.toplevel}/kernel-params) init=${config.system.build.toplevel}/init regInfo=${regInfo}/registration ${consoles} $QEMU_KERNEL_PARAMS" \ - ''} \ - $extraDisks \ - ${qemuGraphics} \ + ${drivesCmdLine config.virtualisation.qemu.drives} \ ${toString config.virtualisation.qemu.options} \ $QEMU_OPTS \ "$@" @@ -367,6 +383,12 @@ in ''; }; + drives = + mkOption { + type = types.listOf (types.submodule driveOpts); + description = "Drives passed to qemu."; + }; + diskInterface = mkOption { default = "virtio"; @@ -476,8 +498,49 @@ in # FIXME: Consolidate this one day. virtualisation.qemu.options = mkMerge [ - (mkIf (pkgs.stdenv.isi686 || pkgs.stdenv.isx86_64) [ "-vga std" "-usb" "-device usb-tablet,bus=usb-bus.0" ]) - (mkIf (pkgs.stdenv.isAarch32 || pkgs.stdenv.isAarch64) [ "-device virtio-gpu-pci" "-device usb-ehci,id=usb0" "-device usb-kbd" "-device usb-tablet" ]) + (mkIf (pkgs.stdenv.isi686 || pkgs.stdenv.isx86_64) [ + "-vga std" "-usb" "-device usb-tablet,bus=usb-bus.0" + ]) + (mkIf (pkgs.stdenv.isAarch32 || pkgs.stdenv.isAarch64) [ + "-device virtio-gpu-pci" "-device usb-ehci,id=usb0" "-device usb-kbd" "-device usb-tablet" + ]) + (mkIf (!cfg.useBootLoader) [ + "-kernel ${config.system.build.toplevel}/kernel" + "-initrd ${config.system.build.toplevel}/initrd" + ''-append "$(cat ${config.system.build.toplevel}/kernel-params) init=${config.system.build.toplevel}/init regInfo=${regInfo}/registration ${consoles} $QEMU_KERNEL_PARAMS"'' + ]) + (mkIf cfg.useEFIBoot [ + "-pflash $TMPDIR/bios.bin" + ]) + (mkIf (!cfg.graphics) [ + "-nographic" + ]) + ]; + + virtualisation.qemu.drives = mkMerge [ + (mkIf cfg.useBootLoader [ + { + file = "$NIX_DISK_IMAGE"; + driveExtraOpts.cache = "writeback"; + driveExtraOpts.werror = "report"; + } + { + file = "$TMPDIR/disk.img"; + driveExtraOpts.media = "disk"; + deviceExtraOpts.bootindex = "1"; + } + ]) + (mkIf (!cfg.useBootLoader) [ + { + file = "$NIX_DISK_IMAGE"; + driveExtraOpts.cache = "writeback"; + driveExtraOpts.werror = "report"; + } + ]) + (imap0 (idx: _: { + file = "$(pwd)/empty${toString idx}.qcow2"; + driveExtraOpts.werror = "report"; + }) cfg.emptyDiskImages) ]; # Mount the host filesystem via 9P, and bind-mount the Nix store diff --git a/nixos/tests/acme.nix b/nixos/tests/acme.nix index 85d32d10944..6bd315ff1ea 100644 --- a/nixos/tests/acme.nix +++ b/nixos/tests/acme.nix @@ -1,6 +1,6 @@ let commonConfig = ./common/letsencrypt/common.nix; -in import ./make-test.nix { +in import ./make-test-python.nix { name = "acme"; nodes = rec { @@ -12,8 +12,11 @@ in import ./make-test.nix { networking.extraHosts = '' ${config.networking.primaryIPAddress} standalone.com ''; - security.acme.certs."standalone.com" = { - webroot = "/var/lib/acme/acme-challenges"; + security.acme = { + server = "https://acme-v02.api.letsencrypt.org/dir"; + certs."standalone.com" = { + webroot = "/var/lib/acme/acme-challenges"; + }; }; systemd.targets."acme-finished-standalone.com" = {}; systemd.services."acme-standalone.com" = { @@ -54,6 +57,8 @@ in import ./make-test.nix { ''; }; + security.acme.server = "https://acme-v02.api.letsencrypt.org/dir"; + nesting.clone = [ ({pkgs, ...}: { @@ -85,39 +90,44 @@ in import ./make-test.nix { newServerSystem = nodes.webserver2.config.system.build.toplevel; switchToNewServer = "${newServerSystem}/bin/switch-to-configuration test"; in - # Note, waitForUnit does not work for oneshot services that do not have RemainAfterExit=true, + # Note, wait_for_unit does not work for oneshot services that do not have RemainAfterExit=true, # this is because a oneshot goes from inactive => activating => inactive, and never # reaches the active state. To work around this, we create some mock target units which # get pulled in by the oneshot units. The target units linger after activation, and hence we # can use them to probe that a oneshot fired. It is a bit ugly, but it is the best we can do '' - $client->start; - $letsencrypt->start; - $acmeStandalone->start; - - $letsencrypt->waitForUnit("default.target"); - $letsencrypt->waitForUnit("pebble.service"); - - subtest "can request certificate with HTTPS-01 challenge", sub { - $acmeStandalone->waitForUnit("default.target"); - $acmeStandalone->succeed("systemctl start acme-standalone.com.service"); - $acmeStandalone->waitForUnit("acme-finished-standalone.com.target"); - }; - - $client->waitForUnit("default.target"); - - $client->succeed('curl https://acme-v02.api.letsencrypt.org:15000/roots/0 > /tmp/ca.crt'); - $client->succeed('curl https://acme-v02.api.letsencrypt.org:15000/intermediate-keys/0 >> /tmp/ca.crt'); - - subtest "Can request certificate for nginx service", sub { - $webserver->waitForUnit("acme-finished-a.example.com.target"); - $client->succeed('curl --cacert /tmp/ca.crt https://a.example.com/ | grep -qF "hello world"'); - }; - - subtest "Can add another certificate for nginx service", sub { - $webserver->succeed("/run/current-system/fine-tune/child-1/bin/switch-to-configuration test"); - $webserver->waitForUnit("acme-finished-b.example.com.target"); - $client->succeed('curl --cacert /tmp/ca.crt https://b.example.com/ | grep -qF "hello world"'); - }; + client.start() + letsencrypt.start() + acmeStandalone.start() + + letsencrypt.wait_for_unit("default.target") + letsencrypt.wait_for_unit("pebble.service") + + with subtest("can request certificate with HTTPS-01 challenge"): + acmeStandalone.wait_for_unit("default.target") + acmeStandalone.succeed("systemctl start acme-standalone.com.service") + acmeStandalone.wait_for_unit("acme-finished-standalone.com.target") + + client.wait_for_unit("default.target") + + client.succeed("curl https://acme-v02.api.letsencrypt.org:15000/roots/0 > /tmp/ca.crt") + client.succeed( + "curl https://acme-v02.api.letsencrypt.org:15000/intermediate-keys/0 >> /tmp/ca.crt" + ) + + with subtest("Can request certificate for nginx service"): + webserver.wait_for_unit("acme-finished-a.example.com.target") + client.succeed( + "curl --cacert /tmp/ca.crt https://a.example.com/ | grep -qF 'hello world'" + ) + + with subtest("Can add another certificate for nginx service"): + webserver.succeed( + "/run/current-system/fine-tune/child-1/bin/switch-to-configuration test" + ) + webserver.wait_for_unit("acme-finished-b.example.com.target") + client.succeed( + "curl --cacert /tmp/ca.crt https://b.example.com/ | grep -qF 'hello world'" + ) ''; } diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 10564e063c6..9db505a27d4 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -39,7 +39,8 @@ in caddy = handleTest ./caddy.nix {}; cadvisor = handleTestOn ["x86_64-linux"] ./cadvisor.nix {}; cassandra = handleTest ./cassandra.nix {}; - ceph = handleTestOn ["x86_64-linux"] ./ceph.nix {}; + ceph-single-node = handleTestOn ["x86_64-linux"] ./ceph-single-node.nix {}; + ceph-multi-node = handleTestOn ["x86_64-linux"] ./ceph-multi-node.nix {}; certmgr = handleTest ./certmgr.nix {}; cfssl = handleTestOn ["x86_64-linux"] ./cfssl.nix {}; chromium = (handleTestOn ["x86_64-linux"] ./chromium.nix {}).stable or {}; @@ -47,7 +48,6 @@ in clickhouse = handleTest ./clickhouse.nix {}; cloud-init = handleTest ./cloud-init.nix {}; codimd = handleTest ./codimd.nix {}; - colord = handleTest ./colord.nix {}; containers-bridge = handleTest ./containers-bridge.nix {}; containers-ephemeral = handleTest ./containers-ephemeral.nix {}; containers-extra_veth = handleTest ./containers-extra_veth.nix {}; @@ -81,32 +81,26 @@ in env = handleTest ./env.nix {}; etcd = handleTestOn ["x86_64-linux"] ./etcd.nix {}; etcd-cluster = handleTestOn ["x86_64-linux"] ./etcd-cluster.nix {}; + fancontrol = handleTest ./fancontrol.nix {}; ferm = handleTest ./ferm.nix {}; firefox = handleTest ./firefox.nix {}; firewall = handleTest ./firewall.nix {}; fish = handleTest ./fish.nix {}; flannel = handleTestOn ["x86_64-linux"] ./flannel.nix {}; - flatpak = handleTest ./flatpak.nix {}; - flatpak-builder = handleTest ./flatpak-builder.nix {}; fluentd = handleTest ./fluentd.nix {}; fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {}; fsck = handleTest ./fsck.nix {}; - fwupd = handleTestOn ["x86_64-linux"] ./fwupd.nix {}; # libsmbios is unsupported on aarch64 - gdk-pixbuf = handleTest ./gdk-pixbuf.nix {}; gotify-server = handleTest ./gotify-server.nix {}; gitea = handleTest ./gitea.nix {}; gitlab = handleTest ./gitlab.nix {}; gitolite = handleTest ./gitolite.nix {}; - gjs = handleTest ./gjs.nix {}; - glib-networking = handleTest ./glib-networking.nix {}; glusterfs = handleTest ./glusterfs.nix {}; gnome3-xorg = handleTest ./gnome3-xorg.nix {}; gnome3 = handleTest ./gnome3.nix {}; - gnome-photos = handleTest ./gnome-photos.nix {}; + installed-tests = pkgs.recurseIntoAttrs (handleTest ./installed-tests {}); gocd-agent = handleTest ./gocd-agent.nix {}; gocd-server = handleTest ./gocd-server.nix {}; google-oslogin = handleTest ./google-oslogin {}; - graphene = handleTest ./graphene.nix {}; grafana = handleTest ./grafana.nix {}; graphite = handleTest ./graphite.nix {}; graylog = handleTest ./graylog.nix {}; @@ -133,7 +127,6 @@ in jackett = handleTest ./jackett.nix {}; jellyfin = handleTest ./jellyfin.nix {}; jenkins = handleTest ./jenkins.nix {}; - jormungandr = handleTest ./jormungandr.nix {}; kafka = handleTest ./kafka.nix {}; kerberos = handleTest ./kerberos/default.nix {}; kernel-latest = handleTest ./kernel-latest.nix {}; @@ -148,8 +141,6 @@ in latestKernel.login = handleTest ./login.nix { latestKernel = true; }; ldap = handleTest ./ldap.nix {}; leaps = handleTest ./leaps.nix {}; - libgdata = handleTest ./libgdata.nix {}; - libxmlb = handleTest ./libxmlb.nix {}; lidarr = handleTest ./lidarr.nix {}; lightdm = handleTest ./lightdm.nix {}; limesurvey = handleTest ./limesurvey.nix {}; @@ -158,6 +149,7 @@ in #logstash = handleTest ./logstash.nix {}; mailcatcher = handleTest ./mailcatcher.nix {}; mathics = handleTest ./mathics.nix {}; + matomo = handleTest ./matomo.nix {}; matrix-synapse = handleTest ./matrix-synapse.nix {}; mediawiki = handleTest ./mediawiki.nix {}; memcached = handleTest ./memcached.nix {}; @@ -167,6 +159,7 @@ in minio = handleTest ./minio.nix {}; minidlna = handleTest ./minidlna.nix {}; misc = handleTest ./misc.nix {}; + moinmoin = handleTest ./moinmoin.nix {}; mongodb = handleTest ./mongodb.nix {}; moodle = handleTest ./moodle.nix {}; morty = handleTest ./morty.nix {}; @@ -209,10 +202,10 @@ in # openstack-image-userdata doesn't work in a sandbox as the simulated openstack instance needs network access #openstack-image-userdata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).userdata or {}; openstack-image-metadata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).metadata or {}; + orangefs = handleTest ./orangefs.nix {}; os-prober = handleTestOn ["x86_64-linux"] ./os-prober.nix {}; osquery = handleTest ./osquery.nix {}; osrm-backend = handleTest ./osrm-backend.nix {}; - ostree = handleTest ./ostree.nix {}; overlayfs = handleTest ./overlayfs.nix {}; packagekit = handleTest ./packagekit.nix {}; pam-oath-login = handleTest ./pam-oath-login.nix {}; @@ -238,7 +231,6 @@ in prosodyMysql = handleTest ./xmpp/prosody-mysql.nix {}; proxy = handleTest ./proxy.nix {}; quagga = handleTest ./quagga.nix {}; - quake3 = handleTest ./quake3.nix {}; rabbitmq = handleTest ./rabbitmq.nix {}; radarr = handleTest ./radarr.nix {}; radicale = handleTest ./radicale.nix {}; @@ -252,6 +244,7 @@ in rxe = handleTest ./rxe.nix {}; samba = handleTest ./samba.nix {}; sddm = handleTest ./sddm.nix {}; + shiori = handleTest ./shiori.nix {}; signal-desktop = handleTest ./signal-desktop.nix {}; simple = handleTest ./simple.nix {}; slim = handleTest ./slim.nix {}; @@ -276,7 +269,9 @@ in tinydns = handleTest ./tinydns.nix {}; tor = handleTest ./tor.nix {}; transmission = handleTest ./transmission.nix {}; + trac = handleTest ./trac.nix {}; trezord = handleTest ./trezord.nix {}; + trickster = handleTest ./trickster.nix {}; udisks2 = handleTest ./udisks2.nix {}; upnp = handleTest ./upnp.nix {}; uwsgi = handleTest ./uwsgi.nix {}; @@ -286,7 +281,6 @@ in wireguard-generated = handleTest ./wireguard/generated.nix {}; wordpress = handleTest ./wordpress.nix {}; xautolock = handleTest ./xautolock.nix {}; - xdg-desktop-portal = handleTest ./xdg-desktop-portal.nix {}; xfce = handleTest ./xfce.nix {}; xfce4-14 = handleTest ./xfce4-14.nix {}; xmonad = handleTest ./xmonad.nix {}; diff --git a/nixos/tests/ammonite.nix b/nixos/tests/ammonite.nix index fedfde233e8..1955e42be5f 100644 --- a/nixos/tests/ammonite.nix +++ b/nixos/tests/ammonite.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "ammonite"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ nequissimus ]; @@ -13,8 +13,8 @@ import ./make-test.nix ({ pkgs, ...} : { }; testScript = '' - startAll; + start_all() - $amm->succeed("amm -c 'val foo = 21; println(foo * 2)' | grep 42") + amm.succeed("amm -c 'val foo = 21; println(foo * 2)' | grep 42") ''; }) diff --git a/nixos/tests/atd.nix b/nixos/tests/atd.nix index 25db7279924..c3abe5c253d 100644 --- a/nixos/tests/atd.nix +++ b/nixos/tests/atd.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: +import ./make-test-python.nix ({ pkgs, ... }: { name = "atd"; @@ -14,18 +14,18 @@ import ./make-test.nix ({ pkgs, ... }: # "at" has a resolution of 1 minute testScript = '' - startAll; + start_all() - $machine->waitForUnit('atd.service'); # wait for atd to start - $machine->fail("test -f ~root/at-1"); - $machine->fail("test -f ~alice/at-1"); + machine.wait_for_unit("atd.service") # wait for atd to start + machine.fail("test -f ~root/at-1") + machine.fail("test -f ~alice/at-1") - $machine->succeed("echo 'touch ~root/at-1' | at now+1min"); - $machine->succeed("su - alice -c \"echo 'touch at-1' | at now+1min\""); + machine.succeed("echo 'touch ~root/at-1' | at now+1min") + machine.succeed("su - alice -c \"echo 'touch at-1' | at now+1min\"") - $machine->succeed("sleep 1.5m"); + machine.succeed("sleep 1.5m") - $machine->succeed("test -f ~root/at-1"); - $machine->succeed("test -f ~alice/at-1"); + machine.succeed("test -f ~root/at-1") + machine.succeed("test -f ~alice/at-1") ''; }) diff --git a/nixos/tests/automysqlbackup.nix b/nixos/tests/automysqlbackup.nix index ada104a34de..224b93862fb 100644 --- a/nixos/tests/automysqlbackup.nix +++ b/nixos/tests/automysqlbackup.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, lib, ... }: +import ./make-test-python.nix ({ pkgs, lib, ... }: { name = "automysqlbackup"; @@ -15,20 +15,24 @@ import ./make-test.nix ({ pkgs, lib, ... }: }; testScript = '' - startAll; + start_all() # Need to have mysql started so that it can be populated with data. - $machine->waitForUnit("mysql.service"); - - # Wait for testdb to be fully populated (5 rows). - $machine->waitUntilSucceeds("mysql -u root -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"); - - # Do a backup and wait for it to start - $machine->startJob("automysqlbackup.service"); - $machine->waitForJob("automysqlbackup.service"); - - # wait for backup file and check that data appears in backup - $machine->waitForFile("/var/backup/mysql/daily/testdb"); - $machine->succeed("${pkgs.gzip}/bin/zcat /var/backup/mysql/daily/testdb/daily_testdb_*.sql.gz | grep hello"); + machine.wait_for_unit("mysql.service") + + with subtest("Wait for testdb to be fully populated (5 rows)."): + machine.wait_until_succeeds( + "mysql -u root -D testdb -N -B -e 'select count(id) from tests' | grep -q 5" + ) + + with subtest("Do a backup and wait for it to start"): + machine.start_job("automysqlbackup.service") + machine.wait_for_job("automysqlbackup.service") + + with subtest("wait for backup file and check that data appears in backup"): + machine.wait_for_file("/var/backup/mysql/daily/testdb") + machine.succeed( + "${pkgs.gzip}/bin/zcat /var/backup/mysql/daily/testdb/daily_testdb_*.sql.gz | grep hello" + ) ''; }) diff --git a/nixos/tests/avahi.nix b/nixos/tests/avahi.nix index ae4f54d5266..fe027c14d5a 100644 --- a/nixos/tests/avahi.nix +++ b/nixos/tests/avahi.nix @@ -1,5 +1,5 @@ # Test whether `avahi-daemon' and `libnss-mdns' work as expected. -import ./make-test.nix ({ pkgs, ... } : { +import ./make-test-python.nix ({ pkgs, ... } : { name = "avahi"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ eelco ]; @@ -23,45 +23,45 @@ import ./make-test.nix ({ pkgs, ... } : { two = cfg; }; - testScript = - '' startAll; + testScript = '' + start_all() - # mDNS. - $one->waitForUnit("network.target"); - $two->waitForUnit("network.target"); + # mDNS. + one.wait_for_unit("network.target") + two.wait_for_unit("network.target") - $one->succeed("avahi-resolve-host-name one.local | tee out >&2"); - $one->succeed("test \"`cut -f1 < out`\" = one.local"); - $one->succeed("avahi-resolve-host-name two.local | tee out >&2"); - $one->succeed("test \"`cut -f1 < out`\" = two.local"); + one.succeed("avahi-resolve-host-name one.local | tee out >&2") + one.succeed('test "`cut -f1 < out`" = one.local') + one.succeed("avahi-resolve-host-name two.local | tee out >&2") + one.succeed('test "`cut -f1 < out`" = two.local') - $two->succeed("avahi-resolve-host-name one.local | tee out >&2"); - $two->succeed("test \"`cut -f1 < out`\" = one.local"); - $two->succeed("avahi-resolve-host-name two.local | tee out >&2"); - $two->succeed("test \"`cut -f1 < out`\" = two.local"); + two.succeed("avahi-resolve-host-name one.local | tee out >&2") + two.succeed('test "`cut -f1 < out`" = one.local') + two.succeed("avahi-resolve-host-name two.local | tee out >&2") + two.succeed('test "`cut -f1 < out`" = two.local') - # Basic DNS-SD. - $one->succeed("avahi-browse -r -t _workstation._tcp | tee out >&2"); - $one->succeed("test `wc -l < out` -gt 0"); - $two->succeed("avahi-browse -r -t _workstation._tcp | tee out >&2"); - $two->succeed("test `wc -l < out` -gt 0"); + # Basic DNS-SD. + one.succeed("avahi-browse -r -t _workstation._tcp | tee out >&2") + one.succeed("test `wc -l < out` -gt 0") + two.succeed("avahi-browse -r -t _workstation._tcp | tee out >&2") + two.succeed("test `wc -l < out` -gt 0") - # More DNS-SD. - $one->execute("avahi-publish -s \"This is a test\" _test._tcp 123 one=1 &"); - $one->sleep(5); - $two->succeed("avahi-browse -r -t _test._tcp | tee out >&2"); - $two->succeed("test `wc -l < out` -gt 0"); + # More DNS-SD. + one.execute('avahi-publish -s "This is a test" _test._tcp 123 one=1 &') + one.sleep(5) + two.succeed("avahi-browse -r -t _test._tcp | tee out >&2") + two.succeed("test `wc -l < out` -gt 0") - # NSS-mDNS. - $one->succeed("getent hosts one.local >&2"); - $one->succeed("getent hosts two.local >&2"); - $two->succeed("getent hosts one.local >&2"); - $two->succeed("getent hosts two.local >&2"); + # NSS-mDNS. + one.succeed("getent hosts one.local >&2") + one.succeed("getent hosts two.local >&2") + two.succeed("getent hosts one.local >&2") + two.succeed("getent hosts two.local >&2") - # extra service definitions - $one->succeed("avahi-browse -r -t _ssh._tcp | tee out >&2"); - $one->succeed("test `wc -l < out` -gt 0"); - $two->succeed("avahi-browse -r -t _ssh._tcp | tee out >&2"); - $two->succeed("test `wc -l < out` -gt 0"); - ''; + # extra service definitions + one.succeed("avahi-browse -r -t _ssh._tcp | tee out >&2") + one.succeed("test `wc -l < out` -gt 0") + two.succeed("avahi-browse -r -t _ssh._tcp | tee out >&2") + two.succeed("test `wc -l < out` -gt 0") + ''; }) diff --git a/nixos/tests/babeld.nix b/nixos/tests/babeld.nix index 5242cf395d7..fafa788ba57 100644 --- a/nixos/tests/babeld.nix +++ b/nixos/tests/babeld.nix @@ -1,5 +1,5 @@ -import ./make-test.nix ({ pkgs, lib, ...} : { +import ./make-test-python.nix ({ pkgs, lib, ...} : { name = "babeld"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ hexa ]; @@ -21,7 +21,7 @@ import ./make-test.nix ({ pkgs, lib, ...} : { }; }; - localRouter = { pkgs, lib, ... }: + local_router = { pkgs, lib, ... }: { virtualisation.vlans = [ 10 20 ]; @@ -70,7 +70,7 @@ import ./make-test.nix ({ pkgs, lib, ...} : { ''; }; }; - remoteRouter = { pkgs, lib, ... }: + remote_router = { pkgs, lib, ... }: { virtualisation.vlans = [ 20 30 ]; @@ -124,25 +124,25 @@ import ./make-test.nix ({ pkgs, lib, ...} : { testScript = '' - startAll; + start_all() - $client->waitForUnit("network-online.target"); - $localRouter->waitForUnit("network-online.target"); - $remoteRouter->waitForUnit("network-online.target"); + client.wait_for_unit("network-online.target") + local_router.wait_for_unit("network-online.target") + remote_router.wait_for_unit("network-online.target") - $localRouter->waitForUnit("babeld.service"); - $remoteRouter->waitForUnit("babeld.service"); + local_router.wait_for_unit("babeld.service") + remote_router.wait_for_unit("babeld.service") - $localRouter->waitUntilSucceeds("ip route get 192.168.30.1"); - $localRouter->waitUntilSucceeds("ip route get 2001:db8:30::1"); + local_router.wait_until_succeeds("ip route get 192.168.30.1") + local_router.wait_until_succeeds("ip route get 2001:db8:30::1") - $remoteRouter->waitUntilSucceeds("ip route get 192.168.10.1"); - $remoteRouter->waitUntilSucceeds("ip route get 2001:db8:10::1"); + remote_router.wait_until_succeeds("ip route get 192.168.10.1") + remote_router.wait_until_succeeds("ip route get 2001:db8:10::1") - $client->succeed("ping -c1 192.168.30.1"); - $client->succeed("ping -c1 2001:db8:30::1"); + client.succeed("ping -c1 192.168.30.1") + client.succeed("ping -c1 2001:db8:30::1") - $remoteRouter->succeed("ping -c1 192.168.10.2"); - $remoteRouter->succeed("ping -c1 2001:db8:10::2"); + remote_router.succeed("ping -c1 192.168.10.2") + remote_router.succeed("ping -c1 2001:db8:10::2") ''; }) diff --git a/nixos/tests/bcachefs.nix b/nixos/tests/bcachefs.nix index 658676ef0ab..0541e580322 100644 --- a/nixos/tests/bcachefs.nix +++ b/nixos/tests/bcachefs.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: { +import ./make-test-python.nix ({ pkgs, ... }: { name = "bcachefs"; meta.maintainers = with pkgs.stdenv.lib.maintainers; [ chiiruno ]; @@ -10,29 +10,25 @@ import ./make-test.nix ({ pkgs, ... }: { }; testScript = '' - $machine->succeed("modprobe bcachefs"); - $machine->succeed("bcachefs version"); - $machine->succeed("ls /dev"); + machine.succeed("modprobe bcachefs") + machine.succeed("bcachefs version") + machine.succeed("ls /dev") - $machine->succeed( - "mkdir /tmp/mnt", - - "udevadm settle", - "parted --script /dev/vdb mklabel msdos", - "parted --script /dev/vdb -- mkpart primary 1024M -1s", - "udevadm settle", - - # Due to #32279, we cannot use encryption for this test yet - # "echo password | bcachefs format --encrypted /dev/vdb1", - # "echo password | bcachefs unlock /dev/vdb1", - "bcachefs format /dev/vdb1", - "mount -t bcachefs /dev/vdb1 /tmp/mnt", - "udevadm settle", - - "bcachefs fs usage /tmp/mnt", - - "umount /tmp/mnt", - "udevadm settle" - ); + machine.succeed( + "mkdir /tmp/mnt", + "udevadm settle", + "parted --script /dev/vdb mklabel msdos", + "parted --script /dev/vdb -- mkpart primary 1024M -1s", + "udevadm settle", + # Due to #32279, we cannot use encryption for this test yet + # "echo password | bcachefs format --encrypted /dev/vdb1", + # "echo password | bcachefs unlock /dev/vdb1", + "bcachefs format /dev/vdb1", + "mount -t bcachefs /dev/vdb1 /tmp/mnt", + "udevadm settle", + "bcachefs fs usage /tmp/mnt", + "umount /tmp/mnt", + "udevadm settle", + ) ''; }) diff --git a/nixos/tests/beanstalkd.nix b/nixos/tests/beanstalkd.nix index fa2fbc2c92a..4f4a454fb47 100644 --- a/nixos/tests/beanstalkd.nix +++ b/nixos/tests/beanstalkd.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, lib, ... }: +import ./make-test-python.nix ({ pkgs, lib, ... }: let pythonEnv = pkgs.python3.withPackages (p: [p.beanstalkc]); @@ -34,12 +34,16 @@ in }; testScript = '' - startAll; + start_all() - $machine->waitForUnit('beanstalkd.service'); + machine.wait_for_unit("beanstalkd.service") - $machine->succeed("${produce}"); - $machine->succeed("${consume}") eq "this is a job\n" or die; - $machine->succeed("${consume}") eq "this is another job\n" or die; + machine.succeed("${produce}") + assert "this is a job\n" == machine.succeed( + "${consume}" + ) + assert "this is another job\n" == machine.succeed( + "${consume}" + ) ''; }) diff --git a/nixos/tests/bind.nix b/nixos/tests/bind.nix index 1f8c1dc7be4..09917b15a8e 100644 --- a/nixos/tests/bind.nix +++ b/nixos/tests/bind.nix @@ -1,4 +1,4 @@ -import ./make-test.nix { +import ./make-test-python.nix { name = "bind"; machine = { pkgs, lib, ... }: { @@ -20,8 +20,8 @@ import ./make-test.nix { }; testScript = '' - $machine->waitForUnit('bind.service'); - $machine->waitForOpenPort(53); - $machine->succeed('host 192.168.0.1 127.0.0.1 | grep -qF ns.example.org'); + machine.wait_for_unit("bind.service") + machine.wait_for_open_port(53) + machine.succeed("host 192.168.0.1 127.0.0.1 | grep -qF ns.example.org") ''; } diff --git a/nixos/tests/bittorrent.nix b/nixos/tests/bittorrent.nix index 3b1169a1b7f..e5be652c711 100644 --- a/nixos/tests/bittorrent.nix +++ b/nixos/tests/bittorrent.nix @@ -6,7 +6,7 @@ # which only works if the first client successfully uses the UPnP-IGD # protocol to poke a hole in the NAT. -import ./make-test.nix ({ pkgs, ... }: +import ./make-test-python.nix ({ pkgs, ... }: let @@ -108,42 +108,56 @@ in testScript = { nodes, ... }: '' - startAll; + start_all() # Wait for network and miniupnpd. - $router->waitForUnit("network-online.target"); - $router->waitForUnit("miniupnpd"); + router.wait_for_unit("network-online.target") + router.wait_for_unit("miniupnpd") # Create the torrent. - $tracker->succeed("mkdir /tmp/data"); - $tracker->succeed("cp ${file} /tmp/data/test.tar.bz2"); - $tracker->succeed("transmission-create /tmp/data/test.tar.bz2 --private --tracker http://${externalTrackerAddress}:6969/announce --outfile /tmp/test.torrent"); - $tracker->succeed("chmod 644 /tmp/test.torrent"); + tracker.succeed("mkdir /tmp/data") + tracker.succeed( + "cp ${file} /tmp/data/test.tar.bz2" + ) + tracker.succeed( + "transmission-create /tmp/data/test.tar.bz2 --private --tracker http://${externalTrackerAddress}:6969/announce --outfile /tmp/test.torrent" + ) + tracker.succeed("chmod 644 /tmp/test.torrent") # Start the tracker. !!! use a less crappy tracker - $tracker->waitForUnit("network-online.target"); - $tracker->waitForUnit("opentracker.service"); - $tracker->waitForOpenPort(6969); + tracker.wait_for_unit("network-online.target") + tracker.wait_for_unit("opentracker.service") + tracker.wait_for_open_port(6969) # Start the initial seeder. - $tracker->succeed("transmission-remote --add /tmp/test.torrent --no-portmap --no-dht --download-dir /tmp/data"); + tracker.succeed( + "transmission-remote --add /tmp/test.torrent --no-portmap --no-dht --download-dir /tmp/data" + ) # Now we should be able to download from the client behind the NAT. - $tracker->waitForUnit("httpd"); - $client1->waitForUnit("network-online.target"); - $client1->succeed("transmission-remote --add http://${externalTrackerAddress}/test.torrent --download-dir /tmp >&2 &"); - $client1->waitForFile("/tmp/test.tar.bz2"); - $client1->succeed("cmp /tmp/test.tar.bz2 ${file}"); + tracker.wait_for_unit("httpd") + client1.wait_for_unit("network-online.target") + client1.succeed( + "transmission-remote --add http://${externalTrackerAddress}/test.torrent --download-dir /tmp >&2 &" + ) + client1.wait_for_file("/tmp/test.tar.bz2") + client1.succeed( + "cmp /tmp/test.tar.bz2 ${file}" + ) # Bring down the initial seeder. - # $tracker->stopJob("transmission"); + # tracker.stop_job("transmission") # Now download from the second client. This can only succeed if # the first client created a NAT hole in the router. - $client2->waitForUnit("network-online.target"); - $client2->succeed("transmission-remote --add http://${externalTrackerAddress}/test.torrent --no-portmap --no-dht --download-dir /tmp >&2 &"); - $client2->waitForFile("/tmp/test.tar.bz2"); - $client2->succeed("cmp /tmp/test.tar.bz2 ${file}"); + client2.wait_for_unit("network-online.target") + client2.succeed( + "transmission-remote --add http://${externalTrackerAddress}/test.torrent --no-portmap --no-dht --download-dir /tmp >&2 &" + ) + client2.wait_for_file("/tmp/test.tar.bz2") + client2.succeed( + "cmp /tmp/test.tar.bz2 ${file}" + ) ''; }) diff --git a/nixos/tests/boot-stage1.nix b/nixos/tests/boot-stage1.nix index b2e74bff6fc..cfb2ccb8285 100644 --- a/nixos/tests/boot-stage1.nix +++ b/nixos/tests/boot-stage1.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: { +import ./make-test-python.nix ({ pkgs, ... }: { name = "boot-stage1"; machine = { config, pkgs, lib, ... }: { @@ -150,12 +150,12 @@ import ./make-test.nix ({ pkgs, ... }: { }; testScript = '' - $machine->waitForUnit("multi-user.target"); - $machine->succeed('test -s /run/canary2.pid'); - $machine->fail('pgrep -a canary1'); - $machine->fail('kill -0 $(< /run/canary2.pid)'); - $machine->succeed('pgrep -a -f \'^@canary3$\'''); - $machine->succeed('pgrep -a -f \'^kcanary$\'''); + machine.wait_for_unit("multi-user.target") + machine.succeed("test -s /run/canary2.pid") + machine.fail("pgrep -a canary1") + machine.fail("kill -0 $(< /run/canary2.pid)") + machine.succeed('pgrep -a -f "^@canary3$"') + machine.succeed('pgrep -a -f "^kcanary$"') ''; meta.maintainers = with pkgs.stdenv.lib.maintainers; [ aszlig ]; diff --git a/nixos/tests/boot.nix b/nixos/tests/boot.nix index 57d8006d7ac..c5040f3b31f 100644 --- a/nixos/tests/boot.nix +++ b/nixos/tests/boot.nix @@ -3,7 +3,7 @@ pkgs ? import ../.. { inherit system config; } }: -with import ../lib/testing.nix { inherit system pkgs; }; +with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let @@ -17,11 +17,11 @@ let ]; }).config.system.build.isoImage; - perlAttrs = params: "{ ${concatStringsSep ", " (mapAttrsToList (name: param: "${name} => ${builtins.toJSON param}") params)} }"; + pythonDict = params: "\n {\n ${concatStringsSep ",\n " (mapAttrsToList (name: param: "\"${name}\": \"${param}\"") params)},\n }\n"; makeBootTest = name: extraConfig: let - machineConfig = perlAttrs ({ qemuFlags = "-m 768"; } // extraConfig); + machineConfig = pythonDict ({ qemuFlags = "-m 768"; } // extraConfig); in makeTest { inherit iso; @@ -29,16 +29,16 @@ let nodes = { }; testScript = '' - my $machine = createMachine(${machineConfig}); - $machine->start; - $machine->waitForUnit("multi-user.target"); - $machine->succeed("nix verify -r --no-trust /run/current-system"); + machine = create_machine(${machineConfig}) + machine.start() + machine.wait_for_unit("multi-user.target") + machine.succeed("nix verify -r --no-trust /run/current-system") - # Test whether the channel got installed correctly. - $machine->succeed("nix-instantiate --dry-run '<nixpkgs>' -A hello"); - $machine->succeed("nix-env --dry-run -iA nixos.procps"); + with subtest("Check whether the channel got installed correctly"): + machine.succeed("nix-instantiate --dry-run '<nixpkgs>' -A hello") + machine.succeed("nix-env --dry-run -iA nixos.procps") - $machine->shutdown; + machine.shutdown() ''; }; @@ -60,7 +60,7 @@ let config.system.build.netbootIpxeScript ]; }; - machineConfig = perlAttrs ({ + machineConfig = pythonDict ({ qemuFlags = "-boot order=n -m 2000"; netBackendArgs = "tftp=${ipxeBootDir},bootfile=netboot.ipxe"; } // extraConfig); @@ -68,12 +68,11 @@ let makeTest { name = "boot-netboot-" + name; nodes = { }; - testScript = - '' - my $machine = createMachine(${machineConfig}); - $machine->start; - $machine->waitForUnit("multi-user.target"); - $machine->shutdown; + testScript = '' + machine = create_machine(${machineConfig}) + machine.start() + machine.wait_for_unit("multi-user.target") + machine.shutdown() ''; }; in { diff --git a/nixos/tests/borgbackup.nix b/nixos/tests/borgbackup.nix index 165f64b0d6d..d97471e293e 100644 --- a/nixos/tests/borgbackup.nix +++ b/nixos/tests/borgbackup.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: +import ./make-test-python.nix ({ pkgs, ... }: let passphrase = "supersecret"; @@ -106,60 +106,70 @@ in { }; testScript = '' - startAll; - - $client->fail('test -d "${remoteRepo}"'); - - $client->succeed("cp ${privateKey} /root/id_ed25519"); - $client->succeed("chmod 0600 /root/id_ed25519"); - $client->succeed("cp ${privateKeyAppendOnly} /root/id_ed25519.appendOnly"); - $client->succeed("chmod 0600 /root/id_ed25519.appendOnly"); - - $client->succeed("mkdir -p ${dataDir}"); - $client->succeed("touch ${dataDir}/${excludeFile}"); - $client->succeed("echo '${keepFileData}' > ${dataDir}/${keepFile}"); - - subtest "local", sub { - my $borg = "BORG_PASSPHRASE='${passphrase}' borg"; - $client->systemctl("start --wait borgbackup-job-local"); - $client->fail("systemctl is-failed borgbackup-job-local"); - # Make sure exactly one archive has been created - $client->succeed("c=\$($borg list '${localRepo}' | wc -l) && [[ \$c == '1' ]]"); - # Make sure excludeFile has been excluded - $client->fail("$borg list '${localRepo}::${archiveName}' | grep -qF '${excludeFile}'"); - # Make sure keepFile has the correct content - $client->succeed("$borg extract '${localRepo}::${archiveName}'"); - $client->succeed('c=$(cat ${dataDir}/${keepFile}) && [[ "$c" == "${keepFileData}" ]]'); - # Make sure the same is true when using `borg mount` - $client->succeed("mkdir -p /mnt/borg && $borg mount '${localRepo}::${archiveName}' /mnt/borg"); - $client->succeed('c=$(cat /mnt/borg/${dataDir}/${keepFile}) && [[ "$c" == "${keepFileData}" ]]'); - }; - - subtest "remote", sub { - my $borg = "BORG_RSH='ssh -oStrictHostKeyChecking=no -i /root/id_ed25519' borg"; - $server->waitForUnit("sshd.service"); - $client->waitForUnit("network.target"); - $client->systemctl("start --wait borgbackup-job-remote"); - $client->fail("systemctl is-failed borgbackup-job-remote"); - - # Make sure we can't access repos other than the specified one - $client->fail("$borg list borg\@server:wrong"); - - #TODO: Make sure that data is actually deleted - }; - - subtest "remoteAppendOnly", sub { - my $borg = "BORG_RSH='ssh -oStrictHostKeyChecking=no -i /root/id_ed25519.appendOnly' borg"; - $server->waitForUnit("sshd.service"); - $client->waitForUnit("network.target"); - $client->systemctl("start --wait borgbackup-job-remoteAppendOnly"); - $client->fail("systemctl is-failed borgbackup-job-remoteAppendOnly"); - - # Make sure we can't access repos other than the specified one - $client->fail("$borg list borg\@server:wrong"); - - #TODO: Make sure that data is not actually deleted - }; - + start_all() + + client.fail('test -d "${remoteRepo}"') + + client.succeed( + "cp ${privateKey} /root/id_ed25519" + ) + client.succeed("chmod 0600 /root/id_ed25519") + client.succeed( + "cp ${privateKeyAppendOnly} /root/id_ed25519.appendOnly" + ) + client.succeed("chmod 0600 /root/id_ed25519.appendOnly") + + client.succeed("mkdir -p ${dataDir}") + client.succeed("touch ${dataDir}/${excludeFile}") + client.succeed("echo '${keepFileData}' > ${dataDir}/${keepFile}") + + with subtest("local"): + borg = "BORG_PASSPHRASE='${passphrase}' borg" + client.systemctl("start --wait borgbackup-job-local") + client.fail("systemctl is-failed borgbackup-job-local") + # Make sure exactly one archive has been created + assert int(client.succeed("{} list '${localRepo}' | wc -l".format(borg))) > 0 + # Make sure excludeFile has been excluded + client.fail( + "{} list '${localRepo}::${archiveName}' | grep -qF '${excludeFile}'".format(borg) + ) + # Make sure keepFile has the correct content + client.succeed("{} extract '${localRepo}::${archiveName}'".format(borg)) + assert "${keepFileData}" in client.succeed("cat ${dataDir}/${keepFile}") + # Make sure the same is true when using `borg mount` + client.succeed( + "mkdir -p /mnt/borg && {} mount '${localRepo}::${archiveName}' /mnt/borg".format( + borg + ) + ) + assert "${keepFileData}" in client.succeed( + "cat /mnt/borg/${dataDir}/${keepFile}" + ) + + with subtest("remote"): + borg = "BORG_RSH='ssh -oStrictHostKeyChecking=no -i /root/id_ed25519' borg" + server.wait_for_unit("sshd.service") + client.wait_for_unit("network.target") + client.systemctl("start --wait borgbackup-job-remote") + client.fail("systemctl is-failed borgbackup-job-remote") + + # Make sure we can't access repos other than the specified one + client.fail("{} list borg\@server:wrong".format(borg)) + + # TODO: Make sure that data is actually deleted + + with subtest("remoteAppendOnly"): + borg = ( + "BORG_RSH='ssh -oStrictHostKeyChecking=no -i /root/id_ed25519.appendOnly' borg" + ) + server.wait_for_unit("sshd.service") + client.wait_for_unit("network.target") + client.systemctl("start --wait borgbackup-job-remoteAppendOnly") + client.fail("systemctl is-failed borgbackup-job-remoteAppendOnly") + + # Make sure we can't access repos other than the specified one + client.fail("{} list borg\@server:wrong".format(borg)) + + # TODO: Make sure that data is not actually deleted ''; }) diff --git a/nixos/tests/caddy.nix b/nixos/tests/caddy.nix index ab9d2fbf4d1..fc10df0c79b 100644 --- a/nixos/tests/caddy.nix +++ b/nixos/tests/caddy.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: { +import ./make-test-python.nix ({ pkgs, ... }: { name = "caddy"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ xfix ]; @@ -50,33 +50,38 @@ import ./make-test.nix ({ pkgs, ... }: { etagSystem = "${nodes.webserver.config.system.build.toplevel}/fine-tune/child-1"; justReloadSystem = "${nodes.webserver.config.system.build.toplevel}/fine-tune/child-2"; in '' - my $url = 'http://localhost/example.html'; - $webserver->waitForUnit("caddy"); - $webserver->waitForOpenPort("80"); + url = "http://localhost/example.html" + webserver.wait_for_unit("caddy") + webserver.wait_for_open_port("80") - sub checkEtag { - my $etag = $webserver->succeed( - 'curl -v '.$url.' 2>&1 | sed -n -e "s/^< [Ee][Tt][Aa][Gg]: *//p"' - ); - $etag =~ s/\r?\n$//; - my $httpCode = $webserver->succeed( - 'curl -w "%{http_code}" -X HEAD -H \'If-None-Match: '.$etag.'\' '.$url - ); - die "HTTP code is not 304" unless $httpCode == 304; - return $etag; - } - subtest "check ETag if serving Nix store paths", sub { - my $oldEtag = checkEtag; - $webserver->succeed("${etagSystem}/bin/switch-to-configuration test >&2"); - $webserver->sleep(1); # race condition - my $newEtag = checkEtag; - die "Old ETag $oldEtag is the same as $newEtag" if $oldEtag eq $newEtag; - }; + def check_etag(url): + etag = webserver.succeed( + "curl -v '{}' 2>&1 | sed -n -e \"s/^< [Ee][Tt][Aa][Gg]: *//p\"".format(url) + ) + etag = etag.replace("\r\n", " ") + http_code = webserver.succeed( + "curl -w \"%{{http_code}}\" -X HEAD -H 'If-None-Match: {}' {}".format(etag, url) + ) + assert int(http_code) == 304, "HTTP code is not 304" + return etag - subtest "config is reloaded on nixos-rebuild switch", sub { - $webserver->succeed("${justReloadSystem}/bin/switch-to-configuration test >&2"); - $webserver->waitForOpenPort("8080"); - }; + + with subtest("check ETag if serving Nix store paths"): + old_etag = check_etag(url) + webserver.succeed( + "${etagSystem}/bin/switch-to-configuration test >&2" + ) + webserver.sleep(1) + new_etag = check_etag(url) + assert old_etag != new_etag, "Old ETag {} is the same as {}".format( + old_etag, new_etag + ) + + with subtest("config is reloaded on nixos-rebuild switch"): + webserver.succeed( + "${justReloadSystem}/bin/switch-to-configuration test >&2" + ) + webserver.wait_for_open_port("8080") ''; }) diff --git a/nixos/tests/cadvisor.nix b/nixos/tests/cadvisor.nix index e60bae4b700..60c04f14780 100644 --- a/nixos/tests/cadvisor.nix +++ b/nixos/tests/cadvisor.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... } : { +import ./make-test-python.nix ({ pkgs, ... } : { name = "cadvisor"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ offline ]; @@ -16,20 +16,19 @@ import ./make-test.nix ({ pkgs, ... } : { }; }; - testScript = - '' - startAll; - $machine->waitForUnit("cadvisor.service"); - $machine->succeed("curl http://localhost:8080/containers/"); + testScript = '' + start_all() + machine.wait_for_unit("cadvisor.service") + machine.succeed("curl http://localhost:8080/containers/") - $influxdb->waitForUnit("influxdb.service"); + influxdb.wait_for_unit("influxdb.service") # create influxdb database - $influxdb->succeed(q~ - curl -XPOST http://localhost:8086/query --data-urlencode "q=CREATE DATABASE root" - ~); + influxdb.succeed( + 'curl -XPOST http://localhost:8086/query --data-urlencode "q=CREATE DATABASE root"' + ) - $influxdb->waitForUnit("cadvisor.service"); - $influxdb->succeed("curl http://localhost:8080/containers/"); + influxdb.wait_for_unit("cadvisor.service") + influxdb.succeed("curl http://localhost:8080/containers/") ''; }) diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix index c55733c9be7..05607956a9d 100644 --- a/nixos/tests/cassandra.nix +++ b/nixos/tests/cassandra.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, lib, ... }: +import ./make-test-python.nix ({ pkgs, lib, ... }: let # Change this to test a different version of Cassandra: testPackage = pkgs.cassandra; @@ -9,13 +9,16 @@ let jmxRolesFile = ./cassandra-jmx-roles; jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}"; jmxPort = 7200; # Non-standard port so it doesn't accidentally work + jmxPortStr = toString jmxPort; - # Would usually be assigned to 512M + # Would usually be assigned to 512M. + # Set it to a different value, so that we can check whether our config + # actually changes it. numMaxHeapSize = "400"; getHeapLimitCommand = '' - nodetool info -p ${toString jmxPort} | grep "^Heap Memory" | awk \'{print $NF}\' + nodetool info -p ${jmxPortStr} | grep "^Heap Memory" | awk '{print $NF}' ''; - checkHeapLimitCommand = '' + checkHeapLimitCommand = pkgs.writeShellScript "check-heap-limit.sh" '' [ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ] ''; @@ -44,7 +47,10 @@ let }; in { - name = "cassandra-ci"; + name = "cassandra"; + meta = { + maintainers = with lib.maintainers; [ johnazoidberg ]; + }; nodes = { cass0 = nodeCfg "192.168.1.1" {}; @@ -52,66 +58,74 @@ in cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; }; }; - testScript = let - jmxPortS = toString jmxPort; - in '' + testScript = '' # Check configuration - subtest "Timers exist", sub { - $cass0->succeed("systemctl list-timers | grep cassandra-full-repair.timer"); - $cass0->succeed("systemctl list-timers | grep cassandra-incremental-repair.timer"); - }; - subtest "Can connect via cqlsh", sub { - $cass0->waitForUnit("cassandra.service"); - $cass0->waitUntilSucceeds("nc -z cass0 9042"); - $cass0->succeed("echo 'show version;' | cqlsh cass0"); - }; - subtest "Nodetool is operational", sub { - $cass0->waitForUnit("cassandra.service"); - $cass0->waitUntilSucceeds("nc -z localhost ${jmxPortS}"); - $cass0->succeed("nodetool status -p ${jmxPortS} --resolve-ip | egrep '^UN[[:space:]]+cass0'"); - }; - subtest "Cluster name was set", sub { - $cass0->waitForUnit("cassandra.service"); - $cass0->waitUntilSucceeds("nc -z localhost ${jmxPortS}"); - $cass0->waitUntilSucceeds("nodetool describecluster -p ${jmxPortS} | grep 'Name: ${clusterName}'"); - }; - subtest "Heap limit set correctly", sub { - # Nodetool takes a while until it can display info - $cass0->waitUntilSucceeds('nodetool info -p ${jmxPortS}'); - $cass0->succeed('${checkHeapLimitCommand}'); - }; + with subtest("Timers exist"): + cass0.succeed("systemctl list-timers | grep cassandra-full-repair.timer") + cass0.succeed("systemctl list-timers | grep cassandra-incremental-repair.timer") + + with subtest("Can connect via cqlsh"): + cass0.wait_for_unit("cassandra.service") + cass0.wait_until_succeeds("nc -z cass0 9042") + cass0.succeed("echo 'show version;' | cqlsh cass0") + + with subtest("Nodetool is operational"): + cass0.wait_for_unit("cassandra.service") + cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}") + cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass0'") + + with subtest("Cluster name was set"): + cass0.wait_for_unit("cassandra.service") + cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}") + cass0.wait_until_succeeds( + "nodetool describecluster -p ${jmxPortStr} | grep 'Name: ${clusterName}'" + ) + + with subtest("Heap limit set correctly"): + # Nodetool takes a while until it can display info + cass0.wait_until_succeeds("nodetool info -p ${jmxPortStr}") + cass0.succeed("${checkHeapLimitCommand}") # Check cluster interaction - subtest "Bring up cluster", sub { - $cass1->waitForUnit("cassandra.service"); - $cass1->waitUntilSucceeds("nodetool -p ${jmxPortS} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"); - $cass0->succeed("nodetool status -p ${jmxPortS} --resolve-ip | egrep '^UN[[:space:]]+cass1'"); - }; + with subtest("Bring up cluster"): + cass1.wait_for_unit("cassandra.service") + cass1.wait_until_succeeds( + "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2" + ) + cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'") '' + lib.optionalString testRemoteAuth '' - subtest "Remote authenticated jmx", sub { - # Doesn't work if not enabled - $cass0->waitUntilSucceeds("nc -z localhost ${jmxPortS}"); - $cass1->fail("nc -z 192.168.1.1 ${toString jmxPort}"); - $cass1->fail("nodetool -p ${jmxPortS} -h 192.168.1.1 status"); + with subtest("Remote authenticated jmx"): + # Doesn't work if not enabled + cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}") + cass1.fail("nc -z 192.168.1.1 ${jmxPortStr}") + cass1.fail("nodetool -p ${jmxPortStr} -h 192.168.1.1 status") - # Works if enabled - $cass1->waitUntilSucceeds("nc -z localhost ${toString jmxPort}"); - $cass0->succeed("nodetool -p ${jmxPortS} -h 192.168.1.2 ${jmxAuthArgs} status"); - }; + # Works if enabled + cass1.wait_until_succeeds("nc -z localhost ${jmxPortStr}") + cass0.succeed("nodetool -p ${jmxPortStr} -h 192.168.1.2 ${jmxAuthArgs} status") '' + '' - subtest "Break and fix node", sub { - $cass1->block; - $cass0->waitUntilSucceeds("nodetool status -p ${jmxPortS} --resolve-ip | egrep -c '^DN[[:space:]]+cass1'"); - $cass0->succeed("nodetool status -p ${jmxPortS} | egrep -c '^UN' | grep 1"); - $cass1->unblock; - $cass1->waitUntilSucceeds("nodetool -p ${jmxPortS} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"); - $cass0->succeed("nodetool status -p ${jmxPortS} | egrep -c '^UN' | grep 2"); - }; - subtest "Replace crashed node", sub { - $cass1->crash; - $cass2->waitForUnit("cassandra.service"); - $cass0->waitUntilFails("nodetool status -p ${jmxPortS} --resolve-ip | egrep '^UN[[:space:]]+cass1'"); - $cass0->waitUntilSucceeds("nodetool status -p ${jmxPortS} --resolve-ip | egrep '^UN[[:space:]]+cass2'"); - }; + with subtest("Break and fix node"): + cass1.block() + cass0.wait_until_succeeds( + "nodetool status -p ${jmxPortStr} --resolve-ip | egrep -c '^DN[[:space:]]+cass1'" + ) + cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 1") + cass1.unblock() + cass1.wait_until_succeeds( + "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2" + ) + cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 2") + + with subtest("Replace crashed node"): + cass1.block() # .crash() waits until it's fully shutdown + cass2.start() + cass0.wait_until_fails( + "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'" + ) + + cass2.wait_for_unit("cassandra.service") + cass0.wait_until_succeeds( + "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass2'" + ) ''; }) diff --git a/nixos/tests/ceph-multi-node.nix b/nixos/tests/ceph-multi-node.nix new file mode 100644 index 00000000000..6698aac3f27 --- /dev/null +++ b/nixos/tests/ceph-multi-node.nix @@ -0,0 +1,247 @@ +import ./make-test.nix ({pkgs, lib, ...}: + +let + cfg = { + clusterId = "066ae264-2a5d-4729-8001-6ad265f50b03"; + monA = { + name = "a"; + ip = "192.168.1.1"; + }; + osd0 = { + name = "0"; + ip = "192.168.1.2"; + key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg=="; + uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9"; + }; + osd1 = { + name = "1"; + ip = "192.168.1.3"; + key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ=="; + uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5"; + }; + }; + generateCephConfig = { daemonConfig }: { + enable = true; + global = { + fsid = cfg.clusterId; + monHost = cfg.monA.ip; + monInitialMembers = cfg.monA.name; + }; + } // daemonConfig; + + generateHost = { pkgs, cephConfig, networkConfig, ... }: { + virtualisation = { + memorySize = 512; + emptyDiskImages = [ 20480 ]; + vlans = [ 1 ]; + }; + + networking = networkConfig; + + environment.systemPackages = with pkgs; [ + bash + sudo + ceph + xfsprogs + netcat-openbsd + ]; + + boot.kernelModules = [ "xfs" ]; + + services.ceph = cephConfig; + + # So that we don't have to battle systemd when bootstraping + systemd.targets.ceph.wantedBy = lib.mkForce []; + }; + + networkMonA = { + dhcpcd.enable = false; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [ + { address = cfg.monA.ip; prefixLength = 24; } + ]; + firewall = { + allowedTCPPorts = [ 6789 3300 ]; + allowedTCPPortRanges = [ { from = 6800; to = 7300; } ]; + }; + }; + cephConfigMonA = generateCephConfig { daemonConfig = { + mon = { + enable = true; + daemons = [ cfg.monA.name ]; + }; + mgr = { + enable = true; + daemons = [ cfg.monA.name ]; + }; + }; }; + + networkOsd0 = { + dhcpcd.enable = false; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [ + { address = cfg.osd0.ip; prefixLength = 24; } + ]; + firewall = { + allowedTCPPortRanges = [ { from = 6800; to = 7300; } ]; + }; + }; + cephConfigOsd0 = generateCephConfig { daemonConfig = { + osd = { + enable = true; + daemons = [ cfg.osd0.name ]; + }; + }; }; + + networkOsd1 = { + dhcpcd.enable = false; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [ + { address = cfg.osd1.ip; prefixLength = 24; } + ]; + firewall = { + allowedTCPPortRanges = [ { from = 6800; to = 7300; } ]; + }; + }; + cephConfigOsd1 = generateCephConfig { daemonConfig = { + osd = { + enable = true; + daemons = [ cfg.osd1.name ]; + }; + }; }; + + testscript = { ... }: '' + startAll; + + $monA->waitForUnit("network.target"); + $osd0->waitForUnit("network.target"); + $osd1->waitForUnit("network.target"); + + # Create the ceph-related directories + $monA->mustSucceed( + "mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}", + "mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}", + "chown ceph:ceph -R /var/lib/ceph/", + "mkdir -p /etc/ceph", + "chown ceph:ceph -R /etc/ceph" + ); + $osd0->mustSucceed( + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}", + "chown ceph:ceph -R /var/lib/ceph/", + "mkdir -p /etc/ceph", + "chown ceph:ceph -R /etc/ceph" + ); + $osd1->mustSucceed( + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}", + "chown ceph:ceph -R /var/lib/ceph/", + "mkdir -p /etc/ceph", + "chown ceph:ceph -R /etc/ceph" + ); + + # Bootstrap ceph-mon daemon + $monA->mustSucceed( + "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'", + "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'", + "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring", + "monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap", + "sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring", + "sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done", + "systemctl start ceph-mon-${cfg.monA.name}" + ); + $monA->waitForUnit("ceph-mon-${cfg.monA.name}"); + $monA->mustSucceed("ceph mon enable-msgr2"); + + # Can't check ceph status until a mon is up + $monA->succeed("ceph -s | grep 'mon: 1 daemons'"); + + # Start the ceph-mgr daemon, it has no deps and hardly any setup + $monA->mustSucceed( + "ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring", + "systemctl start ceph-mgr-${cfg.monA.name}" + ); + $monA->waitForUnit("ceph-mgr-a"); + $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'"); + $monA->waitUntilSucceeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'"); + + # Send the admin keyring to the OSD machines + $monA->mustSucceed("cp /etc/ceph/ceph.client.admin.keyring /tmp/shared"); + $osd0->mustSucceed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph"); + $osd1->mustSucceed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph"); + + # Bootstrap both OSDs + $osd0->mustSucceed( + "mkfs.xfs /dev/vdb", + "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}", + "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}", + "echo '{\"cephx_secret\": \"${cfg.osd0.key}\"}' | ceph osd new ${cfg.osd0.uuid} -i -", + ); + $osd1->mustSucceed( + "mkfs.xfs /dev/vdb", + "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}", + "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}", + "echo '{\"cephx_secret\": \"${cfg.osd1.key}\"}' | ceph osd new ${cfg.osd1.uuid} -i -" + ); + + # Initialize the OSDs with regular filestore + $osd0->mustSucceed( + "ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}", + "chown -R ceph:ceph /var/lib/ceph/osd", + "systemctl start ceph-osd-${cfg.osd0.name}", + ); + $osd1->mustSucceed( + "ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}", + "chown -R ceph:ceph /var/lib/ceph/osd", + "systemctl start ceph-osd-${cfg.osd1.name}" + ); + $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); + $monA->waitUntilSucceeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'"); + $monA->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); + + $monA->mustSucceed( + "ceph osd pool create multi-node-test 100 100", + "ceph osd pool ls | grep 'multi-node-test'", + "ceph osd pool rename multi-node-test multi-node-other-test", + "ceph osd pool ls | grep 'multi-node-other-test'" + ); + $monA->waitUntilSucceeds("ceph -s | grep '1 pools, 100 pgs'"); + $monA->mustSucceed("ceph osd pool set multi-node-other-test size 2"); + $monA->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); + $monA->waitUntilSucceeds("ceph -s | grep '100 active+clean'"); + $monA->mustFail( + "ceph osd pool ls | grep 'multi-node-test'", + "ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it" + ); + + # As we disable the target in the config, we still want to test that it works as intended + $osd0->mustSucceed("systemctl stop ceph-osd-${cfg.osd0.name}"); + $osd1->mustSucceed("systemctl stop ceph-osd-${cfg.osd1.name}"); + $monA->mustSucceed( + "systemctl stop ceph-mgr-${cfg.monA.name}", + "systemctl stop ceph-mon-${cfg.monA.name}" + ); + + $monA->succeed("systemctl start ceph.target"); + $monA->waitForUnit("ceph-mon-${cfg.monA.name}"); + $monA->waitForUnit("ceph-mgr-${cfg.monA.name}"); + $osd0->succeed("systemctl start ceph.target"); + $osd0->waitForUnit("ceph-osd-${cfg.osd0.name}"); + $osd1->succeed("systemctl start ceph.target"); + $osd1->waitForUnit("ceph-osd-${cfg.osd1.name}"); + + $monA->succeed("ceph -s | grep 'mon: 1 daemons'"); + $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'"); + $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); + $monA->waitUntilSucceeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'"); + $monA->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); + ''; +in { + name = "basic-multi-node-ceph-cluster"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ lejonet ]; + }; + + nodes = { + monA = generateHost { pkgs = pkgs; cephConfig = cephConfigMonA; networkConfig = networkMonA; }; + osd0 = generateHost { pkgs = pkgs; cephConfig = cephConfigOsd0; networkConfig = networkOsd0; }; + osd1 = generateHost { pkgs = pkgs; cephConfig = cephConfigOsd1; networkConfig = networkOsd1; }; + }; + + testScript = testscript; +}) diff --git a/nixos/tests/ceph-single-node.nix b/nixos/tests/ceph-single-node.nix new file mode 100644 index 00000000000..10b77cff5a3 --- /dev/null +++ b/nixos/tests/ceph-single-node.nix @@ -0,0 +1,193 @@ +import ./make-test.nix ({pkgs, lib, ...}: + +let + cfg = { + clusterId = "066ae264-2a5d-4729-8001-6ad265f50b03"; + monA = { + name = "a"; + ip = "192.168.1.1"; + }; + osd0 = { + name = "0"; + key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg=="; + uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9"; + }; + osd1 = { + name = "1"; + key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ=="; + uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5"; + }; + }; + generateCephConfig = { daemonConfig }: { + enable = true; + global = { + fsid = cfg.clusterId; + monHost = cfg.monA.ip; + monInitialMembers = cfg.monA.name; + }; + } // daemonConfig; + + generateHost = { pkgs, cephConfig, networkConfig, ... }: { + virtualisation = { + memorySize = 512; + emptyDiskImages = [ 20480 20480 ]; + vlans = [ 1 ]; + }; + + networking = networkConfig; + + environment.systemPackages = with pkgs; [ + bash + sudo + ceph + xfsprogs + ]; + + boot.kernelModules = [ "xfs" ]; + + services.ceph = cephConfig; + + # So that we don't have to battle systemd when bootstraping + systemd.targets.ceph.wantedBy = lib.mkForce []; + }; + + networkMonA = { + dhcpcd.enable = false; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [ + { address = cfg.monA.ip; prefixLength = 24; } + ]; + }; + cephConfigMonA = generateCephConfig { daemonConfig = { + mon = { + enable = true; + daemons = [ cfg.monA.name ]; + }; + mgr = { + enable = true; + daemons = [ cfg.monA.name ]; + }; + osd = { + enable = true; + daemons = [ cfg.osd0.name cfg.osd1.name ]; + }; + }; }; + + testscript = { ... }: '' + startAll; + + $monA->waitForUnit("network.target"); + + # Create the ceph-related directories + $monA->mustSucceed( + "mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}", + "mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}", + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}", + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}", + "mkdir -p /etc/ceph", + "chown ceph:ceph -R /etc/ceph", + "chown ceph:ceph -R /var/lib/ceph/", + ); + + # Bootstrap ceph-mon daemon + $monA->mustSucceed( + "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'", + "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'", + "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring", + "monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap", + "sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring", + "sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done", + "systemctl start ceph-mon-${cfg.monA.name}" + ); + $monA->waitForUnit("ceph-mon-${cfg.monA.name}"); + $monA->mustSucceed("ceph mon enable-msgr2"); + + # Can't check ceph status until a mon is up + $monA->succeed("ceph -s | grep 'mon: 1 daemons'"); + + # Start the ceph-mgr daemon, it has no deps and hardly any setup + $monA->mustSucceed( + "ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring", + "systemctl start ceph-mgr-${cfg.monA.name}" + ); + $monA->waitForUnit("ceph-mgr-a"); + $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'"); + $monA->waitUntilSucceeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'"); + + # Bootstrap both OSDs + $monA->mustSucceed( + "mkfs.xfs /dev/vdb", + "mkfs.xfs /dev/vdc", + "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}", + "mount /dev/vdc /var/lib/ceph/osd/ceph-${cfg.osd1.name}", + "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}", + "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}", + "echo '{\"cephx_secret\": \"${cfg.osd0.key}\"}' | ceph osd new ${cfg.osd0.uuid} -i -", + "echo '{\"cephx_secret\": \"${cfg.osd1.key}\"}' | ceph osd new ${cfg.osd1.uuid} -i -" + ); + + # Initialize the OSDs with regular filestore + $monA->mustSucceed( + "ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}", + "ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}", + "chown -R ceph:ceph /var/lib/ceph/osd", + "systemctl start ceph-osd-${cfg.osd0.name}", + "systemctl start ceph-osd-${cfg.osd1.name}" + ); + $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); + $monA->waitUntilSucceeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'"); + $monA->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); + + $monA->mustSucceed( + "ceph osd pool create single-node-test 100 100", + "ceph osd pool ls | grep 'single-node-test'", + "ceph osd pool rename single-node-test single-node-other-test", + "ceph osd pool ls | grep 'single-node-other-test'" + ); + $monA->waitUntilSucceeds("ceph -s | grep '1 pools, 100 pgs'"); + $monA->mustSucceed( + "ceph osd getcrushmap -o crush", + "crushtool -d crush -o decrushed", + "sed 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' decrushed > modcrush", + "crushtool -c modcrush -o recrushed", + "ceph osd setcrushmap -i recrushed", + "ceph osd pool set single-node-other-test size 2" + ); + $monA->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); + $monA->waitUntilSucceeds("ceph -s | grep '100 active+clean'"); + $monA->mustFail( + "ceph osd pool ls | grep 'multi-node-test'", + "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it" + ); + + # As we disable the target in the config, we still want to test that it works as intended + $monA->mustSucceed( + "systemctl stop ceph-osd-${cfg.osd0.name}", + "systemctl stop ceph-osd-${cfg.osd1.name}", + "systemctl stop ceph-mgr-${cfg.monA.name}", + "systemctl stop ceph-mon-${cfg.monA.name}" + ); + + $monA->succeed("systemctl start ceph.target"); + $monA->waitForUnit("ceph-mon-${cfg.monA.name}"); + $monA->waitForUnit("ceph-mgr-${cfg.monA.name}"); + $monA->waitForUnit("ceph-osd-${cfg.osd0.name}"); + $monA->waitForUnit("ceph-osd-${cfg.osd1.name}"); + + $monA->succeed("ceph -s | grep 'mon: 1 daemons'"); + $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'"); + $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); + $monA->waitUntilSucceeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'"); + $monA->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); + ''; +in { + name = "basic-single-node-ceph-cluster"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ lejonet johanot ]; + }; + + nodes = { + monA = generateHost { pkgs = pkgs; cephConfig = cephConfigMonA; networkConfig = networkMonA; }; + }; + + testScript = testscript; +}) diff --git a/nixos/tests/ceph.nix b/nixos/tests/ceph.nix deleted file mode 100644 index 57120ff978f..00000000000 --- a/nixos/tests/ceph.nix +++ /dev/null @@ -1,161 +0,0 @@ -import ./make-test.nix ({pkgs, lib, ...}: { - name = "All-in-one-basic-ceph-cluster"; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ johanot lejonet ]; - }; - - nodes = { - aio = { pkgs, ... }: { - virtualisation = { - memorySize = 1536; - emptyDiskImages = [ 20480 20480 ]; - vlans = [ 1 ]; - }; - - networking = { - useDHCP = false; - interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [ - { address = "192.168.1.1"; prefixLength = 24; } - ]; - }; - - environment.systemPackages = with pkgs; [ - bash - sudo - ceph - xfsprogs - ]; - - boot.kernelModules = [ "xfs" ]; - - services.ceph.enable = true; - services.ceph.global = { - fsid = "066ae264-2a5d-4729-8001-6ad265f50b03"; - monInitialMembers = "aio"; - monHost = "192.168.1.1"; - }; - - services.ceph.mon = { - enable = true; - daemons = [ "aio" ]; - }; - - services.ceph.mgr = { - enable = true; - daemons = [ "aio" ]; - }; - - services.ceph.osd = { - enable = true; - daemons = [ "0" "1" ]; - }; - - # So that we don't have to battle systemd when bootstraping - systemd.targets.ceph.wantedBy = lib.mkForce []; - }; - }; - - testScript = { ... }: '' - startAll; - - $aio->waitForUnit("network.target"); - - # Create the ceph-related directories - $aio->mustSucceed( - "mkdir -p /var/lib/ceph/mgr/ceph-aio", - "mkdir -p /var/lib/ceph/mon/ceph-aio", - "mkdir -p /var/lib/ceph/osd/ceph-{0,1}", - "chown ceph:ceph -R /var/lib/ceph/", - "mkdir -p /etc/ceph", - "chown ceph:ceph -R /etc/ceph" - ); - - # Bootstrap ceph-mon daemon - $aio->mustSucceed( - "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'", - "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'", - "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring", - "monmaptool --create --add aio 192.168.1.1 --fsid 066ae264-2a5d-4729-8001-6ad265f50b03 /tmp/monmap", - "sudo -u ceph ceph-mon --mkfs -i aio --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring", - "sudo -u ceph touch /var/lib/ceph/mon/ceph-aio/done", - "systemctl start ceph-mon-aio" - ); - $aio->waitForUnit("ceph-mon-aio"); - $aio->mustSucceed("ceph mon enable-msgr2"); - - # Can't check ceph status until a mon is up - $aio->succeed("ceph -s | grep 'mon: 1 daemons'"); - - # Start the ceph-mgr daemon, it has no deps and hardly any setup - $aio->mustSucceed( - "ceph auth get-or-create mgr.aio mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-aio/keyring", - "systemctl start ceph-mgr-aio" - ); - $aio->waitForUnit("ceph-mgr-aio"); - $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'"); - $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'"); - - # Bootstrap both OSDs - $aio->mustSucceed( - "mkfs.xfs /dev/vdb", - "mkfs.xfs /dev/vdc", - "mount /dev/vdb /var/lib/ceph/osd/ceph-0", - "mount /dev/vdc /var/lib/ceph/osd/ceph-1", - "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-0/keyring --name osd.0 --add-key AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==", - "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-1/keyring --name osd.1 --add-key AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==", - "echo '{\"cephx_secret\": \"AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==\"}' | ceph osd new 55ba2294-3e24-478f-bee0-9dca4c231dd9 -i -", - "echo '{\"cephx_secret\": \"AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==\"}' | ceph osd new 5e97a838-85b6-43b0-8950-cb56d554d1e5 -i -" - ); - - # Initialize the OSDs with regular filestore - $aio->mustSucceed( - "ceph-osd -i 0 --mkfs --osd-uuid 55ba2294-3e24-478f-bee0-9dca4c231dd9", - "ceph-osd -i 1 --mkfs --osd-uuid 5e97a838-85b6-43b0-8950-cb56d554d1e5", - "chown -R ceph:ceph /var/lib/ceph/osd", - "systemctl start ceph-osd-0", - "systemctl start ceph-osd-1" - ); - - $aio->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); - $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'"); - $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); - - $aio->mustSucceed( - "ceph osd pool create aio-test 100 100", - "ceph osd pool ls | grep 'aio-test'", - "ceph osd pool rename aio-test aio-other-test", - "ceph osd pool ls | grep 'aio-other-test'", - "ceph -s | grep '1 pools, 100 pgs'", - "ceph osd getcrushmap -o crush", - "crushtool -d crush -o decrushed", - "sed 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' decrushed > modcrush", - "crushtool -c modcrush -o recrushed", - "ceph osd setcrushmap -i recrushed", - "ceph osd pool set aio-other-test size 2" - ); - $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); - $aio->waitUntilSucceeds("ceph -s | grep '100 active+clean'"); - $aio->mustFail( - "ceph osd pool ls | grep 'aio-test'", - "ceph osd pool delete aio-other-test aio-other-test --yes-i-really-really-mean-it" - ); - - # As we disable the target in the config, we still want to test that it works as intended - $aio->mustSucceed( - "systemctl stop ceph-osd-0", - "systemctl stop ceph-osd-1", - "systemctl stop ceph-mgr-aio", - "systemctl stop ceph-mon-aio" - ); - $aio->succeed("systemctl start ceph.target"); - $aio->waitForUnit("ceph-mon-aio"); - $aio->waitForUnit("ceph-mgr-aio"); - $aio->waitForUnit("ceph-osd-0"); - $aio->waitForUnit("ceph-osd-1"); - $aio->succeed("ceph -s | grep 'mon: 1 daemons'"); - $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'"); - $aio->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); - $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'"); - $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); - ''; -}) diff --git a/nixos/tests/certmgr.nix b/nixos/tests/certmgr.nix index fe67833808c..cb69f35e862 100644 --- a/nixos/tests/certmgr.nix +++ b/nixos/tests/certmgr.nix @@ -3,7 +3,7 @@ pkgs ? import ../.. { inherit system config; } }: -with import ../lib/testing.nix { inherit system pkgs; }; +with import ../lib/testing-python.nix { inherit system pkgs; }; let mkSpec = { host, service ? null, action }: { inherit action; @@ -123,17 +123,17 @@ in ))); }; testScript = '' - $machine->waitForUnit('cfssl.service'); - $machine->waitUntilSucceeds('ls /tmp/decl.example.org-ca.pem'); - $machine->waitUntilSucceeds('ls /tmp/decl.example.org-key.pem'); - $machine->waitUntilSucceeds('ls /tmp/decl.example.org-cert.pem'); - $machine->waitUntilSucceeds('ls /tmp/imp.example.org-ca.pem'); - $machine->waitUntilSucceeds('ls /tmp/imp.example.org-key.pem'); - $machine->waitUntilSucceeds('ls /tmp/imp.example.org-cert.pem'); - $machine->waitForUnit('nginx.service'); - $machine->succeed('[ "1" -lt "$(journalctl -u nginx | grep "Starting Nginx" | wc -l)" ]'); - $machine->succeed('curl --cacert /tmp/imp.example.org-ca.pem https://imp.example.org'); - $machine->succeed('curl --cacert /tmp/decl.example.org-ca.pem https://decl.example.org'); + machine.wait_for_unit("cfssl.service") + machine.wait_until_succeeds("ls /tmp/decl.example.org-ca.pem") + machine.wait_until_succeeds("ls /tmp/decl.example.org-key.pem") + machine.wait_until_succeeds("ls /tmp/decl.example.org-cert.pem") + machine.wait_until_succeeds("ls /tmp/imp.example.org-ca.pem") + machine.wait_until_succeeds("ls /tmp/imp.example.org-key.pem") + machine.wait_until_succeeds("ls /tmp/imp.example.org-cert.pem") + machine.wait_for_unit("nginx.service") + assert 1 < int(machine.succeed('journalctl -u nginx | grep "Starting Nginx" | wc -l')) + machine.succeed("curl --cacert /tmp/imp.example.org-ca.pem https://imp.example.org") + machine.succeed("curl --cacert /tmp/decl.example.org-ca.pem https://decl.example.org") ''; }; @@ -143,8 +143,8 @@ in test = mkSpec { host = "command.example.org"; action = "touch /tmp/command.executed"; }; }; testScript = '' - $machine->waitForUnit('cfssl.service'); - $machine->waitUntilSucceeds('stat /tmp/command.executed'); + machine.wait_for_unit("cfssl.service") + machine.wait_until_succeeds("stat /tmp/command.executed") ''; }; diff --git a/nixos/tests/cfssl.nix b/nixos/tests/cfssl.nix index 513ed8c4574..e291fc285fb 100644 --- a/nixos/tests/cfssl.nix +++ b/nixos/tests/cfssl.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "cfssl"; machine = { config, lib, pkgs, ... }: @@ -60,8 +60,8 @@ import ./make-test.nix ({ pkgs, ...} : { }); in '' - $machine->waitForUnit('cfssl.service'); - $machine->waitUntilSucceeds('${cfsslrequest}'); - $machine->succeed('ls /tmp/certificate-key.pem'); + machine.wait_for_unit("cfssl.service") + machine.wait_until_succeeds("${cfsslrequest}") + machine.succeed("ls /tmp/certificate-key.pem") ''; }) diff --git a/nixos/tests/cjdns.nix b/nixos/tests/cjdns.nix index 6660eecf05b..d72236d415d 100644 --- a/nixos/tests/cjdns.nix +++ b/nixos/tests/cjdns.nix @@ -17,7 +17,7 @@ let in -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "cjdns"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ ehmry ]; @@ -83,36 +83,39 @@ import ./make-test.nix ({ pkgs, ...} : { testScript = '' - startAll; + import re - $alice->waitForUnit("cjdns.service"); - $bob->waitForUnit("cjdns.service"); - $carol->waitForUnit("cjdns.service"); + start_all() - sub cjdnsIp { - my ($machine) = @_; - my $ip = (split /[ \/]+/, $machine->succeed("ip -o -6 addr show dev tun0"))[3]; - $machine->log("has ip $ip"); - return $ip; - } + alice.wait_for_unit("cjdns.service") + bob.wait_for_unit("cjdns.service") + carol.wait_for_unit("cjdns.service") - my $aliceIp6 = cjdnsIp $alice; - my $bobIp6 = cjdnsIp $bob; - my $carolIp6 = cjdnsIp $carol; + + def cjdns_ip(machine): + res = machine.succeed("ip -o -6 addr show dev tun0") + ip = re.split("\s+|/", res)[3] + machine.log("has ip {}".format(ip)) + return ip + + + alice_ip6 = cjdns_ip(alice) + bob_ip6 = cjdns_ip(bob) + carol_ip6 = cjdns_ip(carol) # ping a few times each to let the routing table establish itself - $alice->succeed("ping -c 4 $carolIp6"); - $bob->succeed("ping -c 4 $carolIp6"); + alice.succeed("ping -c 4 {}".format(carol_ip6)) + bob.succeed("ping -c 4 {}".format(carol_ip6)) - $carol->succeed("ping -c 4 $aliceIp6"); - $carol->succeed("ping -c 4 $bobIp6"); + carol.succeed("ping -c 4 {}".format(alice_ip6)) + carol.succeed("ping -c 4 {}".format(bob_ip6)) - $alice->succeed("ping -c 4 $bobIp6"); - $bob->succeed("ping -c 4 $aliceIp6"); + alice.succeed("ping -c 4 {}".format(bob_ip6)) + bob.succeed("ping -c 4 {}".format(alice_ip6)) - $alice->waitForUnit("httpd.service"); + alice.wait_for_unit("httpd.service") - $bob->succeed("curl --fail -g http://[$aliceIp6]"); + bob.succeed("curl --fail -g http://[{}]".format(alice_ip6)) ''; }) diff --git a/nixos/tests/cloud-init.nix b/nixos/tests/cloud-init.nix index 516d29c9036..aafa6e24e84 100644 --- a/nixos/tests/cloud-init.nix +++ b/nixos/tests/cloud-init.nix @@ -3,7 +3,7 @@ pkgs ? import ../.. { inherit system config; } }: -with import ../lib/testing.nix { inherit system pkgs; }; +with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let @@ -30,6 +30,7 @@ let ''; }; in makeTest { + name = "cloud-init"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ lewo ]; }; @@ -40,10 +41,12 @@ in makeTest { services.cloud-init.enable = true; }; testScript = '' - $machine->start; - $machine->waitForUnit("cloud-init.service"); - $machine->succeed("cat /tmp/cloudinit-write-file | grep -q 'cloudinit'"); + machine.start() + machine.wait_for_unit("cloud-init.service") + machine.succeed("cat /tmp/cloudinit-write-file | grep -q 'cloudinit'") - $machine->waitUntilSucceeds("cat /root/.ssh/authorized_keys | grep -q 'should be a key!'"); + machine.wait_until_succeeds( + "cat /root/.ssh/authorized_keys | grep -q 'should be a key!'" + ) ''; } diff --git a/nixos/tests/colord.nix b/nixos/tests/colord.nix deleted file mode 100644 index ce38aaca4bf..00000000000 --- a/nixos/tests/colord.nix +++ /dev/null @@ -1,18 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: - -{ - name = "colord"; - - meta = { - maintainers = pkgs.colord.meta.maintainers; - }; - - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.colord.installedTests}/share'"); - ''; -}) diff --git a/nixos/tests/common/letsencrypt/0001-Change-ACME-directory-endpoint-to-directory.patch b/nixos/tests/common/letsencrypt/0001-Change-ACME-directory-endpoint-to-directory.patch deleted file mode 100644 index 9d4a483dd88..00000000000 --- a/nixos/tests/common/letsencrypt/0001-Change-ACME-directory-endpoint-to-directory.patch +++ /dev/null @@ -1,25 +0,0 @@ -From c3b4004386074342d22cab5e129c1f7e623f4272 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= <felix@alternativebit.fr> -Date: Mon, 21 Oct 2019 10:56:13 +0200 -Subject: [PATCH] Change ACME directory endpoint to /directory - ---- - wfe/wfe.go | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/wfe/wfe.go b/wfe/wfe.go -index e24797f..10d29fb 100644 ---- a/wfe/wfe.go -+++ b/wfe/wfe.go -@@ -39,7 +39,7 @@ const ( - // Note: We deliberately pick endpoint paths that differ from Boulder to - // exercise clients processing of the /directory response - // We export the DirectoryPath so that the pebble binary can reference it -- DirectoryPath = "/dir" -+ DirectoryPath = "/directory" - noncePath = "/nonce-plz" - newAccountPath = "/sign-me-up" - acctPath = "/my-account/" --- -2.23.0 - diff --git a/nixos/tests/common/letsencrypt/default.nix b/nixos/tests/common/letsencrypt/default.nix index aaf2896f21c..110a2520971 100644 --- a/nixos/tests/common/letsencrypt/default.nix +++ b/nixos/tests/common/letsencrypt/default.nix @@ -62,17 +62,7 @@ let siteDomain = "letsencrypt.org"; siteCertFile = snakeOilCerts.${siteDomain}.cert; siteKeyFile = snakeOilCerts.${siteDomain}.key; - pebble = pkgs.pebble.overrideAttrs (attrs: { - # The pebble directory endpoint is /dir when the bouder (official - # ACME server) is /directory. Sadly, this endpoint is hardcoded, - # we have to patch it. - # - # Tried to upstream, that said upstream maintainers rather keep - # this custom endpoint to test ACME clients robustness. See - # https://github.com/letsencrypt/pebble/issues/283#issuecomment-545123242 - patches = [ ./0001-Change-ACME-directory-endpoint-to-directory.patch ]; - }); - + pebble = pkgs.pebble; resolver = let message = "You need to define a resolver for the letsencrypt test module."; firstNS = lib.head config.networking.nameservers; diff --git a/nixos/tests/couchdb.nix b/nixos/tests/couchdb.nix index 48ea48eebbb..10e95701acd 100644 --- a/nixos/tests/couchdb.nix +++ b/nixos/tests/couchdb.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, lib, ...}: +import ./make-test-python.nix ({ pkgs, lib, ...}: with lib; @@ -35,22 +35,42 @@ with lib; fi ''; in '' - startAll; - - $couchdb1->waitForUnit("couchdb.service"); - $couchdb1->waitUntilSucceeds("${curlJqCheck "GET" "" ".couchdb" "Welcome"}"); - $couchdb1->waitUntilSucceeds("${curlJqCheck "GET" "_all_dbs" ". | length" "2"}"); - $couchdb1->succeed("${curlJqCheck "PUT" "foo" ".ok" "true"}"); - $couchdb1->succeed("${curlJqCheck "GET" "_all_dbs" ". | length" "3"}"); - $couchdb1->succeed("${curlJqCheck "DELETE" "foo" ".ok" "true"}"); - $couchdb1->succeed("${curlJqCheck "GET" "_all_dbs" ". | length" "2"}"); - - $couchdb2->waitForUnit("couchdb.service"); - $couchdb2->waitUntilSucceeds("${curlJqCheck "GET" "" ".couchdb" "Welcome"}"); - $couchdb2->waitUntilSucceeds("${curlJqCheck "GET" "_all_dbs" ". | length" "0"}"); - $couchdb2->succeed("${curlJqCheck "PUT" "foo" ".ok" "true"}"); - $couchdb2->succeed("${curlJqCheck "GET" "_all_dbs" ". | length" "1"}"); - $couchdb2->succeed("${curlJqCheck "DELETE" "foo" ".ok" "true"}"); - $couchdb2->succeed("${curlJqCheck "GET" "_all_dbs" ". | length" "0"}"); + start_all() + + couchdb1.wait_for_unit("couchdb.service") + couchdb1.wait_until_succeeds( + "${curlJqCheck "GET" "" ".couchdb" "Welcome"}" + ) + couchdb1.wait_until_succeeds( + "${curlJqCheck "GET" "_all_dbs" ". | length" "2"}" + ) + couchdb1.succeed("${curlJqCheck "PUT" "foo" ".ok" "true"}") + couchdb1.succeed( + "${curlJqCheck "GET" "_all_dbs" ". | length" "3"}" + ) + couchdb1.succeed( + "${curlJqCheck "DELETE" "foo" ".ok" "true"}" + ) + couchdb1.succeed( + "${curlJqCheck "GET" "_all_dbs" ". | length" "2"}" + ) + + couchdb2.wait_for_unit("couchdb.service") + couchdb2.wait_until_succeeds( + "${curlJqCheck "GET" "" ".couchdb" "Welcome"}" + ) + couchdb2.wait_until_succeeds( + "${curlJqCheck "GET" "_all_dbs" ". | length" "0"}" + ) + couchdb2.succeed("${curlJqCheck "PUT" "foo" ".ok" "true"}") + couchdb2.succeed( + "${curlJqCheck "GET" "_all_dbs" ". | length" "1"}" + ) + couchdb2.succeed( + "${curlJqCheck "DELETE" "foo" ".ok" "true"}" + ) + couchdb2.succeed( + "${curlJqCheck "GET" "_all_dbs" ". | length" "0"}" + ) ''; }) diff --git a/nixos/tests/dnscrypt-proxy.nix b/nixos/tests/dnscrypt-proxy.nix index 13bc9d3d916..98153d5c904 100644 --- a/nixos/tests/dnscrypt-proxy.nix +++ b/nixos/tests/dnscrypt-proxy.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: { +import ./make-test-python.nix ({ pkgs, ... }: { name = "dnscrypt-proxy"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ joachifm ]; @@ -23,11 +23,13 @@ import ./make-test.nix ({ pkgs, ... }: { }; testScript = '' - $client->waitForUnit("dnsmasq"); + client.wait_for_unit("dnsmasq") # The daemon is socket activated; sending a single ping should activate it. - $client->fail("systemctl is-active dnscrypt-proxy"); - $client->execute("${pkgs.iputils}/bin/ping -c1 example.com"); - $client->waitUntilSucceeds("systemctl is-active dnscrypt-proxy"); + client.fail("systemctl is-active dnscrypt-proxy") + client.execute( + "${pkgs.iputils}/bin/ping -c1 example.com" + ) + client.wait_until_succeeds("systemctl is-active dnscrypt-proxy") ''; }) diff --git a/nixos/tests/docker-edge.nix b/nixos/tests/docker-edge.nix index b306c149be9..96de885a554 100644 --- a/nixos/tests/docker-edge.nix +++ b/nixos/tests/docker-edge.nix @@ -1,6 +1,6 @@ # This test runs docker and checks if simple container starts -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "docker"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ nequissimus offline ]; @@ -31,17 +31,19 @@ import ./make-test.nix ({ pkgs, ...} : { }; testScript = '' - startAll; + start_all() - $docker->waitForUnit("sockets.target"); - $docker->succeed("tar cv --files-from /dev/null | docker import - scratchimg"); - $docker->succeed("docker run -d --name=sleeping -v /nix/store:/nix/store -v /run/current-system/sw/bin:/bin scratchimg /bin/sleep 10"); - $docker->succeed("docker ps | grep sleeping"); - $docker->succeed("sudo -u hasprivs docker ps"); - $docker->fail("sudo -u noprivs docker ps"); - $docker->succeed("docker stop sleeping"); + docker.wait_for_unit("sockets.target") + docker.succeed("tar cv --files-from /dev/null | docker import - scratchimg") + docker.succeed( + "docker run -d --name=sleeping -v /nix/store:/nix/store -v /run/current-system/sw/bin:/bin scratchimg /bin/sleep 10" + ) + docker.succeed("docker ps | grep sleeping") + docker.succeed("sudo -u hasprivs docker ps") + docker.fail("sudo -u noprivs docker ps") + docker.succeed("docker stop sleeping") # Must match version twice to ensure client and server versions are correct - $docker->succeed('[ $(docker version | grep ${pkgs.docker-edge.version} | wc -l) = "2" ]'); + docker.succeed('[ $(docker version | grep ${pkgs.docker-edge.version} | wc -l) = "2" ]') ''; }) diff --git a/nixos/tests/docker.nix b/nixos/tests/docker.nix index d67b2f8743d..8fda7c1395e 100644 --- a/nixos/tests/docker.nix +++ b/nixos/tests/docker.nix @@ -1,6 +1,6 @@ # This test runs docker and checks if simple container starts -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "docker"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ nequissimus offline ]; @@ -31,17 +31,19 @@ import ./make-test.nix ({ pkgs, ...} : { }; testScript = '' - startAll; + start_all() - $docker->waitForUnit("sockets.target"); - $docker->succeed("tar cv --files-from /dev/null | docker import - scratchimg"); - $docker->succeed("docker run -d --name=sleeping -v /nix/store:/nix/store -v /run/current-system/sw/bin:/bin scratchimg /bin/sleep 10"); - $docker->succeed("docker ps | grep sleeping"); - $docker->succeed("sudo -u hasprivs docker ps"); - $docker->fail("sudo -u noprivs docker ps"); - $docker->succeed("docker stop sleeping"); + docker.wait_for_unit("sockets.target") + docker.succeed("tar cv --files-from /dev/null | docker import - scratchimg") + docker.succeed( + "docker run -d --name=sleeping -v /nix/store:/nix/store -v /run/current-system/sw/bin:/bin scratchimg /bin/sleep 10" + ) + docker.succeed("docker ps | grep sleeping") + docker.succeed("sudo -u hasprivs docker ps") + docker.fail("sudo -u noprivs docker ps") + docker.succeed("docker stop sleeping") # Must match version twice to ensure client and server versions are correct - $docker->succeed('[ $(docker version | grep ${pkgs.docker.version} | wc -l) = "2" ]'); + docker.succeed('[ $(docker version | grep ${pkgs.docker.version} | wc -l) = "2" ]') ''; }) diff --git a/nixos/tests/documize.nix b/nixos/tests/documize.nix index 8b852a4f779..3be20a780d3 100644 --- a/nixos/tests/documize.nix +++ b/nixos/tests/documize.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, lib, ...} : { +import ./make-test-python.nix ({ pkgs, lib, ...} : { name = "documize"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ ma27 ]; @@ -29,30 +29,34 @@ import ./make-test.nix ({ pkgs, lib, ...} : { }; testScript = '' - startAll; - - $machine->waitForUnit("documize-server.service"); - $machine->waitForOpenPort(3000); - - my $dbhash = $machine->succeed("curl -f localhost:3000 " - . " | grep 'property=\"dbhash' " - . " | grep -Po 'content=\"\\K[^\"]*'" - ); - - chomp($dbhash); - - $machine->succeed("curl -X POST " - . "--data 'dbname=documize' " - . "--data 'dbhash=$dbhash' " - . "--data 'title=NixOS' " - . "--data 'message=Docs' " - . "--data 'firstname=John' " - . "--data 'lastname=Doe' " - . "--data 'email=john.doe\@nixos.org' " - . "--data 'password=verysafe' " - . "-f localhost:3000/api/setup" - ); - - $machine->succeed('test "$(curl -f localhost:3000/api/public/meta | jq ".title" | xargs echo)" = "NixOS"'); + start_all() + + machine.wait_for_unit("documize-server.service") + machine.wait_for_open_port(3000) + + dbhash = machine.succeed( + "curl -f localhost:3000 | grep 'property=\"dbhash' | grep -Po 'content=\"\\K[^\"]*'" + ) + + dbhash = dbhash.strip() + + machine.succeed( + ( + "curl -X POST" + " --data 'dbname=documize'" + " --data 'dbhash={}'" + " --data 'title=NixOS'" + " --data 'message=Docs'" + " --data 'firstname=John'" + " --data 'lastname=Doe'" + " --data 'email=john.doe@nixos.org'" + " --data 'password=verysafe'" + " -f localhost:3000/api/setup" + ).format(dbhash) + ) + + machine.succeed( + 'test "$(curl -f localhost:3000/api/public/meta | jq ".title" | xargs echo)" = "NixOS"' + ) ''; }) diff --git a/nixos/tests/emacs-daemon.nix b/nixos/tests/emacs-daemon.nix index 3594e35e343..b89d9b1bde6 100644 --- a/nixos/tests/emacs-daemon.nix +++ b/nixos/tests/emacs-daemon.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "emacs-daemon"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ ]; @@ -21,25 +21,28 @@ import ./make-test.nix ({ pkgs, ...} : { environment.variables.TEST_SYSTEM_VARIABLE = "system variable"; }; - testScript = - '' - $machine->waitForUnit("multi-user.target"); + testScript = '' + machine.wait_for_unit("multi-user.target") # checks that the EDITOR environment variable is set - $machine->succeed("test \$(basename \"\$EDITOR\") = emacseditor"); + machine.succeed('test $(basename "$EDITOR") = emacseditor') # waits for the emacs service to be ready - $machine->waitUntilSucceeds("systemctl --user status emacs.service | grep 'Active: active'"); + machine.wait_until_succeeds( + "systemctl --user status emacs.service | grep 'Active: active'" + ) # connects to the daemon - $machine->succeed("emacsclient --create-frame \$EDITOR &"); + machine.succeed("emacsclient --create-frame $EDITOR &") # checks that Emacs shows the edited filename - $machine->waitForText("emacseditor"); + machine.wait_for_text("emacseditor") # makes sure environment variables are accessible from Emacs - $machine->succeed("emacsclient --eval '(getenv \"TEST_SYSTEM_VARIABLE\")'") =~ /system variable/ or die; + machine.succeed( + "emacsclient --eval '(getenv \"TEST_SYSTEM_VARIABLE\")' | grep -q 'system variable'" + ) - $machine->screenshot("emacsclient"); + machine.screenshot("emacsclient") ''; }) diff --git a/nixos/tests/fancontrol.nix b/nixos/tests/fancontrol.nix new file mode 100644 index 00000000000..83ddbb54c5b --- /dev/null +++ b/nixos/tests/fancontrol.nix @@ -0,0 +1,25 @@ +import ./make-test.nix ({ pkgs, ... } : { + name = "fancontrol"; + + machine = + { ... }: + { hardware.fancontrol.enable = true; + hardware.fancontrol.config = '' + INTERVAL=42 + DEVPATH=hwmon1=devices/platform/dummy + DEVNAME=hwmon1=dummy + FCTEMPS=hwmon1/device/pwm1=hwmon1/device/temp1_input + FCFANS=hwmon1/device/pwm1=hwmon1/device/fan1_input + MINTEMP=hwmon1/device/pwm1=25 + MAXTEMP=hwmon1/device/pwm1=65 + MINSTART=hwmon1/device/pwm1=150 + MINSTOP=hwmon1/device/pwm1=0 + ''; + }; + + # This configuration cannot be valid for the test VM, so it's expected to get an 'outdated' error. + testScript = '' + $machine->waitForUnit("fancontrol.service"); + $machine->waitUntilSucceeds("journalctl -eu fancontrol | grep 'Configuration appears to be outdated'"); + ''; +}) diff --git a/nixos/tests/firefox.nix b/nixos/tests/firefox.nix index f5b946a0881..56ddabbae77 100644 --- a/nixos/tests/firefox.nix +++ b/nixos/tests/firefox.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: { +import ./make-test-python.nix ({ pkgs, ... }: { name = "firefox"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ eelco shlevy ]; @@ -11,19 +11,27 @@ import ./make-test.nix ({ pkgs, ... }: { environment.systemPackages = [ pkgs.firefox pkgs.xdotool ]; }; - testScript = - '' - $machine->waitForX; - $machine->execute("xterm -e 'firefox file://${pkgs.valgrind.doc}/share/doc/valgrind/html/index.html' &"); - $machine->waitForWindow(qr/Valgrind/); - $machine->sleep(40); # wait until Firefox has finished loading the page - $machine->execute("xdotool key space"); # do I want to make Firefox the - # default browser? I just want to close the dialog - $machine->sleep(2); # wait until Firefox hides the default browser window - $machine->execute("xdotool key F12"); - $machine->sleep(10); # wait until Firefox draws the developer tool panel - $machine->succeed("xwininfo -root -tree | grep Valgrind"); - $machine->screenshot("screen"); + testScript = '' + machine.wait_for_x() + + with subtest("wait until Firefox has finished loading the Valgrind docs page"): + machine.execute( + "xterm -e 'firefox file://${pkgs.valgrind.doc}/share/doc/valgrind/html/index.html' &" + ) + machine.wait_for_window("Valgrind") + machine.sleep(40) + + with subtest("Close default browser prompt"): + machine.execute("xdotool key space") + + with subtest("Hide default browser window"): + machine.sleep(2) + machine.execute("xdotool key F12") + + with subtest("wait until Firefox draws the developer tool panel"): + machine.sleep(10) + machine.succeed("xwininfo -root -tree | grep Valgrind") + machine.screenshot("screen") ''; }) diff --git a/nixos/tests/flatpak-builder.nix b/nixos/tests/flatpak-builder.nix deleted file mode 100644 index 49b97e8ca99..00000000000 --- a/nixos/tests/flatpak-builder.nix +++ /dev/null @@ -1,20 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: - -{ - name = "flatpak-builder"; - meta = { - maintainers = pkgs.flatpak-builder.meta.maintainers; - }; - - machine = { pkgs, ... }: { - services.flatpak.enable = true; - xdg.portal.enable = true; - environment.systemPackages = with pkgs; [ gnome-desktop-testing flatpak-builder ] ++ flatpak-builder.installedTestsDependencies; - virtualisation.diskSize = 2048; - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.flatpak-builder.installedTests}/share' --timeout 3600"); - ''; -}) diff --git a/nixos/tests/flatpak.nix b/nixos/tests/flatpak.nix deleted file mode 100644 index b0c61830d05..00000000000 --- a/nixos/tests/flatpak.nix +++ /dev/null @@ -1,26 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: - -{ - name = "flatpak"; - meta = { - maintainers = pkgs.flatpak.meta.maintainers; - }; - - machine = { pkgs, ... }: { - imports = [ ./common/x11.nix ]; - services.xserver.desktopManager.gnome3.enable = true; # TODO: figure out minimal environment where the tests work - # common/x11.nix enables the auto display manager (lightdm) - services.xserver.displayManager.gdm.enable = false; - environment.gnome3.excludePackages = pkgs.gnome3.optionalPackages; - services.flatpak.enable = true; - environment.systemPackages = with pkgs; [ gnupg gnome-desktop-testing ostree python2 ]; - virtualisation.memorySize = 2047; - virtualisation.diskSize = 1024; - }; - - testScript = '' - $machine->waitForX(); - $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.flatpak.installedTests}/share' --timeout 3600"); - ''; -}) diff --git a/nixos/tests/fontconfig-default-fonts.nix b/nixos/tests/fontconfig-default-fonts.nix index 1991cec9218..68c6ac9e9c8 100644 --- a/nixos/tests/fontconfig-default-fonts.nix +++ b/nixos/tests/fontconfig-default-fonts.nix @@ -1,7 +1,12 @@ -import ./make-test.nix ({ lib, ... }: +import ./make-test-python.nix ({ lib, ... }: { name = "fontconfig-default-fonts"; + meta.maintainers = with lib.maintainers; [ + jtojnar + worldofpeace + ]; + machine = { config, pkgs, ... }: { fonts.enableDefaultFonts = true; # Background fonts fonts.fonts = with pkgs; [ @@ -20,9 +25,9 @@ import ./make-test.nix ({ lib, ... }: }; testScript = '' - $machine->succeed("fc-match serif | grep '\"Gentium Plus\"'"); - $machine->succeed("fc-match sans-serif | grep '\"Cantarell\"'"); - $machine->succeed("fc-match monospace | grep '\"Source Code Pro\"'"); - $machine->succeed("fc-match emoji | grep '\"Twitter Color Emoji\"'"); + machine.succeed("fc-match serif | grep '\"Gentium Plus\"'") + machine.succeed("fc-match sans-serif | grep '\"Cantarell\"'") + machine.succeed("fc-match monospace | grep '\"Source Code Pro\"'") + machine.succeed("fc-match emoji | grep '\"Twitter Color Emoji\"'") ''; }) diff --git a/nixos/tests/fsck.nix b/nixos/tests/fsck.nix index f943bb7f235..e522419fde2 100644 --- a/nixos/tests/fsck.nix +++ b/nixos/tests/fsck.nix @@ -1,4 +1,4 @@ -import ./make-test.nix { +import ./make-test-python.nix { name = "fsck"; machine = { lib, ... }: { @@ -14,16 +14,18 @@ import ./make-test.nix { }; testScript = '' - $machine->waitForUnit('default.target'); + machine.wait_for_unit("default.target") - subtest "root fs is fsckd", sub { - $machine->succeed('journalctl -b | grep "fsck.ext4.*/dev/vda"'); - }; + with subtest("root fs is fsckd"): + machine.succeed("journalctl -b | grep 'fsck.ext4.*/dev/vda'") - subtest "mnt fs is fsckd", sub { - $machine->succeed('journalctl -b | grep "fsck.*/dev/vdb.*clean"'); - $machine->succeed('grep "Requires=systemd-fsck@dev-vdb.service" /run/systemd/generator/mnt.mount'); - $machine->succeed('grep "After=systemd-fsck@dev-vdb.service" /run/systemd/generator/mnt.mount'); - }; + with subtest("mnt fs is fsckd"): + machine.succeed("journalctl -b | grep 'fsck.*/dev/vdb.*clean'") + machine.succeed( + "grep 'Requires=systemd-fsck@dev-vdb.service' /run/systemd/generator/mnt.mount" + ) + machine.succeed( + "grep 'After=systemd-fsck@dev-vdb.service' /run/systemd/generator/mnt.mount" + ) ''; } diff --git a/nixos/tests/fwupd.nix b/nixos/tests/fwupd.nix deleted file mode 100644 index 88dac8ccbcd..00000000000 --- a/nixos/tests/fwupd.nix +++ /dev/null @@ -1,21 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: { - name = "fwupd"; - - meta = { - maintainers = pkgs.fwupd.meta.maintainers; - }; - - machine = { pkgs, ... }: { - services.fwupd.enable = true; - services.fwupd.blacklistPlugins = []; # don't blacklist test plugin - services.fwupd.enableTestRemote = true; - environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; - environment.variables.XDG_DATA_DIRS = [ "${pkgs.fwupd.installedTests}/share" ]; - virtualisation.memorySize = 768; - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner"); - ''; -}) diff --git a/nixos/tests/gdk-pixbuf.nix b/nixos/tests/gdk-pixbuf.nix deleted file mode 100644 index 9a62b593f46..00000000000 --- a/nixos/tests/gdk-pixbuf.nix +++ /dev/null @@ -1,21 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: { - name = "gdk-pixbuf"; - - meta = { - maintainers = pkgs.gdk-pixbuf.meta.maintainers; - }; - - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; - environment.variables.XDG_DATA_DIRS = [ "${pkgs.gdk-pixbuf.installedTests}/share" ]; - - # Tests allocate a lot of memory trying to exploit a CVE - # but qemu-system-i386 has a 2047M memory limit - virtualisation.memorySize = if pkgs.stdenv.isi686 then 2047 else 4096; - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner -t 1800"); # increase timeout to 1800s - ''; -}) diff --git a/nixos/tests/gitea.nix b/nixos/tests/gitea.nix index b8ab6dabc8c..ffbc07cfbb2 100644 --- a/nixos/tests/gitea.nix +++ b/nixos/tests/gitea.nix @@ -3,7 +3,7 @@ pkgs ? import ../.. { inherit system config; } }: -with import ../lib/testing.nix { inherit system pkgs; }; +with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; { @@ -18,11 +18,11 @@ with pkgs.lib; }; testScript = '' - startAll; + start_all() - $machine->waitForUnit('gitea.service'); - $machine->waitForOpenPort('3000'); - $machine->succeed("curl --fail http://localhost:3000/"); + machine.wait_for_unit("gitea.service") + machine.wait_for_open_port(3000) + machine.succeed("curl --fail http://localhost:3000/") ''; }; @@ -37,11 +37,11 @@ with pkgs.lib; }; testScript = '' - startAll; + start_all() - $machine->waitForUnit('gitea.service'); - $machine->waitForOpenPort('3000'); - $machine->succeed("curl --fail http://localhost:3000/"); + machine.wait_for_unit("gitea.service") + machine.wait_for_open_port(3000) + machine.succeed("curl --fail http://localhost:3000/") ''; }; @@ -56,12 +56,14 @@ with pkgs.lib; }; testScript = '' - startAll; + start_all() - $machine->waitForUnit('gitea.service'); - $machine->waitForOpenPort('3000'); - $machine->succeed("curl --fail http://localhost:3000/"); - $machine->succeed("curl --fail http://localhost:3000/user/sign_up | grep 'Registration is disabled. Please contact your site administrator.'"); + machine.wait_for_unit("gitea.service") + machine.wait_for_open_port(3000) + machine.succeed("curl --fail http://localhost:3000/") + machine.succeed( + "curl --fail http://localhost:3000/user/sign_up | grep 'Registration is disabled. Please contact your site administrator.'" + ) ''; }; } diff --git a/nixos/tests/gjs.nix b/nixos/tests/gjs.nix deleted file mode 100644 index e6002ef98dd..00000000000 --- a/nixos/tests/gjs.nix +++ /dev/null @@ -1,19 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: { - name = "gjs"; - - meta = { - maintainers = pkgs.gnome3.gjs.meta.maintainers; - }; - - machine = { pkgs, ... }: { - imports = [ ./common/x11.nix ]; - environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; - environment.variables.XDG_DATA_DIRS = [ "${pkgs.gnome3.gjs.installedTests}/share" ]; - }; - - testScript = '' - $machine->waitForX; - $machine->succeed("gnome-desktop-testing-runner"); - ''; -}) diff --git a/nixos/tests/glib-networking.nix b/nixos/tests/glib-networking.nix deleted file mode 100644 index c0bbb2b3554..00000000000 --- a/nixos/tests/glib-networking.nix +++ /dev/null @@ -1,17 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: - -{ - name = "glib-networking"; - meta = { - maintainers = pkgs.glib-networking.meta.maintainers; - }; - - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.glib-networking.installedTests}/share'"); - ''; -}) diff --git a/nixos/tests/gnome-photos.nix b/nixos/tests/gnome-photos.nix deleted file mode 100644 index 2ecda1d68ce..00000000000 --- a/nixos/tests/gnome-photos.nix +++ /dev/null @@ -1,42 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, lib, ... }: - -let - - # gsettings tool with access to gsettings-desktop-schemas - desktop-gsettings = with pkgs; stdenv.mkDerivation { - name = "desktop-gsettings"; - dontUnpack = true; - nativeBuildInputs = [ glib wrapGAppsHook ]; - buildInputs = [ gsettings-desktop-schemas ]; - installPhase = '' - runHook preInstall - mkdir -p $out/bin - ln -s ${glib.bin}/bin/gsettings $out/bin/desktop-gsettings - runHook postInstall - ''; - }; - -in - -{ - name = "gnome-photos"; - meta = { - maintainers = pkgs.gnome-photos.meta.maintainers; - }; - - machine = { pkgs, ... }: { - imports = [ ./common/x11.nix ]; - programs.dconf.enable = true; - services.gnome3.at-spi2-core.enable = true; # needed for dogtail - environment.systemPackages = with pkgs; [ gnome-desktop-testing desktop-gsettings ]; - services.dbus.packages = with pkgs; [ gnome-photos ]; - }; - - testScript = '' - $machine->waitForX; - # dogtail needs accessibility enabled - $machine->succeed("desktop-gsettings set org.gnome.desktop.interface toolkit-accessibility true 2>&1"); - $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.gnome-photos.installedTests}/share' 2>&1"); - ''; -}) diff --git a/nixos/tests/grafana.nix b/nixos/tests/grafana.nix index 7a1b4c8ffbb..4b453ece7f1 100644 --- a/nixos/tests/grafana.nix +++ b/nixos/tests/grafana.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ lib, pkgs, ... }: +import ./make-test-python.nix ({ lib, pkgs, ... }: let inherit (lib) mkMerge nameValuePair maintainers; @@ -64,28 +64,34 @@ in { inherit nodes; testScript = '' - startAll(); + start_all() - subtest "Grafana sqlite", sub { - $sqlite->waitForUnit("grafana.service"); - $sqlite->waitForOpenPort(3000); - $sqlite->succeed("curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/org/users | grep -q testadmin\@localhost"); - }; + with subtest("Successful API query as admin user with sqlite db"): + sqlite.wait_for_unit("grafana.service") + sqlite.wait_for_open_port(3000) + sqlite.succeed( + "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/org/users | grep -q testadmin\@localhost" + ) + sqlite.shutdown() - subtest "Grafana postgresql", sub { - $postgresql->waitForUnit("grafana.service"); - $postgresql->waitForUnit("postgresql.service"); - $postgresql->waitForOpenPort(3000); - $postgresql->waitForOpenPort(5432); - $postgresql->succeed("curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/org/users | grep -q testadmin\@localhost"); - }; + with subtest("Successful API query as admin user with postgresql db"): + postgresql.wait_for_unit("grafana.service") + postgresql.wait_for_unit("postgresql.service") + postgresql.wait_for_open_port(3000) + postgresql.wait_for_open_port(5432) + postgresql.succeed( + "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/org/users | grep -q testadmin\@localhost" + ) + postgresql.shutdown() - subtest "Grafana mysql", sub { - $mysql->waitForUnit("grafana.service"); - $mysql->waitForUnit("mysql.service"); - $mysql->waitForOpenPort(3000); - $mysql->waitForOpenPort(3306); - $mysql->succeed("curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/org/users | grep -q testadmin\@localhost"); - }; + with subtest("Successful API query as admin user with mysql db"): + mysql.wait_for_unit("grafana.service") + mysql.wait_for_unit("mysql.service") + mysql.wait_for_open_port(3000) + mysql.wait_for_open_port(3306) + mysql.succeed( + "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/org/users | grep -q testadmin\@localhost" + ) + mysql.shutdown() ''; }) diff --git a/nixos/tests/graphene.nix b/nixos/tests/graphene.nix deleted file mode 100644 index 5591bcc30c0..00000000000 --- a/nixos/tests/graphene.nix +++ /dev/null @@ -1,18 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: - -{ - name = "graphene"; - - meta = { - maintainers = pkgs.graphene.meta.maintainers; - }; - - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.graphene.installedTests}/share'"); - ''; -}) diff --git a/nixos/tests/haproxy.nix b/nixos/tests/haproxy.nix index 22a83e9d1ea..72e77a68193 100644 --- a/nixos/tests/haproxy.nix +++ b/nixos/tests/haproxy.nix @@ -16,6 +16,8 @@ import ./make-test.nix ({ pkgs, ...}: { frontend http bind *:80 mode http + option http-use-htx + http-request use-service prometheus-exporter if { path /metrics } use_backend http_server ''; }; @@ -36,6 +38,6 @@ import ./make-test.nix ({ pkgs, ...}: { $machine->waitForUnit('haproxy.service'); $machine->waitForUnit('httpd.service'); $machine->succeed('curl -k http://localhost:80/index.txt | grep "We are all good!"'); - + $machine->succeed('curl -k http://localhost:80/metrics | grep haproxy_process_pool_allocated_bytes'); ''; }) diff --git a/nixos/tests/initrd-network-ssh/default.nix b/nixos/tests/initrd-network-ssh/default.nix index 796c50c610e..73d9f938e22 100644 --- a/nixos/tests/initrd-network-ssh/default.nix +++ b/nixos/tests/initrd-network-ssh/default.nix @@ -1,4 +1,4 @@ -import ../make-test.nix ({ lib, ... }: +import ../make-test-python.nix ({ lib, ... }: { name = "initrd-network-ssh"; @@ -35,25 +35,31 @@ import ../make-test.nix ({ lib, ... }: client = { config, ... }: { - environment.etc.knownHosts = { - text = concatStrings [ - "server," - "${toString (head (splitString " " ( - toString (elemAt (splitString "\n" config.networking.extraHosts) 2) - )))} " - "${readFile ./dropbear.pub}" - ]; + environment.etc = { + knownHosts = { + text = concatStrings [ + "server," + "${toString (head (splitString " " ( + toString (elemAt (splitString "\n" config.networking.extraHosts) 2) + )))} " + "${readFile ./dropbear.pub}" + ]; + }; + sshKey = { + source = ./openssh.priv; # dont use this anywhere else + mode = "0600"; + }; }; }; }; testScript = '' - startAll; - $client->waitForUnit("network.target"); - $client->copyFileFromHost("${./openssh.priv}","/etc/sshKey"); - $client->succeed("chmod 0600 /etc/sshKey"); - $client->waitUntilSucceeds("ping -c 1 server"); - $client->succeed("ssh -i /etc/sshKey -o UserKnownHostsFile=/etc/knownHosts server 'touch /fnord'"); - $client->shutdown; + start_all() + client.wait_for_unit("network.target") + client.wait_until_succeeds("ping -c 1 server") + client.succeed( + "ssh -i /etc/sshKey -o UserKnownHostsFile=/etc/knownHosts server 'touch /fnord'" + ) + client.shutdown() ''; }) diff --git a/nixos/tests/installed-tests/colord.nix b/nixos/tests/installed-tests/colord.nix new file mode 100644 index 00000000000..77e6b917fe6 --- /dev/null +++ b/nixos/tests/installed-tests/colord.nix @@ -0,0 +1,5 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.colord; +} diff --git a/nixos/tests/installed-tests/default.nix b/nixos/tests/installed-tests/default.nix new file mode 100644 index 00000000000..f4780bdcfc9 --- /dev/null +++ b/nixos/tests/installed-tests/default.nix @@ -0,0 +1,80 @@ +# NixOS tests for gnome-desktop-testing-runner using software +# See https://wiki.gnome.org/Initiatives/GnomeGoals/InstalledTests + +{ system ? builtins.currentSystem, + config ? {}, + pkgs ? import ../../.. { inherit system config; } +}: + +with import ../../lib/testing-python.nix { inherit system pkgs; }; +with pkgs.lib; + +let + + callInstalledTest = pkgs.newScope { inherit makeInstalledTest; }; + + makeInstalledTest = + { # Package to test. Needs to have an installedTests output + tested + + # Config to inject into machine + , testConfig ? {} + + # Test script snippet to inject before gnome-desktop-testing-runner begins. + # This is useful for extra setup the environment may need before the runner begins. + , preTestScript ? "" + + # Does test need X11? + , withX11 ? false + + # Extra flags to pass to gnome-desktop-testing-runner. + , testRunnerFlags ? "" + }: + makeTest rec { + name = tested.name; + + meta = { + maintainers = tested.meta.maintainers; + }; + + machine = { ... }: { + imports = [ + testConfig + ] ++ optional withX11 ../common/x11.nix; + + environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; + + }; + + testScript = + optionalString withX11 '' + machine.wait_for_x() + '' + + optionalString (preTestScript != "") '' + ${preTestScript} + '' + + '' + machine.succeed( + "gnome-desktop-testing-runner ${testRunnerFlags} -d '${tested.installedTests}/share'" + ) + ''; + }; + +in + +{ + colord = callInstalledTest ./colord.nix {}; + flatpak = callInstalledTest ./flatpak.nix {}; + flatpak-builder = callInstalledTest ./flatpak-builder.nix {}; + fwupd = callInstalledTest ./fwupd.nix {}; + gcab = callInstalledTest ./gcab.nix {}; + gdk-pixbuf = callInstalledTest ./gdk-pixbuf.nix {}; + gjs = callInstalledTest ./gjs.nix {}; + glib-networking = callInstalledTest ./glib-networking.nix {}; + gnome-photos = callInstalledTest ./gnome-photos.nix {}; + graphene = callInstalledTest ./graphene.nix {}; + libgdata = callInstalledTest ./libgdata.nix {}; + libxmlb = callInstalledTest ./libxmlb.nix {}; + ostree = callInstalledTest ./ostree.nix {}; + xdg-desktop-portal = callInstalledTest ./xdg-desktop-portal.nix {}; +} diff --git a/nixos/tests/installed-tests/flatpak-builder.nix b/nixos/tests/installed-tests/flatpak-builder.nix new file mode 100644 index 00000000000..31b9f2b258f --- /dev/null +++ b/nixos/tests/installed-tests/flatpak-builder.nix @@ -0,0 +1,14 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.flatpak-builder; + + testConfig = { + services.flatpak.enable = true; + xdg.portal.enable = true; + environment.systemPackages = with pkgs; [ flatpak-builder ] ++ flatpak-builder.installedTestsDependencies; + virtualisation.diskSize = 2048; + }; + + testRunnerFlags = "--timeout 3600"; +} diff --git a/nixos/tests/installed-tests/flatpak.nix b/nixos/tests/installed-tests/flatpak.nix new file mode 100644 index 00000000000..091c9932662 --- /dev/null +++ b/nixos/tests/installed-tests/flatpak.nix @@ -0,0 +1,19 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.flatpak; + withX11 = true; + + testConfig = { + services.xserver.desktopManager.gnome3.enable = true; # TODO: figure out minimal environment where the tests work + # common/x11.nix enables the auto display manager (lightdm) + services.xserver.displayManager.gdm.enable = false; + services.gnome3.core-utilities.enable = false; + services.flatpak.enable = true; + environment.systemPackages = with pkgs; [ gnupg ostree python2 ]; + virtualisation.memorySize = 2047; + virtualisation.diskSize = 1024; + }; + + testRunnerFlags = "--timeout 3600"; +} diff --git a/nixos/tests/installed-tests/fwupd.nix b/nixos/tests/installed-tests/fwupd.nix new file mode 100644 index 00000000000..b9f761e9958 --- /dev/null +++ b/nixos/tests/installed-tests/fwupd.nix @@ -0,0 +1,12 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.fwupd; + + testConfig = { + services.fwupd.enable = true; + services.fwupd.blacklistPlugins = []; # don't blacklist test plugin + services.fwupd.enableTestRemote = true; + virtualisation.memorySize = 768; + }; +} diff --git a/nixos/tests/installed-tests/gcab.nix b/nixos/tests/installed-tests/gcab.nix new file mode 100644 index 00000000000..b24cc2e0126 --- /dev/null +++ b/nixos/tests/installed-tests/gcab.nix @@ -0,0 +1,5 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.gcab; +} diff --git a/nixos/tests/installed-tests/gdk-pixbuf.nix b/nixos/tests/installed-tests/gdk-pixbuf.nix new file mode 100644 index 00000000000..3d0011a427a --- /dev/null +++ b/nixos/tests/installed-tests/gdk-pixbuf.nix @@ -0,0 +1,13 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.gdk-pixbuf; + + testConfig = { + # Tests allocate a lot of memory trying to exploit a CVE + # but qemu-system-i386 has a 2047M memory limit + virtualisation.memorySize = if pkgs.stdenv.isi686 then 2047 else 4096; + }; + + testRunnerFlags = "--timeout 1800"; +} diff --git a/nixos/tests/installed-tests/gjs.nix b/nixos/tests/installed-tests/gjs.nix new file mode 100644 index 00000000000..1656e9de171 --- /dev/null +++ b/nixos/tests/installed-tests/gjs.nix @@ -0,0 +1,6 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.gjs; + withX11 = true; +} diff --git a/nixos/tests/installed-tests/glib-networking.nix b/nixos/tests/installed-tests/glib-networking.nix new file mode 100644 index 00000000000..b58d4df21fc --- /dev/null +++ b/nixos/tests/installed-tests/glib-networking.nix @@ -0,0 +1,5 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.glib-networking; +} diff --git a/nixos/tests/installed-tests/gnome-photos.nix b/nixos/tests/installed-tests/gnome-photos.nix new file mode 100644 index 00000000000..05e7ccb65ad --- /dev/null +++ b/nixos/tests/installed-tests/gnome-photos.nix @@ -0,0 +1,35 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.gnome-photos; + + withX11 = true; + + testConfig = { + programs.dconf.enable = true; + services.gnome3.at-spi2-core.enable = true; # needed for dogtail + environment.systemPackages = with pkgs; [ + # gsettings tool with access to gsettings-desktop-schemas + (stdenv.mkDerivation { + name = "desktop-gsettings"; + dontUnpack = true; + nativeBuildInputs = [ glib wrapGAppsHook ]; + buildInputs = [ gsettings-desktop-schemas ]; + installPhase = '' + runHook preInstall + mkdir -p $out/bin + ln -s ${glib.bin}/bin/gsettings $out/bin/desktop-gsettings + runHook postInstall + ''; + }) + ]; + services.dbus.packages = with pkgs; [ gnome-photos ]; + }; + + preTestScript = '' + # dogtail needs accessibility enabled + machine.succeed( + "desktop-gsettings set org.gnome.desktop.interface toolkit-accessibility true 2>&1" + ) + ''; +} diff --git a/nixos/tests/installed-tests/graphene.nix b/nixos/tests/installed-tests/graphene.nix new file mode 100644 index 00000000000..e43339abd88 --- /dev/null +++ b/nixos/tests/installed-tests/graphene.nix @@ -0,0 +1,5 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.graphene; +} diff --git a/nixos/tests/installed-tests/libgdata.nix b/nixos/tests/installed-tests/libgdata.nix new file mode 100644 index 00000000000..f11a7bc1bc5 --- /dev/null +++ b/nixos/tests/installed-tests/libgdata.nix @@ -0,0 +1,11 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.libgdata; + + testConfig = { + # # GLib-GIO-DEBUG: _g_io_module_get_default: Found default implementation dummy (GDummyTlsBackend) for ‘gio-tls-backend’ + # Bail out! libgdata:ERROR:../gdata/tests/common.c:134:gdata_test_init: assertion failed (child_error == NULL): TLS support is not available (g-tls-error-quark, 0) + services.gnome3.glib-networking.enable = true; + }; +} diff --git a/nixos/tests/installed-tests/libxmlb.nix b/nixos/tests/installed-tests/libxmlb.nix new file mode 100644 index 00000000000..af2bbe9c35e --- /dev/null +++ b/nixos/tests/installed-tests/libxmlb.nix @@ -0,0 +1,5 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.libxmlb; +} diff --git a/nixos/tests/installed-tests/ostree.nix b/nixos/tests/installed-tests/ostree.nix new file mode 100644 index 00000000000..eef7cace54c --- /dev/null +++ b/nixos/tests/installed-tests/ostree.nix @@ -0,0 +1,23 @@ +{ pkgs, lib, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.ostree; + + # TODO: Wrap/patch the tests directly in the package + testConfig = { + environment.systemPackages = with pkgs; [ + (python3.withPackages (p: with p; [ pyyaml ])) + gnupg + ostree + ]; + + # for GJS tests + environment.variables.GI_TYPELIB_PATH = lib.makeSearchPath "lib/girepository-1.0" (with pkgs; [ + gtk3 + pango.out + ostree + gdk-pixbuf + atk + ]); + }; +} diff --git a/nixos/tests/installed-tests/xdg-desktop-portal.nix b/nixos/tests/installed-tests/xdg-desktop-portal.nix new file mode 100644 index 00000000000..b16008ff4ad --- /dev/null +++ b/nixos/tests/installed-tests/xdg-desktop-portal.nix @@ -0,0 +1,5 @@ +{ pkgs, makeInstalledTest, ... }: + +makeInstalledTest { + tested = pkgs.xdg-desktop-portal; +} diff --git a/nixos/tests/jormungandr.nix b/nixos/tests/jormungandr.nix deleted file mode 100644 index 2abafc53ce5..00000000000 --- a/nixos/tests/jormungandr.nix +++ /dev/null @@ -1,77 +0,0 @@ -import ./make-test.nix ({ pkgs, ... }: { - name = "jormungandr"; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ mmahut ]; - }; - - nodes = { - # Testing the Byzantine Fault Tolerant protocol - bft = { ... }: { - environment.systemPackages = [ pkgs.jormungandr ]; - services.jormungandr.enable = true; - services.jormungandr.genesisBlockFile = "/var/lib/jormungandr/block-0.bin"; - services.jormungandr.secretFile = "/etc/secrets/jormungandr.yaml"; - }; - - # Testing the Ouroboros Genesis Praos protocol - genesis = { ... }: { - environment.systemPackages = [ pkgs.jormungandr ]; - services.jormungandr.enable = true; - services.jormungandr.genesisBlockFile = "/var/lib/jormungandr/block-0.bin"; - services.jormungandr.secretFile = "/etc/secrets/jormungandr.yaml"; - }; - }; - - testScript = '' - startAll; - - ## Testing BFT - # Let's wait for the StateDirectory - $bft->waitForFile("/var/lib/jormungandr/"); - - # First, we generate the genesis file for our new blockchain - $bft->succeed("jcli genesis init > /root/genesis.yaml"); - - # We need to generate our secret key - $bft->succeed("jcli key generate --type=Ed25519 > /root/key.prv"); - - # We include the secret key into our services.jormungandr.secretFile - $bft->succeed("mkdir -p /etc/secrets"); - $bft->succeed("echo -e \"bft:\\n signing_key:\" \$(cat /root/key.prv) > /etc/secrets/jormungandr.yaml"); - - # After that, we generate our public key from it - $bft->succeed("cat /root/key.prv | jcli key to-public > /root/key.pub"); - - # We add our public key as a consensus leader in the genesis configration file - $bft->succeed("sed -ie \"s/ed25519_pk1vvwp2s0n5jl5f4xcjurp2e92sj2awehkrydrlas4vgqr7xzt33jsadha32/\$(cat /root/key.pub)/\" /root/genesis.yaml"); - - # Now we can generate the genesis block from it - $bft->succeed("jcli genesis encode --input /root/genesis.yaml --output /var/lib/jormungandr/block-0.bin"); - - # We should have everything to start the service now - $bft->succeed("systemctl restart jormungandr"); - $bft->waitForUnit("jormungandr.service"); - - # Now we can test if we are able to reach the REST API - $bft->waitUntilSucceeds("curl -L http://localhost:8607/api/v0/node/stats | grep uptime"); - - ## Testing Genesis - # Let's wait for the StateDirectory - $genesis->waitForFile("/var/lib/jormungandr/"); - - # Bootstraping the configuration - $genesis->succeed("jormungandr-bootstrap -g -p 8607 -s 1"); - - # Moving generated files in place - $genesis->succeed("mkdir -p /etc/secrets"); - $genesis->succeed("mv pool-secret1.yaml /etc/secrets/jormungandr.yaml"); - $genesis->succeed("mv block-0.bin /var/lib/jormungandr/"); - - # We should have everything to start the service now - $genesis->succeed("systemctl restart jormungandr"); - $genesis->waitForUnit("jormungandr.service"); - - # Now we can create and delegate an account - $genesis->succeed("./create-account-and-delegate.sh | tee -a /tmp/delegate.log"); - ''; -}) diff --git a/nixos/tests/knot.nix b/nixos/tests/knot.nix index e46159836cc..0588cf86ac0 100644 --- a/nixos/tests/knot.nix +++ b/nixos/tests/knot.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, lib, ...} : +import ./make-test-python.nix ({ pkgs, lib, ...} : let common = { networking.firewall.enable = false; @@ -30,6 +30,10 @@ let }; in { name = "knot"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ hexa ]; + }; + nodes = { master = { lib, ... }: { @@ -161,37 +165,35 @@ in { slave4 = (lib.head nodes.slave.config.networking.interfaces.eth1.ipv4.addresses).address; slave6 = (lib.head nodes.slave.config.networking.interfaces.eth1.ipv6.addresses).address; in '' - startAll; - - $client->waitForUnit("network.target"); - $master->waitForUnit("knot.service"); - $slave->waitForUnit("knot.service"); - - sub assertResponse { - my ($knot, $query_type, $query, $expected) = @_; - my $out = $client->succeed("khost -t $query_type $query $knot"); - $client->log("$knot replies with: $out"); - chomp $out; - die "DNS query for $query ($query_type) against $knot gave '$out' instead of '$expected'" - if ($out !~ $expected); - } - - foreach ("${master4}", "${master6}", "${slave4}", "${slave6}") { - subtest $_, sub { - assertResponse($_, "SOA", "example.com", qr/start of authority.*?noc\.example\.com/); - assertResponse($_, "A", "example.com", qr/has no [^ ]+ record/); - assertResponse($_, "AAAA", "example.com", qr/has no [^ ]+ record/); - - assertResponse($_, "A", "www.example.com", qr/address 192.0.2.1$/); - assertResponse($_, "AAAA", "www.example.com", qr/address 2001:db8::1$/); - - assertResponse($_, "NS", "sub.example.com", qr/nameserver is ns\d\.example\.com.$/); - assertResponse($_, "A", "sub.example.com", qr/address 192.0.2.2$/); - assertResponse($_, "AAAA", "sub.example.com", qr/address 2001:db8::2$/); - - assertResponse($_, "RRSIG", "www.example.com", qr/RR set signature is/); - assertResponse($_, "DNSKEY", "example.com", qr/DNSSEC key is/); - }; - } + import re + + start_all() + + client.wait_for_unit("network.target") + master.wait_for_unit("knot.service") + slave.wait_for_unit("knot.service") + + + def test(host, query_type, query, pattern): + out = client.succeed(f"khost -t {query_type} {query} {host}").strip() + client.log(f"{host} replied with: {out}") + assert re.search(pattern, out), f'Did not match "{pattern}"' + + + for host in ("${master4}", "${master6}", "${slave4}", "${slave6}"): + with subtest(f"Interrogate {host}"): + test(host, "SOA", "example.com", r"start of authority.*noc\.example\.com\.") + test(host, "A", "example.com", r"has no [^ ]+ record") + test(host, "AAAA", "example.com", r"has no [^ ]+ record") + + test(host, "A", "www.example.com", r"address 192.0.2.1$") + test(host, "AAAA", "www.example.com", r"address 2001:db8::1$") + + test(host, "NS", "sub.example.com", r"nameserver is ns\d\.example\.com.$") + test(host, "A", "sub.example.com", r"address 192.0.2.2$") + test(host, "AAAA", "sub.example.com", r"address 2001:db8::2$") + + test(host, "RRSIG", "www.example.com", r"RR set signature is") + test(host, "DNSKEY", "example.com", r"DNSSEC key is") ''; }) diff --git a/nixos/tests/libgdata.nix b/nixos/tests/libgdata.nix deleted file mode 100644 index 10a3ca97dd2..00000000000 --- a/nixos/tests/libgdata.nix +++ /dev/null @@ -1,21 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: - -{ - name = "libgdata"; - - meta = { - maintainers = pkgs.libgdata.meta.maintainers; - }; - - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; - # # GLib-GIO-DEBUG: _g_io_module_get_default: Found default implementation dummy (GDummyTlsBackend) for ‘gio-tls-backend’ - # Bail out! libgdata:ERROR:../gdata/tests/common.c:134:gdata_test_init: assertion failed (child_error == NULL): TLS support is not available (g-tls-error-quark, 0) - services.gnome3.glib-networking.enable = true; - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.libgdata.installedTests}/share'"); - ''; -}) diff --git a/nixos/tests/libxmlb.nix b/nixos/tests/libxmlb.nix deleted file mode 100644 index 3bee568ac5a..00000000000 --- a/nixos/tests/libxmlb.nix +++ /dev/null @@ -1,17 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: - -{ - name = "libxmlb"; - meta = { - maintainers = pkgs.libxmlb.meta.maintainers; - }; - - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.libxmlb.installedTests}/share'"); - ''; -}) diff --git a/nixos/tests/login.nix b/nixos/tests/login.nix index bd8ed23a7b8..d36c1a91be4 100644 --- a/nixos/tests/login.nix +++ b/nixos/tests/login.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, latestKernel ? false, ... }: +import ./make-test-python.nix ({ pkgs, latestKernel ? false, ... }: { name = "login"; @@ -12,62 +12,48 @@ import ./make-test.nix ({ pkgs, latestKernel ? false, ... }: sound.enable = true; # needed for the factl test, /dev/snd/* exists without them but udev doesn't care then }; - testScript = - '' - $machine->waitForUnit('multi-user.target'); - $machine->waitUntilSucceeds("pgrep -f 'agetty.*tty1'"); - $machine->screenshot("postboot"); - - subtest "create user", sub { - $machine->succeed("useradd -m alice"); - $machine->succeed("(echo foobar; echo foobar) | passwd alice"); - }; - - # Check whether switching VTs works. - subtest "virtual console switching", sub { - $machine->fail("pgrep -f 'agetty.*tty2'"); - $machine->sendKeys("alt-f2"); - $machine->waitUntilSucceeds("[ \$(fgconsole) = 2 ]"); - $machine->waitForUnit('getty@tty2.service'); - $machine->waitUntilSucceeds("pgrep -f 'agetty.*tty2'"); - }; - - # Log in as alice on a virtual console. - subtest "virtual console login", sub { - $machine->waitUntilTTYMatches(2, "login: "); - $machine->sendChars("alice\n"); - $machine->waitUntilTTYMatches(2, "login: alice"); - $machine->waitUntilSucceeds("pgrep login"); - $machine->waitUntilTTYMatches(2, "Password: "); - $machine->sendChars("foobar\n"); - $machine->waitUntilSucceeds("pgrep -u alice bash"); - $machine->sendChars("touch done\n"); - $machine->waitForFile("/home/alice/done"); - }; - - # Check whether systemd gives and removes device ownership as - # needed. - subtest "device permissions", sub { - $machine->succeed("getfacl -p /dev/snd/timer | grep -q alice"); - $machine->sendKeys("alt-f1"); - $machine->waitUntilSucceeds("[ \$(fgconsole) = 1 ]"); - $machine->fail("getfacl -p /dev/snd/timer | grep -q alice"); - $machine->succeed("chvt 2"); - $machine->waitUntilSucceeds("getfacl -p /dev/snd/timer | grep -q alice"); - }; - - # Log out. - subtest "virtual console logout", sub { - $machine->sendChars("exit\n"); - $machine->waitUntilFails("pgrep -u alice bash"); - $machine->screenshot("mingetty"); - }; - - # Check whether ctrl-alt-delete works. - subtest "ctrl-alt-delete", sub { - $machine->sendKeys("ctrl-alt-delete"); - $machine->waitForShutdown; - }; - ''; - + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.wait_until_succeeds("pgrep -f 'agetty.*tty1'") + machine.screenshot("postboot") + + with subtest("create user"): + machine.succeed("useradd -m alice") + machine.succeed("(echo foobar; echo foobar) | passwd alice") + + with subtest("Check whether switching VTs works"): + machine.fail("pgrep -f 'agetty.*tty2'") + machine.send_key("alt-f2") + machine.wait_until_succeeds("[ $(fgconsole) = 2 ]") + machine.wait_for_unit("getty@tty2.service") + machine.wait_until_succeeds("pgrep -f 'agetty.*tty2'") + + with subtest("Log in as alice on a virtual console"): + machine.wait_until_tty_matches(2, "login: ") + machine.send_chars("alice\n") + machine.wait_until_tty_matches(2, "login: alice") + machine.wait_until_succeeds("pgrep login") + machine.wait_until_tty_matches(2, "Password: ") + machine.send_chars("foobar\n") + machine.wait_until_succeeds("pgrep -u alice bash") + machine.send_chars("touch done\n") + machine.wait_for_file("/home/alice/done") + + with subtest("Systemd gives and removes device ownership as needed"): + machine.succeed("getfacl /dev/snd/timer | grep -q alice") + machine.send_key("alt-f1") + machine.wait_until_succeeds("[ $(fgconsole) = 1 ]") + machine.fail("getfacl /dev/snd/timer | grep -q alice") + machine.succeed("chvt 2") + machine.wait_until_succeeds("getfacl /dev/snd/timer | grep -q alice") + + with subtest("Virtual console logout"): + machine.send_chars("exit\n") + machine.wait_until_fails("pgrep -u alice bash") + machine.screenshot("mingetty") + + with subtest("Check whether ctrl-alt-delete works"): + machine.send_key("ctrl-alt-delete") + machine.wait_for_shutdown() + ''; }) diff --git a/nixos/tests/loki.nix b/nixos/tests/loki.nix index 9c3058d02f8..dbf1e8a650f 100644 --- a/nixos/tests/loki.nix +++ b/nixos/tests/loki.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ lib, pkgs, ... }: +import ./make-test-python.nix ({ lib, pkgs, ... }: { name = "loki"; @@ -26,12 +26,14 @@ import ./make-test.nix ({ lib, pkgs, ... }: }; testScript = '' - $machine->start; - $machine->waitForUnit("loki.service"); - $machine->waitForUnit("promtail.service"); - $machine->waitForOpenPort(3100); - $machine->waitForOpenPort(9080); - $machine->succeed("echo 'Loki Ingestion Test' > /var/log/testlog"); - $machine->waitUntilSucceeds("${pkgs.grafana-loki}/bin/logcli --addr='http://localhost:3100' query --no-labels '{job=\"varlogs\",filename=\"/var/log/testlog\"}' | grep -q 'Loki Ingestion Test'"); + machine.start + machine.wait_for_unit("loki.service") + machine.wait_for_unit("promtail.service") + machine.wait_for_open_port(3100) + machine.wait_for_open_port(9080) + machine.succeed("echo 'Loki Ingestion Test' > /var/log/testlog") + machine.wait_until_succeeds( + "${pkgs.grafana-loki}/bin/logcli --addr='http://localhost:3100' query --no-labels '{job=\"varlogs\",filename=\"/var/log/testlog\"}' | grep -q 'Loki Ingestion Test'" + ) ''; }) diff --git a/nixos/tests/make-test-python.nix b/nixos/tests/make-test-python.nix new file mode 100644 index 00000000000..89897fe7e61 --- /dev/null +++ b/nixos/tests/make-test-python.nix @@ -0,0 +1,9 @@ +f: { + system ? builtins.currentSystem, + pkgs ? import ../.. { inherit system; config = {}; }, + ... +} @ args: + +with import ../lib/testing-python.nix { inherit system pkgs; }; + +makeTest (if pkgs.lib.isFunction f then f (args // { inherit pkgs; inherit (pkgs) lib; }) else f) diff --git a/nixos/tests/matomo.nix b/nixos/tests/matomo.nix new file mode 100644 index 00000000000..4efa65a7b6d --- /dev/null +++ b/nixos/tests/matomo.nix @@ -0,0 +1,43 @@ +{ system ? builtins.currentSystem, config ? { } +, pkgs ? import ../.. { inherit system config; } }: + +with import ../lib/testing.nix { inherit system pkgs; }; +with pkgs.lib; + +let + matomoTest = package: + makeTest { + machine = { config, pkgs, ... }: { + services.matomo = { + package = package; + enable = true; + nginx = { + forceSSL = false; + enableACME = false; + }; + }; + services.mysql = { + enable = true; + package = pkgs.mysql; + }; + services.nginx.enable = true; + }; + + testScript = '' + startAll; + $machine->waitForUnit("mysql.service"); + $machine->waitForUnit("phpfpm-matomo.service"); + $machine->waitForUnit("nginx.service"); + $machine->succeed("curl -sSfL http://localhost/ | grep '<title>Matomo[^<]*Installation'"); + ''; + }; +in { + matomo = matomoTest pkgs.matomo // { + name = "matomo"; + meta.maintainers = with maintainers; [ florianjacob kiwi mmilata ]; + }; + matomo-beta = matomoTest pkgs.matomo-beta // { + name = "matomo-beta"; + meta.maintainers = with maintainers; [ florianjacob kiwi mmilata ]; + }; +} diff --git a/nixos/tests/matrix-synapse.nix b/nixos/tests/matrix-synapse.nix index 882e4b75814..fca53009083 100644 --- a/nixos/tests/matrix-synapse.nix +++ b/nixos/tests/matrix-synapse.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... } : let +import ./make-test-python.nix ({ pkgs, ... } : let runWithOpenSSL = file: cmd: pkgs.runCommand file { @@ -55,13 +55,17 @@ in { }; testScript = '' - startAll; - $serverpostgres->waitForUnit("matrix-synapse.service"); - $serverpostgres->waitUntilSucceeds("curl -L --cacert ${ca_pem} https://localhost:8448/"); - $serverpostgres->requireActiveUnit("postgresql.service"); - $serversqlite->waitForUnit("matrix-synapse.service"); - $serversqlite->waitUntilSucceeds("curl -L --cacert ${ca_pem} https://localhost:8448/"); - $serversqlite->mustSucceed("[ -e /var/lib/matrix-synapse/homeserver.db ]"); + start_all() + serverpostgres.wait_for_unit("matrix-synapse.service") + serverpostgres.wait_until_succeeds( + "curl -L --cacert ${ca_pem} https://localhost:8448/" + ) + serverpostgres.require_unit_state("postgresql.service") + serversqlite.wait_for_unit("matrix-synapse.service") + serversqlite.wait_until_succeeds( + "curl -L --cacert ${ca_pem} https://localhost:8448/" + ) + serversqlite.succeed("[ -e /var/lib/matrix-synapse/homeserver.db ]") ''; }) diff --git a/nixos/tests/metabase.nix b/nixos/tests/metabase.nix index be9e5ed5b1e..1450a4e9086 100644 --- a/nixos/tests/metabase.nix +++ b/nixos/tests/metabase.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: { +import ./make-test-python.nix ({ pkgs, ... }: { name = "metabase"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ mmahut ]; @@ -12,9 +12,9 @@ import ./make-test.nix ({ pkgs, ... }: { }; testScript = '' - startAll; - $machine->waitForUnit("metabase.service"); - $machine->waitForOpenPort(3000); - $machine->waitUntilSucceeds("curl -L http://localhost:3000/setup | grep Metabase"); + start_all() + machine.wait_for_unit("metabase.service") + machine.wait_for_open_port(3000) + machine.wait_until_succeeds("curl -L http://localhost:3000/setup | grep Metabase") ''; }) diff --git a/nixos/tests/moinmoin.nix b/nixos/tests/moinmoin.nix new file mode 100644 index 00000000000..2662b79aa09 --- /dev/null +++ b/nixos/tests/moinmoin.nix @@ -0,0 +1,24 @@ +import ./make-test.nix ({ pkgs, lib, ... }: { + name = "moinmoin"; + meta.maintainers = [ ]; # waiting for https://github.com/NixOS/nixpkgs/pull/65397 + + machine = + { ... }: + { services.moinmoin.enable = true; + services.moinmoin.wikis.ExampleWiki.superUsers = [ "admin" ]; + services.moinmoin.wikis.ExampleWiki.webHost = "localhost"; + + services.nginx.virtualHosts.localhost.enableACME = false; + services.nginx.virtualHosts.localhost.forceSSL = false; + }; + + testScript = '' + startAll; + + $machine->waitForUnit('moin-ExampleWiki.service'); + $machine->waitForUnit('nginx.service'); + $machine->waitForFile('/run/moin/ExampleWiki/gunicorn.sock'); + $machine->succeed('curl -L http://localhost/') =~ /If you have just installed/ or die; + $machine->succeed('moin-ExampleWiki account create --name=admin --email=admin@example.com --password=foo 2>&1') =~ /status success/ or die; + ''; +}) diff --git a/nixos/tests/moodle.nix b/nixos/tests/moodle.nix index 565a6b63694..56aa62596c0 100644 --- a/nixos/tests/moodle.nix +++ b/nixos/tests/moodle.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, lib, ... }: { +import ./make-test-python.nix ({ pkgs, lib, ... }: { name = "moodle"; meta.maintainers = [ lib.maintainers.aanderse ]; @@ -15,8 +15,8 @@ import ./make-test.nix ({ pkgs, lib, ... }: { }; testScript = '' - startAll; - $machine->waitForUnit('phpfpm-moodle.service'); - $machine->succeed('curl http://localhost/') =~ /You are not logged in/ or die; + start_all() + machine.wait_for_unit("phpfpm-moodle.service") + machine.wait_until_succeeds("curl http://localhost/ | grep 'You are not logged in'") ''; }) diff --git a/nixos/tests/morty.nix b/nixos/tests/morty.nix index eab123bd50f..64c5a27665d 100644 --- a/nixos/tests/morty.nix +++ b/nixos/tests/morty.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: +import ./make-test-python.nix ({ pkgs, ... }: { name = "morty"; @@ -22,11 +22,9 @@ import ./make-test.nix ({ pkgs, ... }: testScript = { ... }: '' - $mortyProxyWithKey->waitForUnit("default.target"); - - $mortyProxyWithKey->waitForOpenPort(3001); - $mortyProxyWithKey->succeed("curl -L 127.0.0.1:3001 | grep MortyProxy"); - + mortyProxyWithKey.wait_for_unit("default.target") + mortyProxyWithKey.wait_for_open_port(3001) + mortyProxyWithKey.succeed("curl -L 127.0.0.1:3001 | grep MortyProxy") ''; }) diff --git a/nixos/tests/nixos-generate-config.nix b/nixos/tests/nixos-generate-config.nix index 15a173e024b..6c83ccecc70 100644 --- a/nixos/tests/nixos-generate-config.nix +++ b/nixos/tests/nixos-generate-config.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ lib, ... } : { +import ./make-test-python.nix ({ lib, ... } : { name = "nixos-generate-config"; meta.maintainers = with lib.maintainers; [ basvandijk ]; machine = { @@ -11,14 +11,16 @@ import ./make-test.nix ({ lib, ... } : { ''; }; testScript = '' - startAll; - $machine->waitForUnit("multi-user.target"); - $machine->succeed("nixos-generate-config"); + start_all() + machine.wait_for_unit("multi-user.target") + machine.succeed("nixos-generate-config") # Test if the configuration really is overridden - $machine->succeed("grep 'OVERRIDDEN' /etc/nixos/configuration.nix"); + machine.succeed("grep 'OVERRIDDEN' /etc/nixos/configuration.nix") # Test of if the Perl variable $bootLoaderConfig is spliced correctly: - $machine->succeed("grep 'boot\\.loader\\.grub\\.enable = true;' /etc/nixos/configuration.nix"); + machine.succeed( + "grep 'boot\\.loader\\.grub\\.enable = true;' /etc/nixos/configuration.nix" + ) ''; }) diff --git a/nixos/tests/openssh.nix b/nixos/tests/openssh.nix index 8b9e2170f15..e9692b50327 100644 --- a/nixos/tests/openssh.nix +++ b/nixos/tests/openssh.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: +import ./make-test-python.nix ({ pkgs, ... }: let inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey; @@ -58,47 +58,55 @@ in { }; testScript = '' - startAll; - - my $key=`${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; - - $server->waitForUnit("sshd"); - - subtest "manual-authkey", sub { - $server->succeed("mkdir -m 700 /root/.ssh"); - $server->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - $server_lazy->succeed("mkdir -m 700 /root/.ssh"); - $server_lazy->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - - $client->succeed("mkdir -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); - $client->succeed("chmod 600 /root/.ssh/id_ed25519"); - - $client->waitForUnit("network.target"); - $client->succeed("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server 'echo hello world' >&2"); - $client->succeed("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server 'ulimit -l' | grep 1024"); - - $client->succeed("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server_lazy 'echo hello world' >&2"); - $client->succeed("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server_lazy 'ulimit -l' | grep 1024"); - - }; - - subtest "configured-authkey", sub { - $client->succeed("cat ${snakeOilPrivateKey} > privkey.snakeoil"); - $client->succeed("chmod 600 privkey.snakeoil"); - $client->succeed("ssh -o UserKnownHostsFile=/dev/null" . - " -o StrictHostKeyChecking=no -i privkey.snakeoil" . - " server true"); - - $client->succeed("ssh -o UserKnownHostsFile=/dev/null" . - " -o StrictHostKeyChecking=no -i privkey.snakeoil" . - " server_lazy true"); - - }; - - subtest "localhost-only", sub { - $server_localhost_only->succeed("ss -nlt | grep '127.0.0.1:22'"); - $server_localhost_only_lazy->succeed("ss -nlt | grep '127.0.0.1:22'"); - } + start_all() + + server.wait_for_unit("sshd") + + with subtest("manual-authkey"): + client.succeed("mkdir -m 700 /root/.ssh") + client.succeed( + '${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N ""' + ) + public_key = client.succeed( + "${pkgs.openssh}/bin/ssh-keygen -y -f /root/.ssh/id_ed25519" + ) + public_key = public_key.strip() + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + server.succeed("mkdir -m 700 /root/.ssh") + server.succeed("echo '{}' > /root/.ssh/authorized_keys".format(public_key)) + server_lazy.succeed("mkdir -m 700 /root/.ssh") + server_lazy.succeed("echo '{}' > /root/.ssh/authorized_keys".format(public_key)) + + client.wait_for_unit("network.target") + client.succeed( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server 'echo hello world' >&2" + ) + client.succeed( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server 'ulimit -l' | grep 1024" + ) + + client.succeed( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server_lazy 'echo hello world' >&2" + ) + client.succeed( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server_lazy 'ulimit -l' | grep 1024" + ) + + with subtest("configured-authkey"): + client.succeed( + "cat ${snakeOilPrivateKey} > privkey.snakeoil" + ) + client.succeed("chmod 600 privkey.snakeoil") + client.succeed( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i privkey.snakeoil server true" + ) + client.succeed( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i privkey.snakeoil server_lazy true" + ) + + with subtest("localhost-only"): + server_localhost_only.succeed("ss -nlt | grep '127.0.0.1:22'") + server_localhost_only_lazy.succeed("ss -nlt | grep '127.0.0.1:22'") ''; }) diff --git a/nixos/tests/orangefs.nix b/nixos/tests/orangefs.nix new file mode 100644 index 00000000000..bdf4fc10c44 --- /dev/null +++ b/nixos/tests/orangefs.nix @@ -0,0 +1,88 @@ +import ./make-test.nix ({ ... } : + +let + server = { pkgs, ... } : { + networking.firewall.allowedTCPPorts = [ 3334 ]; + boot.initrd.postDeviceCommands = '' + ${pkgs.e2fsprogs}/bin/mkfs.ext4 -L data /dev/vdb + ''; + + virtualisation.emptyDiskImages = [ 4096 ]; + + fileSystems = pkgs.lib.mkVMOverride + [ { mountPoint = "/data"; + device = "/dev/disk/by-label/data"; + fsType = "ext4"; + } + ]; + + services.orangefs.server = { + enable = true; + dataStorageSpace = "/data/storage"; + metadataStorageSpace = "/data/meta"; + servers = { + server1 = "tcp://server1:3334"; + server2 = "tcp://server2:3334"; + }; + }; + }; + + client = { lib, ... } : { + networking.firewall.enable = true; + + services.orangefs.client = { + enable = true; + fileSystems = [{ + target = "tcp://server1:3334/orangefs"; + mountPoint = "/orangefs"; + }]; + }; + }; + +in { + name = "orangefs"; + + nodes = { + server1 = server; + server2 = server; + + client1 = client; + client2 = client; + }; + + testScript = '' + # format storage + foreach my $server (($server1,$server2)) + { + $server->start(); + $server->waitForUnit("multi-user.target"); + $server->succeed("mkdir -p /data/storage /data/meta"); + $server->succeed("chown orangefs:orangefs /data/storage /data/meta"); + $server->succeed("chmod 0770 /data/storage /data/meta"); + $server->succeed("sudo -g orangefs -u orangefs pvfs2-server -f /etc/orangefs/server.conf"); + } + + # start services after storage is formated on all machines + foreach my $server (($server1,$server2)) + { + $server->succeed("systemctl start orangefs-server.service"); + } + + # Check if clients can reach and mount the FS + foreach my $client (($client1,$client2)) + { + $client->start(); + $client->waitForUnit("orangefs-client.service"); + # Both servers need to be reachable + $client->succeed("pvfs2-check-server -h server1 -f orangefs -n tcp -p 3334"); + $client->succeed("pvfs2-check-server -h server2 -f orangefs -n tcp -p 3334"); + $client->waitForUnit("orangefs.mount"); + + } + + # R/W test between clients + $client1->succeed("echo test > /orangefs/file1"); + $client2->succeed("grep test /orangefs/file1"); + + ''; +}) diff --git a/nixos/tests/ostree.nix b/nixos/tests/ostree.nix deleted file mode 100644 index d7ad84a1a5f..00000000000 --- a/nixos/tests/ostree.nix +++ /dev/null @@ -1,21 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, lib, ... }: { - name = "ostree"; - - meta = { - maintainers = pkgs.ostree.meta.maintainers; - }; - - # TODO: Wrap/patch the tests directly in the package - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ - gnome-desktop-testing ostree gnupg (python3.withPackages (p: with p; [ pyyaml ])) - ]; - - environment.variables.GI_TYPELIB_PATH = lib.makeSearchPath "lib/girepository-1.0" (with pkgs; [ gtk3 pango.out ostree gdk-pixbuf atk ]); # for GJS tests - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner -d ${pkgs.ostree.installedTests}/share"); - ''; -}) diff --git a/nixos/tests/packagekit.nix b/nixos/tests/packagekit.nix index e2d68af661f..7e93ad35e80 100644 --- a/nixos/tests/packagekit.nix +++ b/nixos/tests/packagekit.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: { +import ./make-test-python.nix ({ pkgs, ... }: { name = "packagekit"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ peterhoeg ]; @@ -13,12 +13,14 @@ import ./make-test.nix ({ pkgs, ... }: { }; testScript = '' - startAll; + start_all() # send a dbus message to activate the service - $machine->succeed("dbus-send --system --type=method_call --print-reply --dest=org.freedesktop.PackageKit /org/freedesktop/PackageKit org.freedesktop.DBus.Introspectable.Introspect"); + machine.succeed( + "dbus-send --system --type=method_call --print-reply --dest=org.freedesktop.PackageKit /org/freedesktop/PackageKit org.freedesktop.DBus.Introspectable.Introspect" + ) # so now it should be running - $machine->succeed("systemctl is-active packagekit.service"); + machine.wait_for_unit("packagekit.service") ''; }) diff --git a/nixos/tests/pgjwt.nix b/nixos/tests/pgjwt.nix index a2d81288c81..4793a3e3150 100644 --- a/nixos/tests/pgjwt.nix +++ b/nixos/tests/pgjwt.nix @@ -1,12 +1,5 @@ -import ./make-test.nix ({ pkgs, lib, ...}: -let - test = with pkgs; runCommand "patch-test" { - nativeBuildInputs = [ pgjwt ]; - } - '' - sed -e '12 i CREATE EXTENSION pgcrypto;\nCREATE EXTENSION pgtap;\nSET search_path TO tap,public;' ${pgjwt.src}/test.sql > $out; - ''; -in +import ./make-test-python.nix ({ pkgs, lib, ...}: + with pkgs; { name = "pgjwt"; meta = with lib.maintainers; { @@ -29,9 +22,13 @@ with pkgs; { pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}"; in '' - startAll; - $master->waitForUnit("postgresql"); - $master->copyFileFromHost("${test}","/tmp/test.sql"); - $master->succeed("${pkgs.sudo}/bin/sudo -u ${sqlSU} PGOPTIONS=--search_path=tap,public ${pgProve}/bin/pg_prove -d postgres -v -f /tmp/test.sql"); + start_all() + master.wait_for_unit("postgresql") + master.succeed( + "${pkgs.gnused}/bin/sed -e '12 i CREATE EXTENSION pgcrypto;\\nCREATE EXTENSION pgtap;\\nSET search_path TO tap,public;' ${pgjwt.src}/test.sql > /tmp/test.sql" + ) + master.succeed( + "${pkgs.sudo}/bin/sudo -u ${sqlSU} PGOPTIONS=--search_path=tap,public ${pgProve}/bin/pg_prove -d postgres -v -f /tmp/test.sql" + ) ''; }) diff --git a/nixos/tests/postgresql.nix b/nixos/tests/postgresql.nix index ae5d6d095ea..e71c3888288 100644 --- a/nixos/tests/postgresql.nix +++ b/nixos/tests/postgresql.nix @@ -3,7 +3,7 @@ pkgs ? import ../.. { inherit system config; } }: -with import ../lib/testing.nix { inherit system pkgs; }; +with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let @@ -40,29 +40,33 @@ let backupName = if backup-all then "all" else "postgres"; backupService = if backup-all then "postgresqlBackup" else "postgresqlBackup-postgres"; in '' - sub check_count { - my ($select, $nlines) = @_; - return 'test $(sudo -u postgres psql postgres -tAc "' . $select . '"|wc -l) -eq ' . $nlines; - } + def check_count(statement, lines): + return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( + statement, lines + ) + + + machine.start() + machine.wait_for_unit("postgresql") - $machine->start; - $machine->waitForUnit("postgresql"); # postgresql should be available just after unit start - $machine->succeed("cat ${test-sql} | sudo -u postgres psql"); - $machine->shutdown; # make sure that postgresql survive restart (bug #1735) - sleep(2); - $machine->start; - $machine->waitForUnit("postgresql"); - $machine->fail(check_count("SELECT * FROM sth;", 3)); - $machine->succeed(check_count("SELECT * FROM sth;", 5)); - $machine->fail(check_count("SELECT * FROM sth;", 4)); - $machine->succeed(check_count("SELECT xpath(\'/test/text()\', doc) FROM xmltest;", 1)); + machine.succeed( + "cat ${test-sql} | sudo -u postgres psql" + ) + machine.shutdown() # make sure that postgresql survive restart (bug #1735) + time.sleep(2) + machine.start() + machine.wait_for_unit("postgresql") + machine.fail(check_count("SELECT * FROM sth;", 3)) + machine.succeed(check_count("SELECT * FROM sth;", 5)) + machine.fail(check_count("SELECT * FROM sth;", 4)) + machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1)) # Check backup service - $machine->succeed("systemctl start ${backupService}.service"); - $machine->succeed("zcat /var/backup/postgresql/${backupName}.sql.gz | grep '<test>ok</test>'"); - $machine->succeed("stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600"); - $machine->shutdown; + machine.succeed("systemctl start ${backupService}.service") + machine.succeed("zcat /var/backup/postgresql/${backupName}.sql.gz | grep '<test>ok</test>'") + machine.succeed("stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600") + machine.shutdown() ''; }; diff --git a/nixos/tests/powerdns.nix b/nixos/tests/powerdns.nix index 8addcc78401..75d71315e64 100644 --- a/nixos/tests/powerdns.nix +++ b/nixos/tests/powerdns.nix @@ -1,12 +1,13 @@ -import ./make-test.nix ({ pkgs, ... }: { +import ./make-test-python.nix ({ pkgs, ... }: { name = "powerdns"; nodes.server = { ... }: { services.powerdns.enable = true; + environment.systemPackages = [ pkgs.dnsutils ]; }; testScript = '' - $server->waitForUnit("pdns"); - $server->succeed("${pkgs.dnsutils}/bin/dig version.bind txt chaos \@127.0.0.1"); + server.wait_for_unit("pdns") + server.succeed("dig version.bind txt chaos \@127.0.0.1") ''; }) diff --git a/nixos/tests/pppd.nix b/nixos/tests/pppd.nix index 91f81185909..bda0aa75bb5 100644 --- a/nixos/tests/pppd.nix +++ b/nixos/tests/pppd.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ( +import ./make-test-python.nix ( let chap-secrets = { text = ''"flynn" * "reindeerflotilla" *''; @@ -53,10 +53,10 @@ import ./make-test.nix ( environment.etc."ppp/chap-secrets" = chap-secrets; }; }; - + testScript = '' - startAll; - $client->waitUntilSucceeds("ping -c1 -W1 192.0.2.1"); - $server->waitUntilSucceeds("ping -c1 -W1 192.0.2.2"); + start_all() + client.wait_until_succeeds("ping -c1 -W1 192.0.2.1") + server.wait_until_succeeds("ping -c1 -W1 192.0.2.2") ''; - }) + }) diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix index 676183f6356..76cecb7433a 100644 --- a/nixos/tests/prometheus-exporters.nix +++ b/nixos/tests/prometheus-exporters.nix @@ -4,12 +4,10 @@ }: let - inherit (import ../lib/testing.nix { inherit system pkgs; }) makeTest; + inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest; inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge removeSuffix replaceChars singleton splitString; - escape' = str: replaceChars [''"'' "$" "\n"] [''\\\"'' "\\$" ""] str; - /* * The attrset `exporterTests` contains one attribute * for each exporter test. Each of these attributes @@ -33,9 +31,9 @@ let * services.<metricProvider>.enable = true; * }; * exporterTest = '' - * waitForUnit("prometheus-<exporterName>-exporter.service"); - * waitForOpenPort("1234"); - * succeed("curl -sSf 'localhost:1234/metrics'"); + * wait_for_unit("prometheus-<exporterName>-exporter.service") + * wait_for_open_port("1234") + * succeed("curl -sSf 'localhost:1234/metrics'") * ''; * }; * @@ -49,11 +47,11 @@ let * }; * * testScript = '' - * $<exporterName>->start(); - * $<exporterName>->waitForUnit("prometheus-<exporterName>-exporter.service"); - * $<exporterName>->waitForOpenPort("1234"); - * $<exporterName>->succeed("curl -sSf 'localhost:1234/metrics'"); - * $<exporterName>->shutdown(); + * <exporterName>.start() + * <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service") + * <exporterName>.wait_for_open_port("1234") + * <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'") + * <exporterName>.shutdown() * ''; */ @@ -72,9 +70,11 @@ let ''; }; exporterTest = '' - waitForUnit("prometheus-bind-exporter.service"); - waitForOpenPort(9119); - succeed("curl -sSf http://localhost:9119/metrics | grep -q 'bind_query_recursions_total 0'"); + wait_for_unit("prometheus-bind-exporter.service") + wait_for_open_port(9119) + succeed( + "curl -sSf http://localhost:9119/metrics | grep -q 'bind_query_recursions_total 0'" + ) ''; }; @@ -89,9 +89,11 @@ let }); }; exporterTest = '' - waitForUnit("prometheus-blackbox-exporter.service"); - waitForOpenPort(9115); - succeed("curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep -q 'probe_success 1'"); + wait_for_unit("prometheus-blackbox-exporter.service") + wait_for_open_port(9115) + succeed( + "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep -q 'probe_success 1'" + ) ''; }; @@ -100,7 +102,7 @@ let enable = true; extraFlags = [ "--web.collectd-push-path /collectd" ]; }; - exporterTest =let postData = escape' '' + exporterTest = let postData = replaceChars [ "\n" ] [ "" ] '' [{ "values":[23], "dstypes":["gauge"], @@ -108,13 +110,21 @@ let "interval":1000, "host":"testhost", "plugin":"testplugin", - "time":$(date +%s) + "time":DATE }] ''; in '' - waitForUnit("prometheus-collectd-exporter.service"); - waitForOpenPort(9103); - succeed("curl -sSfH 'Content-Type: application/json' -X POST --data \"${postData}\" localhost:9103/collectd"); - succeed("curl -sSf localhost:9103/metrics | grep -q 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"); + wait_for_unit("prometheus-collectd-exporter.service") + wait_for_open_port(9103) + succeed( + 'echo \'${postData}\'> /tmp/data.json' + ) + succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json') + succeed( + "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd" + ) + succeed( + "curl -sSf localhost:9103/metrics | grep -q 'collectd_testplugin_gauge{instance=\"testhost\"} 23'" + ) ''; }; @@ -127,9 +137,9 @@ let services.dnsmasq.enable = true; }; exporterTest = '' - waitForUnit("prometheus-dnsmasq-exporter.service"); - waitForOpenPort(9153); - succeed("curl -sSf http://localhost:9153/metrics | grep -q 'dnsmasq_leases 0'"); + wait_for_unit("prometheus-dnsmasq-exporter.service") + wait_for_open_port(9153) + succeed("curl -sSf http://localhost:9153/metrics | grep -q 'dnsmasq_leases 0'") ''; }; @@ -144,9 +154,11 @@ let services.dovecot2.enable = true; }; exporterTest = '' - waitForUnit("prometheus-dovecot-exporter.service"); - waitForOpenPort(9166); - succeed("curl -sSf http://localhost:9166/metrics | grep -q 'dovecot_up{scope=\"global\"} 1'"); + wait_for_unit("prometheus-dovecot-exporter.service") + wait_for_open_port(9166) + succeed( + "curl -sSf http://localhost:9166/metrics | grep -q 'dovecot_up{scope=\"global\"} 1'" + ) ''; }; @@ -155,9 +167,11 @@ let enable = true; }; exporterTest = '' - waitForUnit("prometheus-fritzbox-exporter.service"); - waitForOpenPort(9133); - succeed("curl -sSf http://localhost:9133/metrics | grep -q 'fritzbox_exporter_collect_errors 0'"); + wait_for_unit("prometheus-fritzbox-exporter.service") + wait_for_open_port(9133) + succeed( + "curl -sSf http://localhost:9133/metrics | grep -q 'fritzbox_exporter_collect_errors 0'" + ) ''; }; @@ -180,11 +194,11 @@ let }; }; exporterTest = '' - waitForUnit("nginx.service"); - waitForOpenPort(80); - waitForUnit("prometheus-json-exporter.service"); - waitForOpenPort(7979); - succeed("curl -sSf localhost:7979/metrics | grep -q 'json_test_metric 1'"); + wait_for_unit("nginx.service") + wait_for_open_port(80) + wait_for_unit("prometheus-json-exporter.service") + wait_for_open_port(7979) + succeed("curl -sSf localhost:7979/metrics | grep -q 'json_test_metric 1'") ''; }; @@ -222,10 +236,12 @@ let users.users.mailexporter.isSystemUser = true; }; exporterTest = '' - waitForUnit("postfix.service") - waitForUnit("prometheus-mail-exporter.service") - waitForOpenPort(9225) - waitUntilSucceeds("curl -sSf http://localhost:9225/metrics | grep -q 'mail_deliver_success{configname=\"testserver\"} 1'") + wait_for_unit("postfix.service") + wait_for_unit("prometheus-mail-exporter.service") + wait_for_open_port(9225) + wait_until_succeeds( + "curl -sSf http://localhost:9225/metrics | grep -q 'mail_deliver_success{configname=\"testserver\"} 1'" + ) ''; }; @@ -256,9 +272,9 @@ let }; }; exporterTest = '' - waitForUnit("nginx.service") - waitForUnit("prometheus-nextcloud-exporter.service") - waitForOpenPort(9205) + wait_for_unit("nginx.service") + wait_for_unit("prometheus-nextcloud-exporter.service") + wait_for_open_port(9205) succeed("curl -sSf http://localhost:9205/metrics | grep -q 'nextcloud_up 1'") ''; }; @@ -275,9 +291,9 @@ let }; }; exporterTest = '' - waitForUnit("nginx.service") - waitForUnit("prometheus-nginx-exporter.service") - waitForOpenPort(9113) + wait_for_unit("nginx.service") + wait_for_unit("prometheus-nginx-exporter.service") + wait_for_open_port(9113) succeed("curl -sSf http://localhost:9113/metrics | grep -q 'nginx_up 1'") ''; }; @@ -287,9 +303,11 @@ let enable = true; }; exporterTest = '' - waitForUnit("prometheus-node-exporter.service"); - waitForOpenPort(9100); - succeed("curl -sSf http://localhost:9100/metrics | grep -q 'node_exporter_build_info{.\\+} 1'"); + wait_for_unit("prometheus-node-exporter.service") + wait_for_open_port(9100) + succeed( + "curl -sSf http://localhost:9100/metrics | grep -q 'node_exporter_build_info{.\\+} 1'" + ) ''; }; @@ -301,9 +319,11 @@ let services.postfix.enable = true; }; exporterTest = '' - waitForUnit("prometheus-postfix-exporter.service"); - waitForOpenPort(9154); - succeed("curl -sSf http://localhost:9154/metrics | grep -q 'postfix_smtpd_connects_total 0'"); + wait_for_unit("prometheus-postfix-exporter.service") + wait_for_open_port(9154) + succeed( + "curl -sSf http://localhost:9154/metrics | grep -q 'postfix_smtpd_connects_total 0'" + ) ''; }; @@ -316,18 +336,24 @@ let services.postgresql.enable = true; }; exporterTest = '' - waitForUnit("prometheus-postgres-exporter.service"); - waitForOpenPort(9187); - waitForUnit("postgresql.service"); - succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_exporter_last_scrape_error 0'"); - succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 1'"); - systemctl("stop postgresql.service"); - succeed("curl -sSf http://localhost:9187/metrics | grep -qv 'pg_exporter_last_scrape_error 0'"); - succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 0'"); - systemctl("start postgresql.service"); - waitForUnit("postgresql.service"); - succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_exporter_last_scrape_error 0'"); - succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 1'"); + wait_for_unit("prometheus-postgres-exporter.service") + wait_for_open_port(9187) + wait_for_unit("postgresql.service") + succeed( + "curl -sSf http://localhost:9187/metrics | grep -q 'pg_exporter_last_scrape_error 0'" + ) + succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 1'") + systemctl("stop postgresql.service") + succeed( + "curl -sSf http://localhost:9187/metrics | grep -qv 'pg_exporter_last_scrape_error 0'" + ) + succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 0'") + systemctl("start postgresql.service") + wait_for_unit("postgresql.service") + succeed( + "curl -sSf http://localhost:9187/metrics | grep -q 'pg_exporter_last_scrape_error 0'" + ) + succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 1'") ''; }; @@ -339,11 +365,13 @@ let services.rspamd.enable = true; }; exporterTest = '' - waitForUnit("rspamd.service"); - waitForUnit("prometheus-rspamd-exporter.service"); - waitForOpenPort(11334); - waitForOpenPort(7980); - waitUntilSucceeds("curl -sSf localhost:7980/metrics | grep -q 'rspamd_scanned{host=\"rspamd\"} 0'"); + wait_for_unit("rspamd.service") + wait_for_unit("prometheus-rspamd-exporter.service") + wait_for_open_port(11334) + wait_for_open_port(7980) + wait_until_succeeds( + "curl -sSf localhost:7980/metrics | grep -q 'rspamd_scanned{host=\"rspamd\"} 0'" + ) ''; }; @@ -356,9 +384,9 @@ let }; }; exporterTest = '' - waitForUnit("prometheus-snmp-exporter.service"); - waitForOpenPort(9116); - succeed("curl -sSf localhost:9116/metrics | grep -q 'snmp_request_errors_total 0'"); + wait_for_unit("prometheus-snmp-exporter.service") + wait_for_open_port(9116) + succeed("curl -sSf localhost:9116/metrics | grep -q 'snmp_request_errors_total 0'") ''; }; @@ -377,11 +405,11 @@ let }; }; exporterTest = '' - waitForUnit("nginx.service"); - waitForOpenPort(80); - waitForUnit("prometheus-surfboard-exporter.service"); - waitForOpenPort(9239); - succeed("curl -sSf localhost:9239/metrics | grep -q 'surfboard_up 1'"); + wait_for_unit("nginx.service") + wait_for_open_port(80) + wait_for_unit("prometheus-surfboard-exporter.service") + wait_for_open_port(9239) + succeed("curl -sSf localhost:9239/metrics | grep -q 'surfboard_up 1'") ''; }; @@ -396,11 +424,11 @@ let services.tor.controlPort = 9051; }; exporterTest = '' - waitForUnit("tor.service"); - waitForOpenPort(9051); - waitForUnit("prometheus-tor-exporter.service"); - waitForOpenPort(9130); - succeed("curl -sSf localhost:9130/metrics | grep -q 'tor_version{.\\+} 1'"); + wait_for_unit("tor.service") + wait_for_open_port(9051) + wait_for_unit("prometheus-tor-exporter.service") + wait_for_open_port(9130) + succeed("curl -sSf localhost:9130/metrics | grep -q 'tor_version{.\\+} 1'") ''; }; @@ -426,10 +454,12 @@ let }; }; exporterTest = '' - waitForUnit("prometheus-varnish-exporter.service"); - waitForOpenPort(6081); - waitForOpenPort(9131); - succeed("curl -sSf http://localhost:9131/metrics | grep -q 'varnish_up 1'"); + wait_for_unit("prometheus-varnish-exporter.service") + wait_for_open_port(6081) + wait_for_open_port(9131) + succeed( + "curl -sSf http://localhost:9131/metrics | grep -q 'varnish_up 1'" + ) ''; }; @@ -451,9 +481,11 @@ let systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ]; }; exporterTest = '' - waitForUnit("prometheus-wireguard-exporter.service"); - waitForOpenPort(9586); - waitUntilSucceeds("curl -sSf http://localhost:9586/metrics | grep '${snakeoil.peer1.publicKey}'"); + wait_for_unit("prometheus-wireguard-exporter.service") + wait_for_open_port(9586) + wait_until_succeeds( + "curl -sSf http://localhost:9586/metrics | grep '${snakeoil.peer1.publicKey}'" + ) ''; }; }; @@ -466,11 +498,13 @@ mapAttrs (exporter: testConfig: (makeTest { } testConfig.metricProvider or {}]; testScript = '' - ${"$"+exporter}->start(); - ${concatStringsSep " " (map (line: '' - ${"$"+exporter}->${line}; - '') (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))} - ${"$"+exporter}->shutdown(); + ${exporter}.start() + ${concatStringsSep "\n" (map (line: + if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")") + then line + else "${exporter}.${line}" + ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))} + ${exporter}.shutdown() ''; meta = with maintainers; { diff --git a/nixos/tests/quake3.nix b/nixos/tests/quake3.nix deleted file mode 100644 index 4253ce4a867..00000000000 --- a/nixos/tests/quake3.nix +++ /dev/null @@ -1,95 +0,0 @@ -import ./make-test.nix ({ pkgs, ...} : - -let - - # Build Quake with coverage instrumentation. - overrides = pkgs: - { - quake3game = pkgs.quake3game.override (args: { - stdenv = pkgs.stdenvAdapters.addCoverageInstrumentation args.stdenv; - }); - }; - - # Only allow the demo data to be used (only if it's unfreeRedistributable). - unfreePredicate = pkg: with pkgs.lib; let - allowPackageNames = [ "quake3-demodata" "quake3-pointrelease" ]; - allowLicenses = [ pkgs.lib.licenses.unfreeRedistributable ]; - in elem pkg.pname allowPackageNames && - elem (pkg.meta.license or null) allowLicenses; - -in - -rec { - name = "quake3"; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ domenkozar eelco ]; - }; - - # TODO: lcov doesn't work atm - #makeCoverageReport = true; - - client = - { pkgs, ... }: - - { imports = [ ./common/x11.nix ]; - hardware.opengl.driSupport = true; - environment.systemPackages = [ pkgs.quake3demo ]; - nixpkgs.config.packageOverrides = overrides; - nixpkgs.config.allowUnfreePredicate = unfreePredicate; - }; - - nodes = - { server = - { pkgs, ... }: - - { systemd.services.quake3-server = - { wantedBy = [ "multi-user.target" ]; - script = - "${pkgs.quake3demo}/bin/quake3-server +set g_gametype 0 " + - "+map q3dm7 +addbot grunt +addbot daemia 2> /tmp/log"; - }; - nixpkgs.config.packageOverrides = overrides; - nixpkgs.config.allowUnfreePredicate = unfreePredicate; - networking.firewall.allowedUDPPorts = [ 27960 ]; - }; - - client1 = client; - client2 = client; - }; - - testScript = - '' - startAll; - - $server->waitForUnit("quake3-server"); - $client1->waitForX; - $client2->waitForX; - - $client1->execute("quake3 +set r_fullscreen 0 +set name Foo +connect server &"); - $client2->execute("quake3 +set r_fullscreen 0 +set name Bar +connect server &"); - - $server->waitUntilSucceeds("grep -q 'Foo.*entered the game' /tmp/log"); - $server->waitUntilSucceeds("grep -q 'Bar.*entered the game' /tmp/log"); - - $server->sleep(10); # wait for a while to get a nice screenshot - - $client1->block(); - - $server->sleep(20); - - $client1->screenshot("screen1"); - $client2->screenshot("screen2"); - - $client1->unblock(); - - $server->sleep(10); - - $client1->screenshot("screen3"); - $client2->screenshot("screen4"); - - $client1->shutdown(); - $client2->shutdown(); - $server->stopJob("quake3-server"); - ''; - -}) diff --git a/nixos/tests/radarr.nix b/nixos/tests/radarr.nix index 9bc5607ccd5..ed90025ac42 100644 --- a/nixos/tests/radarr.nix +++ b/nixos/tests/radarr.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ lib, ... }: +import ./make-test-python.nix ({ lib, ... }: with lib; @@ -11,8 +11,8 @@ with lib; { services.radarr.enable = true; }; testScript = '' - $machine->waitForUnit('radarr.service'); - $machine->waitForOpenPort('7878'); - $machine->succeed("curl --fail http://localhost:7878/"); + machine.wait_for_unit("radarr.service") + machine.wait_for_open_port("7878") + machine.succeed("curl --fail http://localhost:7878/") ''; }) diff --git a/nixos/tests/redis.nix b/nixos/tests/redis.nix index 325d93424dd..529965d7acd 100644 --- a/nixos/tests/redis.nix +++ b/nixos/tests/redis.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "redis"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ flokli ]; @@ -15,12 +15,10 @@ import ./make-test.nix ({ pkgs, ...} : { }; testScript = '' - startAll; - - $machine->waitForUnit("redis"); - $machine->waitForOpenPort("6379"); - - $machine->succeed("redis-cli ping | grep PONG"); - $machine->succeed("redis-cli -s /run/redis/redis.sock ping | grep PONG"); + start_all() + machine.wait_for_unit("redis") + machine.wait_for_open_port("6379") + machine.succeed("redis-cli ping | grep PONG") + machine.succeed("redis-cli -s /run/redis/redis.sock ping | grep PONG") ''; }) diff --git a/nixos/tests/redmine.nix b/nixos/tests/redmine.nix index 2d4df288b05..f0f4cbf6a21 100644 --- a/nixos/tests/redmine.nix +++ b/nixos/tests/redmine.nix @@ -64,18 +64,13 @@ let }; in { - v3-mysql = mysqlTest pkgs.redmine // { - name = "v3-mysql"; + mysql = mysqlTest pkgs.redmine // { + name = "mysql"; meta.maintainers = [ maintainers.aanderse ]; }; - v4-mysql = mysqlTest pkgs.redmine_4 // { - name = "v4-mysql"; - meta.maintainers = [ maintainers.aanderse ]; - }; - - v4-pgsql = pgsqlTest pkgs.redmine_4 // { - name = "v4-pgsql"; + pgsql = pgsqlTest pkgs.redmine // { + name = "pgsql"; meta.maintainers = [ maintainers.aanderse ]; }; } diff --git a/nixos/tests/roundcube.nix b/nixos/tests/roundcube.nix index ed0ebd7dd19..4f2560ce8c2 100644 --- a/nixos/tests/roundcube.nix +++ b/nixos/tests/roundcube.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "roundcube"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ globin ]; @@ -21,10 +21,10 @@ import ./make-test.nix ({ pkgs, ...} : { }; testScript = '' - $roundcube->start; - $roundcube->waitForUnit("postgresql.service"); - $roundcube->waitForUnit("phpfpm-roundcube.service"); - $roundcube->waitForUnit("nginx.service"); - $roundcube->succeed("curl -sSfL http://roundcube/ | grep 'Keep me logged in'"); + roundcube.start + roundcube.wait_for_unit("postgresql.service") + roundcube.wait_for_unit("phpfpm-roundcube.service") + roundcube.wait_for_unit("nginx.service") + roundcube.succeed("curl -sSfL http://roundcube/ | grep 'Keep me logged in'") ''; }) diff --git a/nixos/tests/rss2email.nix b/nixos/tests/rss2email.nix index 492d47da9f5..d62207a417b 100644 --- a/nixos/tests/rss2email.nix +++ b/nixos/tests/rss2email.nix @@ -1,4 +1,4 @@ -import ./make-test.nix { +import ./make-test-python.nix { name = "opensmtpd"; nodes = { @@ -53,14 +53,14 @@ import ./make-test.nix { }; testScript = '' - startAll; + start_all() - $server->waitForUnit("network-online.target"); - $server->waitForUnit("opensmtpd"); - $server->waitForUnit("dovecot2"); - $server->waitForUnit("nginx"); - $server->waitForUnit("rss2email"); + server.wait_for_unit("network-online.target") + server.wait_for_unit("opensmtpd") + server.wait_for_unit("dovecot2") + server.wait_for_unit("nginx") + server.wait_for_unit("rss2email") - $server->waitUntilSucceeds('check-mail-landed >&2'); + server.wait_until_succeeds("check-mail-landed >&2") ''; } diff --git a/nixos/tests/samba.nix b/nixos/tests/samba.nix index 2802e00a5b1..142269752b3 100644 --- a/nixos/tests/samba.nix +++ b/nixos/tests/samba.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: +import ./make-test-python.nix ({ pkgs, ... }: { name = "samba"; @@ -36,12 +36,12 @@ import ./make-test.nix ({ pkgs, ... }: testScript = '' - $server->start; - $server->waitForUnit("samba.target"); - $server->succeed("mkdir -p /public; echo bar > /public/foo"); + server.start() + server.wait_for_unit("samba.target") + server.succeed("mkdir -p /public; echo bar > /public/foo") - $client->start; - $client->waitForUnit("remote-fs.target"); - $client->succeed("[[ \$(cat /public/foo) = bar ]]"); + client.start() + client.wait_for_unit("remote-fs.target") + client.succeed("[[ $(cat /public/foo) = bar ]]") ''; }) diff --git a/nixos/tests/shiori.nix b/nixos/tests/shiori.nix index 0022a7220fe..a5771262c6f 100644 --- a/nixos/tests/shiori.nix +++ b/nixos/tests/shiori.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ lib, ...}: +import ./make-test-python.nix ({ pkgs, lib, ...}: { name = "shiori"; @@ -8,10 +8,74 @@ import ./make-test.nix ({ lib, ...}: { ... }: { services.shiori.enable = true; }; - testScript = '' - $machine->waitForUnit('shiori.service'); - $machine->waitForOpenPort('8080'); - $machine->succeed("curl --fail http://localhost:8080/"); - $machine->succeed("curl --fail --location http://localhost:8080/ | grep -qi shiori"); + testScript = let + authJSON = pkgs.writeText "auth.json" (builtins.toJSON { + username = "shiori"; + password = "gopher"; + remember = 1; # hour + owner = true; + }); + + insertBookmark = { + url = "http://example.org"; + title = "Example Bookmark"; + }; + + insertBookmarkJSON = pkgs.writeText "insertBookmark.json" (builtins.toJSON insertBookmark); + in '' + import json + + machine.wait_for_unit("shiori.service") + machine.wait_for_open_port(8080) + machine.succeed("curl --fail http://localhost:8080/") + machine.succeed("curl --fail --location http://localhost:8080/ | grep -qi shiori") + + with subtest("login"): + auth_json = machine.succeed( + "curl --fail --location http://localhost:8080/api/login " + "-X POST -H 'Content-Type:application/json' -d @${authJSON}" + ) + auth_ret = json.loads(auth_json) + session_id = auth_ret["session"] + + with subtest("bookmarks"): + with subtest("first use no bookmarks"): + bookmarks_json = machine.succeed( + ( + "curl --fail --location http://localhost:8080/api/bookmarks " + "-H 'X-Session-Id:{}'" + ).format(session_id) + ) + + if json.loads(bookmarks_json)["bookmarks"] != []: + raise Exception("Shiori have a bookmark on first use") + + with subtest("insert bookmark"): + machine.succeed( + ( + "curl --fail --location http://localhost:8080/api/bookmarks " + "-X POST -H 'X-Session-Id:{}' " + "-H 'Content-Type:application/json' -d @${insertBookmarkJSON}" + ).format(session_id) + ) + + with subtest("get inserted bookmark"): + bookmarks_json = machine.succeed( + ( + "curl --fail --location http://localhost:8080/api/bookmarks " + "-H 'X-Session-Id:{}'" + ).format(session_id) + ) + + bookmarks = json.loads(bookmarks_json)["bookmarks"] + if len(bookmarks) != 1: + raise Exception("Shiori didn't save the bookmark") + + bookmark = bookmarks[0] + if ( + bookmark["url"] != "${insertBookmark.url}" + or bookmark["title"] != "${insertBookmark.title}" + ): + raise Exception("Inserted bookmark doesn't have same URL or title") ''; }) diff --git a/nixos/tests/signal-desktop.nix b/nixos/tests/signal-desktop.nix index 605b9c3e130..c746d46dc55 100644 --- a/nixos/tests/signal-desktop.nix +++ b/nixos/tests/signal-desktop.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : +import ./make-test-python.nix ({ pkgs, ...} : { name = "signal-desktop"; @@ -24,14 +24,14 @@ import ./make-test.nix ({ pkgs, ...} : testScript = { nodes, ... }: let user = nodes.machine.config.users.users.alice; in '' - startAll; - $machine->waitForX; + start_all() + machine.wait_for_x() # start signal desktop - $machine->execute("su - alice -c signal-desktop &"); + machine.execute("su - alice -c signal-desktop &") # wait for the "Link your phone to Signal Desktop" message - $machine->waitForText(qr/Link your phone to Signal Desktop/); - $machine->screenshot("signal_desktop"); + machine.wait_for_text("Link your phone to Signal Desktop") + machine.screenshot("signal_desktop") ''; }) diff --git a/nixos/tests/simple.nix b/nixos/tests/simple.nix index 84c5621d962..3810a2cd3a5 100644 --- a/nixos/tests/simple.nix +++ b/nixos/tests/simple.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "simple"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ eelco ]; @@ -10,8 +10,8 @@ import ./make-test.nix ({ pkgs, ...} : { testScript = '' - startAll; - $machine->waitForUnit("multi-user.target"); - $machine->shutdown; + start_all() + machine.wait_for_unit("multi-user.target") + machine.shutdown() ''; }) diff --git a/nixos/tests/smokeping.nix b/nixos/tests/smokeping.nix index 07d22805112..4f8f0fcc9fe 100644 --- a/nixos/tests/smokeping.nix +++ b/nixos/tests/smokeping.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "smokeping"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ cransom ]; @@ -22,12 +22,12 @@ import ./make-test.nix ({ pkgs, ...} : { }; testScript = '' - startAll; - $sm->waitForUnit("smokeping"); - $sm->waitForUnit("thttpd"); - $sm->waitForFile("/var/lib/smokeping/data/Local/LocalMachine.rrd"); - $sm->succeed("curl -s -f localhost:8081/smokeping.fcgi?target=Local"); - $sm->succeed("ls /var/lib/smokeping/cache/Local/LocalMachine_mini.png"); - $sm->succeed("ls /var/lib/smokeping/cache/index.html"); + start_all() + sm.wait_for_unit("smokeping") + sm.wait_for_unit("thttpd") + sm.wait_for_file("/var/lib/smokeping/data/Local/LocalMachine.rrd") + sm.succeed("curl -s -f localhost:8081/smokeping.fcgi?target=Local") + sm.succeed("ls /var/lib/smokeping/cache/Local/LocalMachine_mini.png") + sm.succeed("ls /var/lib/smokeping/cache/index.html") ''; }) diff --git a/nixos/tests/snapper.nix b/nixos/tests/snapper.nix index 74ec22fd349..018102d7f64 100644 --- a/nixos/tests/snapper.nix +++ b/nixos/tests/snapper.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ ... }: +import ./make-test-python.nix ({ ... }: { name = "snapper"; @@ -20,24 +20,16 @@ import ./make-test.nix ({ ... }: }; testScript = '' - $machine->succeed("btrfs subvolume create /home/.snapshots"); - - $machine->succeed("snapper -c home list"); - - $machine->succeed("snapper -c home create --description empty"); - - $machine->succeed("echo test > /home/file"); - $machine->succeed("snapper -c home create --description file"); - - $machine->succeed("snapper -c home status 1..2"); - - $machine->succeed("snapper -c home undochange 1..2"); - $machine->fail("ls /home/file"); - - $machine->succeed("snapper -c home delete 2"); - - $machine->succeed("systemctl --wait start snapper-timeline.service"); - - $machine->succeed("systemctl --wait start snapper-cleanup.service"); + machine.succeed("btrfs subvolume create /home/.snapshots") + machine.succeed("snapper -c home list") + machine.succeed("snapper -c home create --description empty") + machine.succeed("echo test > /home/file") + machine.succeed("snapper -c home create --description file") + machine.succeed("snapper -c home status 1..2") + machine.succeed("snapper -c home undochange 1..2") + machine.fail("ls /home/file") + machine.succeed("snapper -c home delete 2") + machine.succeed("systemctl --wait start snapper-timeline.service") + machine.succeed("systemctl --wait start snapper-cleanup.service") ''; }) diff --git a/nixos/tests/strongswan-swanctl.nix b/nixos/tests/strongswan-swanctl.nix index 9bab9349ea7..152c0d61c54 100644 --- a/nixos/tests/strongswan-swanctl.nix +++ b/nixos/tests/strongswan-swanctl.nix @@ -16,7 +16,7 @@ # See the NixOS manual for how to run this test: # https://nixos.org/nixos/manual/index.html#sec-running-nixos-tests-interactively -import ./make-test.nix ({ pkgs, ...} : +import ./make-test-python.nix ({ pkgs, ...} : let allowESP = "iptables --insert INPUT --protocol ESP --jump ACCEPT"; @@ -142,7 +142,7 @@ in { }; testScript = '' - startAll(); - $carol->waitUntilSucceeds("ping -c 1 alice"); + start_all() + carol.wait_until_succeeds("ping -c 1 alice") ''; }) diff --git a/nixos/tests/telegraf.nix b/nixos/tests/telegraf.nix index 6776f8d8c37..73f741b1135 100644 --- a/nixos/tests/telegraf.nix +++ b/nixos/tests/telegraf.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "telegraf"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ mic92 ]; @@ -22,9 +22,9 @@ import ./make-test.nix ({ pkgs, ...} : { }; testScript = '' - startAll; + start_all() - $machine->waitForUnit("telegraf.service"); - $machine->waitUntilSucceeds("grep -q example /tmp/metrics.out"); + machine.wait_for_unit("telegraf.service") + machine.wait_until_succeeds("grep -q example /tmp/metrics.out") ''; }) diff --git a/nixos/tests/tinydns.nix b/nixos/tests/tinydns.nix index cb7ee0c5fb5..c7740d5ade3 100644 --- a/nixos/tests/tinydns.nix +++ b/nixos/tests/tinydns.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ lib, ...} : { +import ./make-test-python.nix ({ lib, ...} : { name = "tinydns"; meta = { maintainers = with lib.maintainers; [ basvandijk ]; @@ -19,8 +19,8 @@ import ./make-test.nix ({ lib, ...} : { }; }; testScript = '' - $nameserver->start; - $nameserver->waitForUnit("tinydns.service"); - $nameserver->succeed("host bla.foo.bar | grep '1\.2\.3\.4'"); + nameserver.start() + nameserver.wait_for_unit("tinydns.service") + nameserver.succeed("host bla.foo.bar | grep '1\.2\.3\.4'") ''; }) diff --git a/nixos/tests/tor.nix b/nixos/tests/tor.nix index 0cb44ddff24..ad07231557c 100644 --- a/nixos/tests/tor.nix +++ b/nixos/tests/tor.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ lib, ... }: with lib; +import ./make-test-python.nix ({ lib, ... }: with lib; rec { name = "tor"; @@ -21,8 +21,10 @@ rec { }; testScript = '' - $client->waitForUnit("tor.service"); - $client->waitForOpenPort(9051); - $client->succeed("echo GETINFO version | nc 127.0.0.1 9051") =~ /514 Authentication required./ or die; + client.wait_for_unit("tor.service") + client.wait_for_open_port(9051) + assert "514 Authentication required." in client.succeed( + "echo GETINFO version | nc 127.0.0.1 9051" + ) ''; }) diff --git a/nixos/tests/trac.nix b/nixos/tests/trac.nix new file mode 100644 index 00000000000..7953f8d41f7 --- /dev/null +++ b/nixos/tests/trac.nix @@ -0,0 +1,19 @@ +import ./make-test-python.nix ({ pkgs, ... }: { + name = "trac"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ mmahut ]; + }; + + nodes = { + machine = { ... }: { + services.trac.enable = true; + }; + }; + + testScript = '' + start_all() + machine.wait_for_unit("trac.service") + machine.wait_for_open_port(8000) + machine.wait_until_succeeds("curl -L http://localhost:8000/ | grep 'Trac Powered'") + ''; +}) diff --git a/nixos/tests/transmission.nix b/nixos/tests/transmission.nix index f1c238730eb..f4f2186be1f 100644 --- a/nixos/tests/transmission.nix +++ b/nixos/tests/transmission.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "transmission"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ coconnor ]; @@ -14,8 +14,8 @@ import ./make-test.nix ({ pkgs, ...} : { testScript = '' - startAll; - $machine->waitForUnit("transmission"); - $machine->shutdown; + start_all() + machine.wait_for_unit("transmission") + machine.shutdown() ''; }) diff --git a/nixos/tests/trezord.nix b/nixos/tests/trezord.nix index 1c85bf53934..8d908a52249 100644 --- a/nixos/tests/trezord.nix +++ b/nixos/tests/trezord.nix @@ -1,7 +1,7 @@ -import ./make-test.nix ({ pkgs, ... }: { +import ./make-test-python.nix ({ pkgs, ... }: { name = "trezord"; meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ mmahut ]; + maintainers = [ mmahut "1000101" ]; }; nodes = { @@ -12,9 +12,9 @@ import ./make-test.nix ({ pkgs, ... }: { }; testScript = '' - startAll; - $machine->waitForUnit("trezord.service"); - $machine->waitForOpenPort(21325); - $machine->waitUntilSucceeds("curl -L http://localhost:21325/status/ | grep Version"); + start_all() + machine.wait_for_unit("trezord.service") + machine.wait_for_open_port(21325) + machine.wait_until_succeeds("curl -L http://localhost:21325/status/ | grep Version") ''; }) diff --git a/nixos/tests/trickster.nix b/nixos/tests/trickster.nix new file mode 100644 index 00000000000..e2ca00980d5 --- /dev/null +++ b/nixos/tests/trickster.nix @@ -0,0 +1,37 @@ +import ./make-test-python.nix ({ pkgs, ... }: { + name = "trickster"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ "1000101" ]; + }; + + nodes = { + prometheus = { ... }: { + services.prometheus.enable = true; + networking.firewall.allowedTCPPorts = [ 9090 ]; + }; + trickster = { ... }: { + services.trickster.enable = true; + }; + }; + + testScript = '' + start_all() + prometheus.wait_for_unit("prometheus.service") + prometheus.wait_for_open_port(9090) + prometheus.wait_until_succeeds( + "curl -L http://localhost:9090/metrics | grep 'promhttp_metric_handler_requests_total{code=\"500\"} 0'" + ) + trickster.wait_for_unit("trickster.service") + trickster.wait_for_open_port(8082) + trickster.wait_for_open_port(9090) + trickster.wait_until_succeeds( + "curl -L http://localhost:8082/metrics | grep 'promhttp_metric_handler_requests_total{code=\"500\"} 0'" + ) + trickster.wait_until_succeeds( + "curl -L http://prometheus:9090/metrics | grep 'promhttp_metric_handler_requests_total{code=\"500\"} 0'" + ) + trickster.wait_until_succeeds( + "curl -L http://localhost:9090/metrics | grep 'promhttp_metric_handler_requests_total{code=\"500\"} 0'" + ) + ''; +}) \ No newline at end of file diff --git a/nixos/tests/udisks2.nix b/nixos/tests/udisks2.nix index dcf869908d8..0cbfa0c4c7b 100644 --- a/nixos/tests/udisks2.nix +++ b/nixos/tests/udisks2.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: +import ./make-test-python.nix ({ pkgs, ... }: let @@ -30,32 +30,40 @@ in testScript = '' - my $stick = $machine->stateDir . "/usbstick.img"; - system("xz -d < ${stick} > $stick") == 0 or die; + import lzma - $machine->succeed("udisksctl info -b /dev/vda >&2"); - $machine->fail("udisksctl info -b /dev/sda1"); + with lzma.open( + "${stick}" + ) as data, open(machine.state_dir + "/usbstick.img", "wb") as stick: + stick.write(data.read()) + + machine.succeed("udisksctl info -b /dev/vda >&2") + machine.fail("udisksctl info -b /dev/sda1") # Attach a USB stick and wait for it to show up. - $machine->sendMonitorCommand("drive_add 0 id=stick,if=none,file=$stick,format=raw"); - $machine->sendMonitorCommand("device_add usb-storage,id=stick,drive=stick"); - $machine->waitUntilSucceeds("udisksctl info -b /dev/sda1"); - $machine->succeed("udisksctl info -b /dev/sda1 | grep 'IdLabel:.*USBSTICK'"); + machine.send_monitor_command( + f"drive_add 0 id=stick,if=none,file={stick.name},format=raw" + ) + machine.send_monitor_command("device_add usb-storage,id=stick,drive=stick") + machine.wait_until_succeeds("udisksctl info -b /dev/sda1") + machine.succeed("udisksctl info -b /dev/sda1 | grep 'IdLabel:.*USBSTICK'") # Mount the stick as a non-root user and do some stuff with it. - $machine->succeed("su - alice -c 'udisksctl info -b /dev/sda1'"); - $machine->succeed("su - alice -c 'udisksctl mount -b /dev/sda1'"); - $machine->succeed("su - alice -c 'cat /run/media/alice/USBSTICK/test.txt'") =~ /Hello World/ or die; - $machine->succeed("su - alice -c 'echo foo > /run/media/alice/USBSTICK/bar.txt'"); + machine.succeed("su - alice -c 'udisksctl info -b /dev/sda1'") + machine.succeed("su - alice -c 'udisksctl mount -b /dev/sda1'") + machine.succeed( + "su - alice -c 'cat /run/media/alice/USBSTICK/test.txt' | grep -q 'Hello World'" + ) + machine.succeed("su - alice -c 'echo foo > /run/media/alice/USBSTICK/bar.txt'") # Unmounting the stick should make the mountpoint disappear. - $machine->succeed("su - alice -c 'udisksctl unmount -b /dev/sda1'"); - $machine->fail("[ -d /run/media/alice/USBSTICK ]"); + machine.succeed("su - alice -c 'udisksctl unmount -b /dev/sda1'") + machine.fail("[ -d /run/media/alice/USBSTICK ]") # Remove the USB stick. - $machine->sendMonitorCommand("device_del stick"); - $machine->waitUntilFails("udisksctl info -b /dev/sda1"); - $machine->fail("[ -e /dev/sda ]"); + machine.send_monitor_command("device_del stick") + machine.wait_until_fails("udisksctl info -b /dev/sda1") + machine.fail("[ -e /dev/sda ]") ''; }) diff --git a/nixos/tests/upnp.nix b/nixos/tests/upnp.nix index 98344aee3ef..d2e7fdd4fbe 100644 --- a/nixos/tests/upnp.nix +++ b/nixos/tests/upnp.nix @@ -5,7 +5,7 @@ # this succeeds an external client will try to connect to the port # mapping. -import ./make-test.nix ({ pkgs, ... }: +import ./make-test-python.nix ({ pkgs, ... }: let internalRouterAddress = "192.168.3.1"; @@ -75,20 +75,20 @@ in testScript = { nodes, ... }: '' - startAll; + start_all() # Wait for network and miniupnpd. - $router->waitForUnit("network-online.target"); - # $router->waitForUnit("nat"); - $router->waitForUnit("firewall.service"); - $router->waitForUnit("miniupnpd"); + router.wait_for_unit("network-online.target") + # $router.wait_for_unit("nat") + router.wait_for_unit("firewall.service") + router.wait_for_unit("miniupnpd") - $client1->waitForUnit("network-online.target"); + client1.wait_for_unit("network-online.target") - $client1->succeed("upnpc -a ${internalClient1Address} 9000 9000 TCP"); + client1.succeed("upnpc -a ${internalClient1Address} 9000 9000 TCP") - $client1->waitForUnit("httpd"); - $client2->waitUntilSucceeds("curl http://${externalRouterAddress}:9000/"); + client1.wait_for_unit("httpd") + client2.wait_until_succeeds("curl http://${externalRouterAddress}:9000/") ''; }) diff --git a/nixos/tests/vault.nix b/nixos/tests/vault.nix index caf0cbb2abf..ac8cf0703da 100644 --- a/nixos/tests/vault.nix +++ b/nixos/tests/vault.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ... }: +import ./make-test-python.nix ({ pkgs, ... }: { name = "vault"; meta = with pkgs.stdenv.lib.maintainers; { @@ -12,12 +12,12 @@ import ./make-test.nix ({ pkgs, ... }: testScript = '' - startAll; + start_all() - $machine->waitForUnit('multi-user.target'); - $machine->waitForUnit('vault.service'); - $machine->waitForOpenPort(8200); - $machine->succeed('vault operator init'); - $machine->succeed('vault status | grep Sealed | grep true'); + machine.wait_for_unit("multi-user.target") + machine.wait_for_unit("vault.service") + machine.wait_for_open_port(8200) + machine.succeed("vault operator init") + machine.succeed("vault status | grep Sealed | grep true") ''; }) diff --git a/nixos/tests/wireguard/default.nix b/nixos/tests/wireguard/default.nix index b0797b96323..8206823a918 100644 --- a/nixos/tests/wireguard/default.nix +++ b/nixos/tests/wireguard/default.nix @@ -2,7 +2,7 @@ let wg-snakeoil-keys = import ./snakeoil-keys.nix; in -import ../make-test.nix ({ pkgs, ...} : { +import ../make-test-python.nix ({ pkgs, ...} : { name = "wireguard"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ ma27 ]; @@ -86,12 +86,12 @@ import ../make-test.nix ({ pkgs, ...} : { }; testScript = '' - startAll; + start_all() - $peer0->waitForUnit("wireguard-wg0.service"); - $peer1->waitForUnit("wireguard-wg0.service"); + peer0.wait_for_unit("wireguard-wg0.service") + peer1.wait_for_unit("wireguard-wg0.service") - $peer1->succeed("ping -c5 fc00::1"); - $peer1->succeed("ping -c5 10.23.42.1") + peer1.succeed("ping -c5 fc00::1") + peer1.succeed("ping -c5 10.23.42.1") ''; }) diff --git a/nixos/tests/wireguard/generated.nix b/nixos/tests/wireguard/generated.nix index 897feafe3ff..a29afd2d466 100644 --- a/nixos/tests/wireguard/generated.nix +++ b/nixos/tests/wireguard/generated.nix @@ -1,4 +1,4 @@ -import ../make-test.nix ({ pkgs, ...} : { +import ../make-test-python.nix ({ pkgs, ...} : { name = "wireguard-generated"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ ma27 grahamc ]; @@ -28,30 +28,34 @@ import ../make-test.nix ({ pkgs, ...} : { }; testScript = '' - startAll; - - $peer1->waitForUnit("wireguard-wg0.service"); - $peer2->waitForUnit("wireguard-wg0.service"); - - my ($retcode, $peer1pubkey) = $peer1->execute("wg pubkey < /etc/wireguard/private"); - $peer1pubkey =~ s/\s+$//; - if ($retcode != 0) { - die "Could not read public key from peer1"; - } - - my ($retcode, $peer2pubkey) = $peer2->execute("wg pubkey < /etc/wireguard/private"); - $peer2pubkey =~ s/\s+$//; - if ($retcode != 0) { - die "Could not read public key from peer2"; - } - - $peer1->succeed("wg set wg0 peer $peer2pubkey allowed-ips 10.10.10.2/32 endpoint 192.168.1.2:12345 persistent-keepalive 1"); - $peer1->succeed("ip route replace 10.10.10.2/32 dev wg0 table main"); - - $peer2->succeed("wg set wg0 peer $peer1pubkey allowed-ips 10.10.10.1/32 endpoint 192.168.1.1:12345 persistent-keepalive 1"); - $peer2->succeed("ip route replace 10.10.10.1/32 dev wg0 table main"); - - $peer1->succeed("ping -c1 10.10.10.2"); - $peer2->succeed("ping -c1 10.10.10.1"); + start_all() + + peer1.wait_for_unit("wireguard-wg0.service") + peer2.wait_for_unit("wireguard-wg0.service") + + retcode, peer1pubkey = peer1.execute("wg pubkey < /etc/wireguard/private") + if retcode != 0: + raise Exception("Could not read public key from peer1") + + retcode, peer2pubkey = peer2.execute("wg pubkey < /etc/wireguard/private") + if retcode != 0: + raise Exception("Could not read public key from peer2") + + peer1.succeed( + "wg set wg0 peer {} allowed-ips 10.10.10.2/32 endpoint 192.168.1.2:12345 persistent-keepalive 1".format( + peer2pubkey.strip() + ) + ) + peer1.succeed("ip route replace 10.10.10.2/32 dev wg0 table main") + + peer2.succeed( + "wg set wg0 peer {} allowed-ips 10.10.10.1/32 endpoint 192.168.1.1:12345 persistent-keepalive 1".format( + peer1pubkey.strip() + ) + ) + peer2.succeed("ip route replace 10.10.10.1/32 dev wg0 table main") + + peer1.succeed("ping -c1 10.10.10.2") + peer2.succeed("ping -c1 10.10.10.1") ''; }) diff --git a/nixos/tests/xautolock.nix b/nixos/tests/xautolock.nix index ee46d9e05b0..10e92b40e95 100644 --- a/nixos/tests/xautolock.nix +++ b/nixos/tests/xautolock.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, lib, ... }: +import ./make-test-python.nix ({ pkgs, lib, ... }: with lib; @@ -15,10 +15,10 @@ with lib; }; testScript = '' - $machine->start; - $machine->waitForX; - $machine->mustFail("pgrep xlock"); - $machine->sleep(120); - $machine->mustSucceed("pgrep xlock"); + machine.start() + machine.wait_for_x() + machine.fail("pgrep xlock") + machine.sleep(120) + machine.succeed("pgrep xlock") ''; }) diff --git a/nixos/tests/xdg-desktop-portal.nix b/nixos/tests/xdg-desktop-portal.nix deleted file mode 100644 index 79ebb83c49a..00000000000 --- a/nixos/tests/xdg-desktop-portal.nix +++ /dev/null @@ -1,17 +0,0 @@ -# run installed tests -import ./make-test.nix ({ pkgs, ... }: - -{ - name = "xdg-desktop-portal"; - meta = { - maintainers = pkgs.xdg-desktop-portal.meta.maintainers; - }; - - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; - }; - - testScript = '' - $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.xdg-desktop-portal.installedTests}/share'"); - ''; -}) diff --git a/nixos/tests/yabar.nix b/nixos/tests/yabar.nix index bbc0cf4c7dd..9108004d4df 100644 --- a/nixos/tests/yabar.nix +++ b/nixos/tests/yabar.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, lib, ... }: +import ./make-test-python.nix ({ pkgs, lib, ... }: with lib; @@ -20,14 +20,14 @@ with lib; }; testScript = '' - $machine->start; - $machine->waitForX; + machine.start() + machine.wait_for_x() # confirm proper startup - $machine->waitForUnit("yabar.service", "bob"); - $machine->sleep(10); - $machine->waitForUnit("yabar.service", "bob"); + machine.wait_for_unit("yabar.service", "bob") + machine.sleep(10) + machine.wait_for_unit("yabar.service", "bob") - $machine->screenshot("top_bar"); + machine.screenshot("top_bar") ''; }) diff --git a/nixos/tests/zfs.nix b/nixos/tests/zfs.nix index d7a08268e98..8f844aca416 100644 --- a/nixos/tests/zfs.nix +++ b/nixos/tests/zfs.nix @@ -7,7 +7,7 @@ with import ../lib/testing.nix { inherit system pkgs; }; let - makeTest = import ./make-test.nix; + makeTest = import ./make-test-python.nix; makeZfsTest = name: { kernelPackage ? pkgs.linuxPackages_latest @@ -34,12 +34,12 @@ let }; testScript = '' - $machine->succeed("modprobe zfs"); - $machine->succeed("zpool status"); + machine.succeed("modprobe zfs") + machine.succeed("zpool status") - $machine->succeed("ls /dev"); + machine.succeed("ls /dev") - $machine->succeed( + machine.succeed( "mkdir /tmp/mnt", "udevadm settle", @@ -55,9 +55,7 @@ let "umount /tmp/mnt", "zpool destroy rpool", "udevadm settle" - - ); - + ) '' + extraTest; }; @@ -70,8 +68,8 @@ in { unstable = makeZfsTest "unstable" { enableUnstable = true; extraTest = '' - $machine->succeed( - "echo password | zpool create -o altroot='/tmp/mnt' -O encryption=aes-256-gcm -O keyformat=passphrase rpool /dev/vdb1", + machine.succeed( + "echo password | zpool create -o altroot=\"/tmp/mnt\" -O encryption=aes-256-gcm -O keyformat=passphrase rpool /dev/vdb1", "zfs create -o mountpoint=legacy rpool/root", "mount -t zfs rpool/root /tmp/mnt", "udevadm settle", @@ -79,7 +77,7 @@ in { "umount /tmp/mnt", "zpool destroy rpool", "udevadm settle" - ); + ) ''; }; diff --git a/nixos/tests/zookeeper.nix b/nixos/tests/zookeeper.nix index f343ebd39e4..42cf20b39c5 100644 --- a/nixos/tests/zookeeper.nix +++ b/nixos/tests/zookeeper.nix @@ -1,4 +1,4 @@ -import ./make-test.nix ({ pkgs, ...} : { +import ./make-test-python.nix ({ pkgs, ...} : { name = "zookeeper"; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ nequissimus ]; @@ -15,14 +15,20 @@ import ./make-test.nix ({ pkgs, ...} : { }; testScript = '' - startAll; + start_all() - $server->waitForUnit("zookeeper"); - $server->waitForUnit("network.target"); - $server->waitForOpenPort(2181); + server.wait_for_unit("zookeeper") + server.wait_for_unit("network.target") + server.wait_for_open_port(2181) - $server->waitUntilSucceeds("${pkgs.zookeeper}/bin/zkCli.sh -server localhost:2181 create /foo bar"); - $server->waitUntilSucceeds("${pkgs.zookeeper}/bin/zkCli.sh -server localhost:2181 set /foo hello"); - $server->waitUntilSucceeds("${pkgs.zookeeper}/bin/zkCli.sh -server localhost:2181 get /foo | grep hello"); + server.wait_until_succeeds( + "${pkgs.zookeeper}/bin/zkCli.sh -server localhost:2181 create /foo bar" + ) + server.wait_until_succeeds( + "${pkgs.zookeeper}/bin/zkCli.sh -server localhost:2181 set /foo hello" + ) + server.wait_until_succeeds( + "${pkgs.zookeeper}/bin/zkCli.sh -server localhost:2181 get /foo | grep hello" + ) ''; }) |