Merge branch 'master' into staging

8 files changed, 182 insertions, 120 deletions

diff --git a/nixos/doc/manual/release-notes/rl-1903.xml b/nixos/doc/manual/release-notes/rl-1903.xml
index 09abca5a468..e194f96dcd6 100644
--- a/nixos/doc/manual/release-notes/rl-1903.xml
+++ b/nixos/doc/manual/release-notes/rl-1903.xml
@@ -192,6 +192,13 @@
       options can occour more than once in the configuration.
     </para>
    </listitem>
+   <listitem>
+    <para>
+      The <literal>solr</literal> package has been upgraded from 4.10.3 to 7.5.0 and has undergone
+      some major changes. The <literal>services.solr</literal> module has been updated to reflect
+      these changes. Please review http://lucene.apache.org/solr/ carefully before upgrading.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 
diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix
index 16737efb185..6e7f0a007bc 100644
--- a/nixos/modules/misc/ids.nix
+++ b/nixos/modules/misc/ids.nix
@@ -333,6 +333,7 @@
       lidarr = 306;
       slurm = 307;
       kapacitor = 308;
+      solr = 309;
 
       # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
 
@@ -626,6 +627,7 @@
       lidarr = 306;
       slurm = 307;
       kapacitor = 308;
+      solr = 309;
 
       # When adding a gid, make sure it doesn't match an existing
       # uid. Users and groups with the same name should have equal
diff --git a/nixos/modules/programs/bash/bash.nix b/nixos/modules/programs/bash/bash.nix
index 0fbc77ea44c..d325fff6a57 100644
--- a/nixos/modules/programs/bash/bash.nix
+++ b/nixos/modules/programs/bash/bash.nix
@@ -16,7 +16,7 @@ let
     # programmable completion. If we do, enable all modules installed in
     # the system and user profile in obsolete /etc/bash_completion.d/
     # directories. Bash loads completions in all
-    # $XDG_DATA_DIRS/share/bash-completion/completions/
+    # $XDG_DATA_DIRS/bash-completion/completions/
     # on demand, so they do not need to be sourced here.
     if shopt -q progcomp &>/dev/null; then
       . "${pkgs.bash-completion}/etc/profile.d/bash_completion.sh"
diff --git a/nixos/modules/programs/sway-beta.nix b/nixos/modules/programs/sway-beta.nix
index 04f2e0662b8..e651ea4cca3 100644
--- a/nixos/modules/programs/sway-beta.nix
+++ b/nixos/modules/programs/sway-beta.nix
@@ -5,6 +5,15 @@ with lib;
 let
   cfg = config.programs.sway-beta;
   swayPackage = cfg.package;
+
+  swayWrapped = pkgs.writeShellScriptBin "sway" ''
+    ${cfg.extraSessionCommands}
+    exec ${pkgs.dbus.dbus-launch} --exit-with-session ${swayPackage}/bin/sway
+  '';
+  swayJoined = pkgs.symlinkJoin {
+    name = "sway-joined";
+    paths = [ swayWrapped swayPackage ];
+  };
 in {
   options.programs.sway-beta = {
     enable = mkEnableOption ''
@@ -20,13 +29,30 @@ in {
       '';
     };
 
+    extraSessionCommands = mkOption {
+      type = types.lines;
+      default = "";
+      example = ''
+        export SDL_VIDEODRIVER=wayland
+        # needs qt5.qtwayland in systemPackages
+        export QT_QPA_PLATFORM=wayland
+        export QT_WAYLAND_DISABLE_WINDOWDECORATION="1"
+        # Fix for some Java AWT applications (e.g. Android Studio),
+        # use this if they aren't displayed properly:
+        export _JAVA_AWT_WM_NONREPARENTING=1
+      '';
+      description = ''
+        Shell commands executed just before Sway is started.
+      '';
+    };
+
     extraPackages = mkOption {
       type = with types; listOf package;
       default = with pkgs; [
-        xwayland dmenu
+        xwayland rxvt_unicode dmenu
       ];
       defaultText = literalExample ''
-        with pkgs; [ xwayland dmenu ];
+        with pkgs; [ xwayland rxvt_unicode dmenu ];
       '';
       example = literalExample ''
         with pkgs; [
@@ -42,7 +68,7 @@ in {
   };
 
   config = mkIf cfg.enable {
-    environment.systemPackages = [ swayPackage ] ++ cfg.extraPackages;
+    environment.systemPackages = [ swayJoined ] ++ cfg.extraPackages;
     security.pam.services.swaylock = {};
     hardware.opengl.enable = mkDefault true;
     fonts.enableDefaultFonts = mkDefault true;
@@ -51,4 +77,3 @@ in {
 
   meta.maintainers = with lib.maintainers; [ gnidorah primeos colemickens ];
 }
-
diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix
index e2ee995cea8..bf4dfc666bb 100644
--- a/nixos/modules/services/monitoring/prometheus/default.nix
+++ b/nixos/modules/services/monitoring/prometheus/default.nix
@@ -10,6 +10,13 @@ let
   # Get a submodule without any embedded metadata:
   _filter = x: filterAttrs (k: v: k != "_module") x;
 
+  # a wrapper that verifies that the configuration is valid
+  promtoolCheck = what: name: file: pkgs.runCommand "${name}-${what}-checked"
+    { buildInputs = [ cfg.package ]; } ''
+    ln -s ${file} $out
+    promtool ${what} $out
+  '';
+
   # Pretty-print JSON to a file
   writePrettyJSON = name: x:
     pkgs.runCommand name { } ''
@@ -19,18 +26,19 @@ let
   # This becomes the main config file
   promConfig = {
     global = cfg.globalConfig;
-    rule_files = cfg.ruleFiles ++ [
+    rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
       (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg.rules))
-    ];
+    ]);
     scrape_configs = cfg.scrapeConfigs;
   };
 
   generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
 
-  prometheusYml =
-    if cfg.configText != null then
+  prometheusYml = let
+    yml =  if cfg.configText != null then
       pkgs.writeText "prometheus.yml" cfg.configText
-    else generatedPrometheusYml;
+      else generatedPrometheusYml;
+    in promtoolCheck "check-config" "prometheus.yml" yml;
 
   cmdlineArgs = cfg.extraFlags ++ [
     "-storage.local.path=${cfg.dataDir}/metrics"
@@ -376,6 +384,15 @@ in {
         '';
       };
 
+      package = mkOption {
+        type = types.package;
+        default = pkgs.prometheus;
+        defaultText = "pkgs.prometheus";
+        description = ''
+          The prometheus package that should be used.
+        '';
+      };
+
       listenAddress = mkOption {
         type = types.str;
         default = "0.0.0.0:9090";
@@ -495,7 +512,7 @@ in {
       after    = [ "network.target" ];
       script = ''
         #!/bin/sh
-        exec ${pkgs.prometheus}/bin/prometheus \
+        exec ${cfg.package}/bin/prometheus \
           ${concatStringsSep " \\\n  " cmdlineArgs}
       '';
       serviceConfig = {
diff --git a/nixos/modules/services/search/solr.nix b/nixos/modules/services/search/solr.nix
index 90140a337ed..7200c40e89f 100644
--- a/nixos/modules/services/search/solr.nix
+++ b/nixos/modules/services/search/solr.nix
@@ -6,142 +6,105 @@ let
 
   cfg = config.services.solr;
 
-  # Assemble all jars needed for solr
-  solrJars = pkgs.stdenv.mkDerivation {
-    name = "solr-jars";
-
-    src = pkgs.fetchurl {
-      url = http://archive.apache.org/dist/tomcat/tomcat-5/v5.5.36/bin/apache-tomcat-5.5.36.tar.gz;
-      sha256 = "01mzvh53wrs1p2ym765jwd00gl6kn8f9k3nhdrnhdqr8dhimfb2p";
-    };
-
-    installPhase = ''
-      mkdir -p $out/lib
-      cp common/lib/*.jar $out/lib/
-      ln -s ${pkgs.ant}/lib/ant/lib/ant.jar $out/lib/
-      ln -s ${cfg.solrPackage}/lib/ext/* $out/lib/
-      ln -s ${pkgs.jdk.home}/lib/tools.jar $out/lib/
-    '' + optionalString (cfg.extraJars != []) ''
-      for f in ${concatStringsSep " " cfg.extraJars}; do
-         cp $f $out/lib
-      done
-    '';
-  };
-
-in {
+in
 
+{
   options = {
     services.solr = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Enables the solr service.
-        '';
-      };
-
-      javaPackage = mkOption {
-        type = types.package;
-        default = pkgs.jre;
-        defaultText = "pkgs.jre";
-        description = ''
-          Which Java derivation to use for running solr.
-        '';
-      };
+      enable = mkEnableOption "Enables the solr service.";
 
-      solrPackage = mkOption {
+      package = mkOption {
         type = types.package;
         default = pkgs.solr;
         defaultText = "pkgs.solr";
-        description = ''
-          Which solr derivation to use for running solr.
-        '';
+        description = "Which Solr package to use.";
       };
 
-      extraJars = mkOption {
-        type = types.listOf types.path;
-        default = [];
-        description = ''
-          List of paths pointing to jars. Jars are copied to commonLibFolder to be available to java/solr.
-        '';
+      port = mkOption {
+        type = types.int;
+        default = 8983;
+        description = "Port on which Solr is ran.";
       };
 
-      log4jConfiguration = mkOption {
-        type = types.lines;
-        default = ''
-          log4j.rootLogger=INFO, stdout
-          log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-          log4j.appender.stdout.Target=System.out
-          log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-          log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-        '';
-        description = ''
-          Contents of the <literal>log4j.properties</literal> used. By default,
-          everything is logged to stdout (picked up by systemd) with level INFO.
-        '';
+      stateDir = mkOption {
+        type = types.path;
+        default = "/var/lib/solr";
+        description = "The solr home directory containing config, data, and logging files.";
       };
 
-      user = mkOption {
-        type = types.str;
-        description = ''
-          The user that should run the solr process and.
-          the working directories.
-        '';
+      extraJavaOptions = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = "Extra command line options given to the java process running Solr.";
       };
 
-      group = mkOption {
+      user = mkOption {
         type = types.str;
-        description = ''
-          The group that will own the working directory.
-        '';
+        default = "solr";
+        description = "User under which Solr is ran.";
       };
 
-      solrHome = mkOption {
+      group = mkOption {
         type = types.str;
-        description = ''
-          The solr home directory. It is your own responsibility to
-          make sure this directory contains a working solr configuration,
-          and is writeable by the the user running the solr service.
-          Failing to do so, the solr will not start properly.
-        '';
-      };
-
-      extraJavaOptions = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        description = ''
-          Extra command line options given to the java process running
-          solr.
-        '';
-      };
-
-      extraWinstoneOptions = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        description = ''
-          Extra command line options given to the Winstone, which is
-          the servlet container hosting solr.
-        '';
+        default = "solr";
+        description = "Group under which Solr is ran.";
       };
     };
   };
 
   config = mkIf cfg.enable {
 
-    services.winstone.solr = {
-      serviceName = "solr";
-      inherit (cfg) user group javaPackage;
-      warFile = "${cfg.solrPackage}/lib/solr.war";
-      extraOptions = [
-        "--commonLibFolder=${solrJars}/lib"
-        "--useJasper"
-      ] ++ cfg.extraWinstoneOptions;
-      extraJavaOptions = [
-        "-Dsolr.solr.home=${cfg.solrHome}"
-        "-Dlog4j.configuration=file://${pkgs.writeText "log4j.properties" cfg.log4jConfiguration}"
-      ] ++ cfg.extraJavaOptions;
+    environment.systemPackages = [ cfg.package ];
+
+    systemd.services.solr = {
+      after = [ "network.target" "remote-fs.target" "nss-lookup.target" "systemd-journald-dev-log.socket" ];
+      wantedBy = [ "multi-user.target" ];
+
+      environment = {
+        SOLR_HOME = "${cfg.stateDir}/data";
+        LOG4J_PROPS = "${cfg.stateDir}/log4j2.xml";
+        SOLR_LOGS_DIR = "${cfg.stateDir}/logs";
+        SOLR_PORT = "${toString cfg.port}";
+      };
+      path = with pkgs; [
+        gawk
+        procps
+      ];
+      preStart = ''
+        mkdir -p "${cfg.stateDir}/data";
+        mkdir -p "${cfg.stateDir}/logs";
+
+        if ! test -e "${cfg.stateDir}/data/solr.xml"; then
+          install -D -m0640 ${cfg.package}/server/solr/solr.xml "${cfg.stateDir}/data/solr.xml"
+          install -D -m0640 ${cfg.package}/server/solr/zoo.cfg "${cfg.stateDir}/data/zoo.cfg"
+        fi
+
+        if ! test -e "${cfg.stateDir}/log4j2.xml"; then
+          install -D -m0640 ${cfg.package}/server/resources/log4j2.xml "${cfg.stateDir}/log4j2.xml"
+        fi
+      '';
+
+      serviceConfig = {
+        User = cfg.user;
+        Group = cfg.group;
+        ExecStart="${cfg.package}/bin/solr start -f -a \"${concatStringsSep " " cfg.extraJavaOptions}\"";
+        ExecStop="${cfg.package}/bin/solr stop";
+      };
     };
 
+    users.users = optionalAttrs (cfg.user == "solr") (singleton
+      { name = "solr";
+        group = cfg.group;
+        home = cfg.stateDir;
+        createHome = true;
+        uid = config.ids.uids.solr;
+      });
+
+    users.groups = optionalAttrs (cfg.group == "solr") (singleton
+      { name = "solr";
+        gid = config.ids.gids.solr;
+      });
+
   };
 
 }
diff --git a/nixos/release.nix b/nixos/release.nix
index c3a10c9d330..4647f28be18 100644
--- a/nixos/release.nix
+++ b/nixos/release.nix
@@ -410,6 +410,7 @@ in rec {
   tests.slurm = callTest tests/slurm.nix {};
   tests.smokeping = callTest tests/smokeping.nix {};
   tests.snapper = callTest tests/snapper.nix {};
+  tests.solr = callTest tests/solr.nix {};
   #tests.statsd = callTest tests/statsd.nix {}; # statsd is broken: #45946
   tests.strongswan-swanctl = callTest tests/strongswan-swanctl.nix {};
   tests.sudo = callTest tests/sudo.nix {};
diff --git a/nixos/tests/solr.nix b/nixos/tests/solr.nix
new file mode 100644
index 00000000000..9ba3863411e
--- /dev/null
+++ b/nixos/tests/solr.nix
@@ -0,0 +1,47 @@
+import ./make-test.nix ({ pkgs, lib, ... }:
+{
+  name = "solr";
+  meta.maintainers = [ lib.maintainers.aanderse ];
+
+  machine =
+    { config, pkgs, ... }:
+    {
+      # Ensure the virtual machine has enough memory for Solr to avoid the following error:
+      #
+      #   OpenJDK 64-Bit Server VM warning:
+      #     INFO: os::commit_memory(0x00000000e8000000, 402653184, 0)
+      #     failed; error='Cannot allocate memory' (errno=12)
+      #
+      #   There is insufficient memory for the Java Runtime Environment to continue.
+      #   Native memory allocation (mmap) failed to map 402653184 bytes for committing reserved memory.
+      virtualisation.memorySize = 2000;
+
+      services.solr.enable = true;
+    };
+
+  testScript = ''
+    startAll;
+
+    $machine->waitForUnit('solr.service');
+    $machine->waitForOpenPort('8983');
+    $machine->succeed('curl --fail http://localhost:8983/solr/');
+
+    # adapted from pkgs.solr/examples/films/README.txt
+    $machine->succeed('sudo -u solr solr create -c films');
+    $machine->succeed(q(curl http://localhost:8983/solr/films/schema -X POST -H 'Content-type:application/json' --data-binary '{
+      "add-field" : {
+        "name":"name",
+        "type":"text_general",
+        "multiValued":false,
+        "stored":true
+      },
+      "add-field" : {
+        "name":"initial_release_date",
+        "type":"pdate",
+        "stored":true
+      }
+    }')) =~ /"status":0/ or die;
+    $machine->succeed('sudo -u solr post -c films ${pkgs.solr}/example/films/films.json');
+    $machine->succeed('curl http://localhost:8983/solr/films/query?q=name:batman') =~ /"name":"Batman Begins"/ or die;
+  '';
+})