diff options
Diffstat (limited to 'nixos/modules/services/databases')
25 files changed, 6317 insertions, 0 deletions
diff --git a/nixos/modules/services/databases/aerospike.nix b/nixos/modules/services/databases/aerospike.nix new file mode 100644 index 00000000000..8109762aea7 --- /dev/null +++ b/nixos/modules/services/databases/aerospike.nix @@ -0,0 +1,156 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.services.aerospike; + + aerospikeConf = pkgs.writeText "aerospike.conf" '' + # This stanza must come first. + service { + user aerospike + group aerospike + paxos-single-replica-limit 1 # Number of nodes where the replica count is automatically reduced to 1. + proto-fd-max 15000 + work-directory ${cfg.workDir} + } + logging { + console { + context any info + } + } + mod-lua { + system-path ${cfg.package}/share/udf/lua + user-path ${cfg.workDir}/udf/lua + } + network { + ${cfg.networkConfig} + } + ${cfg.extraConfig} + ''; + +in + +{ + + ###### interface + + options = { + + services.aerospike = { + enable = mkEnableOption "Aerospike server"; + + package = mkOption { + default = pkgs.aerospike; + defaultText = literalExpression "pkgs.aerospike"; + type = types.package; + description = "Which Aerospike derivation to use"; + }; + + workDir = mkOption { + type = types.str; + default = "/var/lib/aerospike"; + description = "Location where Aerospike stores its files"; + }; + + networkConfig = mkOption { + type = types.lines; + default = '' + service { + address any + port 3000 + } + + heartbeat { + address any + mode mesh + port 3002 + interval 150 + timeout 10 + } + + fabric { + address any + port 3001 + } + + info { + address any + port 3003 + } + ''; + description = "network section of configuration file"; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + example = '' + namespace test { + replication-factor 2 + memory-size 4G + default-ttl 30d + storage-engine memory + } + ''; + description = "Extra configuration"; + }; + }; + + }; + + + ###### implementation + + config = mkIf config.services.aerospike.enable { + + users.users.aerospike = { + name = "aerospike"; + group = "aerospike"; + uid = config.ids.uids.aerospike; + description = "Aerospike server user"; + }; + users.groups.aerospike.gid = config.ids.gids.aerospike; + + systemd.services.aerospike = rec { + description = "Aerospike server"; + + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + serviceConfig = { + ExecStart = "${cfg.package}/bin/asd --fgdaemon --config-file ${aerospikeConf}"; + User = "aerospike"; + Group = "aerospike"; + LimitNOFILE = 100000; + PermissionsStartOnly = true; + }; + + preStart = '' + if [ $(echo "$(${pkgs.procps}/bin/sysctl -n kernel.shmall) < 4294967296" | ${pkgs.bc}/bin/bc) == "1" ]; then + echo "kernel.shmall too low, setting to 4G pages" + ${pkgs.procps}/bin/sysctl -w kernel.shmall=4294967296 + fi + if [ $(echo "$(${pkgs.procps}/bin/sysctl -n kernel.shmmax) < 1073741824" | ${pkgs.bc}/bin/bc) == "1" ]; then + echo "kernel.shmmax too low, setting to 1GB" + ${pkgs.procps}/bin/sysctl -w kernel.shmmax=1073741824 + fi + if [ $(echo "$(cat /proc/sys/net/core/rmem_max) < 15728640" | ${pkgs.bc}/bin/bc) == "1" ]; then + echo "increasing socket buffer limit (/proc/sys/net/core/rmem_max): $(cat /proc/sys/net/core/rmem_max) -> 15728640" + echo 15728640 > /proc/sys/net/core/rmem_max + fi + if [ $(echo "$(cat /proc/sys/net/core/wmem_max) < 5242880" | ${pkgs.bc}/bin/bc) == "1" ]; then + echo "increasing socket buffer limit (/proc/sys/net/core/wmem_max): $(cat /proc/sys/net/core/wmem_max) -> 5242880" + echo 5242880 > /proc/sys/net/core/wmem_max + fi + install -d -m0700 -o ${serviceConfig.User} -g ${serviceConfig.Group} "${cfg.workDir}" + install -d -m0700 -o ${serviceConfig.User} -g ${serviceConfig.Group} "${cfg.workDir}/smd" + install -d -m0700 -o ${serviceConfig.User} -g ${serviceConfig.Group} "${cfg.workDir}/udf" + install -d -m0700 -o ${serviceConfig.User} -g ${serviceConfig.Group} "${cfg.workDir}/udf/lua" + ''; + }; + + }; + +} diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix new file mode 100644 index 00000000000..b36cac35e7c --- /dev/null +++ b/nixos/modules/services/databases/cassandra.nix @@ -0,0 +1,563 @@ +{ config, lib, pkgs, ... }: + +let + inherit (lib) + concatStringsSep + flip + literalDocBook + literalExpression + optionalAttrs + optionals + recursiveUpdate + mkEnableOption + mkIf + mkOption + types + versionAtLeast + ; + + cfg = config.services.cassandra; + + defaultUser = "cassandra"; + + cassandraConfig = flip recursiveUpdate cfg.extraConfig ( + { + commitlog_sync = "batch"; + commitlog_sync_batch_window_in_ms = 2; + start_native_transport = cfg.allowClients; + cluster_name = cfg.clusterName; + partitioner = "org.apache.cassandra.dht.Murmur3Partitioner"; + endpoint_snitch = "SimpleSnitch"; + data_file_directories = [ "${cfg.homeDir}/data" ]; + commitlog_directory = "${cfg.homeDir}/commitlog"; + saved_caches_directory = "${cfg.homeDir}/saved_caches"; + } // optionalAttrs (cfg.seedAddresses != [ ]) { + seed_provider = [ + { + class_name = "org.apache.cassandra.locator.SimpleSeedProvider"; + parameters = [{ seeds = concatStringsSep "," cfg.seedAddresses; }]; + } + ]; + } // optionalAttrs (versionAtLeast cfg.package.version "3") { + hints_directory = "${cfg.homeDir}/hints"; + } + ); + + cassandraConfigWithAddresses = cassandraConfig // ( + if cfg.listenAddress == null + then { listen_interface = cfg.listenInterface; } + else { listen_address = cfg.listenAddress; } + ) // ( + if cfg.rpcAddress == null + then { rpc_interface = cfg.rpcInterface; } + else { rpc_address = cfg.rpcAddress; } + ); + + cassandraEtc = pkgs.stdenv.mkDerivation { + name = "cassandra-etc"; + + cassandraYaml = builtins.toJSON cassandraConfigWithAddresses; + cassandraEnvPkg = "${cfg.package}/conf/cassandra-env.sh"; + cassandraLogbackConfig = pkgs.writeText "logback.xml" cfg.logbackConfig; + + passAsFile = [ "extraEnvSh" ]; + inherit (cfg) extraEnvSh; + + buildCommand = '' + mkdir -p "$out" + + echo "$cassandraYaml" > "$out/cassandra.yaml" + ln -s "$cassandraLogbackConfig" "$out/logback.xml" + + ( cat "$cassandraEnvPkg" + echo "# lines from services.cassandra.extraEnvSh: " + cat "$extraEnvShPath" + ) > "$out/cassandra-env.sh" + + # Delete default JMX Port, otherwise we can't set it using env variable + sed -i '/JMX_PORT="7199"/d' "$out/cassandra-env.sh" + + # Delete default password file + sed -i '/-Dcom.sun.management.jmxremote.password.file=\/etc\/cassandra\/jmxremote.password/d' "$out/cassandra-env.sh" + ''; + }; + + defaultJmxRolesFile = + builtins.foldl' + (left: right: left + right) "" + (map (role: "${role.username} ${role.password}") cfg.jmxRoles); + + fullJvmOptions = + cfg.jvmOpts + ++ optionals (cfg.jmxRoles != [ ]) [ + "-Dcom.sun.management.jmxremote.authenticate=true" + "-Dcom.sun.management.jmxremote.password.file=${cfg.jmxRolesFile}" + ] ++ optionals cfg.remoteJmx [ + "-Djava.rmi.server.hostname=${cfg.rpcAddress}" + ]; + +in +{ + options.services.cassandra = { + + enable = mkEnableOption '' + Apache Cassandra – Scalable and highly available database. + ''; + + clusterName = mkOption { + type = types.str; + default = "Test Cluster"; + description = '' + The name of the cluster. + This setting prevents nodes in one logical cluster from joining + another. All nodes in a cluster must have the same value. + ''; + }; + + user = mkOption { + type = types.str; + default = defaultUser; + description = "Run Apache Cassandra under this user."; + }; + + group = mkOption { + type = types.str; + default = defaultUser; + description = "Run Apache Cassandra under this group."; + }; + + homeDir = mkOption { + type = types.path; + default = "/var/lib/cassandra"; + description = '' + Home directory for Apache Cassandra. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.cassandra; + defaultText = literalExpression "pkgs.cassandra"; + example = literalExpression "pkgs.cassandra_3_11"; + description = '' + The Apache Cassandra package to use. + ''; + }; + + jvmOpts = mkOption { + type = types.listOf types.str; + default = [ ]; + description = '' + Populate the JVM_OPT environment variable. + ''; + }; + + listenAddress = mkOption { + type = types.nullOr types.str; + default = "127.0.0.1"; + example = null; + description = '' + Address or interface to bind to and tell other Cassandra nodes + to connect to. You _must_ change this if you want multiple + nodes to be able to communicate! + + Set listenAddress OR listenInterface, not both. + + Leaving it blank leaves it up to + InetAddress.getLocalHost(). This will always do the Right + Thing _if_ the node is properly configured (hostname, name + resolution, etc), and the Right Thing is to use the address + associated with the hostname (it might not be). + + Setting listen_address to 0.0.0.0 is always wrong. + ''; + }; + + listenInterface = mkOption { + type = types.nullOr types.str; + default = null; + example = "eth1"; + description = '' + Set listenAddress OR listenInterface, not both. Interfaces + must correspond to a single address, IP aliasing is not + supported. + ''; + }; + + rpcAddress = mkOption { + type = types.nullOr types.str; + default = "127.0.0.1"; + example = null; + description = '' + The address or interface to bind the native transport server to. + + Set rpcAddress OR rpcInterface, not both. + + Leaving rpcAddress blank has the same effect as on + listenAddress (i.e. it will be based on the configured hostname + of the node). + + Note that unlike listenAddress, you can specify 0.0.0.0, but you + must also set extraConfig.broadcast_rpc_address to a value other + than 0.0.0.0. + + For security reasons, you should not expose this port to the + internet. Firewall it if needed. + ''; + }; + + rpcInterface = mkOption { + type = types.nullOr types.str; + default = null; + example = "eth1"; + description = '' + Set rpcAddress OR rpcInterface, not both. Interfaces must + correspond to a single address, IP aliasing is not supported. + ''; + }; + + logbackConfig = mkOption { + type = types.lines; + default = '' + <configuration scan="false"> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%-5level %date{HH:mm:ss,SSS} %msg%n</pattern> + </encoder> + </appender> + + <root level="INFO"> + <appender-ref ref="STDOUT" /> + </root> + + <logger name="com.thinkaurelius.thrift" level="ERROR"/> + </configuration> + ''; + description = '' + XML logback configuration for cassandra + ''; + }; + + seedAddresses = mkOption { + type = types.listOf types.str; + default = [ "127.0.0.1" ]; + description = '' + The addresses of hosts designated as contact points in the cluster. A + joining node contacts one of the nodes in the seeds list to learn the + topology of the ring. + Set to 127.0.0.1 for a single node cluster. + ''; + }; + + allowClients = mkOption { + type = types.bool; + default = true; + description = '' + Enables or disables the native transport server (CQL binary protocol). + This server uses the same address as the <literal>rpcAddress</literal>, + but the port it uses is not <literal>rpc_port</literal> but + <literal>native_transport_port</literal>. See the official Cassandra + docs for more information on these variables and set them using + <literal>extraConfig</literal>. + ''; + }; + + extraConfig = mkOption { + type = types.attrs; + default = { }; + example = + { + commitlog_sync_batch_window_in_ms = 3; + }; + description = '' + Extra options to be merged into cassandra.yaml as nix attribute set. + ''; + }; + + extraEnvSh = mkOption { + type = types.lines; + default = ""; + example = literalExpression ''"CLASSPATH=$CLASSPATH:''${extraJar}"''; + description = '' + Extra shell lines to be appended onto cassandra-env.sh. + ''; + }; + + fullRepairInterval = mkOption { + type = types.nullOr types.str; + default = "3w"; + example = null; + description = '' + Set the interval how often full repairs are run, i.e. + <literal>nodetool repair --full</literal> is executed. See + https://cassandra.apache.org/doc/latest/operating/repair.html + for more information. + + Set to <literal>null</literal> to disable full repairs. + ''; + }; + + fullRepairOptions = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "--partitioner-range" ]; + description = '' + Options passed through to the full repair command. + ''; + }; + + incrementalRepairInterval = mkOption { + type = types.nullOr types.str; + default = "3d"; + example = null; + description = '' + Set the interval how often incremental repairs are run, i.e. + <literal>nodetool repair</literal> is executed. See + https://cassandra.apache.org/doc/latest/operating/repair.html + for more information. + + Set to <literal>null</literal> to disable incremental repairs. + ''; + }; + + incrementalRepairOptions = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "--partitioner-range" ]; + description = '' + Options passed through to the incremental repair command. + ''; + }; + + maxHeapSize = mkOption { + type = types.nullOr types.str; + default = null; + example = "4G"; + description = '' + Must be left blank or set together with heapNewSize. + If left blank a sensible value for the available amount of RAM and CPU + cores is calculated. + + Override to set the amount of memory to allocate to the JVM at + start-up. For production use you may wish to adjust this for your + environment. MAX_HEAP_SIZE is the total amount of memory dedicated + to the Java heap. HEAP_NEWSIZE refers to the size of the young + generation. + + The main trade-off for the young generation is that the larger it + is, the longer GC pause times will be. The shorter it is, the more + expensive GC will be (usually). + ''; + }; + + heapNewSize = mkOption { + type = types.nullOr types.str; + default = null; + example = "800M"; + description = '' + Must be left blank or set together with heapNewSize. + If left blank a sensible value for the available amount of RAM and CPU + cores is calculated. + + Override to set the amount of memory to allocate to the JVM at + start-up. For production use you may wish to adjust this for your + environment. HEAP_NEWSIZE refers to the size of the young + generation. + + The main trade-off for the young generation is that the larger it + is, the longer GC pause times will be. The shorter it is, the more + expensive GC will be (usually). + + The example HEAP_NEWSIZE assumes a modern 8-core+ machine for decent pause + times. If in doubt, and if you do not particularly want to tweak, go with + 100 MB per physical CPU core. + ''; + }; + + mallocArenaMax = mkOption { + type = types.nullOr types.int; + default = null; + example = 4; + description = '' + Set this to control the amount of arenas per-thread in glibc. + ''; + }; + + remoteJmx = mkOption { + type = types.bool; + default = false; + description = '' + Cassandra ships with JMX accessible *only* from localhost. + To enable remote JMX connections set to true. + + Be sure to also enable authentication and/or TLS. + See: https://wiki.apache.org/cassandra/JmxSecurity + ''; + }; + + jmxPort = mkOption { + type = types.int; + default = 7199; + description = '' + Specifies the default port over which Cassandra will be available for + JMX connections. + For security reasons, you should not expose this port to the internet. + Firewall it if needed. + ''; + }; + + jmxRoles = mkOption { + default = [ ]; + description = '' + Roles that are allowed to access the JMX (e.g. nodetool) + BEWARE: The passwords will be stored world readable in the nix-store. + It's recommended to use your own protected file using + <literal>jmxRolesFile</literal> + + Doesn't work in versions older than 3.11 because they don't like that + it's world readable. + ''; + type = types.listOf (types.submodule { + options = { + username = mkOption { + type = types.str; + description = "Username for JMX"; + }; + password = mkOption { + type = types.str; + description = "Password for JMX"; + }; + }; + }); + }; + + jmxRolesFile = mkOption { + type = types.nullOr types.path; + default = + if versionAtLeast cfg.package.version "3.11" + then pkgs.writeText "jmx-roles-file" defaultJmxRolesFile + else null; + defaultText = literalDocBook ''generated configuration file if version is at least 3.11, otherwise <literal>null</literal>''; + example = "/var/lib/cassandra/jmx.password"; + description = '' + Specify your own jmx roles file. + + Make sure the permissions forbid "others" from reading the file if + you're using Cassandra below version 3.11. + ''; + }; + }; + + config = mkIf cfg.enable { + assertions = [ + { + assertion = (cfg.listenAddress == null) != (cfg.listenInterface == null); + message = "You have to set either listenAddress or listenInterface"; + } + { + assertion = (cfg.rpcAddress == null) != (cfg.rpcInterface == null); + message = "You have to set either rpcAddress or rpcInterface"; + } + { + assertion = (cfg.maxHeapSize == null) == (cfg.heapNewSize == null); + message = "If you set either of maxHeapSize or heapNewSize you have to set both"; + } + { + assertion = cfg.remoteJmx -> cfg.jmxRolesFile != null; + message = '' + If you want JMX available remotely you need to set a password using + <literal>jmxRoles</literal> or <literal>jmxRolesFile</literal> if + using Cassandra older than v3.11. + ''; + } + ]; + users = mkIf (cfg.user == defaultUser) { + users.${defaultUser} = { + group = cfg.group; + home = cfg.homeDir; + createHome = true; + uid = config.ids.uids.cassandra; + description = "Cassandra service user"; + }; + groups.${defaultUser}.gid = config.ids.gids.cassandra; + }; + + systemd.services.cassandra = { + description = "Apache Cassandra service"; + after = [ "network.target" ]; + environment = { + CASSANDRA_CONF = "${cassandraEtc}"; + JVM_OPTS = builtins.concatStringsSep " " fullJvmOptions; + MAX_HEAP_SIZE = toString cfg.maxHeapSize; + HEAP_NEWSIZE = toString cfg.heapNewSize; + MALLOC_ARENA_MAX = toString cfg.mallocArenaMax; + LOCAL_JMX = if cfg.remoteJmx then "no" else "yes"; + JMX_PORT = toString cfg.jmxPort; + }; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + ExecStart = "${cfg.package}/bin/cassandra -f"; + SuccessExitStatus = 143; + }; + }; + + systemd.services.cassandra-full-repair = { + description = "Perform a full repair on this Cassandra node"; + after = [ "cassandra.service" ]; + requires = [ "cassandra.service" ]; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + ExecStart = + concatStringsSep " " + ([ + "${cfg.package}/bin/nodetool" + "repair" + "--full" + ] ++ cfg.fullRepairOptions); + }; + }; + + systemd.timers.cassandra-full-repair = + mkIf (cfg.fullRepairInterval != null) { + description = "Schedule full repairs on Cassandra"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = cfg.fullRepairInterval; + OnUnitActiveSec = cfg.fullRepairInterval; + Persistent = true; + }; + }; + + systemd.services.cassandra-incremental-repair = { + description = "Perform an incremental repair on this cassandra node."; + after = [ "cassandra.service" ]; + requires = [ "cassandra.service" ]; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + ExecStart = + concatStringsSep " " + ([ + "${cfg.package}/bin/nodetool" + "repair" + ] ++ cfg.incrementalRepairOptions); + }; + }; + + systemd.timers.cassandra-incremental-repair = + mkIf (cfg.incrementalRepairInterval != null) { + description = "Schedule incremental repairs on Cassandra"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = cfg.incrementalRepairInterval; + OnUnitActiveSec = cfg.incrementalRepairInterval; + Persistent = true; + }; + }; + }; + + meta.maintainers = with lib.maintainers; [ roberth ]; +} diff --git a/nixos/modules/services/databases/clickhouse.nix b/nixos/modules/services/databases/clickhouse.nix new file mode 100644 index 00000000000..3a161d56107 --- /dev/null +++ b/nixos/modules/services/databases/clickhouse.nix @@ -0,0 +1,78 @@ +{ config, lib, pkgs, ... }: +let + cfg = config.services.clickhouse; +in +with lib; +{ + + ###### interface + + options = { + + services.clickhouse = { + + enable = mkEnableOption "ClickHouse database server"; + + package = mkOption { + type = types.package; + default = pkgs.clickhouse; + defaultText = "pkgs.clickhouse"; + description = '' + ClickHouse package to use. + ''; + }; + + }; + + }; + + + ###### implementation + + config = mkIf cfg.enable { + + users.users.clickhouse = { + name = "clickhouse"; + uid = config.ids.uids.clickhouse; + group = "clickhouse"; + description = "ClickHouse server user"; + }; + + users.groups.clickhouse.gid = config.ids.gids.clickhouse; + + systemd.services.clickhouse = { + description = "ClickHouse server"; + + wantedBy = [ "multi-user.target" ]; + + after = [ "network.target" ]; + + serviceConfig = { + User = "clickhouse"; + Group = "clickhouse"; + ConfigurationDirectory = "clickhouse-server"; + AmbientCapabilities = "CAP_SYS_NICE"; + StateDirectory = "clickhouse"; + LogsDirectory = "clickhouse"; + ExecStart = "${cfg.package}/bin/clickhouse-server --config-file=${cfg.package}/etc/clickhouse-server/config.xml"; + }; + }; + + environment.etc = { + "clickhouse-server/config.xml" = { + source = "${cfg.package}/etc/clickhouse-server/config.xml"; + }; + + "clickhouse-server/users.xml" = { + source = "${cfg.package}/etc/clickhouse-server/users.xml"; + }; + }; + + environment.systemPackages = [ cfg.package ]; + + # startup requires a `/etc/localtime` which only if exists if `time.timeZone != null` + time.timeZone = mkDefault "UTC"; + + }; + +} diff --git a/nixos/modules/services/databases/cockroachdb.nix b/nixos/modules/services/databases/cockroachdb.nix new file mode 100644 index 00000000000..eb061af9262 --- /dev/null +++ b/nixos/modules/services/databases/cockroachdb.nix @@ -0,0 +1,217 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.cockroachdb; + crdb = cfg.package; + + escape = builtins.replaceStrings ["%"] ["%%"]; + ifNotNull = v: s: optionalString (v != null) s; + + startupCommand = lib.concatStringsSep " " + [ # Basic startup + "${crdb}/bin/cockroach start" + "--logtostderr" + "--store=/var/lib/cockroachdb" + (ifNotNull cfg.locality "--locality='${cfg.locality}'") + + # WebUI settings + "--http-addr='${cfg.http.address}:${toString cfg.http.port}'" + + # Cluster listen address + "--listen-addr='${cfg.listen.address}:${toString cfg.listen.port}'" + + # Cluster configuration + (ifNotNull cfg.join "--join=${cfg.join}") + + # Cache and memory settings. Must be escaped. + "--cache='${escape cfg.cache}'" + "--max-sql-memory='${escape cfg.maxSqlMemory}'" + + # Certificate/security settings. + (if cfg.insecure then "--insecure" else "--certs-dir=${cfg.certsDir}") + ]; + + addressOption = descr: defaultPort: { + address = mkOption { + type = types.str; + default = "localhost"; + description = "Address to bind to for ${descr}"; + }; + + port = mkOption { + type = types.port; + default = defaultPort; + description = "Port to bind to for ${descr}"; + }; + }; +in + +{ + options = { + services.cockroachdb = { + enable = mkEnableOption "CockroachDB Server"; + + listen = addressOption "intra-cluster communication" 26257; + + http = addressOption "http-based Admin UI" 8080; + + locality = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + An ordered, comma-separated list of key-value pairs that describe the + topography of the machine. Topography might include country, + datacenter or rack designations. Data is automatically replicated to + maximize diversities of each tier. The order of tiers is used to + determine the priority of the diversity, so the more inclusive + localities like country should come before less inclusive localities + like datacenter. The tiers and order must be the same on all nodes. + Including more tiers is better than including fewer. For example: + + <literal> + country=us,region=us-west,datacenter=us-west-1b,rack=12 + country=ca,region=ca-east,datacenter=ca-east-2,rack=4 + + planet=earth,province=manitoba,colo=secondary,power=3 + </literal> + ''; + }; + + join = mkOption { + type = types.nullOr types.str; + default = null; + description = "The addresses for connecting the node to a cluster."; + }; + + insecure = mkOption { + type = types.bool; + default = false; + description = "Run in insecure mode."; + }; + + certsDir = mkOption { + type = types.nullOr types.path; + default = null; + description = "The path to the certificate directory."; + }; + + user = mkOption { + type = types.str; + default = "cockroachdb"; + description = "User account under which CockroachDB runs"; + }; + + group = mkOption { + type = types.str; + default = "cockroachdb"; + description = "User account under which CockroachDB runs"; + }; + + openPorts = mkOption { + type = types.bool; + default = false; + description = "Open firewall ports for cluster communication by default"; + }; + + cache = mkOption { + type = types.str; + default = "25%"; + description = '' + The total size for caches. + + This can be a percentage, expressed with a fraction sign or as a + decimal-point number, or any bytes-based unit. For example, + <literal>"25%"</literal>, <literal>"0.25"</literal> both represent + 25% of the available system memory. The values + <literal>"1000000000"</literal> and <literal>"1GB"</literal> both + represent 1 gigabyte of memory. + + ''; + }; + + maxSqlMemory = mkOption { + type = types.str; + default = "25%"; + description = '' + The maximum in-memory storage capacity available to store temporary + data for SQL queries. + + This can be a percentage, expressed with a fraction sign or as a + decimal-point number, or any bytes-based unit. For example, + <literal>"25%"</literal>, <literal>"0.25"</literal> both represent + 25% of the available system memory. The values + <literal>"1000000000"</literal> and <literal>"1GB"</literal> both + represent 1 gigabyte of memory. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.cockroachdb; + defaultText = literalExpression "pkgs.cockroachdb"; + description = '' + The CockroachDB derivation to use for running the service. + + This would primarily be useful to enable Enterprise Edition features + in your own custom CockroachDB build (Nixpkgs CockroachDB binaries + only contain open source features and open source code). + ''; + }; + }; + }; + + config = mkIf config.services.cockroachdb.enable { + assertions = [ + { assertion = !cfg.insecure -> cfg.certsDir != null; + message = "CockroachDB must have a set of SSL certificates (.certsDir), or run in Insecure Mode (.insecure = true)"; + } + ]; + + environment.systemPackages = [ crdb ]; + + users.users = optionalAttrs (cfg.user == "cockroachdb") { + cockroachdb = { + description = "CockroachDB Server User"; + uid = config.ids.uids.cockroachdb; + group = cfg.group; + }; + }; + + users.groups = optionalAttrs (cfg.group == "cockroachdb") { + cockroachdb.gid = config.ids.gids.cockroachdb; + }; + + networking.firewall.allowedTCPPorts = lib.optionals cfg.openPorts + [ cfg.http.port cfg.listen.port ]; + + systemd.services.cockroachdb = + { description = "CockroachDB Server"; + documentation = [ "man:cockroach(1)" "https://www.cockroachlabs.com" ]; + + after = [ "network.target" "time-sync.target" ]; + requires = [ "time-sync.target" ]; + wantedBy = [ "multi-user.target" ]; + + unitConfig.RequiresMountsFor = "/var/lib/cockroachdb"; + + serviceConfig = + { ExecStart = startupCommand; + Type = "notify"; + User = cfg.user; + StateDirectory = "cockroachdb"; + StateDirectoryMode = "0700"; + + Restart = "always"; + + # A conservative-ish timeout is alright here, because for Type=notify + # cockroach will send systemd pings during startup to keep it alive + TimeoutStopSec = 60; + RestartSec = 10; + }; + }; + }; + + meta.maintainers = with lib.maintainers; [ thoughtpolice ]; +} diff --git a/nixos/modules/services/databases/couchdb.nix b/nixos/modules/services/databases/couchdb.nix new file mode 100644 index 00000000000..742e605d224 --- /dev/null +++ b/nixos/modules/services/databases/couchdb.nix @@ -0,0 +1,225 @@ +{ config, options, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.couchdb; + opt = options.services.couchdb; + configFile = pkgs.writeText "couchdb.ini" ( + '' + [couchdb] + database_dir = ${cfg.databaseDir} + uri_file = ${cfg.uriFile} + view_index_dir = ${cfg.viewIndexDir} + '' + (optionalString (cfg.adminPass != null) '' + [admins] + ${cfg.adminUser} = ${cfg.adminPass} + '' + '' + [chttpd] + '') + + '' + port = ${toString cfg.port} + bind_address = ${cfg.bindAddress} + + [log] + file = ${cfg.logFile} + ''); + executable = "${cfg.package}/bin/couchdb"; + +in { + + ###### interface + + options = { + + services.couchdb = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run CouchDB Server. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.couchdb3; + defaultText = literalExpression "pkgs.couchdb3"; + description = '' + CouchDB package to use. + ''; + }; + + adminUser = mkOption { + type = types.str; + default = "admin"; + description = '' + Couchdb (i.e. fauxton) account with permission for all dbs and + tasks. + ''; + }; + + adminPass = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Couchdb (i.e. fauxton) account with permission for all dbs and + tasks. + ''; + }; + + user = mkOption { + type = types.str; + default = "couchdb"; + description = '' + User account under which couchdb runs. + ''; + }; + + group = mkOption { + type = types.str; + default = "couchdb"; + description = '' + Group account under which couchdb runs. + ''; + }; + + # couchdb options: http://docs.couchdb.org/en/latest/config/index.html + + databaseDir = mkOption { + type = types.path; + default = "/var/lib/couchdb"; + description = '' + Specifies location of CouchDB database files (*.couch named). This + location should be writable and readable for the user the CouchDB + service runs as (couchdb by default). + ''; + }; + + uriFile = mkOption { + type = types.path; + default = "/run/couchdb/couchdb.uri"; + description = '' + This file contains the full URI that can be used to access this + instance of CouchDB. It is used to help discover the port CouchDB is + running on (if it was set to 0 (e.g. automatically assigned any free + one). This file should be writable and readable for the user that + runs the CouchDB service (couchdb by default). + ''; + }; + + viewIndexDir = mkOption { + type = types.path; + default = "/var/lib/couchdb"; + description = '' + Specifies location of CouchDB view index files. This location should + be writable and readable for the user that runs the CouchDB service + (couchdb by default). + ''; + }; + + bindAddress = mkOption { + type = types.str; + default = "127.0.0.1"; + description = '' + Defines the IP address by which CouchDB will be accessible. + ''; + }; + + port = mkOption { + type = types.int; + default = 5984; + description = '' + Defined the port number to listen. + ''; + }; + + logFile = mkOption { + type = types.path; + default = "/var/log/couchdb.log"; + description = '' + Specifies the location of file for logging output. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Extra configuration. Overrides any other cofiguration. + ''; + }; + + argsFile = mkOption { + type = types.path; + default = "${cfg.package}/etc/vm.args"; + defaultText = literalExpression ''"config.${opt.package}/etc/vm.args"''; + description = '' + vm.args configuration. Overrides Couchdb's Erlang VM parameters file. + ''; + }; + + configFile = mkOption { + type = types.path; + description = '' + Configuration file for persisting runtime changes. File + needs to be readable and writable from couchdb user/group. + ''; + }; + + }; + + }; + + ###### implementation + + config = mkIf config.services.couchdb.enable { + + environment.systemPackages = [ cfg.package ]; + + services.couchdb.configFile = mkDefault "/var/lib/couchdb/local.ini"; + + systemd.tmpfiles.rules = [ + "d '${dirOf cfg.uriFile}' - ${cfg.user} ${cfg.group} - -" + "f '${cfg.logFile}' - ${cfg.user} ${cfg.group} - -" + "d '${cfg.databaseDir}' - ${cfg.user} ${cfg.group} - -" + "d '${cfg.viewIndexDir}' - ${cfg.user} ${cfg.group} - -" + ]; + + systemd.services.couchdb = { + description = "CouchDB Server"; + wantedBy = [ "multi-user.target" ]; + + preStart = '' + touch ${cfg.configFile} + ''; + + environment = { + # we are actually specifying 5 configuration files: + # 1. the preinstalled default.ini + # 2. the module configuration + # 3. the extraConfig from the module options + # 4. the locally writable config file, which couchdb itself writes to + ERL_FLAGS= ''-couch_ini ${cfg.package}/etc/default.ini ${configFile} ${pkgs.writeText "couchdb-extra.ini" cfg.extraConfig} ${cfg.configFile}''; + # 5. the vm.args file + COUCHDB_ARGS_FILE=''${cfg.argsFile}''; + }; + + serviceConfig = { + User = cfg.user; + Group = cfg.group; + ExecStart = executable; + }; + }; + + users.users.couchdb = { + description = "CouchDB Server user"; + group = "couchdb"; + uid = config.ids.uids.couchdb; + }; + + users.groups.couchdb.gid = config.ids.gids.couchdb; + + }; +} diff --git a/nixos/modules/services/databases/firebird.nix b/nixos/modules/services/databases/firebird.nix new file mode 100644 index 00000000000..4e3130bea22 --- /dev/null +++ b/nixos/modules/services/databases/firebird.nix @@ -0,0 +1,168 @@ +{ config, lib, pkgs, ... }: + +# TODO: This may file may need additional review, eg which configuartions to +# expose to the user. +# +# I only used it to access some simple databases. + +# test: +# isql, then type the following commands: +# CREATE DATABASE '/var/db/firebird/data/test.fdb' USER 'SYSDBA' PASSWORD 'masterkey'; +# CONNECT '/var/db/firebird/data/test.fdb' USER 'SYSDBA' PASSWORD 'masterkey'; +# CREATE TABLE test ( text varchar(100) ); +# DROP DATABASE; +# +# Be careful, virtuoso-opensource also provides a different isql command ! + +# There are at least two ways to run firebird. superserver has been choosen +# however there are no strong reasons to prefer this or the other one AFAIK +# Eg superserver is said to be most efficiently using resources according to +# http://www.firebirdsql.org/manual/qsg25-classic-or-super.html + +with lib; + +let + + cfg = config.services.firebird; + + firebird = cfg.package; + + dataDir = "${cfg.baseDir}/data"; + systemDir = "${cfg.baseDir}/system"; + +in + +{ + + ###### interface + + options = { + + services.firebird = { + + enable = mkEnableOption "the Firebird super server"; + + package = mkOption { + default = pkgs.firebird; + defaultText = literalExpression "pkgs.firebird"; + type = types.package; + example = literalExpression "pkgs.firebird_3"; + description = '' + Which Firebird package to be installed: <code>pkgs.firebird_3</code> + For SuperServer use override: <code>pkgs.firebird_3.override { superServer = true; };</code> + ''; + }; + + port = mkOption { + default = 3050; + type = types.port; + description = '' + Port Firebird uses. + ''; + }; + + user = mkOption { + default = "firebird"; + type = types.str; + description = '' + User account under which firebird runs. + ''; + }; + + baseDir = mkOption { + default = "/var/lib/firebird"; + type = types.str; + description = '' + Location containing data/ and system/ directories. + data/ stores the databases, system/ stores the password database security2.fdb. + ''; + }; + + }; + + }; + + + ###### implementation + + config = mkIf config.services.firebird.enable { + + environment.systemPackages = [cfg.package]; + + systemd.tmpfiles.rules = [ + "d '${dataDir}' 0700 ${cfg.user} - - -" + "d '${systemDir}' 0700 ${cfg.user} - - -" + ]; + + systemd.services.firebird = + { description = "Firebird Super-Server"; + + wantedBy = [ "multi-user.target" ]; + + # TODO: moving security2.fdb into the data directory works, maybe there + # is a better way + preStart = + '' + if ! test -e "${systemDir}/security2.fdb"; then + cp ${firebird}/security2.fdb "${systemDir}" + fi + + if ! test -e "${systemDir}/security3.fdb"; then + cp ${firebird}/security3.fdb "${systemDir}" + fi + + if ! test -e "${systemDir}/security4.fdb"; then + cp ${firebird}/security4.fdb "${systemDir}" + fi + + chmod -R 700 "${dataDir}" "${systemDir}" /var/log/firebird + ''; + + serviceConfig.User = cfg.user; + serviceConfig.LogsDirectory = "firebird"; + serviceConfig.LogsDirectoryMode = "0700"; + serviceConfig.ExecStart = "${firebird}/bin/fbserver -d"; + + # TODO think about shutdown + }; + + environment.etc."firebird/firebird.msg".source = "${firebird}/firebird.msg"; + + # think about this again - and eventually make it an option + environment.etc."firebird/firebird.conf".text = '' + # RootDirectory = Restrict ${dataDir} + DatabaseAccess = Restrict ${dataDir} + ExternalFileAccess = Restrict ${dataDir} + # what is this? is None allowed? + UdfAccess = None + # "Native" = traditional interbase/firebird, "mixed" is windows only + Authentication = Native + + # defaults to -1 on non Win32 + #MaxUnflushedWrites = 100 + #MaxUnflushedWriteTime = 100 + + # show trace if trouble occurs (does this require debug build?) + # BugcheckAbort = 0 + # ConnectionTimeout = 180 + + #RemoteServiceName = gds_db + RemoteServicePort = ${cfg.port} + + # randomly choose port for server Event Notification + #RemoteAuxPort = 0 + # rsetrict connections to a network card: + #RemoteBindAddress = + # there are some additional settings which should be reviewed + ''; + + users.users.firebird = { + description = "Firebird server user"; + group = "firebird"; + uid = config.ids.uids.firebird; + }; + + users.groups.firebird.gid = config.ids.gids.firebird; + + }; +} diff --git a/nixos/modules/services/databases/foundationdb.nix b/nixos/modules/services/databases/foundationdb.nix new file mode 100644 index 00000000000..e22127403e9 --- /dev/null +++ b/nixos/modules/services/databases/foundationdb.nix @@ -0,0 +1,429 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.foundationdb; + pkg = cfg.package; + + # used for initial cluster configuration + initialIpAddr = if (cfg.publicAddress != "auto") then cfg.publicAddress else "127.0.0.1"; + + fdbServers = n: + concatStringsSep "\n" (map (x: "[fdbserver.${toString (x+cfg.listenPortStart)}]") (range 0 (n - 1))); + + backupAgents = n: + concatStringsSep "\n" (map (x: "[backup_agent.${toString x}]") (range 1 n)); + + configFile = pkgs.writeText "foundationdb.conf" '' + [general] + cluster_file = /etc/foundationdb/fdb.cluster + + [fdbmonitor] + restart_delay = ${toString cfg.restartDelay} + user = ${cfg.user} + group = ${cfg.group} + + [fdbserver] + command = ${pkg}/bin/fdbserver + public_address = ${cfg.publicAddress}:$ID + listen_address = ${cfg.listenAddress} + datadir = ${cfg.dataDir}/$ID + logdir = ${cfg.logDir} + logsize = ${cfg.logSize} + maxlogssize = ${cfg.maxLogSize} + ${optionalString (cfg.class != null) "class = ${cfg.class}"} + memory = ${cfg.memory} + storage_memory = ${cfg.storageMemory} + + ${optionalString (lib.versionAtLeast cfg.package.version "6.1") '' + trace_format = ${cfg.traceFormat} + ''} + + ${optionalString (cfg.tls != null) '' + tls_plugin = ${pkg}/libexec/plugins/FDBLibTLS.so + tls_certificate_file = ${cfg.tls.certificate} + tls_key_file = ${cfg.tls.key} + tls_verify_peers = ${cfg.tls.allowedPeers} + ''} + + ${optionalString (cfg.locality.machineId != null) "locality_machineid=${cfg.locality.machineId}"} + ${optionalString (cfg.locality.zoneId != null) "locality_zoneid=${cfg.locality.zoneId}"} + ${optionalString (cfg.locality.datacenterId != null) "locality_dcid=${cfg.locality.datacenterId}"} + ${optionalString (cfg.locality.dataHall != null) "locality_data_hall=${cfg.locality.dataHall}"} + + ${fdbServers cfg.serverProcesses} + + [backup_agent] + command = ${pkg}/libexec/backup_agent + ${backupAgents cfg.backupProcesses} + ''; +in +{ + options.services.foundationdb = { + + enable = mkEnableOption "FoundationDB Server"; + + package = mkOption { + type = types.package; + description = '' + The FoundationDB package to use for this server. This must be specified by the user + in order to ensure migrations and upgrades are controlled appropriately. + ''; + }; + + publicAddress = mkOption { + type = types.str; + default = "auto"; + description = "Publicly visible IP address of the process. Port is determined by process ID"; + }; + + listenAddress = mkOption { + type = types.str; + default = "public"; + description = "Publicly visible IP address of the process. Port is determined by process ID"; + }; + + listenPortStart = mkOption { + type = types.int; + default = 4500; + description = '' + Starting port number for database listening sockets. Every FDB process binds to a + subsequent port, to this number reflects the start of the overall range. e.g. having + 8 server processes will use all ports between 4500 and 4507. + ''; + }; + + openFirewall = mkOption { + type = types.bool; + default = false; + description = '' + Open the firewall ports corresponding to FoundationDB processes and coordinators + using <option>config.networking.firewall.*</option>. + ''; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/foundationdb"; + description = "Data directory. All cluster data will be put under here."; + }; + + logDir = mkOption { + type = types.path; + default = "/var/log/foundationdb"; + description = "Log directory."; + }; + + user = mkOption { + type = types.str; + default = "foundationdb"; + description = "User account under which FoundationDB runs."; + }; + + group = mkOption { + type = types.str; + default = "foundationdb"; + description = "Group account under which FoundationDB runs."; + }; + + class = mkOption { + type = types.nullOr (types.enum [ "storage" "transaction" "stateless" ]); + default = null; + description = "Process class"; + }; + + restartDelay = mkOption { + type = types.int; + default = 10; + description = "Number of seconds to wait before restarting servers."; + }; + + logSize = mkOption { + type = types.str; + default = "10MiB"; + description = '' + Roll over to a new log file after the current log file + reaches the specified size. + ''; + }; + + maxLogSize = mkOption { + type = types.str; + default = "100MiB"; + description = '' + Delete the oldest log file when the total size of all log + files exceeds the specified size. If set to 0, old log files + will not be deleted. + ''; + }; + + serverProcesses = mkOption { + type = types.int; + default = 1; + description = "Number of fdbserver processes to run."; + }; + + backupProcesses = mkOption { + type = types.int; + default = 1; + description = "Number of backup_agent processes to run for snapshots."; + }; + + memory = mkOption { + type = types.str; + default = "8GiB"; + description = '' + Maximum memory used by the process. The default value is + <literal>8GiB</literal>. When specified without a unit, + <literal>MiB</literal> is assumed. This parameter does not + change the memory allocation of the program. Rather, it sets + a hard limit beyond which the process will kill itself and + be restarted. The default value of <literal>8GiB</literal> + is double the intended memory usage in the default + configuration (providing an emergency buffer to deal with + memory leaks or similar problems). It is not recommended to + decrease the value of this parameter below its default + value. It may be increased if you wish to allocate a very + large amount of storage engine memory or cache. In + particular, when the <literal>storageMemory</literal> + parameter is increased, the <literal>memory</literal> + parameter should be increased by an equal amount. + ''; + }; + + storageMemory = mkOption { + type = types.str; + default = "1GiB"; + description = '' + Maximum memory used for data storage. The default value is + <literal>1GiB</literal>. When specified without a unit, + <literal>MB</literal> is assumed. Clusters using the memory + storage engine will be restricted to using this amount of + memory per process for purposes of data storage. Memory + overhead associated with storing the data is counted against + this total. If you increase the + <literal>storageMemory</literal>, you should also increase + the <literal>memory</literal> parameter by the same amount. + ''; + }; + + tls = mkOption { + default = null; + description = '' + FoundationDB Transport Security Layer (TLS) settings. + ''; + + type = types.nullOr (types.submodule ({ + options = { + certificate = mkOption { + type = types.str; + description = '' + Path to the TLS certificate file. This certificate will + be offered to, and may be verified by, clients. + ''; + }; + + key = mkOption { + type = types.str; + description = "Private key file for the certificate."; + }; + + allowedPeers = mkOption { + type = types.str; + default = "Check.Valid=1,Check.Unexpired=1"; + description = '' + "Peer verification string". This may be used to adjust which TLS + client certificates a server will accept, as a form of user + authorization; for example, it may only accept TLS clients who + offer a certificate abiding by some locality or organization name. + + For more information, please see the FoundationDB documentation. + ''; + }; + }; + })); + }; + + locality = mkOption { + default = { + machineId = null; + zoneId = null; + datacenterId = null; + dataHall = null; + }; + + description = '' + FoundationDB locality settings. + ''; + + type = types.submodule ({ + options = { + machineId = mkOption { + default = null; + type = types.nullOr types.str; + description = '' + Machine identifier key. All processes on a machine should share a + unique id. By default, processes on a machine determine a unique id to share. + This does not generally need to be set. + ''; + }; + + zoneId = mkOption { + default = null; + type = types.nullOr types.str; + description = '' + Zone identifier key. Processes that share a zone id are + considered non-unique for the purposes of data replication. + If unset, defaults to machine id. + ''; + }; + + datacenterId = mkOption { + default = null; + type = types.nullOr types.str; + description = '' + Data center identifier key. All processes physically located in a + data center should share the id. If you are depending on data + center based replication this must be set on all processes. + ''; + }; + + dataHall = mkOption { + default = null; + type = types.nullOr types.str; + description = '' + Data hall identifier key. All processes physically located in a + data hall should share the id. If you are depending on data + hall based replication this must be set on all processes. + ''; + }; + }; + }); + }; + + extraReadWritePaths = mkOption { + default = [ ]; + type = types.listOf types.path; + description = '' + An extra set of filesystem paths that FoundationDB can read to + and write from. By default, FoundationDB runs under a heavily + namespaced systemd environment without write access to most of + the filesystem outside of its data and log directories. By + adding paths to this list, the set of writeable paths will be + expanded. This is useful for allowing e.g. backups to local files, + which must be performed on behalf of the foundationdb service. + ''; + }; + + pidfile = mkOption { + type = types.path; + default = "/run/foundationdb.pid"; + description = "Path to pidfile for fdbmonitor."; + }; + + traceFormat = mkOption { + type = types.enum [ "xml" "json" ]; + default = "xml"; + description = "Trace logging format."; + }; + }; + + config = mkIf cfg.enable { + assertions = [ + { assertion = lib.versionOlder cfg.package.version "6.1" -> cfg.traceFormat == "xml"; + message = '' + Versions of FoundationDB before 6.1 do not support configurable trace formats (only XML is supported). + This option has no effect for version '' + cfg.package.version + '', and enabling it is an error. + ''; + } + ]; + + environment.systemPackages = [ pkg ]; + + users.users = optionalAttrs (cfg.user == "foundationdb") { + foundationdb = { + description = "FoundationDB User"; + uid = config.ids.uids.foundationdb; + group = cfg.group; + }; + }; + + users.groups = optionalAttrs (cfg.group == "foundationdb") { + foundationdb.gid = config.ids.gids.foundationdb; + }; + + networking.firewall.allowedTCPPortRanges = mkIf cfg.openFirewall + [ { from = cfg.listenPortStart; + to = (cfg.listenPortStart + cfg.serverProcesses) - 1; + } + ]; + + systemd.tmpfiles.rules = [ + "d /etc/foundationdb 0755 ${cfg.user} ${cfg.group} - -" + "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -" + "d '${cfg.logDir}' 0770 ${cfg.user} ${cfg.group} - -" + "F '${cfg.pidfile}' - ${cfg.user} ${cfg.group} - -" + ]; + + systemd.services.foundationdb = { + description = "FoundationDB Service"; + + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + unitConfig = + { RequiresMountsFor = "${cfg.dataDir} ${cfg.logDir}"; + }; + + serviceConfig = + let rwpaths = [ cfg.dataDir cfg.logDir cfg.pidfile "/etc/foundationdb" ] + ++ cfg.extraReadWritePaths; + in + { Type = "simple"; + Restart = "always"; + RestartSec = 5; + User = cfg.user; + Group = cfg.group; + PIDFile = "${cfg.pidfile}"; + + PermissionsStartOnly = true; # setup needs root perms + TimeoutSec = 120; # give reasonable time to shut down + + # Security options + NoNewPrivileges = true; + ProtectHome = true; + ProtectSystem = "strict"; + ProtectKernelTunables = true; + ProtectControlGroups = true; + PrivateTmp = true; + PrivateDevices = true; + ReadWritePaths = lib.concatStringsSep " " (map (x: "-" + x) rwpaths); + }; + + path = [ pkg pkgs.coreutils ]; + + preStart = '' + if [ ! -f /etc/foundationdb/fdb.cluster ]; then + cf=/etc/foundationdb/fdb.cluster + desc=$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c8) + rand=$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c8) + echo ''${desc}:''${rand}@${initialIpAddr}:${builtins.toString cfg.listenPortStart} > $cf + chmod 0664 $cf + touch "${cfg.dataDir}/.first_startup" + fi + ''; + + script = "exec fdbmonitor --lockfile ${cfg.pidfile} --conffile ${configFile}"; + + postStart = '' + if [ -e "${cfg.dataDir}/.first_startup" ]; then + fdbcli --exec "configure new single ssd" + rm -f "${cfg.dataDir}/.first_startup"; + fi + ''; + }; + }; + + meta.doc = ./foundationdb.xml; + meta.maintainers = with lib.maintainers; [ thoughtpolice ]; +} diff --git a/nixos/modules/services/databases/foundationdb.xml b/nixos/modules/services/databases/foundationdb.xml new file mode 100644 index 00000000000..b0b1ebeab45 --- /dev/null +++ b/nixos/modules/services/databases/foundationdb.xml @@ -0,0 +1,443 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="module-services-foundationdb"> + <title>FoundationDB</title> + <para> + <emphasis>Source:</emphasis> + <filename>modules/services/databases/foundationdb.nix</filename> + </para> + <para> + <emphasis>Upstream documentation:</emphasis> + <link xlink:href="https://apple.github.io/foundationdb/"/> + </para> + <para> + <emphasis>Maintainer:</emphasis> Austin Seipp + </para> + <para> + <emphasis>Available version(s):</emphasis> 5.1.x, 5.2.x, 6.0.x + </para> + <para> + FoundationDB (or "FDB") is an open source, distributed, transactional + key-value store. + </para> + <section xml:id="module-services-foundationdb-configuring"> + <title>Configuring and basic setup</title> + + <para> + To enable FoundationDB, add the following to your + <filename>configuration.nix</filename>: +<programlisting> +services.foundationdb.enable = true; +services.foundationdb.package = pkgs.foundationdb52; # FoundationDB 5.2.x +</programlisting> + </para> + + <para> + The <option>services.foundationdb.package</option> option is required, and + must always be specified. Due to the fact FoundationDB network protocols and + on-disk storage formats may change between (major) versions, and upgrades + must be explicitly handled by the user, you must always manually specify + this yourself so that the NixOS module will use the proper version. Note + that minor, bugfix releases are always compatible. + </para> + + <para> + After running <command>nixos-rebuild</command>, you can verify whether + FoundationDB is running by executing <command>fdbcli</command> (which is + added to <option>environment.systemPackages</option>): +<screen> +<prompt>$ </prompt>sudo -u foundationdb fdbcli +Using cluster file `/etc/foundationdb/fdb.cluster'. + +The database is available. + +Welcome to the fdbcli. For help, type `help'. +<prompt>fdb> </prompt>status + +Using cluster file `/etc/foundationdb/fdb.cluster'. + +Configuration: + Redundancy mode - single + Storage engine - memory + Coordinators - 1 + +Cluster: + FoundationDB processes - 1 + Machines - 1 + Memory availability - 5.4 GB per process on machine with least available + Fault Tolerance - 0 machines + Server time - 04/20/18 15:21:14 + +... + +<prompt>fdb></prompt> +</screen> + </para> + + <para> + You can also write programs using the available client libraries. For + example, the following Python program can be run in order to grab the + cluster status, as a quick example. (This example uses + <command>nix-shell</command> shebang support to automatically supply the + necessary Python modules). +<screen> +<prompt>a@link> </prompt>cat fdb-status.py +#! /usr/bin/env nix-shell +#! nix-shell -i python -p python pythonPackages.foundationdb52 + +import fdb +import json + +def main(): + fdb.api_version(520) + db = fdb.open() + + @fdb.transactional + def get_status(tr): + return str(tr['\xff\xff/status/json']) + + obj = json.loads(get_status(db)) + print('FoundationDB available: %s' % obj['client']['database_status']['available']) + +if __name__ == "__main__": + main() +<prompt>a@link> </prompt>chmod +x fdb-status.py +<prompt>a@link> </prompt>./fdb-status.py +FoundationDB available: True +<prompt>a@link></prompt> +</screen> + </para> + + <para> + FoundationDB is run under the <command>foundationdb</command> user and group + by default, but this may be changed in the NixOS configuration. The systemd + unit <command>foundationdb.service</command> controls the + <command>fdbmonitor</command> process. + </para> + + <para> + By default, the NixOS module for FoundationDB creates a single SSD-storage + based database for development and basic usage. This storage engine is + designed for SSDs and will perform poorly on HDDs; however it can handle far + more data than the alternative "memory" engine and is a better default + choice for most deployments. (Note that you can change the storage backend + on-the-fly for a given FoundationDB cluster using + <command>fdbcli</command>.) + </para> + + <para> + Furthermore, only 1 server process and 1 backup agent are started in the + default configuration. See below for more on scaling to increase this. + </para> + + <para> + FoundationDB stores all data for all server processes under + <filename>/var/lib/foundationdb</filename>. You can override this using + <option>services.foundationdb.dataDir</option>, e.g. +<programlisting> +services.foundationdb.dataDir = "/data/fdb"; +</programlisting> + </para> + + <para> + Similarly, logs are stored under <filename>/var/log/foundationdb</filename> + by default, and there is a corresponding + <option>services.foundationdb.logDir</option> as well. + </para> + </section> + <section xml:id="module-services-foundationdb-scaling"> + <title>Scaling processes and backup agents</title> + + <para> + Scaling the number of server processes is quite easy; simply specify + <option>services.foundationdb.serverProcesses</option> to be the number of + FoundationDB worker processes that should be started on the machine. + </para> + + <para> + FoundationDB worker processes typically require 4GB of RAM per-process at + minimum for good performance, so this option is set to 1 by default since + the maximum amount of RAM is unknown. You're advised to abide by this + restriction, so pick a number of processes so that each has 4GB or more. + </para> + + <para> + A similar option exists in order to scale backup agent processes, + <option>services.foundationdb.backupProcesses</option>. Backup agents are + not as performance/RAM sensitive, so feel free to experiment with the number + of available backup processes. + </para> + </section> + <section xml:id="module-services-foundationdb-clustering"> + <title>Clustering</title> + + <para> + FoundationDB on NixOS works similarly to other Linux systems, so this + section will be brief. Please refer to the full FoundationDB documentation + for more on clustering. + </para> + + <para> + FoundationDB organizes clusters using a set of + <emphasis>coordinators</emphasis>, which are just specially-designated + worker processes. By default, every installation of FoundationDB on NixOS + will start as its own individual cluster, with a single coordinator: the + first worker process on <command>localhost</command>. + </para> + + <para> + Coordinators are specified globally using the + <command>/etc/foundationdb/fdb.cluster</command> file, which all servers and + client applications will use to find and join coordinators. Note that this + file <emphasis>can not</emphasis> be managed by NixOS so easily: + FoundationDB is designed so that it will rewrite the file at runtime for all + clients and nodes when cluster coordinators change, with clients + transparently handling this without intervention. It is fundamentally a + mutable file, and you should not try to manage it in any way in NixOS. + </para> + + <para> + When dealing with a cluster, there are two main things you want to do: + </para> + + <itemizedlist> + <listitem> + <para> + Add a node to the cluster for storage/compute. + </para> + </listitem> + <listitem> + <para> + Promote an ordinary worker to a coordinator. + </para> + </listitem> + </itemizedlist> + + <para> + A node must already be a member of the cluster in order to properly be + promoted to a coordinator, so you must always add it first if you wish to + promote it. + </para> + + <para> + To add a machine to a FoundationDB cluster: + </para> + + <itemizedlist> + <listitem> + <para> + Choose one of the servers to start as the initial coordinator. + </para> + </listitem> + <listitem> + <para> + Copy the <command>/etc/foundationdb/fdb.cluster</command> file from this + server to all the other servers. Restart FoundationDB on all of these + other servers, so they join the cluster. + </para> + </listitem> + <listitem> + <para> + All of these servers are now connected and working together in the + cluster, under the chosen coordinator. + </para> + </listitem> + </itemizedlist> + + <para> + At this point, you can add as many nodes as you want by just repeating the + above steps. By default there will still be a single coordinator: you can + use <command>fdbcli</command> to change this and add new coordinators. + </para> + + <para> + As a convenience, FoundationDB can automatically assign coordinators based + on the redundancy mode you wish to achieve for the cluster. Once all the + nodes have been joined, simply set the replication policy, and then issue + the <command>coordinators auto</command> command + </para> + + <para> + For example, assuming we have 3 nodes available, we can enable double + redundancy mode, then auto-select coordinators. For double redundancy, 3 + coordinators is ideal: therefore FoundationDB will make + <emphasis>every</emphasis> node a coordinator automatically: + </para> + +<screen> +<prompt>fdbcli> </prompt>configure double ssd +<prompt>fdbcli> </prompt>coordinators auto +</screen> + + <para> + This will transparently update all the servers within seconds, and + appropriately rewrite the <command>fdb.cluster</command> file, as well as + informing all client processes to do the same. + </para> + </section> + <section xml:id="module-services-foundationdb-connectivity"> + <title>Client connectivity</title> + + <para> + By default, all clients must use the current <command>fdb.cluster</command> + file to access a given FoundationDB cluster. This file is located by default + in <command>/etc/foundationdb/fdb.cluster</command> on all machines with the + FoundationDB service enabled, so you may copy the active one from your + cluster to a new node in order to connect, if it is not part of the cluster. + </para> + </section> + <section xml:id="module-services-foundationdb-authorization"> + <title>Client authorization and TLS</title> + + <para> + By default, any user who can connect to a FoundationDB process with the + correct cluster configuration can access anything. FoundationDB uses a + pluggable design to transport security, and out of the box it supports a + LibreSSL-based plugin for TLS support. This plugin not only does in-flight + encryption, but also performs client authorization based on the given + endpoint's certificate chain. For example, a FoundationDB server may be + configured to only accept client connections over TLS, where the client TLS + certificate is from organization <emphasis>Acme Co</emphasis> in the + <emphasis>Research and Development</emphasis> unit. + </para> + + <para> + Configuring TLS with FoundationDB is done using the + <option>services.foundationdb.tls</option> options in order to control the + peer verification string, as well as the certificate and its private key. + </para> + + <para> + Note that the certificate and its private key must be accessible to the + FoundationDB user account that the server runs under. These files are also + NOT managed by NixOS, as putting them into the store may reveal private + information. + </para> + + <para> + After you have a key and certificate file in place, it is not enough to + simply set the NixOS module options -- you must also configure the + <command>fdb.cluster</command> file to specify that a given set of + coordinators use TLS. This is as simple as adding the suffix + <command>:tls</command> to your cluster coordinator configuration, after the + port number. For example, assuming you have a coordinator on localhost with + the default configuration, simply specifying: + </para> + +<programlisting> +XXXXXX:XXXXXX@127.0.0.1:4500:tls +</programlisting> + + <para> + will configure all clients and server processes to use TLS from now on. + </para> + </section> + <section xml:id="module-services-foundationdb-disaster-recovery"> + <title>Backups and Disaster Recovery</title> + + <para> + The usual rules for doing FoundationDB backups apply on NixOS as written in + the FoundationDB manual. However, one important difference is the security + profile for NixOS: by default, the <command>foundationdb</command> systemd + unit uses <emphasis>Linux namespaces</emphasis> to restrict write access to + the system, except for the log directory, data directory, and the + <command>/etc/foundationdb/</command> directory. This is enforced by default + and cannot be disabled. + </para> + + <para> + However, a side effect of this is that the <command>fdbbackup</command> + command doesn't work properly for local filesystem backups: FoundationDB + uses a server process alongside the database processes to perform backups + and copy the backups to the filesystem. As a result, this process is put + under the restricted namespaces above: the backup process can only write to + a limited number of paths. + </para> + + <para> + In order to allow flexible backup locations on local disks, the FoundationDB + NixOS module supports a + <option>services.foundationdb.extraReadWritePaths</option> option. This + option takes a list of paths, and adds them to the systemd unit, allowing + the processes inside the service to write (and read) the specified + directories. + </para> + + <para> + For example, to create backups in <command>/opt/fdb-backups</command>, first + set up the paths in the module options: + </para> + +<programlisting> +services.foundationdb.extraReadWritePaths = [ "/opt/fdb-backups" ]; +</programlisting> + + <para> + Restart the FoundationDB service, and it will now be able to write to this + directory (even if it does not yet exist.) Note: this path + <emphasis>must</emphasis> exist before restarting the unit. Otherwise, + systemd will not include it in the private FoundationDB namespace (and it + will not add it dynamically at runtime). + </para> + + <para> + You can now perform a backup: + </para> + +<screen> +<prompt>$ </prompt>sudo -u foundationdb fdbbackup start -t default -d file:///opt/fdb-backups +<prompt>$ </prompt>sudo -u foundationdb fdbbackup status -t default +</screen> + </section> + <section xml:id="module-services-foundationdb-limitations"> + <title>Known limitations</title> + + <para> + The FoundationDB setup for NixOS should currently be considered beta. + FoundationDB is not new software, but the NixOS compilation and integration + has only undergone fairly basic testing of all the available functionality. + </para> + + <itemizedlist> + <listitem> + <para> + There is no way to specify individual parameters for individual + <command>fdbserver</command> processes. Currently, all server processes + inherit all the global <command>fdbmonitor</command> settings. + </para> + </listitem> + <listitem> + <para> + Ruby bindings are not currently installed. + </para> + </listitem> + <listitem> + <para> + Go bindings are not currently installed. + </para> + </listitem> + </itemizedlist> + </section> + <section xml:id="module-services-foundationdb-options"> + <title>Options</title> + + <para> + NixOS's FoundationDB module allows you to configure all of the most relevant + configuration options for <command>fdbmonitor</command>, matching it quite + closely. A complete list of options for the FoundationDB module may be found + <link linkend="opt-services.foundationdb.enable">here</link>. You should + also read the FoundationDB documentation as well. + </para> + </section> + <section xml:id="module-services-foundationdb-full-docs"> + <title>Full documentation</title> + + <para> + FoundationDB is a complex piece of software, and requires careful + administration to properly use. Full documentation for administration can be + found here: <link xlink:href="https://apple.github.io/foundationdb/"/>. + </para> + </section> +</chapter> diff --git a/nixos/modules/services/databases/hbase.nix b/nixos/modules/services/databases/hbase.nix new file mode 100644 index 00000000000..fe4f05eec64 --- /dev/null +++ b/nixos/modules/services/databases/hbase.nix @@ -0,0 +1,149 @@ +{ config, options, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.hbase; + opt = options.services.hbase; + + buildProperty = configAttr: + (builtins.concatStringsSep "\n" + (lib.mapAttrsToList + (name: value: '' + <property> + <name>${name}</name> + <value>${builtins.toString value}</value> + </property> + '') + configAttr)); + + configFile = pkgs.writeText "hbase-site.xml" + ''<configuration> + ${buildProperty (opt.settings.default // cfg.settings)} + </configuration> + ''; + + configDir = pkgs.runCommand "hbase-config-dir" { preferLocalBuild = true; } '' + mkdir -p $out + cp ${cfg.package}/conf/* $out/ + rm $out/hbase-site.xml + ln -s ${configFile} $out/hbase-site.xml + '' ; + +in { + + ###### interface + + options = { + + services.hbase = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run HBase. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.hbase; + defaultText = literalExpression "pkgs.hbase"; + description = '' + HBase package to use. + ''; + }; + + + user = mkOption { + type = types.str; + default = "hbase"; + description = '' + User account under which HBase runs. + ''; + }; + + group = mkOption { + type = types.str; + default = "hbase"; + description = '' + Group account under which HBase runs. + ''; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/hbase"; + description = '' + Specifies location of HBase database files. This location should be + writable and readable for the user the HBase service runs as + (hbase by default). + ''; + }; + + logDir = mkOption { + type = types.path; + default = "/var/log/hbase"; + description = '' + Specifies the location of HBase log files. + ''; + }; + + settings = mkOption { + type = with lib.types; attrsOf (oneOf [ str int bool ]); + default = { + "hbase.rootdir" = "file://${cfg.dataDir}/hbase"; + "hbase.zookeeper.property.dataDir" = "${cfg.dataDir}/zookeeper"; + }; + defaultText = literalExpression '' + { + "hbase.rootdir" = "file://''${config.${opt.dataDir}}/hbase"; + "hbase.zookeeper.property.dataDir" = "''${config.${opt.dataDir}}/zookeeper"; + } + ''; + description = '' + configurations in hbase-site.xml, see <link xlink:href="https://github.com/apache/hbase/blob/master/hbase-server/src/test/resources/hbase-site.xml"/> for details. + ''; + }; + + }; + + }; + + ###### implementation + + config = mkIf config.services.hbase.enable { + + systemd.tmpfiles.rules = [ + "d '${cfg.dataDir}' - ${cfg.user} ${cfg.group} - -" + "d '${cfg.logDir}' - ${cfg.user} ${cfg.group} - -" + ]; + + systemd.services.hbase = { + description = "HBase Server"; + wantedBy = [ "multi-user.target" ]; + + environment = { + # JRE 15 removed option `UseConcMarkSweepGC` which is needed. + JAVA_HOME = "${pkgs.jre8}"; + HBASE_LOG_DIR = cfg.logDir; + }; + + serviceConfig = { + User = cfg.user; + Group = cfg.group; + ExecStart = "${cfg.package}/bin/hbase --config ${configDir} master start"; + }; + }; + + users.users.hbase = { + description = "HBase Server user"; + group = "hbase"; + uid = config.ids.uids.hbase; + }; + + users.groups.hbase.gid = config.ids.gids.hbase; + + }; +} diff --git a/nixos/modules/services/databases/influxdb.nix b/nixos/modules/services/databases/influxdb.nix new file mode 100644 index 00000000000..f7383b2023a --- /dev/null +++ b/nixos/modules/services/databases/influxdb.nix @@ -0,0 +1,197 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.influxdb; + + configOptions = recursiveUpdate { + meta = { + bind-address = ":8088"; + commit-timeout = "50ms"; + dir = "${cfg.dataDir}/meta"; + election-timeout = "1s"; + heartbeat-timeout = "1s"; + hostname = "localhost"; + leader-lease-timeout = "500ms"; + retention-autocreate = true; + }; + + data = { + dir = "${cfg.dataDir}/data"; + wal-dir = "${cfg.dataDir}/wal"; + max-wal-size = 104857600; + wal-enable-logging = true; + wal-flush-interval = "10m"; + wal-partition-flush-delay = "2s"; + }; + + cluster = { + shard-writer-timeout = "5s"; + write-timeout = "5s"; + }; + + retention = { + enabled = true; + check-interval = "30m"; + }; + + http = { + enabled = true; + auth-enabled = false; + bind-address = ":8086"; + https-enabled = false; + log-enabled = true; + pprof-enabled = false; + write-tracing = false; + }; + + monitor = { + store-enabled = false; + store-database = "_internal"; + store-interval = "10s"; + }; + + admin = { + enabled = true; + bind-address = ":8083"; + https-enabled = false; + }; + + graphite = [{ + enabled = false; + }]; + + udp = [{ + enabled = false; + }]; + + collectd = [{ + enabled = false; + typesdb = "${pkgs.collectd-data}/share/collectd/types.db"; + database = "collectd_db"; + bind-address = ":25826"; + }]; + + opentsdb = [{ + enabled = false; + }]; + + continuous_queries = { + enabled = true; + log-enabled = true; + recompute-previous-n = 2; + recompute-no-older-than = "10m"; + compute-runs-per-interval = 10; + compute-no-more-than = "2m"; + }; + + hinted-handoff = { + enabled = true; + dir = "${cfg.dataDir}/hh"; + max-size = 1073741824; + max-age = "168h"; + retry-rate-limit = 0; + retry-interval = "1s"; + }; + } cfg.extraConfig; + + configFile = pkgs.runCommandLocal "config.toml" { + nativeBuildInputs = [ pkgs.remarshal ]; + } '' + remarshal -if json -of toml \ + < ${pkgs.writeText "config.json" (builtins.toJSON configOptions)} \ + > $out + ''; +in +{ + + ###### interface + + options = { + + services.influxdb = { + + enable = mkOption { + default = false; + description = "Whether to enable the influxdb server"; + type = types.bool; + }; + + package = mkOption { + default = pkgs.influxdb; + defaultText = literalExpression "pkgs.influxdb"; + description = "Which influxdb derivation to use"; + type = types.package; + }; + + user = mkOption { + default = "influxdb"; + description = "User account under which influxdb runs"; + type = types.str; + }; + + group = mkOption { + default = "influxdb"; + description = "Group under which influxdb runs"; + type = types.str; + }; + + dataDir = mkOption { + default = "/var/db/influxdb"; + description = "Data directory for influxd data files."; + type = types.path; + }; + + extraConfig = mkOption { + default = {}; + description = "Extra configuration options for influxdb"; + type = types.attrs; + }; + }; + }; + + + ###### implementation + + config = mkIf config.services.influxdb.enable { + + systemd.tmpfiles.rules = [ + "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -" + ]; + + systemd.services.influxdb = { + description = "InfluxDB Server"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + serviceConfig = { + ExecStart = ''${cfg.package}/bin/influxd -config "${configFile}"''; + User = cfg.user; + Group = cfg.group; + }; + postStart = + let + scheme = if configOptions.http.https-enabled then "-k https" else "http"; + bindAddr = (ba: if hasPrefix ":" ba then "127.0.0.1${ba}" else "${ba}")(toString configOptions.http.bind-address); + in + mkBefore '' + until ${pkgs.curl.bin}/bin/curl -s -o /dev/null ${scheme}://${bindAddr}/ping; do + sleep 1; + done + ''; + }; + + users.users = optionalAttrs (cfg.user == "influxdb") { + influxdb = { + uid = config.ids.uids.influxdb; + group = "influxdb"; + description = "Influxdb daemon user"; + }; + }; + + users.groups = optionalAttrs (cfg.group == "influxdb") { + influxdb.gid = config.ids.gids.influxdb; + }; + }; + +} diff --git a/nixos/modules/services/databases/influxdb2.nix b/nixos/modules/services/databases/influxdb2.nix new file mode 100644 index 00000000000..340c515bbb4 --- /dev/null +++ b/nixos/modules/services/databases/influxdb2.nix @@ -0,0 +1,66 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + format = pkgs.formats.json { }; + cfg = config.services.influxdb2; + configFile = format.generate "config.json" cfg.settings; +in +{ + options = { + services.influxdb2 = { + enable = mkEnableOption "the influxdb2 server"; + + package = mkOption { + default = pkgs.influxdb2-server; + defaultText = literalExpression "pkgs.influxdb2"; + description = "influxdb2 derivation to use."; + type = types.package; + }; + + settings = mkOption { + default = { }; + description = ''configuration options for influxdb2, see <link xlink:href="https://docs.influxdata.com/influxdb/v2.0/reference/config-options"/> for details.''; + type = format.type; + }; + }; + }; + + config = mkIf cfg.enable { + assertions = [{ + assertion = !(builtins.hasAttr "bolt-path" cfg.settings) && !(builtins.hasAttr "engine-path" cfg.settings); + message = "services.influxdb2.config: bolt-path and engine-path should not be set as they are managed by systemd"; + }]; + + systemd.services.influxdb2 = { + description = "InfluxDB is an open-source, distributed, time series database"; + documentation = [ "https://docs.influxdata.com/influxdb/" ]; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + environment = { + INFLUXD_CONFIG_PATH = configFile; + }; + serviceConfig = { + ExecStart = "${cfg.package}/bin/influxd --bolt-path \${STATE_DIRECTORY}/influxd.bolt --engine-path \${STATE_DIRECTORY}/engine"; + StateDirectory = "influxdb2"; + User = "influxdb2"; + Group = "influxdb2"; + CapabilityBoundingSet = ""; + SystemCallFilter = "@system-service"; + LimitNOFILE = 65536; + KillMode = "control-group"; + Restart = "on-failure"; + }; + }; + + users.extraUsers.influxdb2 = { + isSystemUser = true; + group = "influxdb2"; + }; + + users.extraGroups.influxdb2 = {}; + }; + + meta.maintainers = with lib.maintainers; [ nickcao ]; +} diff --git a/nixos/modules/services/databases/memcached.nix b/nixos/modules/services/databases/memcached.nix new file mode 100644 index 00000000000..1c06937e2f3 --- /dev/null +++ b/nixos/modules/services/databases/memcached.nix @@ -0,0 +1,118 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.services.memcached; + + memcached = pkgs.memcached; + +in + +{ + + ###### interface + + options = { + + services.memcached = { + enable = mkEnableOption "Memcached"; + + user = mkOption { + type = types.str; + default = "memcached"; + description = "The user to run Memcached as"; + }; + + listen = mkOption { + type = types.str; + default = "127.0.0.1"; + description = "The IP address to bind to."; + }; + + port = mkOption { + type = types.port; + default = 11211; + description = "The port to bind to."; + }; + + enableUnixSocket = mkEnableOption "unix socket at /run/memcached/memcached.sock"; + + maxMemory = mkOption { + type = types.ints.unsigned; + default = 64; + description = "The maximum amount of memory to use for storage, in megabytes."; + }; + + maxConnections = mkOption { + type = types.ints.unsigned; + default = 1024; + description = "The maximum number of simultaneous connections."; + }; + + extraOptions = mkOption { + type = types.listOf types.str; + default = []; + description = "A list of extra options that will be added as a suffix when running memcached."; + }; + }; + + }; + + ###### implementation + + config = mkIf config.services.memcached.enable { + + users.users = optionalAttrs (cfg.user == "memcached") { + memcached.description = "Memcached server user"; + memcached.isSystemUser = true; + memcached.group = "memcached"; + }; + users.groups = optionalAttrs (cfg.user == "memcached") { memcached = {}; }; + + environment.systemPackages = [ memcached ]; + + systemd.services.memcached = { + description = "Memcached server"; + + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + serviceConfig = { + ExecStart = + let + networking = if cfg.enableUnixSocket + then "-s /run/memcached/memcached.sock" + else "-l ${cfg.listen} -p ${toString cfg.port}"; + in "${memcached}/bin/memcached ${networking} -m ${toString cfg.maxMemory} -c ${toString cfg.maxConnections} ${concatStringsSep " " cfg.extraOptions}"; + + User = cfg.user; + + # Filesystem access + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + RuntimeDirectory = "memcached"; + # Caps + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + # Misc. + LockPersonality = true; + RestrictRealtime = true; + PrivateMounts = true; + MemoryDenyWriteExecute = true; + }; + }; + }; + imports = [ + (mkRemovedOptionModule ["services" "memcached" "socket"] '' + This option was replaced by a fixed unix socket path at /run/memcached/memcached.sock enabled using services.memcached.enableUnixSocket. + '') + ]; + +} diff --git a/nixos/modules/services/databases/monetdb.nix b/nixos/modules/services/databases/monetdb.nix new file mode 100644 index 00000000000..52a2ef041f8 --- /dev/null +++ b/nixos/modules/services/databases/monetdb.nix @@ -0,0 +1,100 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.monetdb; + +in { + meta.maintainers = with maintainers; [ StillerHarpo primeos ]; + + ###### interface + options = { + services.monetdb = { + + enable = mkEnableOption "the MonetDB database server"; + + package = mkOption { + type = types.package; + default = pkgs.monetdb; + defaultText = literalExpression "pkgs.monetdb"; + description = "MonetDB package to use."; + }; + + user = mkOption { + type = types.str; + default = "monetdb"; + description = "User account under which MonetDB runs."; + }; + + group = mkOption { + type = types.str; + default = "monetdb"; + description = "Group under which MonetDB runs."; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/monetdb"; + description = "Data directory for the dbfarm."; + }; + + port = mkOption { + type = types.ints.u16; + default = 50000; + description = "Port to listen on."; + }; + + listenAddress = mkOption { + type = types.str; + default = "127.0.0.1"; + example = "0.0.0.0"; + description = "Address to listen on."; + }; + }; + }; + + ###### implementation + config = mkIf cfg.enable { + + users.users.monetdb = mkIf (cfg.user == "monetdb") { + uid = config.ids.uids.monetdb; + group = cfg.group; + description = "MonetDB user"; + home = cfg.dataDir; + createHome = true; + }; + + users.groups.monetdb = mkIf (cfg.group == "monetdb") { + gid = config.ids.gids.monetdb; + members = [ cfg.user ]; + }; + + environment.systemPackages = [ cfg.package ]; + + systemd.services.monetdb = { + description = "MonetDB database server"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ cfg.package ]; + unitConfig.RequiresMountsFor = "${cfg.dataDir}"; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + ExecStart = "${cfg.package}/bin/monetdbd start -n ${cfg.dataDir}"; + ExecStop = "${cfg.package}/bin/monetdbd stop ${cfg.dataDir}"; + }; + preStart = '' + if [ ! -e ${cfg.dataDir}/.merovingian_properties ]; then + # Create the dbfarm (as cfg.user) + ${cfg.package}/bin/monetdbd create ${cfg.dataDir} + fi + + # Update the properties + ${cfg.package}/bin/monetdbd set port=${toString cfg.port} ${cfg.dataDir} + ${cfg.package}/bin/monetdbd set listenaddr=${cfg.listenAddress} ${cfg.dataDir} + ''; + }; + + }; +} diff --git a/nixos/modules/services/databases/mongodb.nix b/nixos/modules/services/databases/mongodb.nix new file mode 100644 index 00000000000..fccf85d482e --- /dev/null +++ b/nixos/modules/services/databases/mongodb.nix @@ -0,0 +1,197 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.services.mongodb; + + mongodb = cfg.package; + + mongoCnf = cfg: pkgs.writeText "mongodb.conf" + '' + net.bindIp: ${cfg.bind_ip} + ${optionalString cfg.quiet "systemLog.quiet: true"} + systemLog.destination: syslog + storage.dbPath: ${cfg.dbpath} + ${optionalString cfg.enableAuth "security.authorization: enabled"} + ${optionalString (cfg.replSetName != "") "replication.replSetName: ${cfg.replSetName}"} + ${cfg.extraConfig} + ''; + +in + +{ + + ###### interface + + options = { + + services.mongodb = { + + enable = mkEnableOption "the MongoDB server"; + + package = mkOption { + default = pkgs.mongodb; + defaultText = literalExpression "pkgs.mongodb"; + type = types.package; + description = " + Which MongoDB derivation to use. + "; + }; + + user = mkOption { + type = types.str; + default = "mongodb"; + description = "User account under which MongoDB runs"; + }; + + bind_ip = mkOption { + type = types.str; + default = "127.0.0.1"; + description = "IP to bind to"; + }; + + quiet = mkOption { + type = types.bool; + default = false; + description = "quieter output"; + }; + + enableAuth = mkOption { + type = types.bool; + default = false; + description = "Enable client authentication. Creates a default superuser with username root!"; + }; + + initialRootPassword = mkOption { + type = types.nullOr types.str; + default = null; + description = "Password for the root user if auth is enabled."; + }; + + dbpath = mkOption { + type = types.str; + default = "/var/db/mongodb"; + description = "Location where MongoDB stores its files"; + }; + + pidFile = mkOption { + type = types.str; + default = "/run/mongodb.pid"; + description = "Location of MongoDB pid file"; + }; + + replSetName = mkOption { + type = types.str; + default = ""; + description = '' + If this instance is part of a replica set, set its name here. + Otherwise, leave empty to run as single node. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + example = '' + storage.journal.enabled: false + ''; + description = "MongoDB extra configuration in YAML format"; + }; + + initialScript = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + A file containing MongoDB statements to execute on first startup. + ''; + }; + }; + + }; + + + ###### implementation + + config = mkIf config.services.mongodb.enable { + assertions = [ + { assertion = !cfg.enableAuth || cfg.initialRootPassword != null; + message = "`enableAuth` requires `initialRootPassword` to be set."; + } + ]; + + users.users.mongodb = mkIf (cfg.user == "mongodb") + { name = "mongodb"; + isSystemUser = true; + group = "mongodb"; + description = "MongoDB server user"; + }; + users.groups.mongodb = mkIf (cfg.user == "mongodb") {}; + + environment.systemPackages = [ mongodb ]; + + systemd.services.mongodb = + { description = "MongoDB server"; + + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + serviceConfig = { + ExecStart = "${mongodb}/bin/mongod --config ${mongoCnf cfg} --fork --pidfilepath ${cfg.pidFile}"; + User = cfg.user; + PIDFile = cfg.pidFile; + Type = "forking"; + TimeoutStartSec=120; # intial creating of journal can take some time + PermissionsStartOnly = true; + }; + + preStart = let + cfg_ = cfg // { enableAuth = false; bind_ip = "127.0.0.1"; }; + in '' + rm ${cfg.dbpath}/mongod.lock || true + if ! test -e ${cfg.dbpath}; then + install -d -m0700 -o ${cfg.user} ${cfg.dbpath} + # See postStart! + touch ${cfg.dbpath}/.first_startup + fi + if ! test -e ${cfg.pidFile}; then + install -D -o ${cfg.user} /dev/null ${cfg.pidFile} + fi '' + lib.optionalString cfg.enableAuth '' + + if ! test -e "${cfg.dbpath}/.auth_setup_complete"; then + systemd-run --unit=mongodb-for-setup --uid=${cfg.user} ${mongodb}/bin/mongod --config ${mongoCnf cfg_} + # wait for mongodb + while ! ${mongodb}/bin/mongo --eval "db.version()" > /dev/null 2>&1; do sleep 0.1; done + + ${mongodb}/bin/mongo <<EOF + use admin + db.createUser( + { + user: "root", + pwd: "${cfg.initialRootPassword}", + roles: [ + { role: "userAdminAnyDatabase", db: "admin" }, + { role: "dbAdminAnyDatabase", db: "admin" }, + { role: "readWriteAnyDatabase", db: "admin" } + ] + } + ) + EOF + touch "${cfg.dbpath}/.auth_setup_complete" + systemctl stop mongodb-for-setup + fi + ''; + postStart = '' + if test -e "${cfg.dbpath}/.first_startup"; then + ${optionalString (cfg.initialScript != null) '' + ${mongodb}/bin/mongo ${optionalString (cfg.enableAuth) "-u root -p ${cfg.initialRootPassword}"} admin "${cfg.initialScript}" + ''} + rm -f "${cfg.dbpath}/.first_startup" + fi + ''; + }; + + }; + +} diff --git a/nixos/modules/services/databases/mysql.nix b/nixos/modules/services/databases/mysql.nix new file mode 100644 index 00000000000..625b31d081c --- /dev/null +++ b/nixos/modules/services/databases/mysql.nix @@ -0,0 +1,521 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.services.mysql; + + isMariaDB = lib.getName cfg.package == lib.getName pkgs.mariadb; + + mysqldOptions = + "--user=${cfg.user} --datadir=${cfg.dataDir} --basedir=${cfg.package}"; + + format = pkgs.formats.ini { listsAsDuplicateKeys = true; }; + configFile = format.generate "my.cnf" cfg.settings; + +in + +{ + imports = [ + (mkRemovedOptionModule [ "services" "mysql" "pidDir" ] "Don't wait for pidfiles, describe dependencies through systemd.") + (mkRemovedOptionModule [ "services" "mysql" "rootPassword" ] "Use socket authentication or set the password outside of the nix store.") + (mkRemovedOptionModule [ "services" "mysql" "extraOptions" ] "Use services.mysql.settings.mysqld instead.") + (mkRemovedOptionModule [ "services" "mysql" "bind" ] "Use services.mysql.settings.mysqld.bind-address instead.") + (mkRemovedOptionModule [ "services" "mysql" "port" ] "Use services.mysql.settings.mysqld.port instead.") + ]; + + ###### interface + + options = { + + services.mysql = { + + enable = mkEnableOption "MySQL server"; + + package = mkOption { + type = types.package; + example = literalExpression "pkgs.mariadb"; + description = " + Which MySQL derivation to use. MariaDB packages are supported too. + "; + }; + + user = mkOption { + type = types.str; + default = "mysql"; + description = '' + User account under which MySQL runs. + + <note><para> + If left as the default value this user will automatically be created + on system activation, otherwise you are responsible for + ensuring the user exists before the MySQL service starts. + </para></note> + ''; + }; + + group = mkOption { + type = types.str; + default = "mysql"; + description = '' + Group account under which MySQL runs. + + <note><para> + If left as the default value this group will automatically be created + on system activation, otherwise you are responsible for + ensuring the user exists before the MySQL service starts. + </para></note> + ''; + }; + + dataDir = mkOption { + type = types.path; + example = "/var/lib/mysql"; + description = '' + The data directory for MySQL. + + <note><para> + If left as the default value of <literal>/var/lib/mysql</literal> this directory will automatically be created before the MySQL + server starts, otherwise you are responsible for ensuring the directory exists with appropriate ownership and permissions. + </para></note> + ''; + }; + + configFile = mkOption { + type = types.path; + default = configFile; + defaultText = '' + A configuration file automatically generated by NixOS. + ''; + description = '' + Override the configuration file used by MySQL. By default, + NixOS generates one automatically from <option>services.mysql.settings</option>. + ''; + example = literalExpression '' + pkgs.writeText "my.cnf" ''' + [mysqld] + datadir = /var/lib/mysql + bind-address = 127.0.0.1 + port = 3336 + + !includedir /etc/mysql/conf.d/ + '''; + ''; + }; + + settings = mkOption { + type = format.type; + default = {}; + description = '' + MySQL configuration. Refer to + <link xlink:href="https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html"/>, + <link xlink:href="https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html"/>, + and <link xlink:href="https://mariadb.com/kb/en/server-system-variables/"/> + for details on supported values. + + <note> + <para> + MySQL configuration options such as <literal>--quick</literal> should be treated as + boolean options and provided values such as <literal>true</literal>, <literal>false</literal>, + <literal>1</literal>, or <literal>0</literal>. See the provided example below. + </para> + </note> + ''; + example = literalExpression '' + { + mysqld = { + key_buffer_size = "6G"; + table_cache = 1600; + log-error = "/var/log/mysql_err.log"; + plugin-load-add = [ "server_audit" "ed25519=auth_ed25519" ]; + }; + mysqldump = { + quick = true; + max_allowed_packet = "16M"; + }; + } + ''; + }; + + initialDatabases = mkOption { + type = types.listOf (types.submodule { + options = { + name = mkOption { + type = types.str; + description = '' + The name of the database to create. + ''; + }; + schema = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + The initial schema of the database; if null (the default), + an empty database is created. + ''; + }; + }; + }); + default = []; + description = '' + List of database names and their initial schemas that should be used to create databases on the first startup + of MySQL. The schema attribute is optional: If not specified, an empty database is created. + ''; + example = [ + { name = "foodatabase"; schema = literalExpression "./foodatabase.sql"; } + { name = "bardatabase"; } + ]; + }; + + initialScript = mkOption { + type = types.nullOr types.path; + default = null; + description = "A file containing SQL statements to be executed on the first startup. Can be used for granting certain permissions on the database."; + }; + + ensureDatabases = mkOption { + type = types.listOf types.str; + default = []; + description = '' + Ensures that the specified databases exist. + This option will never delete existing databases, especially not when the value of this + option is changed. This means that databases created once through this option or + otherwise have to be removed manually. + ''; + example = [ + "nextcloud" + "matomo" + ]; + }; + + ensureUsers = mkOption { + type = types.listOf (types.submodule { + options = { + name = mkOption { + type = types.str; + description = '' + Name of the user to ensure. + ''; + }; + ensurePermissions = mkOption { + type = types.attrsOf types.str; + default = {}; + description = '' + Permissions to ensure for the user, specified as attribute set. + The attribute names specify the database and tables to grant the permissions for, + separated by a dot. You may use wildcards here. + The attribute values specfiy the permissions to grant. + You may specify one or multiple comma-separated SQL privileges here. + + For more information on how to specify the target + and on which privileges exist, see the + <link xlink:href="https://mariadb.com/kb/en/library/grant/">GRANT syntax</link>. + The attributes are used as <code>GRANT ''${attrName} ON ''${attrValue}</code>. + ''; + example = literalExpression '' + { + "database.*" = "ALL PRIVILEGES"; + "*.*" = "SELECT, LOCK TABLES"; + } + ''; + }; + }; + }); + default = []; + description = '' + Ensures that the specified users exist and have at least the ensured permissions. + The MySQL users will be identified using Unix socket authentication. This authenticates the Unix user with the + same name only, and that without the need for a password. + This option will never delete existing users or remove permissions, especially not when the value of this + option is changed. This means that users created and permissions assigned once through this option or + otherwise have to be removed manually. + ''; + example = literalExpression '' + [ + { + name = "nextcloud"; + ensurePermissions = { + "nextcloud.*" = "ALL PRIVILEGES"; + }; + } + { + name = "backup"; + ensurePermissions = { + "*.*" = "SELECT, LOCK TABLES"; + }; + } + ] + ''; + }; + + replication = { + role = mkOption { + type = types.enum [ "master" "slave" "none" ]; + default = "none"; + description = "Role of the MySQL server instance."; + }; + + serverId = mkOption { + type = types.int; + default = 1; + description = "Id of the MySQL server instance. This number must be unique for each instance."; + }; + + masterHost = mkOption { + type = types.str; + description = "Hostname of the MySQL master server."; + }; + + slaveHost = mkOption { + type = types.str; + description = "Hostname of the MySQL slave server."; + }; + + masterUser = mkOption { + type = types.str; + description = "Username of the MySQL replication user."; + }; + + masterPassword = mkOption { + type = types.str; + description = "Password of the MySQL replication user."; + }; + + masterPort = mkOption { + type = types.port; + default = 3306; + description = "Port number on which the MySQL master server runs."; + }; + }; + }; + + }; + + + ###### implementation + + config = mkIf cfg.enable { + + services.mysql.dataDir = + mkDefault (if versionAtLeast config.system.stateVersion "17.09" then "/var/lib/mysql" + else "/var/mysql"); + + services.mysql.settings.mysqld = mkMerge [ + { + datadir = cfg.dataDir; + port = mkDefault 3306; + } + (mkIf (cfg.replication.role == "master" || cfg.replication.role == "slave") { + log-bin = "mysql-bin-${toString cfg.replication.serverId}"; + log-bin-index = "mysql-bin-${toString cfg.replication.serverId}.index"; + relay-log = "mysql-relay-bin"; + server-id = cfg.replication.serverId; + binlog-ignore-db = [ "information_schema" "performance_schema" "mysql" ]; + }) + (mkIf (!isMariaDB) { + plugin-load-add = "auth_socket.so"; + }) + ]; + + users.users = optionalAttrs (cfg.user == "mysql") { + mysql = { + description = "MySQL server user"; + group = cfg.group; + uid = config.ids.uids.mysql; + }; + }; + + users.groups = optionalAttrs (cfg.group == "mysql") { + mysql.gid = config.ids.gids.mysql; + }; + + environment.systemPackages = [ cfg.package ]; + + environment.etc."my.cnf".source = cfg.configFile; + + systemd.services.mysql = { + description = "MySQL Server"; + + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + restartTriggers = [ cfg.configFile ]; + + unitConfig.RequiresMountsFor = cfg.dataDir; + + path = [ + # Needed for the mysql_install_db command in the preStart script + # which calls the hostname command. + pkgs.nettools + ]; + + preStart = if isMariaDB then '' + if ! test -e ${cfg.dataDir}/mysql; then + ${cfg.package}/bin/mysql_install_db --defaults-file=/etc/my.cnf ${mysqldOptions} + touch ${cfg.dataDir}/mysql_init + fi + '' else '' + if ! test -e ${cfg.dataDir}/mysql; then + ${cfg.package}/bin/mysqld --defaults-file=/etc/my.cnf ${mysqldOptions} --initialize-insecure + touch ${cfg.dataDir}/mysql_init + fi + ''; + + script = '' + # https://mariadb.com/kb/en/getting-started-with-mariadb-galera-cluster/#systemd-and-galera-recovery + if test -n "''${_WSREP_START_POSITION}"; then + if test -e "${cfg.package}/bin/galera_recovery"; then + VAR=$(cd ${cfg.package}/bin/..; ${cfg.package}/bin/galera_recovery); [[ $? -eq 0 ]] && export _WSREP_START_POSITION=$VAR || exit 1 + fi + fi + + # The last two environment variables are used for starting Galera clusters + exec ${cfg.package}/bin/mysqld --defaults-file=/etc/my.cnf ${mysqldOptions} $_WSREP_NEW_CLUSTER $_WSREP_START_POSITION + ''; + + postStart = let + # The super user account to use on *first* run of MySQL server + superUser = if isMariaDB then cfg.user else "root"; + in '' + ${optionalString (!isMariaDB) '' + # Wait until the MySQL server is available for use + count=0 + while [ ! -e /run/mysqld/mysqld.sock ] + do + if [ $count -eq 30 ] + then + echo "Tried 30 times, giving up..." + exit 1 + fi + + echo "MySQL daemon not yet started. Waiting for 1 second..." + count=$((count++)) + sleep 1 + done + ''} + + if [ -f ${cfg.dataDir}/mysql_init ] + then + # While MariaDB comes with a 'mysql' super user account since 10.4.x, MySQL does not + # Since we don't want to run this service as 'root' we need to ensure the account exists on first run + ( echo "CREATE USER IF NOT EXISTS '${cfg.user}'@'localhost' IDENTIFIED WITH ${if isMariaDB then "unix_socket" else "auth_socket"};" + echo "GRANT ALL PRIVILEGES ON *.* TO '${cfg.user}'@'localhost' WITH GRANT OPTION;" + ) | ${cfg.package}/bin/mysql -u ${superUser} -N + + ${concatMapStrings (database: '' + # Create initial databases + if ! test -e "${cfg.dataDir}/${database.name}"; then + echo "Creating initial database: ${database.name}" + ( echo 'create database `${database.name}`;' + + ${optionalString (database.schema != null) '' + echo 'use `${database.name}`;' + + # TODO: this silently falls through if database.schema does not exist, + # we should catch this somehow and exit, but can't do it here because we're in a subshell. + if [ -f "${database.schema}" ] + then + cat ${database.schema} + elif [ -d "${database.schema}" ] + then + cat ${database.schema}/mysql-databases/*.sql + fi + ''} + ) | ${cfg.package}/bin/mysql -u ${superUser} -N + fi + '') cfg.initialDatabases} + + ${optionalString (cfg.replication.role == "master") + '' + # Set up the replication master + + ( echo "use mysql;" + echo "CREATE USER '${cfg.replication.masterUser}'@'${cfg.replication.slaveHost}' IDENTIFIED WITH mysql_native_password;" + echo "SET PASSWORD FOR '${cfg.replication.masterUser}'@'${cfg.replication.slaveHost}' = PASSWORD('${cfg.replication.masterPassword}');" + echo "GRANT REPLICATION SLAVE ON *.* TO '${cfg.replication.masterUser}'@'${cfg.replication.slaveHost}';" + ) | ${cfg.package}/bin/mysql -u ${superUser} -N + ''} + + ${optionalString (cfg.replication.role == "slave") + '' + # Set up the replication slave + + ( echo "stop slave;" + echo "change master to master_host='${cfg.replication.masterHost}', master_user='${cfg.replication.masterUser}', master_password='${cfg.replication.masterPassword}';" + echo "start slave;" + ) | ${cfg.package}/bin/mysql -u ${superUser} -N + ''} + + ${optionalString (cfg.initialScript != null) + '' + # Execute initial script + # using toString to avoid copying the file to nix store if given as path instead of string, + # as it might contain credentials + cat ${toString cfg.initialScript} | ${cfg.package}/bin/mysql -u ${superUser} -N + ''} + + rm ${cfg.dataDir}/mysql_init + fi + + ${optionalString (cfg.ensureDatabases != []) '' + ( + ${concatMapStrings (database: '' + echo "CREATE DATABASE IF NOT EXISTS \`${database}\`;" + '') cfg.ensureDatabases} + ) | ${cfg.package}/bin/mysql -N + ''} + + ${concatMapStrings (user: + '' + ( echo "CREATE USER IF NOT EXISTS '${user.name}'@'localhost' IDENTIFIED WITH ${if isMariaDB then "unix_socket" else "auth_socket"};" + ${concatStringsSep "\n" (mapAttrsToList (database: permission: '' + echo "GRANT ${permission} ON ${database} TO '${user.name}'@'localhost';" + '') user.ensurePermissions)} + ) | ${cfg.package}/bin/mysql -N + '') cfg.ensureUsers} + ''; + + serviceConfig = mkMerge [ + { + Type = if isMariaDB then "notify" else "simple"; + Restart = "on-abort"; + RestartSec = "5s"; + + # User and group + User = cfg.user; + Group = cfg.group; + # Runtime directory and mode + RuntimeDirectory = "mysqld"; + RuntimeDirectoryMode = "0755"; + # Access write directories + ReadWritePaths = [ cfg.dataDir ]; + # Capabilities + CapabilityBoundingSet = ""; + # Security + NoNewPrivileges = true; + # Sandboxing + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectHostname = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ]; + LockPersonality = true; + MemoryDenyWriteExecute = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + # System Call Filtering + SystemCallArchitectures = "native"; + } + (mkIf (cfg.dataDir == "/var/lib/mysql") { + StateDirectory = "mysql"; + StateDirectoryMode = "0700"; + }) + ]; + }; + }; +} diff --git a/nixos/modules/services/databases/neo4j.nix b/nixos/modules/services/databases/neo4j.nix new file mode 100644 index 00000000000..8816f3b2e4b --- /dev/null +++ b/nixos/modules/services/databases/neo4j.nix @@ -0,0 +1,673 @@ +{ config, options, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.neo4j; + opt = options.services.neo4j; + certDirOpt = options.services.neo4j.directories.certificates; + isDefaultPathOption = opt: isOption opt && opt.type == types.path && opt.highestPrio >= 1500; + + sslPolicies = mapAttrsToList ( + name: conf: '' + dbms.ssl.policy.${name}.allow_key_generation=${boolToString conf.allowKeyGeneration} + dbms.ssl.policy.${name}.base_directory=${conf.baseDirectory} + ${optionalString (conf.ciphers != null) '' + dbms.ssl.policy.${name}.ciphers=${concatStringsSep "," conf.ciphers} + ''} + dbms.ssl.policy.${name}.client_auth=${conf.clientAuth} + ${if length (splitString "/" conf.privateKey) > 1 then + "dbms.ssl.policy.${name}.private_key=${conf.privateKey}" + else + "dbms.ssl.policy.${name}.private_key=${conf.baseDirectory}/${conf.privateKey}" + } + ${if length (splitString "/" conf.privateKey) > 1 then + "dbms.ssl.policy.${name}.public_certificate=${conf.publicCertificate}" + else + "dbms.ssl.policy.${name}.public_certificate=${conf.baseDirectory}/${conf.publicCertificate}" + } + dbms.ssl.policy.${name}.revoked_dir=${conf.revokedDir} + dbms.ssl.policy.${name}.tls_versions=${concatStringsSep "," conf.tlsVersions} + dbms.ssl.policy.${name}.trust_all=${boolToString conf.trustAll} + dbms.ssl.policy.${name}.trusted_dir=${conf.trustedDir} + '' + ) cfg.ssl.policies; + + serverConfig = pkgs.writeText "neo4j.conf" '' + # General + dbms.allow_upgrade=${boolToString cfg.allowUpgrade} + dbms.connectors.default_listen_address=${cfg.defaultListenAddress} + dbms.read_only=${boolToString cfg.readOnly} + ${optionalString (cfg.workerCount > 0) '' + dbms.threads.worker_count=${toString cfg.workerCount} + ''} + + # Directories + dbms.directories.certificates=${cfg.directories.certificates} + dbms.directories.data=${cfg.directories.data} + dbms.directories.logs=${cfg.directories.home}/logs + dbms.directories.plugins=${cfg.directories.plugins} + ${optionalString (cfg.constrainLoadCsv) '' + dbms.directories.import=${cfg.directories.imports} + ''} + + # HTTP Connector + ${optionalString (cfg.http.enable) '' + dbms.connector.http.enabled=${boolToString cfg.http.enable} + dbms.connector.http.listen_address=${cfg.http.listenAddress} + ''} + ${optionalString (!cfg.http.enable) '' + # It is not possible to disable the HTTP connector. To fully prevent + # clients from connecting to HTTP, block the HTTP port (7474 by default) + # via firewall. listen_address is set to the loopback interface to + # prevent remote clients from connecting. + dbms.connector.http.listen_address=127.0.0.1 + ''} + + # HTTPS Connector + dbms.connector.https.enabled=${boolToString cfg.https.enable} + dbms.connector.https.listen_address=${cfg.https.listenAddress} + https.ssl_policy=${cfg.https.sslPolicy} + + # BOLT Connector + dbms.connector.bolt.enabled=${boolToString cfg.bolt.enable} + dbms.connector.bolt.listen_address=${cfg.bolt.listenAddress} + bolt.ssl_policy=${cfg.bolt.sslPolicy} + dbms.connector.bolt.tls_level=${cfg.bolt.tlsLevel} + + # neo4j-shell + dbms.shell.enabled=${boolToString cfg.shell.enable} + + # SSL Policies + ${concatStringsSep "\n" sslPolicies} + + # Default retention policy from neo4j.conf + dbms.tx_log.rotation.retention_policy=1 days + + # Default JVM parameters from neo4j.conf + dbms.jvm.additional=-XX:+UseG1GC + dbms.jvm.additional=-XX:-OmitStackTraceInFastThrow + dbms.jvm.additional=-XX:+AlwaysPreTouch + dbms.jvm.additional=-XX:+UnlockExperimentalVMOptions + dbms.jvm.additional=-XX:+TrustFinalNonStaticFields + dbms.jvm.additional=-XX:+DisableExplicitGC + dbms.jvm.additional=-Djdk.tls.ephemeralDHKeySize=2048 + dbms.jvm.additional=-Djdk.tls.rejectClientInitiatedRenegotiation=true + dbms.jvm.additional=-Dunsupported.dbms.udc.source=tarball + + # Usage Data Collector + dbms.udc.enabled=${boolToString cfg.udc.enable} + + # Extra Configuration + ${cfg.extraServerConfig} + ''; + +in { + + imports = [ + (mkRenamedOptionModule [ "services" "neo4j" "host" ] [ "services" "neo4j" "defaultListenAddress" ]) + (mkRenamedOptionModule [ "services" "neo4j" "listenAddress" ] [ "services" "neo4j" "defaultListenAddress" ]) + (mkRenamedOptionModule [ "services" "neo4j" "enableBolt" ] [ "services" "neo4j" "bolt" "enable" ]) + (mkRenamedOptionModule [ "services" "neo4j" "enableHttps" ] [ "services" "neo4j" "https" "enable" ]) + (mkRenamedOptionModule [ "services" "neo4j" "certDir" ] [ "services" "neo4j" "directories" "certificates" ]) + (mkRenamedOptionModule [ "services" "neo4j" "dataDir" ] [ "services" "neo4j" "directories" "home" ]) + (mkRemovedOptionModule [ "services" "neo4j" "port" ] "Use services.neo4j.http.listenAddress instead.") + (mkRemovedOptionModule [ "services" "neo4j" "boltPort" ] "Use services.neo4j.bolt.listenAddress instead.") + (mkRemovedOptionModule [ "services" "neo4j" "httpsPort" ] "Use services.neo4j.https.listenAddress instead.") + ]; + + ###### interface + + options.services.neo4j = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable Neo4j Community Edition. + ''; + }; + + allowUpgrade = mkOption { + type = types.bool; + default = false; + description = '' + Allow upgrade of Neo4j database files from an older version. + ''; + }; + + constrainLoadCsv = mkOption { + type = types.bool; + default = true; + description = '' + Sets the root directory for file URLs used with the Cypher + <literal>LOAD CSV</literal> clause to be that defined by + <option>directories.imports</option>. It restricts + access to only those files within that directory and its + subdirectories. + </para> + <para> + Setting this option to <literal>false</literal> introduces + possible security problems. + ''; + }; + + defaultListenAddress = mkOption { + type = types.str; + default = "127.0.0.1"; + description = '' + Default network interface to listen for incoming connections. To + listen for connections on all interfaces, use "0.0.0.0". + </para> + <para> + Specifies the default IP address and address part of connector + specific <option>listenAddress</option> options. To bind specific + connectors to a specific network interfaces, specify the entire + <option>listenAddress</option> option for that connector. + ''; + }; + + extraServerConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Extra configuration for Neo4j Community server. Refer to the + <link xlink:href="https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/">complete reference</link> + of Neo4j configuration settings. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.neo4j; + defaultText = literalExpression "pkgs.neo4j"; + description = '' + Neo4j package to use. + ''; + }; + + readOnly = mkOption { + type = types.bool; + default = false; + description = '' + Only allow read operations from this Neo4j instance. + ''; + }; + + workerCount = mkOption { + type = types.ints.between 0 44738; + default = 0; + description = '' + Number of Neo4j worker threads, where the default of + <literal>0</literal> indicates a worker count equal to the number of + available processors. + ''; + }; + + bolt = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + Enable the BOLT connector for Neo4j. Setting this option to + <literal>false</literal> will stop Neo4j from listening for incoming + connections on the BOLT port (7687 by default). + ''; + }; + + listenAddress = mkOption { + type = types.str; + default = ":7687"; + description = '' + Neo4j listen address for BOLT traffic. The listen address is + expressed in the format <literal><ip-address>:<port-number></literal>. + ''; + }; + + sslPolicy = mkOption { + type = types.str; + default = "legacy"; + description = '' + Neo4j SSL policy for BOLT traffic. + </para> + <para> + The legacy policy is a special policy which is not defined in + the policy configuration section, but rather derives from + <option>directories.certificates</option> and + associated files (by default: <filename>neo4j.key</filename> and + <filename>neo4j.cert</filename>). Its use will be deprecated. + </para> + <para> + Note: This connector must be configured to support/require + SSL/TLS for the legacy policy to actually be utilized. See + <option>bolt.tlsLevel</option>. + ''; + }; + + tlsLevel = mkOption { + type = types.enum [ "REQUIRED" "OPTIONAL" "DISABLED" ]; + default = "OPTIONAL"; + description = '' + SSL/TSL requirement level for BOLT traffic. + ''; + }; + }; + + directories = { + certificates = mkOption { + type = types.path; + default = "${cfg.directories.home}/certificates"; + defaultText = literalExpression ''"''${config.${opt.directories.home}}/certificates"''; + description = '' + Directory for storing certificates to be used by Neo4j for + TLS connections. + </para> + <para> + When setting this directory to something other than its default, + ensure the directory's existence, and that read/write permissions are + given to the Neo4j daemon user <literal>neo4j</literal>. + </para> + <para> + Note that changing this directory from its default will prevent + the directory structure required for each SSL policy from being + automatically generated. A policy's directory structure as defined by + its <option>baseDirectory</option>,<option>revokedDir</option> and + <option>trustedDir</option> must then be setup manually. The + existence of these directories is mandatory, as well as the presence + of the certificate file and the private key. Ensure the correct + permissions are set on these directories and files. + ''; + }; + + data = mkOption { + type = types.path; + default = "${cfg.directories.home}/data"; + defaultText = literalExpression ''"''${config.${opt.directories.home}}/data"''; + description = '' + Path of the data directory. You must not configure more than one + Neo4j installation to use the same data directory. + </para> + <para> + When setting this directory to something other than its default, + ensure the directory's existence, and that read/write permissions are + given to the Neo4j daemon user <literal>neo4j</literal>. + ''; + }; + + home = mkOption { + type = types.path; + default = "/var/lib/neo4j"; + description = '' + Path of the Neo4j home directory. Other default directories are + subdirectories of this path. This directory will be created if + non-existent, and its ownership will be <command>chown</command> to + the Neo4j daemon user <literal>neo4j</literal>. + ''; + }; + + imports = mkOption { + type = types.path; + default = "${cfg.directories.home}/import"; + defaultText = literalExpression ''"''${config.${opt.directories.home}}/import"''; + description = '' + The root directory for file URLs used with the Cypher + <literal>LOAD CSV</literal> clause. Only meaningful when + <option>constrainLoadCvs</option> is set to + <literal>true</literal>. + </para> + <para> + When setting this directory to something other than its default, + ensure the directory's existence, and that read permission is + given to the Neo4j daemon user <literal>neo4j</literal>. + ''; + }; + + plugins = mkOption { + type = types.path; + default = "${cfg.directories.home}/plugins"; + defaultText = literalExpression ''"''${config.${opt.directories.home}}/plugins"''; + description = '' + Path of the database plugin directory. Compiled Java JAR files that + contain database procedures will be loaded if they are placed in + this directory. + </para> + <para> + When setting this directory to something other than its default, + ensure the directory's existence, and that read permission is + given to the Neo4j daemon user <literal>neo4j</literal>. + ''; + }; + }; + + http = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + The HTTP connector is required for Neo4j, and cannot be disabled. + Setting this option to <literal>false</literal> will force the HTTP + connector's <option>listenAddress</option> to the loopback + interface to prevent connection of remote clients. To prevent all + clients from connecting, block the HTTP port (7474 by default) by + firewall. + ''; + }; + + listenAddress = mkOption { + type = types.str; + default = ":7474"; + description = '' + Neo4j listen address for HTTP traffic. The listen address is + expressed in the format <literal><ip-address>:<port-number></literal>. + ''; + }; + }; + + https = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + Enable the HTTPS connector for Neo4j. Setting this option to + <literal>false</literal> will stop Neo4j from listening for incoming + connections on the HTTPS port (7473 by default). + ''; + }; + + listenAddress = mkOption { + type = types.str; + default = ":7473"; + description = '' + Neo4j listen address for HTTPS traffic. The listen address is + expressed in the format <literal><ip-address>:<port-number></literal>. + ''; + }; + + sslPolicy = mkOption { + type = types.str; + default = "legacy"; + description = '' + Neo4j SSL policy for HTTPS traffic. + </para> + <para> + The legacy policy is a special policy which is not defined in the + policy configuration section, but rather derives from + <option>directories.certificates</option> and + associated files (by default: <filename>neo4j.key</filename> and + <filename>neo4j.cert</filename>). Its use will be deprecated. + ''; + }; + }; + + shell = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Enable a remote shell server which Neo4j Shell clients can log in to. + Only applicable to <command>neo4j-shell</command>. + ''; + }; + }; + + ssl.policies = mkOption { + type = with types; attrsOf (submodule ({ name, config, options, ... }: { + options = { + + allowKeyGeneration = mkOption { + type = types.bool; + default = false; + description = '' + Allows the generation of a private key and associated self-signed + certificate. Only performed when both objects cannot be found for + this policy. It is recommended to turn this off again after keys + have been generated. + </para> + <para> + The public certificate is required to be duplicated to the + directory holding trusted certificates as defined by the + <option>trustedDir</option> option. + </para> + <para> + Keys should in general be generated and distributed offline by a + trusted certificate authority and not by utilizing this mode. + ''; + }; + + baseDirectory = mkOption { + type = types.path; + default = "${cfg.directories.certificates}/${name}"; + defaultText = literalExpression ''"''${config.${opt.directories.certificates}}/''${name}"''; + description = '' + The mandatory base directory for cryptographic objects of this + policy. This path is only automatically generated when this + option as well as <option>directories.certificates</option> are + left at their default. Ensure read/write permissions are given + to the Neo4j daemon user <literal>neo4j</literal>. + </para> + <para> + It is also possible to override each individual + configuration with absolute paths. See the + <option>privateKey</option> and <option>publicCertificate</option> + policy options. + ''; + }; + + ciphers = mkOption { + type = types.nullOr (types.listOf types.str); + default = null; + description = '' + Restrict the allowed ciphers of this policy to those defined + here. The default ciphers are those of the JVM platform. + ''; + }; + + clientAuth = mkOption { + type = types.enum [ "NONE" "OPTIONAL" "REQUIRE" ]; + default = "REQUIRE"; + description = '' + The client authentication stance for this policy. + ''; + }; + + privateKey = mkOption { + type = types.str; + default = "private.key"; + description = '' + The name of private PKCS #8 key file for this policy to be found + in the <option>baseDirectory</option>, or the absolute path to + the key file. It is mandatory that a key can be found or generated. + ''; + }; + + publicCertificate = mkOption { + type = types.str; + default = "public.crt"; + description = '' + The name of public X.509 certificate (chain) file in PEM format + for this policy to be found in the <option>baseDirectory</option>, + or the absolute path to the certificate file. It is mandatory + that a certificate can be found or generated. + </para> + <para> + The public certificate is required to be duplicated to the + directory holding trusted certificates as defined by the + <option>trustedDir</option> option. + ''; + }; + + revokedDir = mkOption { + type = types.path; + default = "${config.baseDirectory}/revoked"; + defaultText = literalExpression ''"''${config.${options.baseDirectory}}/revoked"''; + description = '' + Path to directory of CRLs (Certificate Revocation Lists) in + PEM format. Must be an absolute path. The existence of this + directory is mandatory and will need to be created manually when: + setting this option to something other than its default; setting + either this policy's <option>baseDirectory</option> or + <option>directories.certificates</option> to something other than + their default. Ensure read/write permissions are given to the + Neo4j daemon user <literal>neo4j</literal>. + ''; + }; + + tlsVersions = mkOption { + type = types.listOf types.str; + default = [ "TLSv1.2" ]; + description = '' + Restrict the TLS protocol versions of this policy to those + defined here. + ''; + }; + + trustAll = mkOption { + type = types.bool; + default = false; + description = '' + Makes this policy trust all remote parties. Enabling this is not + recommended and the policy's trusted directory will be ignored. + Use of this mode is discouraged. It would offer encryption but + no security. + ''; + }; + + trustedDir = mkOption { + type = types.path; + default = "${config.baseDirectory}/trusted"; + defaultText = literalExpression ''"''${config.${options.baseDirectory}}/trusted"''; + description = '' + Path to directory of X.509 certificates in PEM format for + trusted parties. Must be an absolute path. The existence of this + directory is mandatory and will need to be created manually when: + setting this option to something other than its default; setting + either this policy's <option>baseDirectory</option> or + <option>directories.certificates</option> to something other than + their default. Ensure read/write permissions are given to the + Neo4j daemon user <literal>neo4j</literal>. + </para> + <para> + The public certificate as defined by + <option>publicCertificate</option> is required to be duplicated + to this directory. + ''; + }; + + directoriesToCreate = mkOption { + type = types.listOf types.path; + internal = true; + readOnly = true; + description = '' + Directories of this policy that will be created automatically + when the certificates directory is left at its default value. + This includes all options of type path that are left at their + default value. + ''; + }; + + }; + + config.directoriesToCreate = optionals + (certDirOpt.highestPrio >= 1500 && options.baseDirectory.highestPrio >= 1500) + (map (opt: opt.value) (filter isDefaultPathOption (attrValues options))); + + })); + default = {}; + description = '' + Defines the SSL policies for use with Neo4j connectors. Each attribute + of this set defines a policy, with the attribute name defining the name + of the policy and its namespace. Refer to the operations manual section + on Neo4j's + <link xlink:href="https://neo4j.com/docs/operations-manual/current/security/ssl-framework/">SSL Framework</link> + for further details. + ''; + }; + + udc = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Enable the Usage Data Collector which Neo4j uses to collect usage + data. Refer to the operations manual section on the + <link xlink:href="https://neo4j.com/docs/operations-manual/current/configuration/usage-data-collector/">Usage Data Collector</link> + for more information. + ''; + }; + }; + + }; + + ###### implementation + + config = + let + # Assertion helpers + policyNameList = attrNames cfg.ssl.policies; + validPolicyNameList = [ "legacy" ] ++ policyNameList; + validPolicyNameString = concatStringsSep ", " validPolicyNameList; + + # Capture various directories left at their default so they can be created. + defaultDirectoriesToCreate = map (opt: opt.value) (filter isDefaultPathOption (attrValues options.services.neo4j.directories)); + policyDirectoriesToCreate = concatMap (pol: pol.directoriesToCreate) (attrValues cfg.ssl.policies); + in + + mkIf cfg.enable { + assertions = [ + { assertion = !elem "legacy" policyNameList; + message = "The policy 'legacy' is special to Neo4j, and its name is reserved."; } + { assertion = elem cfg.bolt.sslPolicy validPolicyNameList; + message = "Invalid policy assigned: `services.neo4j.bolt.sslPolicy = \"${cfg.bolt.sslPolicy}\"`, defined policies are: ${validPolicyNameString}"; } + { assertion = elem cfg.https.sslPolicy validPolicyNameList; + message = "Invalid policy assigned: `services.neo4j.https.sslPolicy = \"${cfg.https.sslPolicy}\"`, defined policies are: ${validPolicyNameString}"; } + ]; + + systemd.services.neo4j = { + description = "Neo4j Daemon"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + environment = { + NEO4J_HOME = "${cfg.package}/share/neo4j"; + NEO4J_CONF = "${cfg.directories.home}/conf"; + }; + serviceConfig = { + ExecStart = "${cfg.package}/bin/neo4j console"; + User = "neo4j"; + PermissionsStartOnly = true; + LimitNOFILE = 40000; + }; + + preStart = '' + # Directories Setup + # Always ensure home exists with nested conf, logs directories. + mkdir -m 0700 -p ${cfg.directories.home}/{conf,logs} + + # Create other sub-directories and policy directories that have been left at their default. + ${concatMapStringsSep "\n" ( + dir: '' + mkdir -m 0700 -p ${dir} + '') (defaultDirectoriesToCreate ++ policyDirectoriesToCreate)} + + # Place the configuration where Neo4j can find it. + ln -fs ${serverConfig} ${cfg.directories.home}/conf/neo4j.conf + + # Ensure neo4j user ownership + chown -R neo4j ${cfg.directories.home} + ''; + }; + + environment.systemPackages = [ cfg.package ]; + + users.users.neo4j = { + isSystemUser = true; + group = "neo4j"; + description = "Neo4j daemon user"; + home = cfg.directories.home; + }; + users.groups.neo4j = {}; + }; + + meta = { + maintainers = with lib.maintainers; [ patternspandemic ]; + }; +} diff --git a/nixos/modules/services/databases/openldap.nix b/nixos/modules/services/databases/openldap.nix new file mode 100644 index 00000000000..2c1e25d4308 --- /dev/null +++ b/nixos/modules/services/databases/openldap.nix @@ -0,0 +1,325 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.openldap; + legacyOptions = [ "rootpwFile" "suffix" "dataDir" "rootdn" "rootpw" ]; + openldap = cfg.package; + configDir = if cfg.configDir != null then cfg.configDir else "/etc/openldap/slapd.d"; + + ldapValueType = let + # Can't do types.either with multiple non-overlapping submodules, so define our own + singleLdapValueType = lib.mkOptionType rec { + name = "LDAP"; + description = "LDAP value"; + check = x: lib.isString x || (lib.isAttrs x && (x ? path || x ? base64)); + merge = lib.mergeEqualOption; + }; + # We don't coerce to lists of single values, as some values must be unique + in types.either singleLdapValueType (types.listOf singleLdapValueType); + + ldapAttrsType = + let + options = { + attrs = mkOption { + type = types.attrsOf ldapValueType; + default = {}; + description = "Attributes of the parent entry."; + }; + children = mkOption { + # Hide the child attributes, to avoid infinite recursion in e.g. documentation + # Actual Nix evaluation is lazy, so this is not an issue there + type = let + hiddenOptions = lib.mapAttrs (name: attr: attr // { visible = false; }) options; + in types.attrsOf (types.submodule { options = hiddenOptions; }); + default = {}; + description = "Child entries of the current entry, with recursively the same structure."; + example = lib.literalExpression '' + { + "cn=schema" = { + # The attribute used in the DN must be defined + attrs = { cn = "schema"; }; + children = { + # This entry's DN is expanded to "cn=foo,cn=schema" + "cn=foo" = { ... }; + }; + # These includes are inserted after "cn=schema", but before "cn=foo,cn=schema" + includes = [ ... ]; + }; + } + ''; + }; + includes = mkOption { + type = types.listOf types.path; + default = []; + description = '' + LDIF files to include after the parent's attributes but before its children. + ''; + }; + }; + in types.submodule { inherit options; }; + + valueToLdif = attr: values: let + listValues = if lib.isList values then values else lib.singleton values; + in map (value: + if lib.isAttrs value then + if lib.hasAttr "path" value + then "${attr}:< file://${value.path}" + else "${attr}:: ${value.base64}" + else "${attr}: ${lib.replaceStrings [ "\n" ] [ "\n " ] value}" + ) listValues; + + attrsToLdif = dn: { attrs, children, includes, ... }: ['' + dn: ${dn} + ${lib.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList valueToLdif attrs))} + ''] ++ (map (path: "include: file://${path}\n") includes) ++ ( + lib.flatten (lib.mapAttrsToList (name: value: attrsToLdif "${name},${dn}" value) children) + ); +in { + imports = let + deprecationNote = "This option is removed due to the deprecation of `slapd.conf` upstream. Please migrate to `services.openldap.settings`, see the release notes for advice with this process."; + mkDatabaseOption = old: new: + lib.mkChangedOptionModule [ "services" "openldap" old ] [ "services" "openldap" "settings" "children" ] + (config: let + database = lib.getAttrFromPath [ "services" "openldap" "database" ] config; + value = lib.getAttrFromPath [ "services" "openldap" old ] config; + in lib.setAttrByPath ([ "olcDatabase={1}${database}" "attrs" ] ++ new) value); + in [ + (lib.mkRemovedOptionModule [ "services" "openldap" "extraConfig" ] deprecationNote) + (lib.mkRemovedOptionModule [ "services" "openldap" "extraDatabaseConfig" ] deprecationNote) + + (lib.mkChangedOptionModule [ "services" "openldap" "logLevel" ] [ "services" "openldap" "settings" "attrs" "olcLogLevel" ] + (config: lib.splitString " " (lib.getAttrFromPath [ "services" "openldap" "logLevel" ] config))) + (lib.mkChangedOptionModule [ "services" "openldap" "defaultSchemas" ] [ "services" "openldap" "settings" "children" "cn=schema" "includes"] + (config: lib.optionals (lib.getAttrFromPath [ "services" "openldap" "defaultSchemas" ] config) ( + map (schema: "${openldap}/etc/schema/${schema}.ldif") [ "core" "cosine" "inetorgperson" "nis" ]))) + + (lib.mkChangedOptionModule [ "services" "openldap" "database" ] [ "services" "openldap" "settings" "children" ] + (config: let + database = lib.getAttrFromPath [ "services" "openldap" "database" ] config; + in { + "olcDatabase={1}${database}".attrs = { + # objectClass is case-insensitive, so don't need to capitalize ${database} + objectClass = [ "olcdatabaseconfig" "olc${database}config" ]; + olcDatabase = "{1}${database}"; + olcDbDirectory = lib.mkDefault "/var/db/openldap"; + }; + "cn=schema".includes = lib.mkDefault ( + map (schema: "${openldap}/etc/schema/${schema}.ldif") [ "core" "cosine" "inetorgperson" "nis" ] + ); + })) + (mkDatabaseOption "rootpwFile" [ "olcRootPW" "path" ]) + (mkDatabaseOption "suffix" [ "olcSuffix" ]) + (mkDatabaseOption "dataDir" [ "olcDbDirectory" ]) + (mkDatabaseOption "rootdn" [ "olcRootDN" ]) + (mkDatabaseOption "rootpw" [ "olcRootPW" ]) + ]; + options = { + services.openldap = { + enable = mkOption { + type = types.bool; + default = false; + description = " + Whether to enable the ldap server. + "; + }; + + package = mkOption { + type = types.package; + default = pkgs.openldap; + defaultText = literalExpression "pkgs.openldap"; + description = '' + OpenLDAP package to use. + + This can be used to, for example, set an OpenLDAP package + with custom overrides to enable modules or other + functionality. + ''; + }; + + user = mkOption { + type = types.str; + default = "openldap"; + description = "User account under which slapd runs."; + }; + + group = mkOption { + type = types.str; + default = "openldap"; + description = "Group account under which slapd runs."; + }; + + urlList = mkOption { + type = types.listOf types.str; + default = [ "ldap:///" ]; + description = "URL list slapd should listen on."; + example = [ "ldaps:///" ]; + }; + + settings = mkOption { + type = ldapAttrsType; + description = "Configuration for OpenLDAP, in OLC format"; + example = lib.literalExpression '' + { + attrs.olcLogLevel = [ "stats" ]; + children = { + "cn=schema".includes = [ + "''${pkgs.openldap}/etc/schema/core.ldif" + "''${pkgs.openldap}/etc/schema/cosine.ldif" + "''${pkgs.openldap}/etc/schema/inetorgperson.ldif" + ]; + "olcDatabase={-1}frontend" = { + attrs = { + objectClass = "olcDatabaseConfig"; + olcDatabase = "{-1}frontend"; + olcAccess = [ "{0}to * by dn.exact=uidNumber=0+gidNumber=0,cn=peercred,cn=external,cn=auth manage stop by * none stop" ]; + }; + }; + "olcDatabase={0}config" = { + attrs = { + objectClass = "olcDatabaseConfig"; + olcDatabase = "{0}config"; + olcAccess = [ "{0}to * by * none break" ]; + }; + }; + "olcDatabase={1}mdb" = { + attrs = { + objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ]; + olcDatabase = "{1}mdb"; + olcDbDirectory = "/var/db/ldap"; + olcDbIndex = [ + "objectClass eq" + "cn pres,eq" + "uid pres,eq" + "sn pres,eq,subany" + ]; + olcSuffix = "dc=example,dc=com"; + olcAccess = [ "{0}to * by * read break" ]; + }; + }; + }; + }; + ''; + }; + + # This option overrides settings + configDir = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Use this config directory instead of generating one from the + <literal>settings</literal> option. Overrides all NixOS settings. If + you use this option,ensure `olcPidFile` is set to `/run/slapd/slapd.conf`. + ''; + example = "/var/db/slapd.d"; + }; + + declarativeContents = mkOption { + type = with types; attrsOf lines; + default = {}; + description = '' + Declarative contents for the LDAP database, in LDIF format by suffix. + + All data will be erased when starting the LDAP server. Modifications + to the database are not prevented, they are just dropped on the next + reboot of the server. Performance-wise the database and indexes are + rebuilt on each server startup, so this will slow down server startup, + especially with large databases. + ''; + example = lib.literalExpression '' + { + "dc=example,dc=org" = ''' + dn= dn: dc=example,dc=org + objectClass: domain + dc: example + + dn: ou=users,dc=example,dc=org + objectClass = organizationalUnit + ou: users + + # ... + '''; + } + ''; + }; + }; + }; + + meta.maintainers = with lib.maintainers; [ mic92 kwohlfahrt ]; + + config = mkIf cfg.enable { + assertions = map (opt: { + assertion = ((getAttr opt cfg) != "_mkMergedOptionModule") -> (cfg.database != "_mkMergedOptionModule"); + message = "Legacy OpenLDAP option `services.openldap.${opt}` requires `services.openldap.database` (use value \"mdb\" if unsure)"; + }) legacyOptions; + environment.systemPackages = [ openldap ]; + + # Literal attributes must always be set + services.openldap.settings = { + attrs = { + objectClass = "olcGlobal"; + cn = "config"; + olcPidFile = "/run/slapd/slapd.pid"; + }; + children."cn=schema".attrs = { + cn = "schema"; + objectClass = "olcSchemaConfig"; + }; + }; + + systemd.services.openldap = { + description = "LDAP server"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + preStart = let + settingsFile = pkgs.writeText "config.ldif" (lib.concatStringsSep "\n" (attrsToLdif "cn=config" cfg.settings)); + + dbSettings = lib.filterAttrs (name: value: lib.hasPrefix "olcDatabase=" name) cfg.settings.children; + dataDirs = lib.mapAttrs' (name: value: lib.nameValuePair value.attrs.olcSuffix value.attrs.olcDbDirectory) + (lib.filterAttrs (_: value: value.attrs ? olcDbDirectory) dbSettings); + dataFiles = lib.mapAttrs (dn: contents: pkgs.writeText "${dn}.ldif" contents) cfg.declarativeContents; + mkLoadScript = dn: let + dataDir = lib.escapeShellArg (getAttr dn dataDirs); + in '' + rm -rf ${dataDir}/* + ${openldap}/bin/slapadd -F ${lib.escapeShellArg configDir} -b ${dn} -l ${getAttr dn dataFiles} + chown -R "${cfg.user}:${cfg.group}" ${dataDir} + ''; + in '' + mkdir -p /run/slapd + chown -R "${cfg.user}:${cfg.group}" /run/slapd + + mkdir -p ${lib.escapeShellArg configDir} ${lib.escapeShellArgs (lib.attrValues dataDirs)} + chown "${cfg.user}:${cfg.group}" ${lib.escapeShellArg configDir} ${lib.escapeShellArgs (lib.attrValues dataDirs)} + + ${lib.optionalString (cfg.configDir == null) ('' + rm -Rf ${configDir}/* + ${openldap}/bin/slapadd -F ${configDir} -bcn=config -l ${settingsFile} + '')} + chown -R "${cfg.user}:${cfg.group}" ${lib.escapeShellArg configDir} + + ${lib.concatStrings (map mkLoadScript (lib.attrNames cfg.declarativeContents))} + ${openldap}/bin/slaptest -u -F ${lib.escapeShellArg configDir} + ''; + serviceConfig = { + ExecStart = lib.escapeShellArgs ([ + "${openldap}/libexec/slapd" "-u" cfg.user "-g" cfg.group "-F" configDir + "-h" (lib.concatStringsSep " " cfg.urlList) + ]); + Type = "forking"; + PIDFile = cfg.settings.attrs.olcPidFile; + }; + }; + + users.users = lib.optionalAttrs (cfg.user == "openldap") { + openldap = { + group = cfg.group; + isSystemUser = true; + }; + }; + + users.groups = lib.optionalAttrs (cfg.group == "openldap") { + openldap = {}; + }; + }; +} diff --git a/nixos/modules/services/databases/opentsdb.nix b/nixos/modules/services/databases/opentsdb.nix new file mode 100644 index 00000000000..e873b2f7011 --- /dev/null +++ b/nixos/modules/services/databases/opentsdb.nix @@ -0,0 +1,108 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.opentsdb; + + configFile = pkgs.writeText "opentsdb.conf" cfg.config; + +in { + + ###### interface + + options = { + + services.opentsdb = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run OpenTSDB. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.opentsdb; + defaultText = literalExpression "pkgs.opentsdb"; + description = '' + OpenTSDB package to use. + ''; + }; + + user = mkOption { + type = types.str; + default = "opentsdb"; + description = '' + User account under which OpenTSDB runs. + ''; + }; + + group = mkOption { + type = types.str; + default = "opentsdb"; + description = '' + Group account under which OpenTSDB runs. + ''; + }; + + port = mkOption { + type = types.int; + default = 4242; + description = '' + Which port OpenTSDB listens on. + ''; + }; + + config = mkOption { + type = types.lines; + default = '' + tsd.core.auto_create_metrics = true + tsd.http.request.enable_chunked = true + ''; + description = '' + The contents of OpenTSDB's configuration file + ''; + }; + + }; + + }; + + ###### implementation + + config = mkIf config.services.opentsdb.enable { + + systemd.services.opentsdb = { + description = "OpenTSDB Server"; + wantedBy = [ "multi-user.target" ]; + requires = [ "hbase.service" ]; + + environment.JAVA_HOME = "${pkgs.jre}"; + path = [ pkgs.gnuplot ]; + + preStart = + '' + COMPRESSION=NONE HBASE_HOME=${config.services.hbase.package} ${cfg.package}/share/opentsdb/tools/create_table.sh + ''; + + serviceConfig = { + PermissionsStartOnly = true; + User = cfg.user; + Group = cfg.group; + ExecStart = "${cfg.package}/bin/tsdb tsd --staticroot=${cfg.package}/share/opentsdb/static --cachedir=/tmp/opentsdb --port=${toString cfg.port} --config=${configFile}"; + }; + }; + + users.users.opentsdb = { + description = "OpenTSDB Server user"; + group = "opentsdb"; + uid = config.ids.uids.opentsdb; + }; + + users.groups.opentsdb.gid = config.ids.gids.opentsdb; + + }; +} diff --git a/nixos/modules/services/databases/pgmanage.nix b/nixos/modules/services/databases/pgmanage.nix new file mode 100644 index 00000000000..f30f71866af --- /dev/null +++ b/nixos/modules/services/databases/pgmanage.nix @@ -0,0 +1,207 @@ +{ lib, pkgs, config, ... } : + +with lib; + +let + cfg = config.services.pgmanage; + + confFile = pkgs.writeTextFile { + name = "pgmanage.conf"; + text = '' + connection_file = ${pgmanageConnectionsFile} + + allow_custom_connections = ${builtins.toJSON cfg.allowCustomConnections} + + pgmanage_port = ${toString cfg.port} + + super_only = ${builtins.toJSON cfg.superOnly} + + ${optionalString (cfg.loginGroup != null) "login_group = ${cfg.loginGroup}"} + + login_timeout = ${toString cfg.loginTimeout} + + web_root = ${cfg.package}/etc/pgmanage/web_root + + sql_root = ${cfg.sqlRoot} + + ${optionalString (cfg.tls != null) '' + tls_cert = ${cfg.tls.cert} + tls_key = ${cfg.tls.key} + ''} + + log_level = ${cfg.logLevel} + ''; + }; + + pgmanageConnectionsFile = pkgs.writeTextFile { + name = "pgmanage-connections.conf"; + text = concatStringsSep "\n" + (mapAttrsToList (name : conn : "${name}: ${conn}") cfg.connections); + }; + + pgmanage = "pgmanage"; + +in { + + options.services.pgmanage = { + enable = mkEnableOption "PostgreSQL Administration for the web"; + + package = mkOption { + type = types.package; + default = pkgs.pgmanage; + defaultText = literalExpression "pkgs.pgmanage"; + description = '' + The pgmanage package to use. + ''; + }; + + connections = mkOption { + type = types.attrsOf types.str; + default = {}; + example = { + nuc-server = "hostaddr=192.168.0.100 port=5432 dbname=postgres"; + mini-server = "hostaddr=127.0.0.1 port=5432 dbname=postgres sslmode=require"; + }; + description = '' + pgmanage requires at least one PostgreSQL server be defined. + </para><para> + Detailed information about PostgreSQL connection strings is available at: + <link xlink:href="http://www.postgresql.org/docs/current/static/libpq-connect.html"/> + </para><para> + Note that you should not specify your user name or password. That + information will be entered on the login screen. If you specify a + username or password, it will be removed by pgmanage before attempting to + connect to a database. + ''; + }; + + allowCustomConnections = mkOption { + type = types.bool; + default = false; + description = '' + This tells pgmanage whether or not to allow anyone to use a custom + connection from the login screen. + ''; + }; + + port = mkOption { + type = types.int; + default = 8080; + description = '' + This tells pgmanage what port to listen on for browser requests. + ''; + }; + + localOnly = mkOption { + type = types.bool; + default = true; + description = '' + This tells pgmanage whether or not to set the listening socket to local + addresses only. + ''; + }; + + superOnly = mkOption { + type = types.bool; + default = true; + description = '' + This tells pgmanage whether or not to only allow super users to + login. The recommended value is true and will restrict users who are not + super users from logging in to any PostgreSQL instance through + pgmanage. Note that a connection will be made to PostgreSQL in order to + test if the user is a superuser. + ''; + }; + + loginGroup = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + This tells pgmanage to only allow users in a certain PostgreSQL group to + login to pgmanage. Note that a connection will be made to PostgreSQL in + order to test if the user is a member of the login group. + ''; + }; + + loginTimeout = mkOption { + type = types.int; + default = 3600; + description = '' + Number of seconds of inactivity before user is automatically logged + out. + ''; + }; + + sqlRoot = mkOption { + type = types.str; + default = "/var/lib/pgmanage"; + description = '' + This tells pgmanage where to put the SQL file history. All tabs are saved + to this location so that if you get disconnected from pgmanage you + don't lose your work. + ''; + }; + + tls = mkOption { + type = types.nullOr (types.submodule { + options = { + cert = mkOption { + type = types.str; + description = "TLS certificate"; + }; + key = mkOption { + type = types.str; + description = "TLS key"; + }; + }; + }); + default = null; + description = '' + These options tell pgmanage where the TLS Certificate and Key files + reside. If you use these options then you'll only be able to access + pgmanage through a secure TLS connection. These options are only + necessary if you wish to connect directly to pgmanage using a secure TLS + connection. As an alternative, you can set up pgmanage in a reverse proxy + configuration. This allows your web server to terminate the secure + connection and pass on the request to pgmanage. You can find help to set + up this configuration in: + <link xlink:href="https://github.com/pgManage/pgManage/blob/master/INSTALL_NGINX.md"/> + ''; + }; + + logLevel = mkOption { + type = types.enum ["error" "warn" "notice" "info"]; + default = "error"; + description = '' + Verbosity of logs + ''; + }; + }; + + config = mkIf cfg.enable { + systemd.services.pgmanage = { + description = "pgmanage - PostgreSQL Administration for the web"; + wants = [ "postgresql.service" ]; + after = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + User = pgmanage; + Group = pgmanage; + ExecStart = "${pkgs.pgmanage}/sbin/pgmanage -c ${confFile}" + + optionalString cfg.localOnly " --local-only=true"; + }; + }; + users = { + users.${pgmanage} = { + name = pgmanage; + group = pgmanage; + home = cfg.sqlRoot; + createHome = true; + isSystemUser = true; + }; + groups.${pgmanage} = { + name = pgmanage; + }; + }; + }; +} diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix new file mode 100644 index 00000000000..2919022496a --- /dev/null +++ b/nixos/modules/services/databases/postgresql.nix @@ -0,0 +1,424 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.services.postgresql; + + postgresql = + if cfg.extraPlugins == [] + then cfg.package + else cfg.package.withPackages (_: cfg.extraPlugins); + + toStr = value: + if true == value then "yes" + else if false == value then "no" + else if isString value then "'${lib.replaceStrings ["'"] ["''"] value}'" + else toString value; + + # The main PostgreSQL configuration file. + configFile = pkgs.writeTextDir "postgresql.conf" (concatStringsSep "\n" (mapAttrsToList (n: v: "${n} = ${toStr v}") cfg.settings)); + + configFileCheck = pkgs.runCommand "postgresql-configfile-check" {} '' + ${cfg.package}/bin/postgres -D${configFile} -C config_file >/dev/null + touch $out + ''; + + groupAccessAvailable = versionAtLeast postgresql.version "11.0"; + +in + +{ + imports = [ + (mkRemovedOptionModule [ "services" "postgresql" "extraConfig" ] "Use services.postgresql.settings instead.") + ]; + + ###### interface + + options = { + + services.postgresql = { + + enable = mkEnableOption "PostgreSQL Server"; + + package = mkOption { + type = types.package; + example = literalExpression "pkgs.postgresql_11"; + description = '' + PostgreSQL package to use. + ''; + }; + + port = mkOption { + type = types.int; + default = 5432; + description = '' + The port on which PostgreSQL listens. + ''; + }; + + checkConfig = mkOption { + type = types.bool; + default = true; + description = "Check the syntax of the configuration file at compile time"; + }; + + dataDir = mkOption { + type = types.path; + defaultText = literalExpression ''"/var/lib/postgresql/''${config.services.postgresql.package.psqlSchema}"''; + example = "/var/lib/postgresql/11"; + description = '' + The data directory for PostgreSQL. If left as the default value + this directory will automatically be created before the PostgreSQL server starts, otherwise + the sysadmin is responsible for ensuring the directory exists with appropriate ownership + and permissions. + ''; + }; + + authentication = mkOption { + type = types.lines; + default = ""; + description = '' + Defines how users authenticate themselves to the server. See the + <link xlink:href="https://www.postgresql.org/docs/current/auth-pg-hba-conf.html"> + PostgreSQL documentation for pg_hba.conf</link> + for details on the expected format of this option. By default, + peer based authentication will be used for users connecting + via the Unix socket, and md5 password authentication will be + used for users connecting via TCP. Any added rules will be + inserted above the default rules. If you'd like to replace the + default rules entirely, you can use <function>lib.mkForce</function> in your + module. + ''; + }; + + identMap = mkOption { + type = types.lines; + default = ""; + description = '' + Defines the mapping from system users to database users. + + The general form is: + + map-name system-username database-username + ''; + }; + + initdbArgs = mkOption { + type = with types; listOf str; + default = []; + example = [ "--data-checksums" "--allow-group-access" ]; + description = '' + Additional arguments passed to <literal>initdb</literal> during data dir + initialisation. + ''; + }; + + initialScript = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + A file containing SQL statements to execute on first startup. + ''; + }; + + ensureDatabases = mkOption { + type = types.listOf types.str; + default = []; + description = '' + Ensures that the specified databases exist. + This option will never delete existing databases, especially not when the value of this + option is changed. This means that databases created once through this option or + otherwise have to be removed manually. + ''; + example = [ + "gitea" + "nextcloud" + ]; + }; + + ensureUsers = mkOption { + type = types.listOf (types.submodule { + options = { + name = mkOption { + type = types.str; + description = '' + Name of the user to ensure. + ''; + }; + ensurePermissions = mkOption { + type = types.attrsOf types.str; + default = {}; + description = '' + Permissions to ensure for the user, specified as an attribute set. + The attribute names specify the database and tables to grant the permissions for. + The attribute values specify the permissions to grant. You may specify one or + multiple comma-separated SQL privileges here. + + For more information on how to specify the target + and on which privileges exist, see the + <link xlink:href="https://www.postgresql.org/docs/current/sql-grant.html">GRANT syntax</link>. + The attributes are used as <code>GRANT ''${attrValue} ON ''${attrName}</code>. + ''; + example = literalExpression '' + { + "DATABASE \"nextcloud\"" = "ALL PRIVILEGES"; + "ALL TABLES IN SCHEMA public" = "ALL PRIVILEGES"; + } + ''; + }; + }; + }); + default = []; + description = '' + Ensures that the specified users exist and have at least the ensured permissions. + The PostgreSQL users will be identified using peer authentication. This authenticates the Unix user with the + same name only, and that without the need for a password. + This option will never delete existing users or remove permissions, especially not when the value of this + option is changed. This means that users created and permissions assigned once through this option or + otherwise have to be removed manually. + ''; + example = literalExpression '' + [ + { + name = "nextcloud"; + ensurePermissions = { + "DATABASE nextcloud" = "ALL PRIVILEGES"; + }; + } + { + name = "superuser"; + ensurePermissions = { + "ALL TABLES IN SCHEMA public" = "ALL PRIVILEGES"; + }; + } + ] + ''; + }; + + enableTCPIP = mkOption { + type = types.bool; + default = false; + description = '' + Whether PostgreSQL should listen on all network interfaces. + If disabled, the database can only be accessed via its Unix + domain socket or via TCP connections to localhost. + ''; + }; + + logLinePrefix = mkOption { + type = types.str; + default = "[%p] "; + example = "%m [%p] "; + description = '' + A printf-style string that is output at the beginning of each log line. + Upstream default is <literal>'%m [%p] '</literal>, i.e. it includes the timestamp. We do + not include the timestamp, because journal has it anyway. + ''; + }; + + extraPlugins = mkOption { + type = types.listOf types.path; + default = []; + example = literalExpression "with pkgs.postgresql_11.pkgs; [ postgis pg_repack ]"; + description = '' + List of PostgreSQL plugins. PostgreSQL version for each plugin should + match version for <literal>services.postgresql.package</literal> value. + ''; + }; + + settings = mkOption { + type = with types; attrsOf (oneOf [ bool float int str ]); + default = {}; + description = '' + PostgreSQL configuration. Refer to + <link xlink:href="https://www.postgresql.org/docs/11/config-setting.html#CONFIG-SETTING-CONFIGURATION-FILE"/> + for an overview of <literal>postgresql.conf</literal>. + + <note><para> + String values will automatically be enclosed in single quotes. Single quotes will be + escaped with two single quotes as described by the upstream documentation linked above. + </para></note> + ''; + example = literalExpression '' + { + log_connections = true; + log_statement = "all"; + logging_collector = true + log_disconnections = true + log_destination = lib.mkForce "syslog"; + } + ''; + }; + + recoveryConfig = mkOption { + type = types.nullOr types.lines; + default = null; + description = '' + Contents of the <filename>recovery.conf</filename> file. + ''; + }; + + superUser = mkOption { + type = types.str; + default = "postgres"; + internal = true; + readOnly = true; + description = '' + PostgreSQL superuser account to use for various operations. Internal since changing + this value would lead to breakage while setting up databases. + ''; + }; + }; + + }; + + + ###### implementation + + config = mkIf cfg.enable { + + services.postgresql.settings = + { + hba_file = "${pkgs.writeText "pg_hba.conf" cfg.authentication}"; + ident_file = "${pkgs.writeText "pg_ident.conf" cfg.identMap}"; + log_destination = "stderr"; + log_line_prefix = cfg.logLinePrefix; + listen_addresses = if cfg.enableTCPIP then "*" else "localhost"; + port = cfg.port; + }; + + services.postgresql.package = let + mkThrow = ver: throw "postgresql_${ver} was removed, please upgrade your postgresql version."; + in + # Note: when changing the default, make it conditional on + # ‘system.stateVersion’ to maintain compatibility with existing + # systems! + mkDefault (if versionAtLeast config.system.stateVersion "21.11" then pkgs.postgresql_13 + else if versionAtLeast config.system.stateVersion "20.03" then pkgs.postgresql_11 + else if versionAtLeast config.system.stateVersion "17.09" then mkThrow "9_6" + else mkThrow "9_5"); + + services.postgresql.dataDir = mkDefault "/var/lib/postgresql/${cfg.package.psqlSchema}"; + + services.postgresql.authentication = mkAfter + '' + # Generated file; do not edit! + local all all peer + host all all 127.0.0.1/32 md5 + host all all ::1/128 md5 + ''; + + users.users.postgres = + { name = "postgres"; + uid = config.ids.uids.postgres; + group = "postgres"; + description = "PostgreSQL server user"; + home = "${cfg.dataDir}"; + useDefaultShell = true; + }; + + users.groups.postgres.gid = config.ids.gids.postgres; + + environment.systemPackages = [ postgresql ]; + + environment.pathsToLink = [ + "/share/postgresql" + ]; + + system.extraDependencies = lib.optional (cfg.checkConfig && pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) configFileCheck; + + systemd.services.postgresql = + { description = "PostgreSQL Server"; + + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + environment.PGDATA = cfg.dataDir; + + path = [ postgresql ]; + + preStart = + '' + if ! test -e ${cfg.dataDir}/PG_VERSION; then + # Cleanup the data directory. + rm -f ${cfg.dataDir}/*.conf + + # Initialise the database. + initdb -U ${cfg.superUser} ${concatStringsSep " " cfg.initdbArgs} + + # See postStart! + touch "${cfg.dataDir}/.first_startup" + fi + + ln -sfn "${configFile}/postgresql.conf" "${cfg.dataDir}/postgresql.conf" + ${optionalString (cfg.recoveryConfig != null) '' + ln -sfn "${pkgs.writeText "recovery.conf" cfg.recoveryConfig}" \ + "${cfg.dataDir}/recovery.conf" + ''} + ''; + + # Wait for PostgreSQL to be ready to accept connections. + postStart = + '' + PSQL="psql --port=${toString cfg.port}" + + while ! $PSQL -d postgres -c "" 2> /dev/null; do + if ! kill -0 "$MAINPID"; then exit 1; fi + sleep 0.1 + done + + if test -e "${cfg.dataDir}/.first_startup"; then + ${optionalString (cfg.initialScript != null) '' + $PSQL -f "${cfg.initialScript}" -d postgres + ''} + rm -f "${cfg.dataDir}/.first_startup" + fi + '' + optionalString (cfg.ensureDatabases != []) '' + ${concatMapStrings (database: '' + $PSQL -tAc "SELECT 1 FROM pg_database WHERE datname = '${database}'" | grep -q 1 || $PSQL -tAc 'CREATE DATABASE "${database}"' + '') cfg.ensureDatabases} + '' + '' + ${concatMapStrings (user: '' + $PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' + ${concatStringsSep "\n" (mapAttrsToList (database: permission: '' + $PSQL -tAc 'GRANT ${permission} ON ${database} TO "${user.name}"' + '') user.ensurePermissions)} + '') cfg.ensureUsers} + ''; + + serviceConfig = mkMerge [ + { ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + User = "postgres"; + Group = "postgres"; + RuntimeDirectory = "postgresql"; + Type = if versionAtLeast cfg.package.version "9.6" + then "notify" + else "simple"; + + # Shut down Postgres using SIGINT ("Fast Shutdown mode"). See + # http://www.postgresql.org/docs/current/static/server-shutdown.html + KillSignal = "SIGINT"; + KillMode = "mixed"; + + # Give Postgres a decent amount of time to clean up after + # receiving systemd's SIGINT. + TimeoutSec = 120; + + ExecStart = "${postgresql}/bin/postgres"; + } + (mkIf (cfg.dataDir == "/var/lib/postgresql/${cfg.package.psqlSchema}") { + StateDirectory = "postgresql postgresql/${cfg.package.psqlSchema}"; + StateDirectoryMode = if groupAccessAvailable then "0750" else "0700"; + }) + ]; + + unitConfig.RequiresMountsFor = "${cfg.dataDir}"; + }; + + }; + + meta.doc = ./postgresql.xml; + meta.maintainers = with lib.maintainers; [ thoughtpolice danbst ]; +} diff --git a/nixos/modules/services/databases/postgresql.xml b/nixos/modules/services/databases/postgresql.xml new file mode 100644 index 00000000000..0ca9f3faed2 --- /dev/null +++ b/nixos/modules/services/databases/postgresql.xml @@ -0,0 +1,214 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="module-postgresql"> + <title>PostgreSQL</title> +<!-- FIXME: render nicely --> +<!-- FIXME: source can be added automatically --> + <para> + <emphasis>Source:</emphasis> <filename>modules/services/databases/postgresql.nix</filename> + </para> + <para> + <emphasis>Upstream documentation:</emphasis> <link xlink:href="http://www.postgresql.org/docs/"/> + </para> +<!-- FIXME: more stuff, like maintainer? --> + <para> + PostgreSQL is an advanced, free relational database. +<!-- MORE --> + </para> + <section xml:id="module-services-postgres-configuring"> + <title>Configuring</title> + + <para> + To enable PostgreSQL, add the following to your <filename>configuration.nix</filename>: +<programlisting> +<xref linkend="opt-services.postgresql.enable"/> = true; +<xref linkend="opt-services.postgresql.package"/> = pkgs.postgresql_11; +</programlisting> + Note that you are required to specify the desired version of PostgreSQL (e.g. <literal>pkgs.postgresql_11</literal>). Since upgrading your PostgreSQL version requires a database dump and reload (see below), NixOS cannot provide a default value for <xref linkend="opt-services.postgresql.package"/> such as the most recent release of PostgreSQL. + </para> + +<!-- +<para>After running <command>nixos-rebuild</command>, you can verify +whether PostgreSQL works by running <command>psql</command>: + +<screen> +<prompt>$ </prompt>psql +psql (9.2.9) +Type "help" for help. + +<prompt>alice=></prompt> +</screen> +--> + + <para> + By default, PostgreSQL stores its databases in <filename>/var/lib/postgresql/$psqlSchema</filename>. You can override this using <xref linkend="opt-services.postgresql.dataDir"/>, e.g. +<programlisting> +<xref linkend="opt-services.postgresql.dataDir"/> = "/data/postgresql"; +</programlisting> + </para> + </section> + <section xml:id="module-services-postgres-upgrading"> + <title>Upgrading</title> + + <note> + <para> + The steps below demonstrate how to upgrade from an older version to <package>pkgs.postgresql_13</package>. + These instructions are also applicable to other versions. + </para> + </note> + <para> + Major PostgreSQL upgrades require a downtime and a few imperative steps to be called. This is the case because + each major version has some internal changes in the databases' state during major releases. Because of that, + NixOS places the state into <filename>/var/lib/postgresql/<version></filename> where each <literal>version</literal> + can be obtained like this: +<programlisting> +<prompt>$ </prompt>nix-instantiate --eval -A postgresql_13.psqlSchema +"13" +</programlisting> + For an upgrade, a script like this can be used to simplify the process: +<programlisting> +{ config, pkgs, ... }: +{ + <xref linkend="opt-environment.systemPackages" /> = [ + (pkgs.writeScriptBin "upgrade-pg-cluster" '' + set -eux + # XXX it's perhaps advisable to stop all services that depend on postgresql + systemctl stop postgresql + + # XXX replace `<new version>` with the psqlSchema here + export NEWDATA="/var/lib/postgresql/<new version>" + + # XXX specify the postgresql package you'd like to upgrade to + export NEWBIN="${pkgs.postgresql_13}/bin" + + export OLDDATA="${config.<xref linkend="opt-services.postgresql.dataDir"/>}" + export OLDBIN="${config.<xref linkend="opt-services.postgresql.package"/>}/bin" + + install -d -m 0700 -o postgres -g postgres "$NEWDATA" + cd "$NEWDATA" + sudo -u postgres $NEWBIN/initdb -D "$NEWDATA" + + sudo -u postgres $NEWBIN/pg_upgrade \ + --old-datadir "$OLDDATA" --new-datadir "$NEWDATA" \ + --old-bindir $OLDBIN --new-bindir $NEWBIN \ + "$@" + '') + ]; +} +</programlisting> + </para> + + <para> + The upgrade process is: + </para> + + <orderedlist> + <listitem> + <para> + Rebuild nixos configuration with the configuration above added to your <filename>configuration.nix</filename>. Alternatively, add that into separate file and reference it in <literal>imports</literal> list. + </para> + </listitem> + <listitem> + <para> + Login as root (<literal>sudo su -</literal>) + </para> + </listitem> + <listitem> + <para> + Run <literal>upgrade-pg-cluster</literal>. It will stop old postgresql, initialize a new one and migrate the old one to the new one. You may supply arguments like <literal>--jobs 4</literal> and <literal>--link</literal> to speedup migration process. See <link xlink:href="https://www.postgresql.org/docs/current/pgupgrade.html" /> for details. + </para> + </listitem> + <listitem> + <para> + Change postgresql package in NixOS configuration to the one you were upgrading to via <xref linkend="opt-services.postgresql.package" />. Rebuild NixOS. This should start new postgres using upgraded data directory and all services you stopped during the upgrade. + </para> + </listitem> + <listitem> + <para> + After the upgrade it's advisable to analyze the new cluster (as <literal>su -l postgres</literal> in the + <xref linkend="opt-services.postgresql.dataDir" />, in this example <filename>/var/lib/postgresql/13</filename>): +<programlisting> +<prompt>$ </prompt>./analyze_new_cluster.sh +</programlisting> + <warning><para>The next step removes the old state-directory!</para></warning> +<programlisting> +<prompt>$ </prompt>./delete_old_cluster.sh +</programlisting> + </para> + </listitem> + </orderedlist> + </section> + <section xml:id="module-services-postgres-options"> + <title>Options</title> + + <para> + A complete list of options for the PostgreSQL module may be found <link linkend="opt-services.postgresql.enable">here</link>. + </para> + </section> + <section xml:id="module-services-postgres-plugins"> + <title>Plugins</title> + + <para> + Plugins collection for each PostgreSQL version can be accessed with <literal>.pkgs</literal>. For example, for <literal>pkgs.postgresql_11</literal> package, its plugin collection is accessed by <literal>pkgs.postgresql_11.pkgs</literal>: +<screen> +<prompt>$ </prompt>nix repl '<nixpkgs>' + +Loading '<nixpkgs>'... +Added 10574 variables. + +<prompt>nix-repl> </prompt>postgresql_11.pkgs.<TAB><TAB> +postgresql_11.pkgs.cstore_fdw postgresql_11.pkgs.pg_repack +postgresql_11.pkgs.pg_auto_failover postgresql_11.pkgs.pg_safeupdate +postgresql_11.pkgs.pg_bigm postgresql_11.pkgs.pg_similarity +postgresql_11.pkgs.pg_cron postgresql_11.pkgs.pg_topn +postgresql_11.pkgs.pg_hll postgresql_11.pkgs.pgjwt +postgresql_11.pkgs.pg_partman postgresql_11.pkgs.pgroonga +... +</screen> + </para> + + <para> + To add plugins via NixOS configuration, set <literal>services.postgresql.extraPlugins</literal>: +<programlisting> +<xref linkend="opt-services.postgresql.package"/> = pkgs.postgresql_11; +<xref linkend="opt-services.postgresql.extraPlugins"/> = with pkgs.postgresql_11.pkgs; [ + pg_repack + postgis +]; +</programlisting> + </para> + + <para> + You can build custom PostgreSQL-with-plugins (to be used outside of NixOS) using function <literal>.withPackages</literal>. For example, creating a custom PostgreSQL package in an overlay can look like: +<programlisting> +self: super: { + postgresql_custom = self.postgresql_11.withPackages (ps: [ + ps.pg_repack + ps.postgis + ]); +} +</programlisting> + </para> + + <para> + Here's a recipe on how to override a particular plugin through an overlay: +<programlisting> +self: super: { + postgresql_11 = super.postgresql_11.override { this = self.postgresql_11; } // { + pkgs = super.postgresql_11.pkgs // { + pg_repack = super.postgresql_11.pkgs.pg_repack.overrideAttrs (_: { + name = "pg_repack-v20181024"; + src = self.fetchzip { + url = "https://github.com/reorg/pg_repack/archive/923fa2f3c709a506e111cc963034bf2fd127aa00.tar.gz"; + sha256 = "17k6hq9xaax87yz79j773qyigm4fwk8z4zh5cyp6z0sxnwfqxxw5"; + }; + }); + }; + }; +} +</programlisting> + </para> + </section> +</chapter> diff --git a/nixos/modules/services/databases/redis.nix b/nixos/modules/services/databases/redis.nix new file mode 100644 index 00000000000..a1bd73c9e37 --- /dev/null +++ b/nixos/modules/services/databases/redis.nix @@ -0,0 +1,391 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.redis; + + mkValueString = value: + if value == true then "yes" + else if value == false then "no" + else generators.mkValueStringDefault { } value; + + redisConfig = settings: pkgs.writeText "redis.conf" (generators.toKeyValue { + listsAsDuplicateKeys = true; + mkKeyValue = generators.mkKeyValueDefault { inherit mkValueString; } " "; + } settings); + + redisName = name: "redis" + optionalString (name != "") ("-"+name); + enabledServers = filterAttrs (name: conf: conf.enable) config.services.redis.servers; + +in { + imports = [ + (mkRemovedOptionModule [ "services" "redis" "user" ] "The redis module now is hardcoded to the redis user.") + (mkRemovedOptionModule [ "services" "redis" "dbpath" ] "The redis module now uses /var/lib/redis as data directory.") + (mkRemovedOptionModule [ "services" "redis" "dbFilename" ] "The redis module now uses /var/lib/redis/dump.rdb as database dump location.") + (mkRemovedOptionModule [ "services" "redis" "appendOnlyFilename" ] "This option was never used.") + (mkRemovedOptionModule [ "services" "redis" "pidFile" ] "This option was removed.") + (mkRemovedOptionModule [ "services" "redis" "extraConfig" ] "Use services.redis.servers.*.settings instead.") + (mkRenamedOptionModule [ "services" "redis" "enable"] [ "services" "redis" "servers" "" "enable" ]) + (mkRenamedOptionModule [ "services" "redis" "port"] [ "services" "redis" "servers" "" "port" ]) + (mkRenamedOptionModule [ "services" "redis" "openFirewall"] [ "services" "redis" "servers" "" "openFirewall" ]) + (mkRenamedOptionModule [ "services" "redis" "bind"] [ "services" "redis" "servers" "" "bind" ]) + (mkRenamedOptionModule [ "services" "redis" "unixSocket"] [ "services" "redis" "servers" "" "unixSocket" ]) + (mkRenamedOptionModule [ "services" "redis" "unixSocketPerm"] [ "services" "redis" "servers" "" "unixSocketPerm" ]) + (mkRenamedOptionModule [ "services" "redis" "logLevel"] [ "services" "redis" "servers" "" "logLevel" ]) + (mkRenamedOptionModule [ "services" "redis" "logfile"] [ "services" "redis" "servers" "" "logfile" ]) + (mkRenamedOptionModule [ "services" "redis" "syslog"] [ "services" "redis" "servers" "" "syslog" ]) + (mkRenamedOptionModule [ "services" "redis" "databases"] [ "services" "redis" "servers" "" "databases" ]) + (mkRenamedOptionModule [ "services" "redis" "maxclients"] [ "services" "redis" "servers" "" "maxclients" ]) + (mkRenamedOptionModule [ "services" "redis" "save"] [ "services" "redis" "servers" "" "save" ]) + (mkRenamedOptionModule [ "services" "redis" "slaveOf"] [ "services" "redis" "servers" "" "slaveOf" ]) + (mkRenamedOptionModule [ "services" "redis" "masterAuth"] [ "services" "redis" "servers" "" "masterAuth" ]) + (mkRenamedOptionModule [ "services" "redis" "requirePass"] [ "services" "redis" "servers" "" "requirePass" ]) + (mkRenamedOptionModule [ "services" "redis" "requirePassFile"] [ "services" "redis" "servers" "" "requirePassFile" ]) + (mkRenamedOptionModule [ "services" "redis" "appendOnly"] [ "services" "redis" "servers" "" "appendOnly" ]) + (mkRenamedOptionModule [ "services" "redis" "appendFsync"] [ "services" "redis" "servers" "" "appendFsync" ]) + (mkRenamedOptionModule [ "services" "redis" "slowLogLogSlowerThan"] [ "services" "redis" "servers" "" "slowLogLogSlowerThan" ]) + (mkRenamedOptionModule [ "services" "redis" "slowLogMaxLen"] [ "services" "redis" "servers" "" "slowLogMaxLen" ]) + (mkRenamedOptionModule [ "services" "redis" "settings"] [ "services" "redis" "servers" "" "settings" ]) + ]; + + ###### interface + + options = { + + services.redis = { + package = mkOption { + type = types.package; + default = pkgs.redis; + defaultText = literalExpression "pkgs.redis"; + description = "Which Redis derivation to use."; + }; + + vmOverCommit = mkEnableOption '' + setting of vm.overcommit_memory to 1 + (Suggested for Background Saving: http://redis.io/topics/faq) + ''; + + servers = mkOption { + type = with types; attrsOf (submodule ({config, name, ...}@args: { + options = { + enable = mkEnableOption '' + Redis server. + + Note that the NixOS module for Redis disables kernel support + for Transparent Huge Pages (THP), + because this features causes major performance problems for Redis, + e.g. (https://redis.io/topics/latency). + ''; + + user = mkOption { + type = types.str; + default = redisName name; + defaultText = literalExpression '' + if name == "" then "redis" else "redis-''${name}" + ''; + description = "The username and groupname for redis-server."; + }; + + port = mkOption { + type = types.port; + default = if name == "" then 6379 else 0; + defaultText = literalExpression ''if name == "" then 6379 else 0''; + description = '' + The TCP port to accept connections. + If port 0 is specified Redis will not listen on a TCP socket. + ''; + }; + + openFirewall = mkOption { + type = types.bool; + default = false; + description = '' + Whether to open ports in the firewall for the server. + ''; + }; + + bind = mkOption { + type = with types; nullOr str; + default = "127.0.0.1"; + description = '' + The IP interface to bind to. + <literal>null</literal> means "all interfaces". + ''; + example = "192.0.2.1"; + }; + + unixSocket = mkOption { + type = with types; nullOr path; + default = "/run/${redisName name}/redis.sock"; + defaultText = literalExpression '' + if name == "" then "/run/redis/redis.sock" else "/run/redis-''${name}/redis.sock" + ''; + description = "The path to the socket to bind to."; + }; + + unixSocketPerm = mkOption { + type = types.int; + default = 660; + description = "Change permissions for the socket"; + example = 600; + }; + + logLevel = mkOption { + type = types.str; + default = "notice"; # debug, verbose, notice, warning + example = "debug"; + description = "Specify the server verbosity level, options: debug, verbose, notice, warning."; + }; + + logfile = mkOption { + type = types.str; + default = "/dev/null"; + description = "Specify the log file name. Also 'stdout' can be used to force Redis to log on the standard output."; + example = "/var/log/redis.log"; + }; + + syslog = mkOption { + type = types.bool; + default = true; + description = "Enable logging to the system logger."; + }; + + databases = mkOption { + type = types.int; + default = 16; + description = "Set the number of databases."; + }; + + maxclients = mkOption { + type = types.int; + default = 10000; + description = "Set the max number of connected clients at the same time."; + }; + + save = mkOption { + type = with types; listOf (listOf int); + default = [ [900 1] [300 10] [60 10000] ]; + description = "The schedule in which data is persisted to disk, represented as a list of lists where the first element represent the amount of seconds and the second the number of changes."; + }; + + slaveOf = mkOption { + type = with types; nullOr (submodule ({ ... }: { + options = { + ip = mkOption { + type = str; + description = "IP of the Redis master"; + example = "192.168.1.100"; + }; + + port = mkOption { + type = port; + description = "port of the Redis master"; + default = 6379; + }; + }; + })); + + default = null; + description = "IP and port to which this redis instance acts as a slave."; + example = { ip = "192.168.1.100"; port = 6379; }; + }; + + masterAuth = mkOption { + type = with types; nullOr str; + default = null; + description = ''If the master is password protected (using the requirePass configuration) + it is possible to tell the slave to authenticate before starting the replication synchronization + process, otherwise the master will refuse the slave request. + (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE)''; + }; + + requirePass = mkOption { + type = with types; nullOr str; + default = null; + description = '' + Password for database (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE). + Use requirePassFile to store it outside of the nix store in a dedicated file. + ''; + example = "letmein!"; + }; + + requirePassFile = mkOption { + type = with types; nullOr path; + default = null; + description = "File with password for the database."; + example = "/run/keys/redis-password"; + }; + + appendOnly = mkOption { + type = types.bool; + default = false; + description = "By default data is only periodically persisted to disk, enable this option to use an append-only file for improved persistence."; + }; + + appendFsync = mkOption { + type = types.str; + default = "everysec"; # no, always, everysec + description = "How often to fsync the append-only log, options: no, always, everysec."; + }; + + slowLogLogSlowerThan = mkOption { + type = types.int; + default = 10000; + description = "Log queries whose execution take longer than X in milliseconds."; + example = 1000; + }; + + slowLogMaxLen = mkOption { + type = types.int; + default = 128; + description = "Maximum number of items to keep in slow log."; + }; + + settings = mkOption { + # TODO: this should be converted to freeformType + type = with types; attrsOf (oneOf [ bool int str (listOf str) ]); + default = {}; + description = '' + Redis configuration. Refer to + <link xlink:href="https://redis.io/topics/config"/> + for details on supported values. + ''; + example = literalExpression '' + { + loadmodule = [ "/path/to/my_module.so" "/path/to/other_module.so" ]; + } + ''; + }; + }; + config.settings = mkMerge [ + { + port = config.port; + daemonize = false; + supervised = "systemd"; + loglevel = config.logLevel; + logfile = config.logfile; + syslog-enabled = config.syslog; + databases = config.databases; + maxclients = config.maxclients; + save = map (d: "${toString (builtins.elemAt d 0)} ${toString (builtins.elemAt d 1)}") config.save; + dbfilename = "dump.rdb"; + dir = "/var/lib/${redisName name}"; + appendOnly = config.appendOnly; + appendfsync = config.appendFsync; + slowlog-log-slower-than = config.slowLogLogSlowerThan; + slowlog-max-len = config.slowLogMaxLen; + } + (mkIf (config.bind != null) { bind = config.bind; }) + (mkIf (config.unixSocket != null) { + unixsocket = config.unixSocket; + unixsocketperm = toString config.unixSocketPerm; + }) + (mkIf (config.slaveOf != null) { slaveof = "${config.slaveOf.ip} ${toString config.slaveOf.port}"; }) + (mkIf (config.masterAuth != null) { masterauth = config.masterAuth; }) + (mkIf (config.requirePass != null) { requirepass = config.requirePass; }) + ]; + })); + description = "Configuration of multiple <literal>redis-server</literal> instances."; + default = {}; + }; + }; + + }; + + + ###### implementation + + config = mkIf (enabledServers != {}) { + + assertions = attrValues (mapAttrs (name: conf: { + assertion = conf.requirePass != null -> conf.requirePassFile == null; + message = '' + You can only set one services.redis.servers.${name}.requirePass + or services.redis.servers.${name}.requirePassFile + ''; + }) enabledServers); + + boot.kernel.sysctl = mkMerge [ + { "vm.nr_hugepages" = "0"; } + ( mkIf cfg.vmOverCommit { "vm.overcommit_memory" = "1"; } ) + ]; + + networking.firewall.allowedTCPPorts = concatMap (conf: + optional conf.openFirewall conf.port + ) (attrValues enabledServers); + + environment.systemPackages = [ cfg.package ]; + + users.users = mapAttrs' (name: conf: nameValuePair (redisName name) { + description = "System user for the redis-server instance ${name}"; + isSystemUser = true; + group = redisName name; + }) enabledServers; + users.groups = mapAttrs' (name: conf: nameValuePair (redisName name) { + }) enabledServers; + + systemd.services = mapAttrs' (name: conf: nameValuePair (redisName name) { + description = "Redis Server - ${redisName name}"; + + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + serviceConfig = { + ExecStart = "${cfg.package}/bin/redis-server /run/${redisName name}/redis.conf"; + ExecStartPre = [("+"+pkgs.writeShellScript "${redisName name}-credentials" ('' + install -o '${conf.user}' -m 600 ${redisConfig conf.settings} /run/${redisName name}/redis.conf + '' + optionalString (conf.requirePassFile != null) '' + { + printf requirePass' ' + cat ${escapeShellArg conf.requirePassFile} + } >>/run/${redisName name}/redis.conf + '') + )]; + Type = "notify"; + # User and group + User = conf.user; + Group = conf.user; + # Runtime directory and mode + RuntimeDirectory = redisName name; + RuntimeDirectoryMode = "0750"; + # State directory and mode + StateDirectory = redisName name; + StateDirectoryMode = "0700"; + # Access write directories + UMask = "0077"; + # Capabilities + CapabilityBoundingSet = ""; + # Security + NoNewPrivileges = true; + # Process Properties + LimitNOFILE = mkDefault "${toString (conf.maxclients + 32)}"; + # Sandboxing + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + PrivateUsers = true; + ProtectClock = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictAddressFamilies = + optionals (conf.port != 0) ["AF_INET" "AF_INET6"] ++ + optional (conf.unixSocket != null) "AF_UNIX"; + RestrictNamespaces = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + # System Call Filtering + SystemCallArchitectures = "native"; + SystemCallFilter = "~@cpu-emulation @debug @keyring @memlock @mount @obsolete @privileged @resources @setuid"; + }; + }) enabledServers; + + }; +} diff --git a/nixos/modules/services/databases/rethinkdb.nix b/nixos/modules/services/databases/rethinkdb.nix new file mode 100644 index 00000000000..c764d6c21c6 --- /dev/null +++ b/nixos/modules/services/databases/rethinkdb.nix @@ -0,0 +1,108 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.rethinkdb; + rethinkdb = cfg.package; +in + +{ + + ###### interface + + options = { + + services.rethinkdb = { + + enable = mkEnableOption "RethinkDB server"; + + #package = mkOption { + # default = pkgs.rethinkdb; + # description = "Which RethinkDB derivation to use."; + #}; + + user = mkOption { + default = "rethinkdb"; + description = "User account under which RethinkDB runs."; + }; + + group = mkOption { + default = "rethinkdb"; + description = "Group which rethinkdb user belongs to."; + }; + + dbpath = mkOption { + default = "/var/db/rethinkdb"; + description = "Location where RethinkDB stores its data, 1 data directory per instance."; + }; + + pidpath = mkOption { + default = "/run/rethinkdb"; + description = "Location where each instance's pid file is located."; + }; + + #cfgpath = mkOption { + # default = "/etc/rethinkdb/instances.d"; + # description = "Location where RethinkDB stores it config files, 1 config file per instance."; + #}; + + # TODO: currently not used by our implementation. + #instances = mkOption { + # type = types.attrsOf types.str; + # default = {}; + # description = "List of named RethinkDB instances in our cluster."; + #}; + + }; + + }; + + ###### implementation + config = mkIf config.services.rethinkdb.enable { + + environment.systemPackages = [ rethinkdb ]; + + systemd.services.rethinkdb = { + description = "RethinkDB server"; + + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + serviceConfig = { + # TODO: abstract away 'default', which is a per-instance directory name + # allowing end user of this nix module to provide multiple instances, + # and associated directory per instance + ExecStart = "${rethinkdb}/bin/rethinkdb -d ${cfg.dbpath}/default"; + ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + User = cfg.user; + Group = cfg.group; + PIDFile = "${cfg.pidpath}/default.pid"; + PermissionsStartOnly = true; + }; + + preStart = '' + if ! test -e ${cfg.dbpath}; then + install -d -m0755 -o ${cfg.user} -g ${cfg.group} ${cfg.dbpath} + install -d -m0755 -o ${cfg.user} -g ${cfg.group} ${cfg.dbpath}/default + chown -R ${cfg.user}:${cfg.group} ${cfg.dbpath} + fi + if ! test -e "${cfg.pidpath}/default.pid"; then + install -D -o ${cfg.user} -g ${cfg.group} /dev/null "${cfg.pidpath}/default.pid" + fi + ''; + }; + + users.users.rethinkdb = mkIf (cfg.user == "rethinkdb") + { name = "rethinkdb"; + description = "RethinkDB server user"; + isSystemUser = true; + }; + + users.groups = optionalAttrs (cfg.group == "rethinkdb") (singleton + { name = "rethinkdb"; + }); + + }; + +} diff --git a/nixos/modules/services/databases/riak.nix b/nixos/modules/services/databases/riak.nix new file mode 100644 index 00000000000..cc4237d038c --- /dev/null +++ b/nixos/modules/services/databases/riak.nix @@ -0,0 +1,162 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.services.riak; + +in + +{ + + ###### interface + + options = { + + services.riak = { + + enable = mkEnableOption "riak"; + + package = mkOption { + type = types.package; + default = pkgs.riak; + defaultText = literalExpression "pkgs.riak"; + description = '' + Riak package to use. + ''; + }; + + nodeName = mkOption { + type = types.str; + default = "riak@127.0.0.1"; + description = '' + Name of the Erlang node. + ''; + }; + + distributedCookie = mkOption { + type = types.str; + default = "riak"; + description = '' + Cookie for distributed node communication. All nodes in the + same cluster should use the same cookie or they will not be able to + communicate. + ''; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/db/riak"; + description = '' + Data directory for Riak. + ''; + }; + + logDir = mkOption { + type = types.path; + default = "/var/log/riak"; + description = '' + Log directory for Riak. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Additional text to be appended to <filename>riak.conf</filename>. + ''; + }; + + extraAdvancedConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Additional text to be appended to <filename>advanced.config</filename>. + ''; + }; + + }; + + }; + + ###### implementation + + config = mkIf cfg.enable { + + environment.systemPackages = [ cfg.package ]; + environment.etc."riak/riak.conf".text = '' + nodename = ${cfg.nodeName} + distributed_cookie = ${cfg.distributedCookie} + + platform_log_dir = ${cfg.logDir} + platform_etc_dir = /etc/riak + platform_data_dir = ${cfg.dataDir} + + ${cfg.extraConfig} + ''; + + environment.etc."riak/advanced.config".text = '' + ${cfg.extraAdvancedConfig} + ''; + + users.users.riak = { + name = "riak"; + uid = config.ids.uids.riak; + group = "riak"; + description = "Riak server user"; + }; + + users.groups.riak.gid = config.ids.gids.riak; + + systemd.services.riak = { + description = "Riak Server"; + + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + path = [ + pkgs.util-linux # for `logger` + pkgs.bash + ]; + + environment.HOME = "${cfg.dataDir}"; + environment.RIAK_DATA_DIR = "${cfg.dataDir}"; + environment.RIAK_LOG_DIR = "${cfg.logDir}"; + environment.RIAK_ETC_DIR = "/etc/riak"; + + preStart = '' + if ! test -e ${cfg.logDir}; then + mkdir -m 0755 -p ${cfg.logDir} + chown -R riak ${cfg.logDir} + fi + + if ! test -e ${cfg.dataDir}; then + mkdir -m 0700 -p ${cfg.dataDir} + chown -R riak ${cfg.dataDir} + fi + ''; + + serviceConfig = { + ExecStart = "${cfg.package}/bin/riak console"; + ExecStop = "${cfg.package}/bin/riak stop"; + StandardInput = "tty"; + User = "riak"; + Group = "riak"; + PermissionsStartOnly = true; + # Give Riak a decent amount of time to clean up. + TimeoutStopSec = 120; + LimitNOFILE = 65536; + }; + + unitConfig.RequiresMountsFor = [ + "${cfg.dataDir}" + "${cfg.logDir}" + "/etc/riak" + ]; + }; + + }; + +} diff --git a/nixos/modules/services/databases/victoriametrics.nix b/nixos/modules/services/databases/victoriametrics.nix new file mode 100644 index 00000000000..0513dcff172 --- /dev/null +++ b/nixos/modules/services/databases/victoriametrics.nix @@ -0,0 +1,78 @@ +{ config, pkgs, lib, ... }: +let cfg = config.services.victoriametrics; in +{ + options.services.victoriametrics = with lib; { + enable = mkEnableOption "victoriametrics"; + package = mkOption { + type = types.package; + default = pkgs.victoriametrics; + defaultText = literalExpression "pkgs.victoriametrics"; + description = '' + The VictoriaMetrics distribution to use. + ''; + }; + listenAddress = mkOption { + default = ":8428"; + type = types.str; + description = '' + The listen address for the http interface. + ''; + }; + retentionPeriod = mkOption { + type = types.int; + default = 1; + description = '' + Retention period in months. + ''; + }; + extraOptions = mkOption { + type = types.listOf types.str; + default = []; + description = '' + Extra options to pass to VictoriaMetrics. See the README: <link + xlink:href="https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md" /> + or <command>victoriametrics -help</command> for more + information. + ''; + }; + }; + config = lib.mkIf cfg.enable { + systemd.services.victoriametrics = { + description = "VictoriaMetrics time series database"; + after = [ "network.target" ]; + startLimitBurst = 5; + serviceConfig = { + Restart = "on-failure"; + RestartSec = 1; + StateDirectory = "victoriametrics"; + DynamicUser = true; + ExecStart = '' + ${cfg.package}/bin/victoria-metrics \ + -storageDataPath=/var/lib/victoriametrics \ + -httpListenAddr ${cfg.listenAddress} \ + -retentionPeriod ${toString cfg.retentionPeriod} \ + ${lib.escapeShellArgs cfg.extraOptions} + ''; + # victoriametrics 1.59 with ~7GB of data seems to eventually panic when merging files and then + # begins restart-looping forever. Set LimitNOFILE= to a large number to work around this issue. + # + # panic: FATAL: unrecoverable error when merging small parts in the partition "/var/lib/victoriametrics/data/small/2021_08": + # cannot open source part for merging: cannot open values file in stream mode: + # cannot open file "/var/lib/victoriametrics/data/small/2021_08/[...]/values.bin": + # open /var/lib/victoriametrics/data/small/2021_08/[...]/values.bin: too many open files + LimitNOFILE = 1048576; + }; + wantedBy = [ "multi-user.target" ]; + + postStart = + let + bindAddr = (lib.optionalString (lib.hasPrefix ":" cfg.listenAddress) "127.0.0.1") + cfg.listenAddress; + in + lib.mkBefore '' + until ${lib.getBin pkgs.curl}/bin/curl -s -o /dev/null http://${bindAddr}/ping; do + sleep 1; + done + ''; + }; + }; +} |