diff options
Diffstat (limited to 'nixos/modules/services/logging')
18 files changed, 2486 insertions, 0 deletions
diff --git a/nixos/modules/services/logging/SystemdJournal2Gelf.nix b/nixos/modules/services/logging/SystemdJournal2Gelf.nix new file mode 100644 index 00000000000..f28ecab8ac2 --- /dev/null +++ b/nixos/modules/services/logging/SystemdJournal2Gelf.nix @@ -0,0 +1,60 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let cfg = config.services.SystemdJournal2Gelf; +in + +{ options = { + services.SystemdJournal2Gelf = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable SystemdJournal2Gelf. + ''; + }; + + graylogServer = mkOption { + type = types.str; + example = "graylog2.example.com:11201"; + description = '' + Host and port of your graylog2 input. This should be a GELF + UDP input. + ''; + }; + + extraOptions = mkOption { + type = types.separatedString " "; + default = ""; + description = '' + Any extra flags to pass to SystemdJournal2Gelf. Note that + these are basically <literal>journalctl</literal> flags. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.systemd-journal2gelf; + defaultText = literalExpression "pkgs.systemd-journal2gelf"; + description = '' + SystemdJournal2Gelf package to use. + ''; + }; + + }; + }; + + config = mkIf cfg.enable { + systemd.services.SystemdJournal2Gelf = { + description = "SystemdJournal2Gelf"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = "${cfg.package}/bin/SystemdJournal2Gelf ${cfg.graylogServer} --follow ${cfg.extraOptions}"; + Restart = "on-failure"; + RestartSec = "30"; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/awstats.nix b/nixos/modules/services/logging/awstats.nix new file mode 100644 index 00000000000..df0124380ff --- /dev/null +++ b/nixos/modules/services/logging/awstats.nix @@ -0,0 +1,257 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.awstats; + package = pkgs.awstats; + configOpts = {name, config, ...}: { + options = { + type = mkOption{ + type = types.enum [ "mail" "web" ]; + default = "web"; + example = "mail"; + description = '' + The type of log being collected. + ''; + }; + domain = mkOption { + type = types.str; + default = name; + description = "The domain name to collect stats for."; + example = "example.com"; + }; + + logFile = mkOption { + type = types.str; + example = "/var/log/nginx/access.log"; + description = '' + The log file to be scanned. + + For mail, set this to + <literal> + journalctl $OLD_CURSOR -u postfix.service | ''${pkgs.perl}/bin/perl ''${pkgs.awstats.out}/share/awstats/tools/maillogconvert.pl standard | + </literal> + ''; + }; + + logFormat = mkOption { + type = types.str; + default = "1"; + description = '' + The log format being used. + + For mail, set this to + <literal> + %time2 %email %email_r %host %host_r %method %url %code %bytesd + </literal> + ''; + }; + + hostAliases = mkOption { + type = types.listOf types.str; + default = []; + example = [ "www.example.org" ]; + description = '' + List of aliases the site has. + ''; + }; + + extraConfig = mkOption { + type = types.attrsOf types.str; + default = {}; + example = literalExpression '' + { + "ValidHTTPCodes" = "404"; + } + ''; + description = "Extra configuration to be appended to awstats.\${name}.conf."; + }; + + webService = { + enable = mkEnableOption "awstats web service"; + + hostname = mkOption { + type = types.str; + default = config.domain; + description = "The hostname the web service appears under."; + }; + + urlPrefix = mkOption { + type = types.str; + default = "/awstats"; + description = "The URL prefix under which the awstats pages appear."; + }; + }; + }; + }; + webServices = filterAttrs (name: value: value.webService.enable) cfg.configs; +in +{ + imports = [ + (mkRemovedOptionModule [ "services" "awstats" "service" "enable" ] "Please enable per domain with `services.awstats.configs.<name>.webService.enable`") + (mkRemovedOptionModule [ "services" "awstats" "service" "urlPrefix" ] "Please set per domain with `services.awstats.configs.<name>.webService.urlPrefix`") + (mkRenamedOptionModule [ "services" "awstats" "vardir" ] [ "services" "awstats" "dataDir" ]) + ]; + + options.services.awstats = { + enable = mkEnableOption "awstats"; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/awstats"; + description = "The directory where awstats data will be stored."; + }; + + configs = mkOption { + type = types.attrsOf (types.submodule configOpts); + default = {}; + example = literalExpression '' + { + "mysite" = { + domain = "example.com"; + logFile = "/var/log/nginx/access.log"; + }; + } + ''; + description = "Attribute set of domains to collect stats for."; + }; + + updateAt = mkOption { + type = types.nullOr types.str; + default = null; + example = "hourly"; + description = '' + Specification of the time at which awstats will get updated. + (in the format described by <citerefentry> + <refentrytitle>systemd.time</refentrytitle> + <manvolnum>7</manvolnum></citerefentry>) + ''; + }; + }; + + + config = mkIf cfg.enable { + environment.systemPackages = [ package.bin ]; + + environment.etc = mapAttrs' (name: opts: + nameValuePair "awstats/awstats.${name}.conf" { + source = pkgs.runCommand "awstats.${name}.conf" + { preferLocalBuild = true; } + ('' + sed \ + '' + # set up mail stats + + optionalString (opts.type == "mail") + '' + -e 's|^\(LogType\)=.*$|\1=M|' \ + -e 's|^\(LevelForBrowsersDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForOSDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForRefererAnalyze\)=.*$|\1=0|' \ + -e 's|^\(LevelForRobotsDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForSearchEnginesDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForFileTypesDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForWormsDetection\)=.*$|\1=0|' \ + -e 's|^\(ShowMenu\)=.*$|\1=1|' \ + -e 's|^\(ShowSummary\)=.*$|\1=HB|' \ + -e 's|^\(ShowMonthStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowDaysOfMonthStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowDaysOfWeekStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowHoursStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowDomainsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowHostsStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowAuthenticatedUsers\)=.*$|\1=0|' \ + -e 's|^\(ShowRobotsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowEMailSenders\)=.*$|\1=HBML|' \ + -e 's|^\(ShowEMailReceivers\)=.*$|\1=HBML|' \ + -e 's|^\(ShowSessionsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowPagesStats\)=.*$|\1=0|' \ + -e 's|^\(ShowFileTypesStats\)=.*$|\1=0|' \ + -e 's|^\(ShowFileSizesStats\)=.*$|\1=0|' \ + -e 's|^\(ShowBrowsersStats\)=.*$|\1=0|' \ + -e 's|^\(ShowOSStats\)=.*$|\1=0|' \ + -e 's|^\(ShowOriginStats\)=.*$|\1=0|' \ + -e 's|^\(ShowKeyphrasesStats\)=.*$|\1=0|' \ + -e 's|^\(ShowKeywordsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowMiscStats\)=.*$|\1=0|' \ + -e 's|^\(ShowHTTPErrorsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowSMTPErrorsStats\)=.*$|\1=1|' \ + '' + + + # common options + '' + -e 's|^\(DirData\)=.*$|\1="${cfg.dataDir}/${name}"|' \ + -e 's|^\(DirIcons\)=.*$|\1="icons"|' \ + -e 's|^\(CreateDirDataIfNotExists\)=.*$|\1=1|' \ + -e 's|^\(SiteDomain\)=.*$|\1="${name}"|' \ + -e 's|^\(LogFile\)=.*$|\1="${opts.logFile}"|' \ + -e 's|^\(LogFormat\)=.*$|\1="${opts.logFormat}"|' \ + '' + + + # extra config + concatStringsSep "\n" (mapAttrsToList (n: v: '' + -e 's|^\(${n}\)=.*$|\1="${v}"|' \ + '') opts.extraConfig) + + + '' + < '${package.out}/wwwroot/cgi-bin/awstats.model.conf' > "$out" + ''); + }) cfg.configs; + + # create data directory with the correct permissions + systemd.tmpfiles.rules = + [ "d '${cfg.dataDir}' 755 root root - -" ] ++ + mapAttrsToList (name: opts: "d '${cfg.dataDir}/${name}' 755 root root - -") cfg.configs ++ + [ "Z '${cfg.dataDir}' 755 root root - -" ]; + + # nginx options + services.nginx.virtualHosts = mapAttrs'(name: opts: { + name = opts.webService.hostname; + value = { + locations = { + "${opts.webService.urlPrefix}/css/" = { + alias = "${package.out}/wwwroot/css/"; + }; + "${opts.webService.urlPrefix}/icons/" = { + alias = "${package.out}/wwwroot/icon/"; + }; + "${opts.webService.urlPrefix}/" = { + alias = "${cfg.dataDir}/${name}/"; + extraConfig = '' + autoindex on; + ''; + }; + }; + }; + }) webServices; + + # update awstats + systemd.services = mkIf (cfg.updateAt != null) (mapAttrs' (name: opts: + nameValuePair "awstats-${name}-update" { + description = "update awstats for ${name}"; + script = optionalString (opts.type == "mail") + '' + if [[ -f "${cfg.dataDir}/${name}-cursor" ]]; then + CURSOR="$(cat "${cfg.dataDir}/${name}-cursor" | tr -d '\n')" + if [[ -n "$CURSOR" ]]; then + echo "Using cursor: $CURSOR" + export OLD_CURSOR="--cursor $CURSOR" + fi + fi + NEW_CURSOR="$(journalctl $OLD_CURSOR -u postfix.service --show-cursor | tail -n 1 | tr -d '\n' | sed -e 's#^-- cursor: \(.*\)#\1#')" + echo "New cursor: $NEW_CURSOR" + ${package.bin}/bin/awstats -update -config=${name} + if [ -n "$NEW_CURSOR" ]; then + echo -n "$NEW_CURSOR" > ${cfg.dataDir}/${name}-cursor + fi + '' + '' + ${package.out}/share/awstats/tools/awstats_buildstaticpages.pl \ + -config=${name} -update -dir=${cfg.dataDir}/${name} \ + -awstatsprog=${package.bin}/bin/awstats + ''; + startAt = cfg.updateAt; + }) cfg.configs); + }; + +} + diff --git a/nixos/modules/services/logging/filebeat.nix b/nixos/modules/services/logging/filebeat.nix new file mode 100644 index 00000000000..223a993c505 --- /dev/null +++ b/nixos/modules/services/logging/filebeat.nix @@ -0,0 +1,253 @@ +{ config, lib, utils, pkgs, ... }: + +let + inherit (lib) + attrValues + literalExpression + mkEnableOption + mkIf + mkOption + types; + + cfg = config.services.filebeat; + + json = pkgs.formats.json {}; +in +{ + options = { + + services.filebeat = { + + enable = mkEnableOption "filebeat"; + + package = mkOption { + type = types.package; + default = pkgs.filebeat; + defaultText = literalExpression "pkgs.filebeat"; + example = literalExpression "pkgs.filebeat7"; + description = '' + The filebeat package to use. + ''; + }; + + inputs = mkOption { + description = '' + Inputs specify how Filebeat locates and processes input data. + + This is like <literal>services.filebeat.settings.filebeat.inputs</literal>, + but structured as an attribute set. This has the benefit + that multiple NixOS modules can contribute settings to a + single filebeat input. + + An input type can be specified multiple times by choosing a + different <literal><name></literal> for each, but setting + <xref linkend="opt-services.filebeat.inputs._name_.type"/> + to the same value. + + See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>. + ''; + default = {}; + type = types.attrsOf (types.submodule ({ name, ... }: { + freeformType = json.type; + options = { + type = mkOption { + type = types.str; + default = name; + description = '' + The input type. + + Look for the value after <literal>type:</literal> on + the individual input pages linked from + <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>. + ''; + }; + }; + })); + example = literalExpression '' + { + journald.id = "everything"; # Only for filebeat7 + log = { + enabled = true; + paths = [ + "/var/log/*.log" + ]; + }; + }; + ''; + }; + + modules = mkOption { + description = '' + Filebeat modules provide a quick way to get started + processing common log formats. They contain default + configurations, Elasticsearch ingest pipeline definitions, + and Kibana dashboards to help you implement and deploy a log + monitoring solution. + + This is like <literal>services.filebeat.settings.filebeat.modules</literal>, + but structured as an attribute set. This has the benefit + that multiple NixOS modules can contribute settings to a + single filebeat module. + + A module can be specified multiple times by choosing a + different <literal><name></literal> for each, but setting + <xref linkend="opt-services.filebeat.modules._name_.module"/> + to the same value. + + See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>. + ''; + default = {}; + type = types.attrsOf (types.submodule ({ name, ... }: { + freeformType = json.type; + options = { + module = mkOption { + type = types.str; + default = name; + description = '' + The name of the module. + + Look for the value after <literal>module:</literal> on + the individual input pages linked from + <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>. + ''; + }; + }; + })); + example = literalExpression '' + { + nginx = { + access = { + enabled = true; + var.paths = [ "/path/to/log/nginx/access.log*" ]; + }; + error = { + enabled = true; + var.paths = [ "/path/to/log/nginx/error.log*" ]; + }; + }; + }; + ''; + }; + + settings = mkOption { + type = types.submodule { + freeformType = json.type; + + options = { + + output.elasticsearch.hosts = mkOption { + type = with types; listOf str; + default = [ "127.0.0.1:9200" ]; + example = [ "myEShost:9200" ]; + description = '' + The list of Elasticsearch nodes to connect to. + + The events are distributed to these nodes in round + robin order. If one node becomes unreachable, the + event is automatically sent to another node. Each + Elasticsearch node can be defined as a URL or + IP:PORT. For example: + <literal>http://192.15.3.2</literal>, + <literal>https://es.found.io:9230</literal> or + <literal>192.24.3.2:9300</literal>. If no port is + specified, <literal>9200</literal> is used. + ''; + }; + + filebeat = { + inputs = mkOption { + type = types.listOf json.type; + default = []; + internal = true; + description = '' + Inputs specify how Filebeat locates and processes + input data. Use <xref + linkend="opt-services.filebeat.inputs"/> instead. + + See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>. + ''; + }; + modules = mkOption { + type = types.listOf json.type; + default = []; + internal = true; + description = '' + Filebeat modules provide a quick way to get started + processing common log formats. They contain default + configurations, Elasticsearch ingest pipeline + definitions, and Kibana dashboards to help you + implement and deploy a log monitoring solution. + + Use <xref linkend="opt-services.filebeat.modules"/> instead. + + See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>. + ''; + }; + }; + }; + }; + default = {}; + example = literalExpression '' + { + settings = { + output.elasticsearch = { + hosts = [ "myEShost:9200" ]; + username = "filebeat_internal"; + password = { _secret = "/var/keys/elasticsearch_password"; }; + }; + logging.level = "info"; + }; + }; + ''; + + description = '' + Configuration for filebeat. See + <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html"/> + for supported values. + + Options containing secret data should be set to an attribute + set containing the attribute <literal>_secret</literal> - a + string pointing to a file containing the value the option + should be set to. See the example to get a better picture of + this: in the resulting + <filename>filebeat.yml</filename> file, the + <literal>output.elasticsearch.password</literal> + key will be set to the contents of the + <filename>/var/keys/elasticsearch_password</filename> file. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + + services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs; + services.filebeat.settings.filebeat.modules = attrValues cfg.modules; + + systemd.services.filebeat = { + description = "Filebeat log shipper"; + wantedBy = [ "multi-user.target" ]; + wants = [ "elasticsearch.service" ]; + after = [ "elasticsearch.service" ]; + serviceConfig = { + ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" '' + set -euo pipefail + + umask u=rwx,g=,o= + + ${utils.genJqSecretsReplacementSnippet + cfg.settings + "/var/lib/filebeat/filebeat.yml" + } + ''; + ExecStart = '' + ${cfg.package}/bin/filebeat -e \ + -c "/var/lib/filebeat/filebeat.yml" \ + --path.data "/var/lib/filebeat" + ''; + Restart = "always"; + StateDirectory = "filebeat"; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/fluentd.nix b/nixos/modules/services/logging/fluentd.nix new file mode 100644 index 00000000000..dd19617a13f --- /dev/null +++ b/nixos/modules/services/logging/fluentd.nix @@ -0,0 +1,58 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.fluentd; + + pluginArgs = concatStringsSep " " (map (x: "-p ${x}") cfg.plugins); +in { + ###### interface + + options = { + + services.fluentd = { + enable = mkOption { + type = types.bool; + default = false; + description = "Whether to enable fluentd."; + }; + + config = mkOption { + type = types.lines; + default = ""; + description = "Fluentd config."; + }; + + package = mkOption { + type = types.path; + default = pkgs.fluentd; + defaultText = literalExpression "pkgs.fluentd"; + description = "The fluentd package to use."; + }; + + plugins = mkOption { + type = types.listOf types.path; + default = []; + description = '' + A list of plugin paths to pass into fluentd. It will make plugins defined in ruby files + there available in your config. + ''; + }; + }; + }; + + + ###### implementation + + config = mkIf cfg.enable { + systemd.services.fluentd = with pkgs; { + description = "Fluentd Daemon"; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = "${cfg.package}/bin/fluentd -c ${pkgs.writeText "fluentd.conf" cfg.config} ${pluginArgs}"; + ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/graylog.nix b/nixos/modules/services/logging/graylog.nix new file mode 100644 index 00000000000..e6a23233ba2 --- /dev/null +++ b/nixos/modules/services/logging/graylog.nix @@ -0,0 +1,169 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.graylog; + + confFile = pkgs.writeText "graylog.conf" '' + is_master = ${boolToString cfg.isMaster} + node_id_file = ${cfg.nodeIdFile} + password_secret = ${cfg.passwordSecret} + root_username = ${cfg.rootUsername} + root_password_sha2 = ${cfg.rootPasswordSha2} + elasticsearch_hosts = ${concatStringsSep "," cfg.elasticsearchHosts} + message_journal_dir = ${cfg.messageJournalDir} + mongodb_uri = ${cfg.mongodbUri} + plugin_dir = /var/lib/graylog/plugins + + ${cfg.extraConfig} + ''; + + glPlugins = pkgs.buildEnv { + name = "graylog-plugins"; + paths = cfg.plugins; + }; + +in + +{ + ###### interface + + options = { + + services.graylog = { + + enable = mkEnableOption "Graylog"; + + package = mkOption { + type = types.package; + default = pkgs.graylog; + defaultText = literalExpression "pkgs.graylog"; + description = "Graylog package to use."; + }; + + user = mkOption { + type = types.str; + default = "graylog"; + description = "User account under which graylog runs"; + }; + + isMaster = mkOption { + type = types.bool; + default = true; + description = "Whether this is the master instance of your Graylog cluster"; + }; + + nodeIdFile = mkOption { + type = types.str; + default = "/var/lib/graylog/server/node-id"; + description = "Path of the file containing the graylog node-id"; + }; + + passwordSecret = mkOption { + type = types.str; + description = '' + You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters. + Generate one by using for example: pwgen -N 1 -s 96 + ''; + }; + + rootUsername = mkOption { + type = types.str; + default = "admin"; + description = "Name of the default administrator user"; + }; + + rootPasswordSha2 = mkOption { + type = types.str; + example = "e3c652f0ba0b4801205814f8b6bc49672c4c74e25b497770bb89b22cdeb4e952"; + description = '' + You MUST specify a hash password for the root user (which you only need to initially set up the + system and in case you lose connectivity to your authentication backend) + This password cannot be changed using the API or via the web interface. If you need to change it, + modify it here. + Create one by using for example: echo -n yourpassword | shasum -a 256 + and use the resulting hash value as string for the option + ''; + }; + + elasticsearchHosts = mkOption { + type = types.listOf types.str; + example = literalExpression ''[ "http://node1:9200" "http://user:password@node2:19200" ]''; + description = "List of valid URIs of the http ports of your elastic nodes. If one or more of your elasticsearch hosts require authentication, include the credentials in each node URI that requires authentication"; + }; + + messageJournalDir = mkOption { + type = types.str; + default = "/var/lib/graylog/data/journal"; + description = "The directory which will be used to store the message journal. The directory must be exclusively used by Graylog and must not contain any other files than the ones created by Graylog itself"; + }; + + mongodbUri = mkOption { + type = types.str; + default = "mongodb://localhost/graylog"; + description = "MongoDB connection string. See http://docs.mongodb.org/manual/reference/connection-string/ for details"; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = "Any other configuration options you might want to add"; + }; + + plugins = mkOption { + description = "Extra graylog plugins"; + default = [ ]; + type = types.listOf types.package; + }; + + }; + }; + + + ###### implementation + + config = mkIf cfg.enable { + + users.users = mkIf (cfg.user == "graylog") { + graylog = { + isSystemUser = true; + group = "graylog"; + description = "Graylog server daemon user"; + }; + }; + users.groups = mkIf (cfg.user == "graylog") {}; + + systemd.tmpfiles.rules = [ + "d '${cfg.messageJournalDir}' - ${cfg.user} - - -" + ]; + + systemd.services.graylog = { + description = "Graylog Server"; + wantedBy = [ "multi-user.target" ]; + environment = { + GRAYLOG_CONF = "${confFile}"; + }; + path = [ pkgs.which pkgs.procps ]; + preStart = '' + rm -rf /var/lib/graylog/plugins || true + mkdir -p /var/lib/graylog/plugins -m 755 + + mkdir -p "$(dirname ${cfg.nodeIdFile})" + chown -R ${cfg.user} "$(dirname ${cfg.nodeIdFile})" + + for declarativeplugin in `ls ${glPlugins}/bin/`; do + ln -sf ${glPlugins}/bin/$declarativeplugin /var/lib/graylog/plugins/$declarativeplugin + done + for includedplugin in `ls ${cfg.package}/plugin/`; do + ln -s ${cfg.package}/plugin/$includedplugin /var/lib/graylog/plugins/$includedplugin || true + done + ''; + serviceConfig = { + User="${cfg.user}"; + StateDirectory = "graylog"; + ExecStart = "${cfg.package}/bin/graylogctl run"; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/heartbeat.nix b/nixos/modules/services/logging/heartbeat.nix new file mode 100644 index 00000000000..56fb4deabda --- /dev/null +++ b/nixos/modules/services/logging/heartbeat.nix @@ -0,0 +1,74 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.heartbeat; + + heartbeatYml = pkgs.writeText "heartbeat.yml" '' + name: ${cfg.name} + tags: ${builtins.toJSON cfg.tags} + + ${cfg.extraConfig} + ''; + +in +{ + options = { + + services.heartbeat = { + + enable = mkEnableOption "heartbeat"; + + name = mkOption { + type = types.str; + default = "heartbeat"; + description = "Name of the beat"; + }; + + tags = mkOption { + type = types.listOf types.str; + default = []; + description = "Tags to place on the shipped log messages"; + }; + + stateDir = mkOption { + type = types.str; + default = "/var/lib/heartbeat"; + description = "The state directory. heartbeat's own logs and other data are stored here."; + }; + + extraConfig = mkOption { + type = types.lines; + default = '' + heartbeat.monitors: + - type: http + urls: ["http://localhost:9200"] + schedule: '@every 10s' + ''; + description = "Any other configuration options you want to add"; + }; + + }; + }; + + config = mkIf cfg.enable { + + systemd.tmpfiles.rules = [ + "d '${cfg.stateDir}' - nobody nogroup - -" + ]; + + systemd.services.heartbeat = with pkgs; { + description = "heartbeat log shipper"; + wantedBy = [ "multi-user.target" ]; + preStart = '' + mkdir -p "${cfg.stateDir}"/{data,logs} + ''; + serviceConfig = { + User = "nobody"; + AmbientCapabilities = "cap_net_raw"; + ExecStart = "${pkgs.heartbeat}/bin/heartbeat -c \"${heartbeatYml}\" -path.data \"${cfg.stateDir}/data\" -path.logs \"${cfg.stateDir}/logs\""; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix new file mode 100644 index 00000000000..4035ab48b4b --- /dev/null +++ b/nixos/modules/services/logging/journalbeat.nix @@ -0,0 +1,94 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.journalbeat; + + journalbeatYml = pkgs.writeText "journalbeat.yml" '' + name: ${cfg.name} + tags: ${builtins.toJSON cfg.tags} + + ${cfg.extraConfig} + ''; + +in +{ + options = { + + services.journalbeat = { + + enable = mkEnableOption "journalbeat"; + + package = mkOption { + type = types.package; + default = pkgs.journalbeat; + defaultText = literalExpression "pkgs.journalbeat"; + description = '' + The journalbeat package to use + ''; + }; + + name = mkOption { + type = types.str; + default = "journalbeat"; + description = "Name of the beat"; + }; + + tags = mkOption { + type = types.listOf types.str; + default = []; + description = "Tags to place on the shipped log messages"; + }; + + stateDir = mkOption { + type = types.str; + default = "journalbeat"; + description = '' + Directory below <literal>/var/lib/</literal> to store journalbeat's + own logs and other data. This directory will be created automatically + using systemd's StateDirectory mechanism. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = "Any other configuration options you want to add"; + }; + + }; + }; + + config = mkIf cfg.enable { + + assertions = [ + { + assertion = !hasPrefix "/" cfg.stateDir; + message = + "The option services.journalbeat.stateDir shouldn't be an absolute directory." + + " It should be a directory relative to /var/lib/."; + } + ]; + + systemd.services.journalbeat = { + description = "Journalbeat log shipper"; + wantedBy = [ "multi-user.target" ]; + wants = [ "elasticsearch.service" ]; + after = [ "elasticsearch.service" ]; + preStart = '' + mkdir -p ${cfg.stateDir}/data + mkdir -p ${cfg.stateDir}/logs + ''; + serviceConfig = { + StateDirectory = cfg.stateDir; + ExecStart = '' + ${cfg.package}/bin/journalbeat \ + -c ${journalbeatYml} \ + -path.data /var/lib/${cfg.stateDir}/data \ + -path.logs /var/lib/${cfg.stateDir}/logs''; + Restart = "always"; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/journaldriver.nix b/nixos/modules/services/logging/journaldriver.nix new file mode 100644 index 00000000000..9bd581e9ec0 --- /dev/null +++ b/nixos/modules/services/logging/journaldriver.nix @@ -0,0 +1,112 @@ +# This module implements a systemd service for running journaldriver, +# a log forwarding agent that sends logs from journald to Stackdriver +# Logging. +# +# It can be enabled without extra configuration when running on GCP. +# On machines hosted elsewhere, the other configuration options need +# to be set. +# +# For further information please consult the documentation in the +# upstream repository at: https://github.com/tazjin/journaldriver/ + +{ config, lib, pkgs, ...}: + +with lib; let cfg = config.services.journaldriver; +in { + options.services.journaldriver = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable journaldriver to forward journald logs to + Stackdriver Logging. + ''; + }; + + logLevel = mkOption { + type = types.str; + default = "info"; + description = '' + Log level at which journaldriver logs its own output. + ''; + }; + + logName = mkOption { + type = with types; nullOr str; + default = null; + description = '' + Configures the name of the target log in Stackdriver Logging. + This option can be set to, for example, the hostname of a + machine to improve the user experience in the logging + overview. + ''; + }; + + googleCloudProject = mkOption { + type = with types; nullOr str; + default = null; + description = '' + Configures the name of the Google Cloud project to which to + forward journald logs. + + This option is required on non-GCP machines, but should not be + set on GCP instances. + ''; + }; + + logStream = mkOption { + type = with types; nullOr str; + default = null; + description = '' + Configures the name of the Stackdriver Logging log stream into + which to write journald entries. + + This option is required on non-GCP machines, but should not be + set on GCP instances. + ''; + }; + + applicationCredentials = mkOption { + type = with types; nullOr path; + default = null; + description = '' + Path to the service account private key (in JSON-format) used + to forward log entries to Stackdriver Logging on non-GCP + instances. + + This option is required on non-GCP machines, but should not be + set on GCP instances. + ''; + }; + }; + + config = mkIf cfg.enable { + systemd.services.journaldriver = { + description = "Stackdriver Logging journal forwarder"; + script = "${pkgs.journaldriver}/bin/journaldriver"; + after = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Restart = "always"; + DynamicUser = true; + + # This directive lets systemd automatically configure + # permissions on /var/lib/journaldriver, the directory in + # which journaldriver persists its cursor state. + StateDirectory = "journaldriver"; + + # This group is required for accessing journald. + SupplementaryGroups = "systemd-journal"; + }; + + environment = { + RUST_LOG = cfg.logLevel; + LOG_NAME = cfg.logName; + LOG_STREAM = cfg.logStream; + GOOGLE_CLOUD_PROJECT = cfg.googleCloudProject; + GOOGLE_APPLICATION_CREDENTIALS = cfg.applicationCredentials; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/journalwatch.nix b/nixos/modules/services/logging/journalwatch.nix new file mode 100644 index 00000000000..fb86904d1ea --- /dev/null +++ b/nixos/modules/services/logging/journalwatch.nix @@ -0,0 +1,265 @@ +{ config, lib, pkgs, ... }: +with lib; + +let + cfg = config.services.journalwatch; + user = "journalwatch"; + # for journal access + group = "systemd-journal"; + dataDir = "/var/lib/${user}"; + + journalwatchConfig = pkgs.writeText "config" ('' + # (File Generated by NixOS journalwatch module.) + [DEFAULT] + mail_binary = ${cfg.mailBinary} + priority = ${toString cfg.priority} + mail_from = ${cfg.mailFrom} + '' + + optionalString (cfg.mailTo != null) '' + mail_to = ${cfg.mailTo} + '' + + cfg.extraConfig); + + journalwatchPatterns = pkgs.writeText "patterns" '' + # (File Generated by NixOS journalwatch module.) + + ${mkPatterns cfg.filterBlocks} + ''; + + # empty line at the end needed to to separate the blocks + mkPatterns = filterBlocks: concatStringsSep "\n" (map (block: '' + ${block.match} + ${block.filters} + + '') filterBlocks); + + # can't use joinSymlinks directly, because when we point $XDG_CONFIG_HOME + # to the /nix/store path, we still need the subdirectory "journalwatch" inside that + # to match journalwatch's expectations + journalwatchConfigDir = pkgs.runCommand "journalwatch-config" + { preferLocalBuild = true; allowSubstitutes = false; } + '' + mkdir -p $out/journalwatch + ln -sf ${journalwatchConfig} $out/journalwatch/config + ln -sf ${journalwatchPatterns} $out/journalwatch/patterns + ''; + + +in { + options = { + services.journalwatch = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + If enabled, periodically check the journal with journalwatch and report the results by mail. + ''; + }; + + priority = mkOption { + type = types.int; + default = 6; + description = '' + Lowest priority of message to be considered. + A value between 7 ("debug"), and 0 ("emerg"). Defaults to 6 ("info"). + If you don't care about anything with "info" priority, you can reduce + this to e.g. 5 ("notice") to considerably reduce the amount of + messages without needing many <option>filterBlocks</option>. + ''; + }; + + # HACK: this is a workaround for journalwatch's usage of socket.getfqdn() which always returns localhost if + # there's an alias for the localhost on a separate line in /etc/hosts, or take for ages if it's not present and + # then return something right-ish in the direction of /etc/hostname. Just bypass it completely. + mailFrom = mkOption { + type = types.str; + default = "journalwatch@${config.networking.hostName}"; + defaultText = literalExpression ''"journalwatch@''${config.networking.hostName}"''; + description = '' + Mail address to send journalwatch reports from. + ''; + }; + + mailTo = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Mail address to send journalwatch reports to. + ''; + }; + + mailBinary = mkOption { + type = types.path; + default = "/run/wrappers/bin/sendmail"; + description = '' + Sendmail-compatible binary to be used to send the messages. + ''; + }; + + extraConfig = mkOption { + type = types.str; + default = ""; + description = '' + Extra lines to be added verbatim to the journalwatch/config configuration file. + You can add any commandline argument to the config, without the '--'. + See <literal>journalwatch --help</literal> for all arguments and their description. + ''; + }; + + filterBlocks = mkOption { + type = types.listOf (types.submodule { + options = { + match = mkOption { + type = types.str; + example = "SYSLOG_IDENTIFIER = systemd"; + description = '' + Syntax: <literal>field = value</literal> + Specifies the log entry <literal>field</literal> this block should apply to. + If the <literal>field</literal> of a message matches this <literal>value</literal>, + this patternBlock's <option>filters</option> are applied. + If <literal>value</literal> starts and ends with a slash, it is interpreted as + an extended python regular expression, if not, it's an exact match. + The journal fields are explained in systemd.journal-fields(7). + ''; + }; + + filters = mkOption { + type = types.str; + example = '' + (Stopped|Stopping|Starting|Started) .* + (Reached target|Stopped target) .* + ''; + description = '' + The filters to apply on all messages which satisfy <option>match</option>. + Any of those messages that match any specified filter will be removed from journalwatch's output. + Each filter is an extended Python regular expression. + You can specify multiple filters and separate them by newlines. + Lines starting with '#' are comments. Inline-comments are not permitted. + ''; + }; + }; + }); + + example = [ + # examples taken from upstream + { + match = "_SYSTEMD_UNIT = systemd-logind.service"; + filters = '' + New session [a-z]?\d+ of user \w+\. + Removed session [a-z]?\d+\. + ''; + } + + { + match = "SYSLOG_IDENTIFIER = /(CROND|crond)/"; + filters = '' + pam_unix\(crond:session\): session (opened|closed) for user \w+ + \(\w+\) CMD .* + ''; + } + ]; + + # another example from upstream. + # very useful on priority = 6, and required as journalwatch throws an error when no pattern is defined at all. + default = [ + { + match = "SYSLOG_IDENTIFIER = systemd"; + filters = '' + (Stopped|Stopping|Starting|Started) .* + (Created slice|Removed slice) user-\d*\.slice\. + Received SIGRTMIN\+24 from PID .* + (Reached target|Stopped target) .* + Startup finished in \d*ms\. + ''; + } + ]; + + + description = '' + filterBlocks can be defined to blacklist journal messages which are not errors. + Each block matches on a log entry field, and the filters in that block then are matched + against all messages with a matching log entry field. + + All messages whose PRIORITY is at least 6 (INFO) are processed by journalwatch. + If you don't specify any filterBlocks, PRIORITY is reduced to 5 (NOTICE) by default. + + All regular expressions are extended Python regular expressions, for details + see: http://doc.pyschools.com/html/regex.html + ''; + }; + + interval = mkOption { + type = types.str; + default = "hourly"; + description = '' + How often to run journalwatch. + + The format is described in systemd.time(7). + ''; + }; + accuracy = mkOption { + type = types.str; + default = "10min"; + description = '' + The time window around the interval in which the journalwatch run will be scheduled. + + The format is described in systemd.time(7). + ''; + }; + }; + }; + + config = mkIf cfg.enable { + + users.users.${user} = { + isSystemUser = true; + home = dataDir; + group = group; + }; + + systemd.tmpfiles.rules = [ + # present since NixOS 19.09: remove old stateful symlink join directory, + # which has been replaced with the journalwatchConfigDir store path + "R ${dataDir}/config" + ]; + + systemd.services.journalwatch = { + + environment = { + # journalwatch stores the last processed timpestamp here + # the share subdirectory is historic now that config home lives in /nix/store, + # but moving this in a backwards-compatible way is much more work than what's justified + # for cleaning that up. + XDG_DATA_HOME = "${dataDir}/share"; + XDG_CONFIG_HOME = journalwatchConfigDir; + }; + serviceConfig = { + User = user; + Group = group; + Type = "oneshot"; + # requires a relative directory name to create beneath /var/lib + StateDirectory = user; + StateDirectoryMode = 0750; + ExecStart = "${pkgs.python3Packages.journalwatch}/bin/journalwatch mail"; + # lowest CPU and IO priority, but both still in best-effort class to prevent starvation + Nice=19; + IOSchedulingPriority=7; + }; + }; + + systemd.timers.journalwatch = { + description = "Periodic journalwatch run"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.interval; + AccuracySec = cfg.accuracy; + Persistent = true; + }; + }; + + }; + + meta = { + maintainers = with lib.maintainers; [ florianjacob ]; + }; +} diff --git a/nixos/modules/services/logging/klogd.nix b/nixos/modules/services/logging/klogd.nix new file mode 100644 index 00000000000..8d371c161eb --- /dev/null +++ b/nixos/modules/services/logging/klogd.nix @@ -0,0 +1,38 @@ +{ config, lib, pkgs, ... }: + +with lib; + +{ + ###### interface + + options = { + + services.klogd.enable = mkOption { + type = types.bool; + default = versionOlder (getVersion config.boot.kernelPackages.kernel) "3.5"; + defaultText = literalExpression ''versionOlder (getVersion config.boot.kernelPackages.kernel) "3.5"''; + description = '' + Whether to enable klogd, the kernel log message processing + daemon. Since systemd handles logging of kernel messages on + Linux 3.5 and later, this is only useful if you're running an + older kernel. + ''; + }; + + }; + + + ###### implementation + + config = mkIf config.services.klogd.enable { + systemd.services.klogd = { + description = "Kernel Log Daemon"; + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.sysklogd ]; + unitConfig.ConditionVirtualization = "!systemd-nspawn"; + script = + "klogd -c 1 -2 -n " + + "-k $(dirname $(readlink -f /run/booted-system/kernel))/System.map"; + }; + }; +} diff --git a/nixos/modules/services/logging/logcheck.nix b/nixos/modules/services/logging/logcheck.nix new file mode 100644 index 00000000000..c8738b734f9 --- /dev/null +++ b/nixos/modules/services/logging/logcheck.nix @@ -0,0 +1,242 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.logcheck; + + defaultRules = pkgs.runCommand "logcheck-default-rules" { preferLocalBuild = true; } '' + cp -prd ${pkgs.logcheck}/etc/logcheck $out + chmod u+w $out + rm -r $out/logcheck.* + ''; + + rulesDir = pkgs.symlinkJoin + { name = "logcheck-rules-dir"; + paths = ([ defaultRules ] ++ cfg.extraRulesDirs); + }; + + configFile = pkgs.writeText "logcheck.conf" cfg.config; + + logFiles = pkgs.writeText "logcheck.logfiles" cfg.files; + + flags = "-r ${rulesDir} -c ${configFile} -L ${logFiles} -${levelFlag} -m ${cfg.mailTo}"; + + levelFlag = getAttrFromPath [cfg.level] + { paranoid = "p"; + server = "s"; + workstation = "w"; + }; + + cronJob = '' + @reboot logcheck env PATH=/run/wrappers/bin:$PATH nice -n10 ${pkgs.logcheck}/sbin/logcheck -R ${flags} + 2 ${cfg.timeOfDay} * * * logcheck env PATH=/run/wrappers/bin:$PATH nice -n10 ${pkgs.logcheck}/sbin/logcheck ${flags} + ''; + + writeIgnoreRule = name: {level, regex, ...}: + pkgs.writeTextFile + { inherit name; + destination = "/ignore.d.${level}/${name}"; + text = '' + ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ ${regex} + ''; + }; + + writeIgnoreCronRule = name: {level, user, regex, cmdline, ...}: + let escapeRegex = escape (stringToCharacters "\\[]{}()^$?*+|."); + cmdline_ = builtins.unsafeDiscardStringContext cmdline; + re = if regex != "" then regex else if cmdline_ == "" then ".*" else escapeRegex cmdline_; + in writeIgnoreRule "cron-${name}" { + inherit level; + regex = '' + (/usr/bin/)?cron\[[0-9]+\]: \(${user}\) CMD \(${re}\)$ + ''; + }; + + levelOption = mkOption { + default = "server"; + type = types.enum [ "workstation" "server" "paranoid" ]; + description = '' + Set the logcheck level. + ''; + }; + + ignoreOptions = { + options = { + level = levelOption; + + regex = mkOption { + default = ""; + type = types.str; + description = '' + Regex specifying which log lines to ignore. + ''; + }; + }; + }; + + ignoreCronOptions = { + options = { + user = mkOption { + default = "root"; + type = types.str; + description = '' + User that runs the cronjob. + ''; + }; + + cmdline = mkOption { + default = ""; + type = types.str; + description = '' + Command line for the cron job. Will be turned into a regex for the logcheck ignore rule. + ''; + }; + + timeArgs = mkOption { + default = null; + type = types.nullOr (types.str); + example = "02 06 * * *"; + description = '' + "min hr dom mon dow" crontab time args, to auto-create a cronjob too. + Leave at null to not do this and just add a logcheck ignore rule. + ''; + }; + }; + }; + +in +{ + options = { + services.logcheck = { + enable = mkOption { + default = false; + type = types.bool; + description = '' + Enable the logcheck cron job. + ''; + }; + + user = mkOption { + default = "logcheck"; + type = types.str; + description = '' + Username for the logcheck user. + ''; + }; + + timeOfDay = mkOption { + default = "*"; + example = "6"; + type = types.str; + description = '' + Time of day to run logcheck. A logcheck will be scheduled at xx:02 each day. + Leave default (*) to run every hour. Of course when nothing special was logged, + logcheck will be silent. + ''; + }; + + mailTo = mkOption { + default = "root"; + example = "you@domain.com"; + type = types.str; + description = '' + Email address to send reports to. + ''; + }; + + level = mkOption { + default = "server"; + type = types.str; + description = '' + Set the logcheck level. Either "workstation", "server", or "paranoid". + ''; + }; + + config = mkOption { + default = "FQDN=1"; + type = types.lines; + description = '' + Config options that you would like in logcheck.conf. + ''; + }; + + files = mkOption { + default = [ "/var/log/messages" ]; + type = types.listOf types.path; + example = [ "/var/log/messages" "/var/log/mail" ]; + description = '' + Which log files to check. + ''; + }; + + extraRulesDirs = mkOption { + default = []; + example = [ "/etc/logcheck" ]; + type = types.listOf types.path; + description = '' + Directories with extra rules. + ''; + }; + + ignore = mkOption { + default = {}; + description = '' + This option defines extra ignore rules. + ''; + type = with types; attrsOf (submodule ignoreOptions); + }; + + ignoreCron = mkOption { + default = {}; + description = '' + This option defines extra ignore rules for cronjobs. + ''; + type = with types; attrsOf (submodule ignoreCronOptions); + }; + + extraGroups = mkOption { + default = []; + type = types.listOf types.str; + example = [ "postdrop" "mongodb" ]; + description = '' + Extra groups for the logcheck user, for example to be able to use sendmail, + or to access certain log files. + ''; + }; + + }; + }; + + config = mkIf cfg.enable { + services.logcheck.extraRulesDirs = + mapAttrsToList writeIgnoreRule cfg.ignore + ++ mapAttrsToList writeIgnoreCronRule cfg.ignoreCron; + + users.users = optionalAttrs (cfg.user == "logcheck") { + logcheck = { + group = "logcheck"; + isSystemUser = true; + shell = "/bin/sh"; + description = "Logcheck user account"; + extraGroups = cfg.extraGroups; + }; + }; + users.groups = optionalAttrs (cfg.user == "logcheck") { + logcheck = {}; + }; + + system.activationScripts.logcheck = '' + mkdir -m 700 -p /var/{lib,lock}/logcheck + chown ${cfg.user} /var/{lib,lock}/logcheck + ''; + + services.cron.systemCronJobs = + let withTime = name: {timeArgs, ...}: timeArgs != null; + mkCron = name: {user, cmdline, timeArgs, ...}: '' + ${timeArgs} ${user} ${cmdline} + ''; + in mapAttrsToList mkCron (filterAttrs withTime cfg.ignoreCron) + ++ [ cronJob ]; + }; +} diff --git a/nixos/modules/services/logging/logrotate.nix b/nixos/modules/services/logging/logrotate.nix new file mode 100644 index 00000000000..082cf92ff4e --- /dev/null +++ b/nixos/modules/services/logging/logrotate.nix @@ -0,0 +1,179 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.logrotate; + + pathOpts = { name, ... }: { + options = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + Whether to enable log rotation for this path. This can be used to explicitly disable + logging that has been configured by NixOS. + ''; + }; + + name = mkOption { + type = types.str; + internal = true; + }; + + path = mkOption { + type = with types; either str (listOf str); + default = name; + defaultText = "attribute name"; + description = '' + The path to log files to be rotated. + Spaces are allowed and normal shell quoting rules apply, + with ', ", and \ characters supported. + ''; + }; + + user = mkOption { + type = with types; nullOr str; + default = null; + description = '' + The user account to use for rotation. + ''; + }; + + group = mkOption { + type = with types; nullOr str; + default = null; + description = '' + The group to use for rotation. + ''; + }; + + frequency = mkOption { + type = types.enum [ "hourly" "daily" "weekly" "monthly" "yearly" ]; + default = "daily"; + description = '' + How often to rotate the logs. + ''; + }; + + keep = mkOption { + type = types.int; + default = 20; + description = '' + How many rotations to keep. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Extra logrotate config options for this path. Refer to + <link xlink:href="https://linux.die.net/man/8/logrotate"/> for details. + ''; + }; + + priority = mkOption { + type = types.int; + default = 1000; + description = '' + Order of this logrotate block in relation to the others. The semantics are + the same as with `lib.mkOrder`. Smaller values have a greater priority. + ''; + }; + }; + + config.name = name; + }; + + mkConf = pathOpts: '' + # generated by NixOS using the `services.logrotate.paths.${pathOpts.name}` attribute set + ${concatMapStringsSep " " (path: ''"${path}"'') (toList pathOpts.path)} { + ${optionalString (pathOpts.user != null || pathOpts.group != null) "su ${pathOpts.user} ${pathOpts.group}"} + ${pathOpts.frequency} + rotate ${toString pathOpts.keep} + ${pathOpts.extraConfig} + } + ''; + + paths = sortProperties (attrValues (filterAttrs (_: pathOpts: pathOpts.enable) cfg.paths)); + configFile = pkgs.writeText "logrotate.conf" ( + concatStringsSep "\n" ( + [ "missingok" "notifempty" cfg.extraConfig ] ++ (map mkConf paths) + ) + ); + +in +{ + imports = [ + (mkRenamedOptionModule [ "services" "logrotate" "config" ] [ "services" "logrotate" "extraConfig" ]) + ]; + + options = { + services.logrotate = { + enable = mkEnableOption "the logrotate systemd service" // { + default = foldr (n: a: a || n.enable) false (attrValues cfg.paths); + defaultText = literalExpression "cfg.paths != {}"; + }; + + paths = mkOption { + type = with types; attrsOf (submodule pathOpts); + default = {}; + description = '' + Attribute set of paths to rotate. The order each block appears in the generated configuration file + can be controlled by the <link linkend="opt-services.logrotate.paths._name_.priority">priority</link> option + using the same semantics as `lib.mkOrder`. Smaller values have a greater priority. + ''; + example = literalExpression '' + { + httpd = { + path = "/var/log/httpd/*.log"; + user = config.services.httpd.user; + group = config.services.httpd.group; + keep = 7; + }; + + myapp = { + path = "/var/log/myapp/*.log"; + user = "myuser"; + group = "mygroup"; + frequency = "weekly"; + keep = 5; + priority = 1; + }; + } + ''; + }; + + extraConfig = mkOption { + default = ""; + type = types.lines; + description = '' + Extra contents to append to the logrotate configuration file. Refer to + <link xlink:href="https://linux.die.net/man/8/logrotate"/> for details. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + assertions = mapAttrsToList (name: pathOpts: + { assertion = (pathOpts.user != null) == (pathOpts.group != null); + message = '' + If either of `services.logrotate.paths.${name}.user` or `services.logrotate.paths.${name}.group` are specified then *both* must be specified. + ''; + } + ) cfg.paths; + + systemd.services.logrotate = { + description = "Logrotate Service"; + startAt = "hourly"; + + serviceConfig = { + Restart = "no"; + User = "root"; + ExecStart = "${pkgs.logrotate}/sbin/logrotate ${configFile}"; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/logstash.nix b/nixos/modules/services/logging/logstash.nix new file mode 100644 index 00000000000..a08203dffe7 --- /dev/null +++ b/nixos/modules/services/logging/logstash.nix @@ -0,0 +1,194 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.logstash; + ops = lib.optionalString; + verbosityFlag = "--log.level " + cfg.logLevel; + + logstashConf = pkgs.writeText "logstash.conf" '' + input { + ${cfg.inputConfig} + } + + filter { + ${cfg.filterConfig} + } + + output { + ${cfg.outputConfig} + } + ''; + + logstashSettingsYml = pkgs.writeText "logstash.yml" cfg.extraSettings; + + logstashJvmOptionsFile = pkgs.writeText "jvm.options" cfg.extraJvmOptions; + + logstashSettingsDir = pkgs.runCommand "logstash-settings" { + inherit logstashJvmOptionsFile; + inherit logstashSettingsYml; + preferLocalBuild = true; + } '' + mkdir -p $out + ln -s $logstashSettingsYml $out/logstash.yml + ln -s $logstashJvmOptionsFile $out/jvm.options + ''; +in + +{ + imports = [ + (mkRenamedOptionModule [ "services" "logstash" "address" ] [ "services" "logstash" "listenAddress" ]) + (mkRemovedOptionModule [ "services" "logstash" "enableWeb" ] "The web interface was removed from logstash") + ]; + + ###### interface + + options = { + + services.logstash = { + + enable = mkOption { + type = types.bool; + default = false; + description = "Enable logstash."; + }; + + package = mkOption { + type = types.package; + default = pkgs.logstash; + defaultText = literalExpression "pkgs.logstash"; + description = "Logstash package to use."; + }; + + plugins = mkOption { + type = types.listOf types.path; + default = [ ]; + example = literalExpression "[ pkgs.logstash-contrib ]"; + description = "The paths to find other logstash plugins in."; + }; + + dataDir = mkOption { + type = types.str; + default = "/var/lib/logstash"; + description = '' + A path to directory writable by logstash that it uses to store data. + Plugins will also have access to this path. + ''; + }; + + logLevel = mkOption { + type = types.enum [ "debug" "info" "warn" "error" "fatal" ]; + default = "warn"; + description = "Logging verbosity level."; + }; + + filterWorkers = mkOption { + type = types.int; + default = 1; + description = "The quantity of filter workers to run."; + }; + + listenAddress = mkOption { + type = types.str; + default = "127.0.0.1"; + description = "Address on which to start webserver."; + }; + + port = mkOption { + type = types.str; + default = "9292"; + description = "Port on which to start webserver."; + }; + + inputConfig = mkOption { + type = types.lines; + default = "generator { }"; + description = "Logstash input configuration."; + example = literalExpression '' + ''' + # Read from journal + pipe { + command => "''${pkgs.systemd}/bin/journalctl -f -o json" + type => "syslog" codec => json {} + } + ''' + ''; + }; + + filterConfig = mkOption { + type = types.lines; + default = ""; + description = "logstash filter configuration."; + example = '' + if [type] == "syslog" { + # Keep only relevant systemd fields + # http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html + prune { + whitelist_names => [ + "type", "@timestamp", "@version", + "MESSAGE", "PRIORITY", "SYSLOG_FACILITY" + ] + } + } + ''; + }; + + outputConfig = mkOption { + type = types.lines; + default = "stdout { codec => rubydebug }"; + description = "Logstash output configuration."; + example = '' + redis { host => ["localhost"] data_type => "list" key => "logstash" codec => json } + elasticsearch { } + ''; + }; + + extraSettings = mkOption { + type = types.lines; + default = ""; + description = "Extra Logstash settings in YAML format."; + example = '' + pipeline: + batch: + size: 125 + delay: 5 + ''; + }; + + extraJvmOptions = mkOption { + type = types.lines; + default = ""; + description = "Extra JVM options, one per line (jvm.options format)."; + example = '' + -Xms2g + -Xmx2g + ''; + }; + + }; + }; + + + ###### implementation + + config = mkIf cfg.enable { + systemd.services.logstash = { + description = "Logstash Daemon"; + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.bash ]; + serviceConfig = { + ExecStartPre = ''${pkgs.coreutils}/bin/mkdir -p "${cfg.dataDir}" ; ${pkgs.coreutils}/bin/chmod 700 "${cfg.dataDir}"''; + ExecStart = concatStringsSep " " (filter (s: stringLength s != 0) [ + "${cfg.package}/bin/logstash" + "-w ${toString cfg.filterWorkers}" + (concatMapStringsSep " " (x: "--path.plugins ${x}") cfg.plugins) + "${verbosityFlag}" + "-f ${logstashConf}" + "--path.settings ${logstashSettingsDir}" + "--path.data ${cfg.dataDir}" + ]); + }; + }; + }; +} diff --git a/nixos/modules/services/logging/promtail.nix b/nixos/modules/services/logging/promtail.nix new file mode 100644 index 00000000000..a34bc07b6ab --- /dev/null +++ b/nixos/modules/services/logging/promtail.nix @@ -0,0 +1,91 @@ +{ config, lib, pkgs, ... }: with lib; +let + cfg = config.services.promtail; + + prettyJSON = conf: pkgs.runCommandLocal "promtail-config.json" {} '' + echo '${builtins.toJSON conf}' | ${pkgs.buildPackages.jq}/bin/jq 'del(._module)' > $out + ''; + + allowSystemdJournal = cfg.configuration ? scrape_configs && lib.any (v: v ? journal) cfg.configuration.scrape_configs; + + allowPositionsFile = !lib.hasPrefix "/var/cache/promtail" positionsFile; + positionsFile = cfg.configuration.positions.filename; +in { + options.services.promtail = with types; { + enable = mkEnableOption "the Promtail ingresser"; + + + configuration = mkOption { + type = (pkgs.formats.json {}).type; + description = '' + Specify the configuration for Promtail in Nix. + ''; + }; + + extraFlags = mkOption { + type = listOf str; + default = []; + example = [ "--server.http-listen-port=3101" ]; + description = '' + Specify a list of additional command line flags, + which get escaped and are then passed to Loki. + ''; + }; + }; + + config = mkIf cfg.enable { + services.promtail.configuration.positions.filename = mkDefault "/var/cache/promtail/positions.yaml"; + + systemd.services.promtail = { + description = "Promtail log ingress"; + wantedBy = [ "multi-user.target" ]; + stopIfChanged = false; + + serviceConfig = { + Restart = "on-failure"; + TimeoutStopSec = 10; + + ExecStart = "${pkgs.promtail}/bin/promtail -config.file=${prettyJSON cfg.configuration} ${escapeShellArgs cfg.extraFlags}"; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + CacheDirectory = "promtail"; + ReadWritePaths = lib.optional allowPositionsFile (builtins.dirOf positionsFile); + + User = "promtail"; + Group = "promtail"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + + SupplementaryGroups = lib.optional (allowSystemdJournal) "systemd-journal"; + } // (optionalAttrs (!pkgs.stdenv.isAarch64) { # FIXME: figure out why this breaks on aarch64 + SystemCallFilter = "@system-service"; + }); + }; + + users.groups.promtail = {}; + users.users.promtail = { + description = "Promtail service user"; + isSystemUser = true; + group = "promtail"; + }; + }; +} diff --git a/nixos/modules/services/logging/rsyslogd.nix b/nixos/modules/services/logging/rsyslogd.nix new file mode 100644 index 00000000000..b924d94e0b0 --- /dev/null +++ b/nixos/modules/services/logging/rsyslogd.nix @@ -0,0 +1,105 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.services.rsyslogd; + + syslogConf = pkgs.writeText "syslog.conf" '' + $ModLoad imuxsock + $SystemLogSocketName /run/systemd/journal/syslog + $WorkDirectory /var/spool/rsyslog + + ${cfg.defaultConfig} + ${cfg.extraConfig} + ''; + + defaultConf = '' + # "local1" is used for dhcpd messages. + local1.* -/var/log/dhcpd + + mail.* -/var/log/mail + + *.=warning;*.=err -/var/log/warn + *.crit /var/log/warn + + *.*;mail.none;local1.none -/var/log/messages + ''; + +in + +{ + ###### interface + + options = { + + services.rsyslogd = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable syslogd. Note that systemd also logs + syslog messages, so you normally don't need to run syslogd. + ''; + }; + + defaultConfig = mkOption { + type = types.lines; + default = defaultConf; + description = '' + The default <filename>syslog.conf</filename> file configures a + fairly standard setup of log files, which can be extended by + means of <varname>extraConfig</varname>. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + example = "news.* -/var/log/news"; + description = '' + Additional text appended to <filename>syslog.conf</filename>, + i.e. the contents of <varname>defaultConfig</varname>. + ''; + }; + + extraParams = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "-m 0" ]; + description = '' + Additional parameters passed to <command>rsyslogd</command>. + ''; + }; + + }; + + }; + + + ###### implementation + + config = mkIf cfg.enable { + + environment.systemPackages = [ pkgs.rsyslog ]; + + systemd.services.syslog = + { description = "Syslog Daemon"; + + requires = [ "syslog.socket" ]; + + wantedBy = [ "multi-user.target" ]; + + serviceConfig = + { ExecStart = "${pkgs.rsyslog}/sbin/rsyslogd ${toString cfg.extraParams} -f ${syslogConf} -n"; + ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/spool/rsyslog"; + # Prevent syslogd output looping back through journald. + StandardOutput = "null"; + }; + }; + + }; + +} diff --git a/nixos/modules/services/logging/syslog-ng.nix b/nixos/modules/services/logging/syslog-ng.nix new file mode 100644 index 00000000000..0a57bf20bd0 --- /dev/null +++ b/nixos/modules/services/logging/syslog-ng.nix @@ -0,0 +1,101 @@ +{ config, pkgs, lib, ... }: + +with lib; + +let + + cfg = config.services.syslog-ng; + + syslogngConfig = pkgs.writeText "syslog-ng.conf" '' + ${cfg.configHeader} + ${cfg.extraConfig} + ''; + + ctrlSocket = "/run/syslog-ng/syslog-ng.ctl"; + pidFile = "/run/syslog-ng/syslog-ng.pid"; + persistFile = "/var/syslog-ng/syslog-ng.persist"; + + syslogngOptions = [ + "--foreground" + "--module-path=${concatStringsSep ":" (["${cfg.package}/lib/syslog-ng"] ++ cfg.extraModulePaths)}" + "--cfgfile=${syslogngConfig}" + "--control=${ctrlSocket}" + "--persist-file=${persistFile}" + "--pidfile=${pidFile}" + ]; + +in { + imports = [ + (mkRemovedOptionModule [ "services" "syslog-ng" "serviceName" ] "") + (mkRemovedOptionModule [ "services" "syslog-ng" "listenToJournal" ] "") + ]; + + options = { + + services.syslog-ng = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable the syslog-ng daemon. + ''; + }; + package = mkOption { + type = types.package; + default = pkgs.syslogng; + defaultText = literalExpression "pkgs.syslogng"; + description = '' + The package providing syslog-ng binaries. + ''; + }; + extraModulePaths = mkOption { + type = types.listOf types.str; + default = []; + example = literalExpression '' + [ "''${pkgs.syslogng_incubator}/lib/syslog-ng" ] + ''; + description = '' + A list of paths that should be included in syslog-ng's + <literal>--module-path</literal> option. They should usually + end in <literal>/lib/syslog-ng</literal> + ''; + }; + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Configuration added to the end of <literal>syslog-ng.conf</literal>. + ''; + }; + configHeader = mkOption { + type = types.lines; + default = '' + @version: 3.6 + @include "scl.conf" + ''; + description = '' + The very first lines of the configuration file. Should usually contain + the syslog-ng version header. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + systemd.services.syslog-ng = { + description = "syslog-ng daemon"; + preStart = "mkdir -p /{var,run}/syslog-ng"; + wantedBy = [ "multi-user.target" ]; + after = [ "multi-user.target" ]; # makes sure hostname etc is set + serviceConfig = { + Type = "notify"; + PIDFile = pidFile; + StandardOutput = "null"; + Restart = "on-failure"; + ExecStart = "${cfg.package}/sbin/syslog-ng ${concatStringsSep " " syslogngOptions}"; + ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + }; + }; + }; + +} diff --git a/nixos/modules/services/logging/syslogd.nix b/nixos/modules/services/logging/syslogd.nix new file mode 100644 index 00000000000..fe0b0490811 --- /dev/null +++ b/nixos/modules/services/logging/syslogd.nix @@ -0,0 +1,130 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.services.syslogd; + + syslogConf = pkgs.writeText "syslog.conf" '' + ${if (cfg.tty != "") then "kern.warning;*.err;authpriv.none /dev/${cfg.tty}" else ""} + ${cfg.defaultConfig} + ${cfg.extraConfig} + ''; + + defaultConf = '' + # Send emergency messages to all users. + *.emerg * + + # "local1" is used for dhcpd messages. + local1.* -/var/log/dhcpd + + mail.* -/var/log/mail + + *.=warning;*.=err -/var/log/warn + *.crit /var/log/warn + + *.*;mail.none;local1.none -/var/log/messages + ''; + +in + +{ + ###### interface + + options = { + + services.syslogd = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable syslogd. Note that systemd also logs + syslog messages, so you normally don't need to run syslogd. + ''; + }; + + tty = mkOption { + type = types.str; + default = "tty10"; + description = '' + The tty device on which syslogd will print important log + messages. Leave this option blank to disable tty logging. + ''; + }; + + defaultConfig = mkOption { + type = types.lines; + default = defaultConf; + description = '' + The default <filename>syslog.conf</filename> file configures a + fairly standard setup of log files, which can be extended by + means of <varname>extraConfig</varname>. + ''; + }; + + enableNetworkInput = mkOption { + type = types.bool; + default = false; + description = '' + Accept logging through UDP. Option -r of syslogd(8). + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + example = "news.* -/var/log/news"; + description = '' + Additional text appended to <filename>syslog.conf</filename>, + i.e. the contents of <varname>defaultConfig</varname>. + ''; + }; + + extraParams = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "-m 0" ]; + description = '' + Additional parameters passed to <command>syslogd</command>. + ''; + }; + + }; + + }; + + + ###### implementation + + config = mkIf cfg.enable { + + assertions = + [ { assertion = !config.services.rsyslogd.enable; + message = "rsyslogd conflicts with syslogd"; + } + ]; + + environment.systemPackages = [ pkgs.sysklogd ]; + + services.syslogd.extraParams = optional cfg.enableNetworkInput "-r"; + + # FIXME: restarting syslog seems to break journal logging. + systemd.services.syslog = + { description = "Syslog Daemon"; + + requires = [ "syslog.socket" ]; + + wantedBy = [ "multi-user.target" ]; + + serviceConfig = + { ExecStart = "${pkgs.sysklogd}/sbin/syslogd ${toString cfg.extraParams} -f ${syslogConf} -n"; + # Prevent syslogd output looping back through journald. + StandardOutput = "null"; + }; + }; + + }; + +} diff --git a/nixos/modules/services/logging/vector.nix b/nixos/modules/services/logging/vector.nix new file mode 100644 index 00000000000..be36b2a41bb --- /dev/null +++ b/nixos/modules/services/logging/vector.nix @@ -0,0 +1,64 @@ +{ config, lib, pkgs, ... }: + +with lib; +let cfg = config.services.vector; + +in +{ + options.services.vector = { + enable = mkEnableOption "Vector"; + + journaldAccess = mkOption { + type = types.bool; + default = false; + description = '' + Enable Vector to access journald. + ''; + }; + + settings = mkOption { + type = (pkgs.formats.json { }).type; + default = { }; + description = '' + Specify the configuration for Vector in Nix. + ''; + }; + }; + + config = mkIf cfg.enable { + + users.groups.vector = { }; + users.users.vector = { + description = "Vector service user"; + group = "vector"; + isSystemUser = true; + }; + systemd.services.vector = { + description = "Vector event and log aggregator"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" ]; + requires = [ "network-online.target" ]; + serviceConfig = + let + format = pkgs.formats.toml { }; + conf = format.generate "vector.toml" cfg.settings; + validateConfig = file: + pkgs.runCommand "validate-vector-conf" { } '' + ${pkgs.vector}/bin/vector validate --no-environment "${file}" + ln -s "${file}" "$out" + ''; + in + { + ExecStart = "${pkgs.vector}/bin/vector --config ${validateConfig conf}"; + User = "vector"; + Group = "vector"; + Restart = "no"; + StateDirectory = "vector"; + ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + AmbientCapabilities = "CAP_NET_BIND_SERVICE"; + # This group is required for accessing journald. + SupplementaryGroups = mkIf cfg.journaldAccess "systemd-journal"; + }; + }; + }; +} |