diff options
Diffstat (limited to 'nixos/modules/services/backup/znapzend.nix')
| -rw-r--r-- | nixos/modules/services/backup/znapzend.nix | 469 |
1 files changed, 469 insertions, 0 deletions
diff --git a/nixos/modules/services/backup/znapzend.nix b/nixos/modules/services/backup/znapzend.nix new file mode 100644 index 00000000000..09e60177c39 --- /dev/null +++ b/nixos/modules/services/backup/znapzend.nix @@ -0,0 +1,469 @@ +{ config, lib, pkgs, ... }: + +with lib; +with types; + +let + + planDescription = '' + The znapzend backup plan to use for the source. + + The plan specifies how often to backup and for how long to keep the + backups. It consists of a series of retention periodes to interval + associations: + + <literal> + retA=>intA,retB=>intB,... + </literal> + + Both intervals and retention periods are expressed in standard units + of time or multiples of them. You can use both the full name or a + shortcut according to the following listing: + + <literal> + second|sec|s, minute|min, hour|h, day|d, week|w, month|mon|m, year|y + </literal> + + See <citerefentry><refentrytitle>znapzendzetup</refentrytitle><manvolnum>1</manvolnum></citerefentry> for more info. + ''; + planExample = "1h=>10min,1d=>1h,1w=>1d,1m=>1w,1y=>1m"; + + # A type for a string of the form number{b|k|M|G} + mbufferSizeType = str // { + check = x: str.check x && builtins.isList (builtins.match "^[0-9]+[bkMG]$" x); + description = "string of the form number{b|k|M|G}"; + }; + + enabledFeatures = concatLists (mapAttrsToList (name: enabled: optional enabled name) cfg.features); + + # Type for a string that must contain certain other strings (the list parameter). + # Note that these would need regex escaping. + stringContainingStrings = list: let + matching = s: map (str: builtins.match ".*${str}.*" s) list; + in str // { + check = x: str.check x && all isList (matching x); + description = "string containing all of the characters ${concatStringsSep ", " list}"; + }; + + timestampType = stringContainingStrings [ "%Y" "%m" "%d" "%H" "%M" "%S" ]; + + destType = srcConfig: submodule ({ name, ... }: { + options = { + + label = mkOption { + type = str; + description = "Label for this destination. Defaults to the attribute name."; + }; + + plan = mkOption { + type = str; + description = planDescription; + example = planExample; + }; + + dataset = mkOption { + type = str; + description = "Dataset name to send snapshots to."; + example = "tank/main"; + }; + + host = mkOption { + type = nullOr str; + description = '' + Host to use for the destination dataset. Can be prefixed with + <literal>user@</literal> to specify the ssh user. + ''; + default = null; + example = "john@example.com"; + }; + + presend = mkOption { + type = nullOr str; + description = '' + Command to run before sending the snapshot to the destination. + Intended to run a remote script via <command>ssh</command> on the + destination, e.g. to bring up a backup disk or server or to put a + zpool online/offline. See also <option>postsend</option>. + ''; + default = null; + example = "ssh root@bserv zpool import -Nf tank"; + }; + + postsend = mkOption { + type = nullOr str; + description = '' + Command to run after sending the snapshot to the destination. + Intended to run a remote script via <command>ssh</command> on the + destination, e.g. to bring up a backup disk or server or to put a + zpool online/offline. See also <option>presend</option>. + ''; + default = null; + example = "ssh root@bserv zpool export tank"; + }; + }; + + config = { + label = mkDefault name; + plan = mkDefault srcConfig.plan; + }; + }); + + + + srcType = submodule ({ name, config, ... }: { + options = { + + enable = mkOption { + type = bool; + description = "Whether to enable this source."; + default = true; + }; + + recursive = mkOption { + type = bool; + description = "Whether to do recursive snapshots."; + default = false; + }; + + mbuffer = { + enable = mkOption { + type = bool; + description = "Whether to use <command>mbuffer</command>."; + default = false; + }; + + port = mkOption { + type = nullOr ints.u16; + description = '' + Port to use for <command>mbuffer</command>. + + If this is null, it will run <command>mbuffer</command> through + ssh. + + If this is not null, it will run <command>mbuffer</command> + directly through TCP, which is not encrypted but faster. In that + case the given port needs to be open on the destination host. + ''; + default = null; + }; + + size = mkOption { + type = mbufferSizeType; + description = '' + The size for <command>mbuffer</command>. + Supports the units b, k, M, G. + ''; + default = "1G"; + example = "128M"; + }; + }; + + presnap = mkOption { + type = nullOr str; + description = '' + Command to run before snapshots are taken on the source dataset, + e.g. for database locking/flushing. See also + <option>postsnap</option>. + ''; + default = null; + example = literalExpression '' + '''''${pkgs.mariadb}/bin/mysql -e "set autocommit=0;flush tables with read lock;\\! ''${pkgs.coreutils}/bin/sleep 600" & ''${pkgs.coreutils}/bin/echo $! > /tmp/mariadblock.pid ; sleep 10''' + ''; + }; + + postsnap = mkOption { + type = nullOr str; + description = '' + Command to run after snapshots are taken on the source dataset, + e.g. for database unlocking. See also <option>presnap</option>. + ''; + default = null; + example = literalExpression '' + "''${pkgs.coreutils}/bin/kill `''${pkgs.coreutils}/bin/cat /tmp/mariadblock.pid`;''${pkgs.coreutils}/bin/rm /tmp/mariadblock.pid" + ''; + }; + + timestampFormat = mkOption { + type = timestampType; + description = '' + The timestamp format to use for constructing snapshot names. + The syntax is <literal>strftime</literal>-like. The string must + consist of the mandatory <literal>%Y %m %d %H %M %S</literal>. + Optionally <literal>- _ . :</literal> characters as well as any + alphanumeric character are allowed. If suffixed by a + <literal>Z</literal>, times will be in UTC. + ''; + default = "%Y-%m-%d-%H%M%S"; + example = "znapzend-%m.%d.%Y-%H%M%SZ"; + }; + + sendDelay = mkOption { + type = int; + description = '' + Specify delay (in seconds) before sending snaps to the destination. + May be useful if you want to control sending time. + ''; + default = 0; + example = 60; + }; + + plan = mkOption { + type = str; + description = planDescription; + example = planExample; + }; + + dataset = mkOption { + type = str; + description = "The dataset to use for this source."; + example = "tank/home"; + }; + + destinations = mkOption { + type = attrsOf (destType config); + description = "Additional destinations."; + default = {}; + example = literalExpression '' + { + local = { + dataset = "btank/backup"; + presend = "zpool import -N btank"; + postsend = "zpool export btank"; + }; + remote = { + host = "john@example.com"; + dataset = "tank/john"; + }; + }; + ''; + }; + }; + + config = { + dataset = mkDefault name; + }; + + }); + + ### Generating the configuration from here + + cfg = config.services.znapzend; + + onOff = b: if b then "on" else "off"; + nullOff = b: if b == null then "off" else toString b; + stripSlashes = replaceStrings [ "/" ] [ "." ]; + + attrsToFile = config: concatStringsSep "\n" (builtins.attrValues ( + mapAttrs (n: v: "${n}=${v}") config)); + + mkDestAttrs = dst: with dst; + mapAttrs' (n: v: nameValuePair "dst_${label}${n}" v) ({ + "" = optionalString (host != null) "${host}:" + dataset; + _plan = plan; + } // optionalAttrs (presend != null) { + _precmd = presend; + } // optionalAttrs (postsend != null) { + _pstcmd = postsend; + }); + + mkSrcAttrs = srcCfg: with srcCfg; { + enabled = onOff enable; + # mbuffer is not referenced by its full path to accomodate non-NixOS systems or differing mbuffer versions between source and target + mbuffer = with mbuffer; if enable then "mbuffer" + + optionalString (port != null) ":${toString port}" else "off"; + mbuffer_size = mbuffer.size; + post_znap_cmd = nullOff postsnap; + pre_znap_cmd = nullOff presnap; + recursive = onOff recursive; + src = dataset; + src_plan = plan; + tsformat = timestampFormat; + zend_delay = toString sendDelay; + } // foldr (a: b: a // b) {} ( + map mkDestAttrs (builtins.attrValues destinations) + ); + + files = mapAttrs' (n: srcCfg: let + fileText = attrsToFile (mkSrcAttrs srcCfg); + in { + name = srcCfg.dataset; + value = pkgs.writeText (stripSlashes srcCfg.dataset) fileText; + }) cfg.zetup; + +in +{ + options = { + services.znapzend = { + enable = mkEnableOption "ZnapZend ZFS backup daemon"; + + logLevel = mkOption { + default = "debug"; + example = "warning"; + type = enum ["debug" "info" "warning" "err" "alert"]; + description = '' + The log level when logging to file. Any of debug, info, warning, err, + alert. Default in daemonized form is debug. + ''; + }; + + logTo = mkOption { + type = str; + default = "syslog::daemon"; + example = "/var/log/znapzend.log"; + description = '' + Where to log to (syslog::<facility> or <filepath>). + ''; + }; + + noDestroy = mkOption { + type = bool; + default = false; + description = "Does all changes to the filesystem except destroy."; + }; + + autoCreation = mkOption { + type = bool; + default = false; + description = "Automatically create the destination dataset if it does not exist."; + }; + + zetup = mkOption { + type = attrsOf srcType; + description = "Znapzend configuration."; + default = {}; + example = literalExpression '' + { + "tank/home" = { + # Make snapshots of tank/home every hour, keep those for 1 day, + # keep every days snapshot for 1 month, etc. + plan = "1d=>1h,1m=>1d,1y=>1m"; + recursive = true; + # Send all those snapshots to john@example.com:rtank/john as well + destinations.remote = { + host = "john@example.com"; + dataset = "rtank/john"; + }; + }; + }; + ''; + }; + + pure = mkOption { + type = bool; + description = '' + Do not persist any stateful znapzend setups. If this option is + enabled, your previously set znapzend setups will be cleared and only + the ones defined with this module will be applied. + ''; + default = false; + }; + + features.oracleMode = mkEnableOption '' + Destroy snapshots one by one instead of using one long argument list. + If source and destination are out of sync for a long time, you may have + so many snapshots to destroy that the argument gets is too long and the + command fails. + ''; + features.recvu = mkEnableOption '' + recvu feature which uses <literal>-u</literal> on the receiving end to keep the destination + filesystem unmounted. + ''; + features.compressed = mkEnableOption '' + compressed feature which adds the options <literal>-Lce</literal> to + the <command>zfs send</command> command. When this is enabled, make + sure that both the sending and receiving pool have the same relevant + features enabled. Using <literal>-c</literal> will skip unneccessary + decompress-compress stages, <literal>-L</literal> is for large block + support and -e is for embedded data support. see + <citerefentry><refentrytitle>znapzend</refentrytitle><manvolnum>1</manvolnum></citerefentry> + and <citerefentry><refentrytitle>zfs</refentrytitle><manvolnum>8</manvolnum></citerefentry> + for more info. + ''; + features.sendRaw = mkEnableOption '' + sendRaw feature which adds the options <literal>-w</literal> to the + <command>zfs send</command> command. For encrypted source datasets this + instructs zfs not to decrypt before sending which results in a remote + backup that can't be read without the encryption key/passphrase, useful + when the remote isn't fully trusted or not physically secure. This + option must be used consistently, raw incrementals cannot be based on + non-raw snapshots and vice versa. + ''; + features.skipIntermediates = mkEnableOption '' + Enable the skipIntermediates feature to send a single increment + between latest common snapshot and the newly made one. It may skip + several source snaps if the destination was offline for some time, and + it should skip snapshots not managed by znapzend. Normally for online + destinations, the new snapshot is sent as soon as it is created on the + source, so there are no automatic increments to skip. + ''; + features.lowmemRecurse = mkEnableOption '' + use lowmemRecurse on systems where you have too many datasets, so a + recursive listing of attributes to find backup plans exhausts the + memory available to <command>znapzend</command>: instead, go the slower + way to first list all impacted dataset names, and then query their + configs one by one. + ''; + features.zfsGetType = mkEnableOption '' + use zfsGetType if your <command>zfs get</command> supports a + <literal>-t</literal> argument for filtering by dataset type at all AND + lists properties for snapshots by default when recursing, so that there + is too much data to process while searching for backup plans. + If these two conditions apply to your system, the time needed for a + <literal>--recursive</literal> search for backup plans can literally + differ by hundreds of times (depending on the amount of snapshots in + that dataset tree... and a decent backup plan will ensure you have a lot + of those), so you would benefit from requesting this feature. + ''; + }; + }; + + config = mkIf cfg.enable { + environment.systemPackages = [ pkgs.znapzend ]; + + systemd.services = { + znapzend = { + description = "ZnapZend - ZFS Backup System"; + wantedBy = [ "zfs.target" ]; + after = [ "zfs.target" ]; + + path = with pkgs; [ zfs mbuffer openssh ]; + + preStart = optionalString cfg.pure '' + echo Resetting znapzend zetups + ${pkgs.znapzend}/bin/znapzendzetup list \ + | grep -oP '(?<=\*\*\* backup plan: ).*(?= \*\*\*)' \ + | xargs -I{} ${pkgs.znapzend}/bin/znapzendzetup delete "{}" + '' + concatStringsSep "\n" (mapAttrsToList (dataset: config: '' + echo Importing znapzend zetup ${config} for dataset ${dataset} + ${pkgs.znapzend}/bin/znapzendzetup import --write ${dataset} ${config} & + '') files) + '' + wait + ''; + + serviceConfig = { + # znapzendzetup --import apparently tries to connect to the backup + # host 3 times with a timeout of 30 seconds, leading to a startup + # delay of >90s when the host is down, which is just above the default + # service timeout of 90 seconds. Increase the timeout so it doesn't + # make the service fail in that case. + TimeoutStartSec = 180; + # Needs to have write access to ZFS + User = "root"; + ExecStart = let + args = concatStringsSep " " [ + "--logto=${cfg.logTo}" + "--loglevel=${cfg.logLevel}" + (optionalString cfg.noDestroy "--nodestroy") + (optionalString cfg.autoCreation "--autoCreation") + (optionalString (enabledFeatures != []) + "--features=${concatStringsSep "," enabledFeatures}") + ]; in "${pkgs.znapzend}/bin/znapzend ${args}"; + ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + Restart = "on-failure"; + }; + }; + }; + }; + + meta.maintainers = with maintainers; [ infinisil SlothOfAnarchy ]; +} |