Merge pull request #72060 from lopsided98/sanoid-init

sanoid: add package, NixOS module and test

Changed files
+551
nixos
pkgs
tools
backup
sanoid
top-level
+2
nixos/modules/module-list.nix
···
./services/backup/restic.nix
./services/backup/restic-rest-server.nix
./services/backup/rsnapshot.nix
./services/backup/tarsnap.nix
./services/backup/tsm.nix
./services/backup/zfs-replication.nix
···
./services/backup/restic.nix
./services/backup/restic-rest-server.nix
./services/backup/rsnapshot.nix
+
./services/backup/sanoid.nix
+
./services/backup/syncoid.nix
./services/backup/tarsnap.nix
./services/backup/tsm.nix
./services/backup/zfs-replication.nix
+213
nixos/modules/services/backup/sanoid.nix
···
···
+
{ config, lib, pkgs, ... }:
+
+
with lib;
+
+
let
+
cfg = config.services.sanoid;
+
+
datasetSettingsType = with types;
+
(attrsOf (nullOr (oneOf [ str int bool (listOf str) ]))) // {
+
description = "dataset/template options";
+
};
+
+
# Default values from https://github.com/jimsalterjrs/sanoid/blob/master/sanoid.defaults.conf
+
+
commonOptions = {
+
hourly = mkOption {
+
description = "Number of hourly snapshots.";
+
type = types.ints.unsigned;
+
default = 48;
+
};
+
+
daily = mkOption {
+
description = "Number of daily snapshots.";
+
type = types.ints.unsigned;
+
default = 90;
+
};
+
+
monthly = mkOption {
+
description = "Number of monthly snapshots.";
+
type = types.ints.unsigned;
+
default = 6;
+
};
+
+
yearly = mkOption {
+
description = "Number of yearly snapshots.";
+
type = types.ints.unsigned;
+
default = 0;
+
};
+
+
autoprune = mkOption {
+
description = "Whether to automatically prune old snapshots.";
+
type = types.bool;
+
default = true;
+
};
+
+
autosnap = mkOption {
+
description = "Whether to automatically take snapshots.";
+
type = types.bool;
+
default = true;
+
};
+
+
settings = mkOption {
+
description = ''
+
Free-form settings for this template/dataset. See
+
<link xlink:href="https://github.com/jimsalterjrs/sanoid/blob/master/sanoid.defaults.conf"/>
+
for allowed values.
+
'';
+
type = datasetSettingsType;
+
};
+
};
+
+
commonConfig = config: {
+
settings = {
+
hourly = mkDefault config.hourly;
+
daily = mkDefault config.daily;
+
monthly = mkDefault config.monthly;
+
yearly = mkDefault config.yearly;
+
autoprune = mkDefault config.autoprune;
+
autosnap = mkDefault config.autosnap;
+
};
+
};
+
+
datasetOptions = {
+
useTemplate = mkOption {
+
description = "Names of the templates to use for this dataset.";
+
type = (types.listOf (types.enum (attrNames cfg.templates))) // {
+
description = "list of template names";
+
};
+
default = [];
+
};
+
+
recursive = mkOption {
+
description = "Whether to recursively snapshot dataset children.";
+
type = types.bool;
+
default = false;
+
};
+
+
processChildrenOnly = mkOption {
+
description = "Whether to only snapshot child datasets if recursing.";
+
type = types.bool;
+
default = false;
+
};
+
};
+
+
datasetConfig = config: {
+
settings = {
+
use_template = mkDefault config.useTemplate;
+
recursive = mkDefault config.recursive;
+
process_children_only = mkDefault config.processChildrenOnly;
+
};
+
};
+
+
# Extract pool names from configured datasets
+
pools = unique (map (d: head (builtins.match "([^/]+).*" d)) (attrNames cfg.datasets));
+
+
configFile = let
+
mkValueString = v:
+
if builtins.isList v then concatStringsSep "," v
+
else generators.mkValueStringDefault {} v;
+
+
mkKeyValue = k: v: if v == null then ""
+
else generators.mkKeyValueDefault { inherit mkValueString; } "=" k v;
+
in generators.toINI { inherit mkKeyValue; } cfg.settings;
+
+
configDir = pkgs.writeTextDir "sanoid.conf" configFile;
+
+
in {
+
+
# Interface
+
+
options.services.sanoid = {
+
enable = mkEnableOption "Sanoid ZFS snapshotting service";
+
+
interval = mkOption {
+
type = types.str;
+
default = "hourly";
+
example = "daily";
+
description = ''
+
Run sanoid at this interval. The default is to run hourly.
+
+
The format is described in
+
<citerefentry><refentrytitle>systemd.time</refentrytitle>
+
<manvolnum>7</manvolnum></citerefentry>.
+
'';
+
};
+
+
datasets = mkOption {
+
type = types.attrsOf (types.submodule ({ config, ... }: {
+
options = commonOptions // datasetOptions;
+
config = mkMerge [ (commonConfig config) (datasetConfig config) ];
+
}));
+
default = {};
+
description = "Datasets to snapshot.";
+
};
+
+
templates = mkOption {
+
type = types.attrsOf (types.submodule ({ config, ... }: {
+
options = commonOptions;
+
config = commonConfig config;
+
}));
+
default = {};
+
description = "Templates for datasets.";
+
};
+
+
settings = mkOption {
+
type = types.attrsOf datasetSettingsType;
+
description = ''
+
Free-form settings written directly to the config file. See
+
<link xlink:href="https://github.com/jimsalterjrs/sanoid/blob/master/sanoid.defaults.conf"/>
+
for allowed values.
+
'';
+
};
+
+
extraArgs = mkOption {
+
type = types.listOf types.str;
+
default = [];
+
example = [ "--verbose" "--readonly" "--debug" ];
+
description = ''
+
Extra arguments to pass to sanoid. See
+
<link xlink:href="https://github.com/jimsalterjrs/sanoid/#sanoid-command-line-options"/>
+
for allowed options.
+
'';
+
};
+
};
+
+
# Implementation
+
+
config = mkIf cfg.enable {
+
services.sanoid.settings = mkMerge [
+
(mapAttrs' (d: v: nameValuePair ("template_" + d) v.settings) cfg.templates)
+
(mapAttrs (d: v: v.settings) cfg.datasets)
+
];
+
+
systemd.services.sanoid = {
+
description = "Sanoid snapshot service";
+
serviceConfig = {
+
ExecStartPre = map (pool: lib.escapeShellArgs [
+
"+/run/booted-system/sw/bin/zfs" "allow"
+
"sanoid" "snapshot,mount,destroy" pool
+
]) pools;
+
ExecStart = lib.escapeShellArgs ([
+
"${pkgs.sanoid}/bin/sanoid"
+
"--cron"
+
"--configdir" configDir
+
] ++ cfg.extraArgs);
+
ExecStopPost = map (pool: lib.escapeShellArgs [
+
"+/run/booted-system/sw/bin/zfs" "unallow" "sanoid" pool
+
]) pools;
+
User = "sanoid";
+
Group = "sanoid";
+
DynamicUser = true;
+
RuntimeDirectory = "sanoid";
+
CacheDirectory = "sanoid";
+
};
+
# Prevents missing snapshots during DST changes
+
environment.TZ = "UTC";
+
after = [ "zfs.target" ];
+
startAt = cfg.interval;
+
};
+
};
+
+
meta.maintainers = with maintainers; [ lopsided98 ];
+
}
+168
nixos/modules/services/backup/syncoid.nix
···
···
+
{ config, lib, pkgs, ... }:
+
+
with lib;
+
+
let
+
cfg = config.services.syncoid;
+
in {
+
+
# Interface
+
+
options.services.syncoid = {
+
enable = mkEnableOption "Syncoid ZFS synchronization service";
+
+
interval = mkOption {
+
type = types.str;
+
default = "hourly";
+
example = "*-*-* *:15:00";
+
description = ''
+
Run syncoid at this interval. The default is to run hourly.
+
+
The format is described in
+
<citerefentry><refentrytitle>systemd.time</refentrytitle>
+
<manvolnum>7</manvolnum></citerefentry>.
+
'';
+
};
+
+
user = mkOption {
+
type = types.str;
+
default = "root";
+
example = "backup";
+
description = ''
+
The user for the service. Sudo or ZFS privilege delegation must be
+
configured to use a user other than root.
+
'';
+
};
+
+
sshKey = mkOption {
+
type = types.nullOr types.path;
+
# Prevent key from being copied to store
+
apply = mapNullable toString;
+
default = null;
+
description = ''
+
SSH private key file to use to login to the remote system. Can be
+
overridden in individual commands.
+
'';
+
};
+
+
commonArgs = mkOption {
+
type = types.listOf types.str;
+
default = [];
+
example = [ "--no-sync-snap" ];
+
description = ''
+
Arguments to add to every syncoid command, unless disabled for that
+
command. See
+
<link xlink:href="https://github.com/jimsalterjrs/sanoid/#syncoid-command-line-options"/>
+
for available options.
+
'';
+
};
+
+
commands = mkOption {
+
type = types.attrsOf (types.submodule ({ name, ... }: {
+
options = {
+
source = mkOption {
+
type = types.str;
+
example = "pool/dataset";
+
description = ''
+
Source ZFS dataset. Can be either local or remote. Defaults to
+
the attribute name.
+
'';
+
};
+
+
target = mkOption {
+
type = types.str;
+
example = "user@server:pool/dataset";
+
description = ''
+
Target ZFS dataset. Can be either local
+
(<replaceable>pool/dataset</replaceable>) or remote
+
(<replaceable>user@server:pool/dataset</replaceable>).
+
'';
+
};
+
+
recursive = mkOption {
+
type = types.bool;
+
default = false;
+
description = ''
+
Whether to also transfer child datasets.
+
'';
+
};
+
+
sshKey = mkOption {
+
type = types.nullOr types.path;
+
# Prevent key from being copied to store
+
apply = mapNullable toString;
+
description = ''
+
SSH private key file to use to login to the remote system.
+
Defaults to <option>services.syncoid.sshKey</option> option.
+
'';
+
};
+
+
sendOptions = mkOption {
+
type = types.separatedString " ";
+
default = "";
+
example = "Lc e";
+
description = ''
+
Advanced options to pass to zfs send. Options are specified
+
without their leading dashes and separated by spaces.
+
'';
+
};
+
+
recvOptions = mkOption {
+
type = types.separatedString " ";
+
default = "";
+
example = "ux recordsize o compression=lz4";
+
description = ''
+
Advanced options to pass to zfs recv. Options are specified
+
without their leading dashes and separated by spaces.
+
'';
+
};
+
+
useCommonArgs = mkOption {
+
type = types.bool;
+
default = true;
+
description = ''
+
Whether to add the configured common arguments to this command.
+
'';
+
};
+
+
extraArgs = mkOption {
+
type = types.listOf types.str;
+
default = [];
+
example = [ "--sshport 2222" ];
+
description = "Extra syncoid arguments for this command.";
+
};
+
};
+
config = {
+
source = mkDefault name;
+
sshKey = mkDefault cfg.sshKey;
+
};
+
}));
+
default = {};
+
example."pool/test".target = "root@target:pool/test";
+
description = "Syncoid commands to run.";
+
};
+
};
+
+
# Implementation
+
+
config = mkIf cfg.enable {
+
systemd.services.syncoid = {
+
description = "Syncoid ZFS synchronization service";
+
script = concatMapStringsSep "\n" (c: lib.escapeShellArgs
+
([ "${pkgs.sanoid}/bin/syncoid" ]
+
++ (optionals c.useCommonArgs cfg.commonArgs)
+
++ (optional c.recursive "-r")
+
++ (optionals (c.sshKey != null) [ "--sshkey" c.sshKey ])
+
++ c.extraArgs
+
++ [ "--sendoptions" c.sendOptions
+
"--recvoptions" c.recvOptions
+
c.source c.target
+
])) (attrValues cfg.commands);
+
after = [ "zfs.target" ];
+
serviceConfig.User = cfg.user;
+
startAt = cfg.interval;
+
};
+
};
+
+
meta.maintainers = with maintainers; [ lopsided98 ];
+
}
+1
nixos/tests/all-tests.nix
···
runInMachine = handleTest ./run-in-machine.nix {};
rxe = handleTest ./rxe.nix {};
samba = handleTest ./samba.nix {};
sddm = handleTest ./sddm.nix {};
shiori = handleTest ./shiori.nix {};
signal-desktop = handleTest ./signal-desktop.nix {};
···
runInMachine = handleTest ./run-in-machine.nix {};
rxe = handleTest ./rxe.nix {};
samba = handleTest ./samba.nix {};
+
sanoid = handleTest ./sanoid.nix {};
sddm = handleTest ./sddm.nix {};
shiori = handleTest ./shiori.nix {};
signal-desktop = handleTest ./signal-desktop.nix {};
+90
nixos/tests/sanoid.nix
···
···
+
import ./make-test-python.nix ({ pkgs, ... }: let
+
inherit (import ./ssh-keys.nix pkgs)
+
snakeOilPrivateKey snakeOilPublicKey;
+
+
commonConfig = { pkgs, ... }: {
+
virtualisation.emptyDiskImages = [ 2048 ];
+
boot.supportedFilesystems = [ "zfs" ];
+
environment.systemPackages = [ pkgs.parted ];
+
};
+
in {
+
name = "sanoid";
+
meta = with pkgs.stdenv.lib.maintainers; {
+
maintainers = [ lopsided98 ];
+
};
+
+
nodes = {
+
source = { ... }: {
+
imports = [ commonConfig ];
+
networking.hostId = "daa82e91";
+
+
programs.ssh.extraConfig = ''
+
UserKnownHostsFile=/dev/null
+
StrictHostKeyChecking=no
+
'';
+
+
services.sanoid = {
+
enable = true;
+
templates.test = {
+
hourly = 12;
+
daily = 1;
+
monthly = 1;
+
yearly = 1;
+
+
autosnap = true;
+
};
+
datasets."pool/test".useTemplate = [ "test" ];
+
};
+
+
services.syncoid = {
+
enable = true;
+
sshKey = "/root/.ssh/id_ecdsa";
+
commonArgs = [ "--no-sync-snap" ];
+
commands."pool/test".target = "root@target:pool/test";
+
};
+
};
+
target = { ... }: {
+
imports = [ commonConfig ];
+
networking.hostId = "dcf39d36";
+
+
services.openssh.enable = true;
+
users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
+
};
+
};
+
+
testScript = ''
+
source.succeed(
+
"mkdir /tmp/mnt",
+
"parted --script /dev/vdb -- mklabel msdos mkpart primary 1024M -1s",
+
"udevadm settle",
+
"zpool create pool /dev/vdb1",
+
"zfs create -o mountpoint=legacy pool/test",
+
"mount -t zfs pool/test /tmp/mnt",
+
"udevadm settle",
+
)
+
target.succeed(
+
"parted --script /dev/vdb -- mklabel msdos mkpart primary 1024M -1s",
+
"udevadm settle",
+
"zpool create pool /dev/vdb1",
+
"udevadm settle",
+
)
+
+
source.succeed("mkdir -m 700 /root/.ssh")
+
source.succeed(
+
"cat '${snakeOilPrivateKey}' > /root/.ssh/id_ecdsa"
+
)
+
source.succeed("chmod 600 /root/.ssh/id_ecdsa")
+
+
source.succeed("touch /tmp/mnt/test.txt")
+
source.systemctl("start --wait sanoid.service")
+
+
target.wait_for_open_port(22)
+
source.systemctl("start --wait syncoid.service")
+
target.succeed(
+
"mkdir /tmp/mnt",
+
"zfs set mountpoint=legacy pool/test",
+
"mount -t zfs pool/test /tmp/mnt",
+
)
+
target.succeed("cat /tmp/mnt/test.txt")
+
'';
+
})
+75
pkgs/tools/backup/sanoid/default.nix
···
···
+
{ lib, stdenv, fetchFromGitHub, fetchpatch, makeWrapper, coreutils, zfs
+
, perlPackages, procps, which, openssh, sudo, mbuffer, pv, lzop, gzip, pigz }:
+
+
with lib;
+
+
stdenv.mkDerivation rec {
+
pname = "sanoid";
+
version = "2.0.3";
+
+
src = fetchFromGitHub {
+
owner = "jimsalterjrs";
+
repo = pname;
+
rev = "v${version}";
+
sha256 = "1wmymzqg503nmhw8hrblfs67is1l3ljbk2fjvrqwyb01b7mbn80x";
+
};
+
+
patches = [
+
# Make sanoid look for programs in PATH
+
(fetchpatch {
+
url = "https://github.com/jimsalterjrs/sanoid/commit/dc2371775afe08af799d3097d47b48182d1716eb.patch";
+
sha256 = "16hlwcbcb8h3ar1ywd2bzr3h3whgbcfk6walmp8z6j74wbx81aav";
+
})
+
# Make findoid look for programs in PATH
+
(fetchpatch {
+
url = "https://github.com/jimsalterjrs/sanoid/commit/44bcd21f269e17765acd1ad0d45161902a205c7b.patch";
+
sha256 = "0zqyl8q5sfscqcc07acw68ysnlnh3nb57cigjfwbccsm0zwlwham";
+
})
+
# Add --cache-dir option
+
(fetchpatch {
+
url = "https://github.com/jimsalterjrs/sanoid/commit/a1f5e4c0c006e16a5047a16fc65c9b3663adb81e.patch";
+
sha256 = "1bb4g2zxrbvf7fvcgzzxsr1cvxzrxg5dzh89sx3h7qlrd6grqhdy";
+
})
+
# Add --run-dir option
+
(fetchpatch {
+
url = "https://github.com/jimsalterjrs/sanoid/commit/59a07f92b4920952cc9137b03c1533656f48b121.patch";
+
sha256 = "11v4jhc36v839gppzvhvzp5jd22904k8xqdhhpx6ghl75yyh4f4s";
+
})
+
];
+
+
nativeBuildInputs = [ makeWrapper ];
+
buildInputs = with perlPackages; [ perl ConfigIniFiles CaptureTiny ];
+
+
installPhase = ''
+
mkdir -p "$out/bin"
+
mkdir -p "$out/etc/sanoid"
+
cp sanoid.defaults.conf "$out/etc/sanoid/sanoid.defaults.conf"
+
# Hardcode path to default config
+
substitute sanoid "$out/bin/sanoid" \
+
--replace "\$args{'configdir'}/sanoid.defaults.conf" "$out/etc/sanoid/sanoid.defaults.conf"
+
chmod +x "$out/bin/sanoid"
+
# Prefer ZFS userspace tools from /run/booted-system/sw/bin to avoid
+
# incompatibilities with the ZFS kernel module.
+
wrapProgram "$out/bin/sanoid" \
+
--prefix PERL5LIB : "$PERL5LIB" \
+
--prefix PATH : "${makeBinPath [ procps "/run/booted-system/sw" zfs ]}"
+
+
install -m755 syncoid "$out/bin/syncoid"
+
wrapProgram "$out/bin/syncoid" \
+
--prefix PERL5LIB : "$PERL5LIB" \
+
--prefix PATH : "${makeBinPath [ openssh procps which pv mbuffer lzop gzip pigz "/run/booted-system/sw" zfs ]}"
+
+
install -m755 findoid "$out/bin/findoid"
+
wrapProgram "$out/bin/findoid" \
+
--prefix PERL5LIB : "$PERL5LIB" \
+
--prefix PATH : "${makeBinPath [ "/run/booted-system/sw" zfs ]}"
+
'';
+
+
meta = {
+
description = "A policy-driven snapshot management tool for ZFS filesystems";
+
homepage = "https://github.com/jimsalterjrs/sanoid";
+
license = licenses.gpl3;
+
maintainers = with maintainers; [ lopsided98 ];
+
platforms = platforms.all;
+
};
+
}
+2
pkgs/top-level/all-packages.nix
···
sane-frontends = callPackage ../applications/graphics/sane/frontends.nix { };
satysfi = callPackage ../tools/typesetting/satysfi { };
sc-controller = pythonPackages.callPackage ../misc/drivers/sc-controller {
···
sane-frontends = callPackage ../applications/graphics/sane/frontends.nix { };
+
sanoid = callPackage ../tools/backup/sanoid { };
+
satysfi = callPackage ../tools/typesetting/satysfi { };
sc-controller = pythonPackages.callPackage ../misc/drivers/sc-controller {