Merge pull request #127479 from symphorien/btrbk-module

nixos/btrbk: add module and test

Changed files
+346 -1
nixos
doc
manual
from_md
release-notes
release-notes
modules
services
backup
tests
pkgs
tools
backup
btrbk
+9
nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
···
<itemizedlist>
<listitem>
<para>
+
<link xlink:href="https://digint.ch/btrbk/index.html">btrbk</link>,
+
a backup tool for btrfs subvolumes, taking advantage of btrfs
+
specific capabilities to create atomic snapshots and transfer
+
them incrementally to your backup locations. Available as
+
<link xlink:href="options.html#opt-services.brtbk.instances">services.btrbk</link>.
+
</para>
+
</listitem>
+
<listitem>
+
<para>
<link xlink:href="https://github.com/maxmind/geoipupdate">geoipupdate</link>,
a GeoIP database updater from MaxMind. Available as
<link xlink:href="options.html#opt-services.geoipupdate.enable">services.geoipupdate</link>.
+2
nixos/doc/manual/release-notes/rl-2111.section.md
···
## New Services {#sec-release-21.11-new-services}
+
- [btrbk](https://digint.ch/btrbk/index.html), a backup tool for btrfs subvolumes, taking advantage of btrfs specific capabilities to create atomic snapshots and transfer them incrementally to your backup locations. Available as [services.btrbk](options.html#opt-services.brtbk.instances).
+
- [geoipupdate](https://github.com/maxmind/geoipupdate), a GeoIP database updater from MaxMind. Available as [services.geoipupdate](options.html#opt-services.geoipupdate.enable).
- [sourcehut](https://sr.ht), a collection of tools useful for software development. Available as [services.sourcehut](options.html#opt-services.sourcehut.enable).
+1
nixos/modules/module-list.nix
···
./services/backup/bacula.nix
./services/backup/borgbackup.nix
./services/backup/borgmatic.nix
+
./services/backup/btrbk.nix
./services/backup/duplicati.nix
./services/backup/duplicity.nix
./services/backup/mysql-backup.nix
+220
nixos/modules/services/backup/btrbk.nix
···
+
{ config, pkgs, lib, ... }:
+
let
+
cfg = config.services.btrbk;
+
sshEnabled = cfg.sshAccess != [ ];
+
serviceEnabled = cfg.instances != { };
+
attr2Lines = attr:
+
let
+
pairs = lib.attrsets.mapAttrsToList (name: value: { inherit name value; }) attr;
+
isSubsection = value:
+
if builtins.isAttrs value then true
+
else if builtins.isString value then false
+
else throw "invalid type in btrbk config ${builtins.typeOf value}";
+
sortedPairs = lib.lists.partition (x: isSubsection x.value) pairs;
+
in
+
lib.flatten (
+
# non subsections go first
+
(
+
map (pair: [ "${pair.name} ${pair.value}" ]) sortedPairs.wrong
+
)
+
++ # subsections go last
+
(
+
map
+
(
+
pair:
+
lib.mapAttrsToList
+
(
+
childname: value:
+
[ "${pair.name} ${childname}" ] ++ (map (x: " " + x) (attr2Lines value))
+
)
+
pair.value
+
)
+
sortedPairs.right
+
)
+
)
+
;
+
addDefaults = settings: { backend = "btrfs-progs-sudo"; } // settings;
+
mkConfigFile = settings: lib.concatStringsSep "\n" (attr2Lines (addDefaults settings));
+
mkTestedConfigFile = name: settings:
+
let
+
configFile = pkgs.writeText "btrbk-${name}.conf" (mkConfigFile settings);
+
in
+
pkgs.runCommand "btrbk-${name}-tested.conf" { } ''
+
mkdir foo
+
cp ${configFile} $out
+
if (set +o pipefail; ${pkgs.btrbk}/bin/btrbk -c $out ls foo 2>&1 | grep $out);
+
then
+
echo btrbk configuration is invalid
+
cat $out
+
exit 1
+
fi;
+
'';
+
in
+
{
+
options = {
+
services.btrbk = {
+
extraPackages = lib.mkOption {
+
description = "Extra packages for btrbk, like compression utilities for <literal>stream_compress</literal>";
+
type = lib.types.listOf lib.types.package;
+
default = [ ];
+
example = lib.literalExample "[ pkgs.xz ]";
+
};
+
niceness = lib.mkOption {
+
description = "Niceness for local instances of btrbk. Also applies to remote ones connecting via ssh when positive.";
+
type = lib.types.ints.between (-20) 19;
+
default = 10;
+
};
+
ioSchedulingClass = lib.mkOption {
+
description = "IO scheduling class for btrbk (see ionice(1) for a quick description). Applies to local instances, and remote ones connecting by ssh if set to idle.";
+
type = lib.types.enum [ "idle" "best-effort" "realtime" ];
+
default = "best-effort";
+
};
+
instances = lib.mkOption {
+
description = "Set of btrbk instances. The instance named <literal>btrbk</literal> is the default one.";
+
type = with lib.types;
+
attrsOf (
+
submodule {
+
options = {
+
onCalendar = lib.mkOption {
+
type = lib.types.str;
+
default = "daily";
+
description = "How often this btrbk instance is started. See systemd.time(7) for more information about the format.";
+
};
+
settings = lib.mkOption {
+
type = let t = lib.types.attrsOf (lib.types.either lib.types.str (t // { description = "instances of this type recursively"; })); in t;
+
default = { };
+
example = {
+
snapshot_preserve_min = "2d";
+
snapshot_preserve = "14d";
+
volume = {
+
"/mnt/btr_pool" = {
+
target = "/mnt/btr_backup/mylaptop";
+
subvolume = {
+
"rootfs" = { };
+
"home" = { snapshot_create = "always"; };
+
};
+
};
+
};
+
};
+
description = "configuration options for btrbk. Nested attrsets translate to subsections.";
+
};
+
};
+
}
+
);
+
default = { };
+
};
+
sshAccess = lib.mkOption {
+
description = "SSH keys that should be able to make or push snapshots on this system remotely with btrbk";
+
type = with lib.types; listOf (
+
submodule {
+
options = {
+
key = lib.mkOption {
+
type = str;
+
description = "SSH public key allowed to login as user <literal>btrbk</literal> to run remote backups.";
+
};
+
roles = lib.mkOption {
+
type = listOf (enum [ "info" "source" "target" "delete" "snapshot" "send" "receive" ]);
+
example = [ "source" "info" "send" ];
+
description = "What actions can be performed with this SSH key. See ssh_filter_btrbk(1) for details";
+
};
+
};
+
}
+
);
+
default = [ ];
+
};
+
};
+
+
};
+
config = lib.mkIf (sshEnabled || serviceEnabled) {
+
environment.systemPackages = [ pkgs.btrbk ] ++ cfg.extraPackages;
+
security.sudo.extraRules = [
+
{
+
users = [ "btrbk" ];
+
commands = [
+
{ command = "${pkgs.btrfs-progs}/bin/btrfs"; options = [ "NOPASSWD" ]; }
+
{ command = "${pkgs.coreutils}/bin/mkdir"; options = [ "NOPASSWD" ]; }
+
{ command = "${pkgs.coreutils}/bin/readlink"; options = [ "NOPASSWD" ]; }
+
# for ssh, they are not the same than the one hard coded in ${pkgs.btrbk}
+
{ command = "/run/current-system/bin/btrfs"; options = [ "NOPASSWD" ]; }
+
{ command = "/run/current-system/sw/bin/mkdir"; options = [ "NOPASSWD" ]; }
+
{ command = "/run/current-system/sw/bin/readlink"; options = [ "NOPASSWD" ]; }
+
];
+
}
+
];
+
users.users.btrbk = {
+
isSystemUser = true;
+
# ssh needs a home directory
+
home = "/var/lib/btrbk";
+
createHome = true;
+
shell = "${pkgs.bash}/bin/bash";
+
group = "btrbk";
+
openssh.authorizedKeys.keys = map
+
(
+
v:
+
let
+
options = lib.concatMapStringsSep " " (x: "--" + x) v.roles;
+
ioniceClass = {
+
"idle" = 3;
+
"best-effort" = 2;
+
"realtime" = 1;
+
}.${cfg.ioSchedulingClass};
+
in
+
''command="${pkgs.util-linux}/bin/ionice -t -c ${toString ioniceClass} ${lib.optionalString (cfg.niceness >= 1) "${pkgs.coreutils}/bin/nice -n ${toString cfg.niceness}"} ${pkgs.btrbk}/share/btrbk/scripts/ssh_filter_btrbk.sh --sudo ${options}" ${v.key}''
+
)
+
cfg.sshAccess;
+
};
+
users.groups.btrbk = { };
+
systemd.tmpfiles.rules = [
+
"d /var/lib/btrbk 0750 btrbk btrbk"
+
"d /var/lib/btrbk/.ssh 0700 btrbk btrbk"
+
"f /var/lib/btrbk/.ssh/config 0700 btrbk btrbk - StrictHostKeyChecking=accept-new"
+
];
+
environment.etc = lib.mapAttrs'
+
(
+
name: instance: {
+
name = "btrbk/${name}.conf";
+
value.source = mkTestedConfigFile name instance.settings;
+
}
+
)
+
cfg.instances;
+
systemd.services = lib.mapAttrs'
+
(
+
name: _: {
+
name = "btrbk-${name}";
+
value = {
+
description = "Takes BTRFS snapshots and maintains retention policies.";
+
unitConfig.Documentation = "man:btrbk(1)";
+
path = [ "/run/wrappers" ] ++ cfg.extraPackages;
+
serviceConfig = {
+
User = "btrbk";
+
Group = "btrbk";
+
Type = "oneshot";
+
ExecStart = "${pkgs.btrbk}/bin/btrbk -c /etc/btrbk/${name}.conf run";
+
Nice = cfg.niceness;
+
IOSchedulingClass = cfg.ioSchedulingClass;
+
StateDirectory = "btrbk";
+
};
+
};
+
}
+
)
+
cfg.instances;
+
+
systemd.timers = lib.mapAttrs'
+
(
+
name: instance: {
+
name = "btrbk-${name}";
+
value = {
+
description = "Timer to take BTRFS snapshots and maintain retention policies.";
+
wantedBy = [ "timers.target" ];
+
timerConfig = {
+
OnCalendar = instance.onCalendar;
+
AccuracySec = "10min";
+
Persistent = true;
+
};
+
};
+
}
+
)
+
cfg.instances;
+
};
+
+
}
+1
nixos/tests/all-tests.nix
···
boot-stage1 = handleTest ./boot-stage1.nix {};
borgbackup = handleTest ./borgbackup.nix {};
botamusique = handleTest ./botamusique.nix {};
+
btrbk = handleTest ./btrbk.nix {};
buildbot = handleTest ./buildbot.nix {};
buildkite-agents = handleTest ./buildkite-agents.nix {};
caddy = handleTest ./caddy.nix {};
+110
nixos/tests/btrbk.nix
···
+
import ./make-test-python.nix ({ pkgs, ... }:
+
+
let
+
privateKey = ''
+
-----BEGIN OPENSSH PRIVATE KEY-----
+
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+
QyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrwAAAJB+cF5HfnBe
+
RwAAAAtzc2gtZWQyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrw
+
AAAEBN75NsJZSpt63faCuaD75Unko0JjlSDxMhYHAPJk2/xXHxQHThDpD9/AMWNqQer3Tg
+
9gXMb2lTZMn0pelo8xyvAAAADXJzY2h1ZXR6QGt1cnQ=
+
-----END OPENSSH PRIVATE KEY-----
+
'';
+
publicKey = ''
+
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHxQHThDpD9/AMWNqQer3Tg9gXMb2lTZMn0pelo8xyv
+
'';
+
in
+
{
+
name = "btrbk";
+
meta = with pkgs.lib; {
+
maintainers = with maintainers; [ symphorien ];
+
};
+
+
nodes = {
+
archive = { ... }: {
+
environment.systemPackages = with pkgs; [ btrfs-progs ];
+
# note: this makes the privateKey world readable.
+
# don't do it with real ssh keys.
+
environment.etc."btrbk_key".text = privateKey;
+
services.btrbk = {
+
extraPackages = [ pkgs.lz4 ];
+
instances = {
+
remote = {
+
onCalendar = "minutely";
+
settings = {
+
ssh_identity = "/etc/btrbk_key";
+
ssh_user = "btrbk";
+
stream_compress = "lz4";
+
volume = {
+
"ssh://main/mnt" = {
+
target = "/mnt";
+
snapshot_dir = "btrbk/remote";
+
subvolume = "to_backup";
+
};
+
};
+
};
+
};
+
};
+
};
+
};
+
+
main = { ... }: {
+
environment.systemPackages = with pkgs; [ btrfs-progs ];
+
services.openssh = {
+
enable = true;
+
passwordAuthentication = false;
+
challengeResponseAuthentication = false;
+
};
+
services.btrbk = {
+
extraPackages = [ pkgs.lz4 ];
+
sshAccess = [
+
{
+
key = publicKey;
+
roles = [ "source" "send" "info" "delete" ];
+
}
+
];
+
instances = {
+
local = {
+
onCalendar = "minutely";
+
settings = {
+
volume = {
+
"/mnt" = {
+
snapshot_dir = "btrbk/local";
+
subvolume = "to_backup";
+
};
+
};
+
};
+
};
+
};
+
};
+
};
+
};
+
+
testScript = ''
+
start_all()
+
+
# create btrfs partition at /mnt
+
for machine in (archive, main):
+
machine.succeed("dd if=/dev/zero of=/data_fs bs=120M count=1")
+
machine.succeed("mkfs.btrfs /data_fs")
+
machine.succeed("mkdir /mnt")
+
machine.succeed("mount /data_fs /mnt")
+
+
# what to backup and where
+
main.succeed("btrfs subvolume create /mnt/to_backup")
+
main.succeed("mkdir -p /mnt/btrbk/{local,remote}")
+
+
# check that local snapshots work
+
with subtest("local"):
+
main.succeed("echo foo > /mnt/to_backup/bar")
+
main.wait_until_succeeds("cat /mnt/btrbk/local/*/bar | grep foo")
+
main.succeed("echo bar > /mnt/to_backup/bar")
+
main.succeed("cat /mnt/btrbk/local/*/bar | grep foo")
+
+
# check that btrfs send/receive works and ssh access works
+
with subtest("remote"):
+
archive.wait_until_succeeds("cat /mnt/*/bar | grep bar")
+
main.succeed("echo baz > /mnt/to_backup/bar")
+
archive.succeed("cat /mnt/*/bar | grep bar")
+
'';
+
})
+3 -1
pkgs/tools/backup/btrbk/default.nix
···
{ lib, stdenv, fetchurl, bash, btrfs-progs, openssh, perl, perlPackages
-
, util-linux, asciidoc, asciidoctor, mbuffer, makeWrapper }:
+
, util-linux, asciidoc, asciidoctor, mbuffer, makeWrapper, nixosTests }:
stdenv.mkDerivation rec {
pname = "btrbk";
···
--set PERL5LIB $PERL5LIB \
--prefix PATH ':' "${lib.makeBinPath [ btrfs-progs bash mbuffer openssh ]}"
'';
+
+
passthru.tests.btrbk = nixosTests.btrbk;
meta = with lib; {
description = "A backup tool for btrfs subvolumes";