pyrox.dev
1# Systemd services for lxd.
2
3{ config, lib, pkgs, ... }:
4
5with lib;
6
7let
8 cfg = config.virtualisation.lxd;
9in {
10 imports = [
11 (mkRemovedOptionModule [ "virtualisation" "lxd" "zfsPackage" ] "Override zfs in an overlay instead to override it globally")
12 ];
13
14 ###### interface
15
16 options = {
17 virtualisation.lxd = {
18 enable = mkOption {
19 type = types.bool;
20 default = false;
21 description = ''
22 This option enables lxd, a daemon that manages
23 containers. Users in the "lxd" group can interact with
24 the daemon (e.g. to start or stop containers) using the
25 <command>lxc</command> command line tool, among others.
26
27 Most of the time, you'll also want to start lxcfs, so
28 that containers can "see" the limits:
29 <code>
30 virtualisation.lxc.lxcfs.enable = true;
31 </code>
32 '';
33 };
34
35 package = mkOption {
36 type = types.package;
37 default = pkgs.lxd;
38 defaultText = literalExpression "pkgs.lxd";
39 description = ''
40 The LXD package to use.
41 '';
42 };
43
44 lxcPackage = mkOption {
45 type = types.package;
46 default = pkgs.lxc;
47 defaultText = literalExpression "pkgs.lxc";
48 description = ''
49 The LXC package to use with LXD (required for AppArmor profiles).
50 '';
51 };
52
53 zfsSupport = mkOption {
54 type = types.bool;
55 default = config.boot.zfs.enabled;
56 defaultText = literalExpression "config.boot.zfs.enabled";
57 description = ''
58 Enables lxd to use zfs as a storage for containers.
59
60 This option is enabled by default if a zfs pool is configured
61 with nixos.
62 '';
63 };
64
65 recommendedSysctlSettings = mkOption {
66 type = types.bool;
67 default = false;
68 description = ''
69 Enables various settings to avoid common pitfalls when
70 running containers requiring many file operations.
71 Fixes errors like "Too many open files" or
72 "neighbour: ndisc_cache: neighbor table overflow!".
73 See https://lxd.readthedocs.io/en/latest/production-setup/
74 for details.
75 '';
76 };
77
78 startTimeout = mkOption {
79 type = types.int;
80 default = 600;
81 apply = toString;
82 description = ''
83 Time to wait (in seconds) for LXD to become ready to process requests.
84 If LXD does not reply within the configured time, lxd.service will be
85 considered failed and systemd will attempt to restart it.
86 '';
87 };
88 };
89 };
90
91 ###### implementation
92 config = mkIf cfg.enable {
93 environment.systemPackages = [ cfg.package ];
94
95 # Note: the following options are also declared in virtualisation.lxc, but
96 # the latter can't be simply enabled to reuse the formers, because it
97 # does a bunch of unrelated things.
98 systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ];
99
100 security.apparmor = {
101 packages = [ cfg.lxcPackage ];
102 policies = {
103 "bin.lxc-start".profile = ''
104 include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start
105 '';
106 "lxc-containers".profile = ''
107 include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers
108 '';
109 };
110 };
111
112 # TODO: remove once LXD gets proper support for cgroupsv2
113 # (currently most of the e.g. CPU accounting stuff doesn't work)
114 systemd.enableUnifiedCgroupHierarchy = false;
115
116 systemd.sockets.lxd = {
117 description = "LXD UNIX socket";
118 wantedBy = [ "sockets.target" ];
119
120 socketConfig = {
121 ListenStream = "/var/lib/lxd/unix.socket";
122 SocketMode = "0660";
123 SocketGroup = "lxd";
124 Service = "lxd.service";
125 };
126 };
127
128 systemd.services.lxd = {
129 description = "LXD Container Management Daemon";
130
131 wantedBy = [ "multi-user.target" ];
132 after = [ "network-online.target" "lxcfs.service" ];
133 requires = [ "network-online.target" "lxd.socket" "lxcfs.service" ];
134 documentation = [ "man:lxd(1)" ];
135
136 path = optional cfg.zfsSupport config.boot.zfs.package;
137
138 serviceConfig = {
139 ExecStart = "@${cfg.package}/bin/lxd lxd --group lxd";
140 ExecStartPost = "${cfg.package}/bin/lxd waitready --timeout=${cfg.startTimeout}";
141 ExecStop = "${cfg.package}/bin/lxd shutdown";
142
143 KillMode = "process"; # when stopping, leave the containers alone
144 LimitMEMLOCK = "infinity";
145 LimitNOFILE = "1048576";
146 LimitNPROC = "infinity";
147 TasksMax = "infinity";
148
149 Restart = "on-failure";
150 TimeoutStartSec = "${cfg.startTimeout}s";
151 TimeoutStopSec = "30s";
152
153 # By default, `lxd` loads configuration files from hard-coded
154 # `/usr/share/lxc/config` - since this is a no-go for us, we have to
155 # explicitly tell it where the actual configuration files are
156 Environment = mkIf (config.virtualisation.lxc.lxcfs.enable)
157 "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config";
158 };
159 };
160
161 users.groups.lxd = {};
162
163 users.users.root = {
164 subUidRanges = [ { startUid = 1000000; count = 65536; } ];
165 subGidRanges = [ { startGid = 1000000; count = 65536; } ];
166 };
167
168 boot.kernel.sysctl = mkIf cfg.recommendedSysctlSettings {
169 "fs.inotify.max_queued_events" = 1048576;
170 "fs.inotify.max_user_instances" = 1048576;
171 "fs.inotify.max_user_watches" = 1048576;
172 "vm.max_map_count" = 262144;
173 "kernel.dmesg_restrict" = 1;
174 "net.ipv4.neigh.default.gc_thresh3" = 8192;
175 "net.ipv6.neigh.default.gc_thresh3" = 8192;
176 "kernel.keys.maxkeys" = 2000;
177 };
178
179 boot.kernelModules = [ "veth" "xt_comment" "xt_CHECKSUM" "xt_MASQUERADE" ]
180 ++ optionals (!config.networking.nftables.enable) [ "iptable_mangle" ];
181 };
182}