pyrox.dev
1# Systemd services for lxd.
2
3{ config, lib, pkgs, ... }:
4
5let
6 cfg = config.virtualisation.lxd;
7 preseedFormat = pkgs.formats.yaml {};
8in {
9 imports = [
10 (lib.mkRemovedOptionModule [ "virtualisation" "lxd" "zfsPackage" ] "Override zfs in an overlay instead to override it globally")
11 ];
12
13 ###### interface
14
15 options = {
16 virtualisation.lxd = {
17 enable = lib.mkOption {
18 type = lib.types.bool;
19 default = false;
20 description = lib.mdDoc ''
21 This option enables lxd, a daemon that manages
22 containers. Users in the "lxd" group can interact with
23 the daemon (e.g. to start or stop containers) using the
24 {command}`lxc` command line tool, among others.
25
26 Most of the time, you'll also want to start lxcfs, so
27 that containers can "see" the limits:
28 ```
29 virtualisation.lxc.lxcfs.enable = true;
30 ```
31 '';
32 };
33
34 package = lib.mkOption {
35 type = lib.types.package;
36 default = pkgs.lxd;
37 defaultText = lib.literalExpression "pkgs.lxd";
38 description = lib.mdDoc ''
39 The LXD package to use.
40 '';
41 };
42
43 lxcPackage = lib.mkOption {
44 type = lib.types.package;
45 default = pkgs.lxc;
46 defaultText = lib.literalExpression "pkgs.lxc";
47 description = lib.mdDoc ''
48 The LXC package to use with LXD (required for AppArmor profiles).
49 '';
50 };
51
52 zfsSupport = lib.mkOption {
53 type = lib.types.bool;
54 default = config.boot.zfs.enabled;
55 defaultText = lib.literalExpression "config.boot.zfs.enabled";
56 description = lib.mdDoc ''
57 Enables lxd to use zfs as a storage for containers.
58
59 This option is enabled by default if a zfs pool is configured
60 with nixos.
61 '';
62 };
63
64 recommendedSysctlSettings = lib.mkOption {
65 type = lib.types.bool;
66 default = false;
67 description = lib.mdDoc ''
68 Enables various settings to avoid common pitfalls when
69 running containers requiring many file operations.
70 Fixes errors like "Too many open files" or
71 "neighbour: ndisc_cache: neighbor table overflow!".
72 See https://lxd.readthedocs.io/en/latest/production-setup/
73 for details.
74 '';
75 };
76
77 preseed = lib.mkOption {
78 type = lib.types.nullOr (lib.types.submodule {
79 freeformType = preseedFormat.type;
80 });
81
82 default = null;
83
84 description = lib.mdDoc ''
85 Configuration for LXD preseed, see
86 <https://documentation.ubuntu.com/lxd/en/latest/howto/initialize/#initialize-preseed>
87 for supported values.
88
89 Changes to this will be re-applied to LXD which will overwrite existing entities or create missing ones,
90 but entities will *not* be removed by preseed.
91 '';
92
93 example = lib.literalExpression ''
94 {
95 networks = [
96 {
97 name = "lxdbr0";
98 type = "bridge";
99 config = {
100 "ipv4.address" = "10.0.100.1/24";
101 "ipv4.nat" = "true";
102 };
103 }
104 ];
105 profiles = [
106 {
107 name = "default";
108 devices = {
109 eth0 = {
110 name = "eth0";
111 network = "lxdbr0";
112 type = "nic";
113 };
114 root = {
115 path = "/";
116 pool = "default";
117 size = "35GiB";
118 type = "disk";
119 };
120 };
121 }
122 ];
123 storage_pools = [
124 {
125 name = "default";
126 driver = "dir";
127 config = {
128 source = "/var/lib/lxd/storage-pools/default";
129 };
130 }
131 ];
132 }
133 '';
134 };
135
136 startTimeout = lib.mkOption {
137 type = lib.types.int;
138 default = 600;
139 apply = toString;
140 description = lib.mdDoc ''
141 Time to wait (in seconds) for LXD to become ready to process requests.
142 If LXD does not reply within the configured time, lxd.service will be
143 considered failed and systemd will attempt to restart it.
144 '';
145 };
146
147 ui = {
148 enable = lib.mkEnableOption (lib.mdDoc "(experimental) LXD UI");
149
150 package = lib.mkPackageOption pkgs.lxd-unwrapped "ui" { };
151 };
152 };
153 };
154
155 ###### implementation
156 config = lib.mkIf cfg.enable {
157 environment.systemPackages = [ cfg.package ];
158
159 # Note: the following options are also declared in virtualisation.lxc, but
160 # the latter can't be simply enabled to reuse the formers, because it
161 # does a bunch of unrelated things.
162 systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ];
163
164 security.apparmor = {
165 packages = [ cfg.lxcPackage ];
166 policies = {
167 "bin.lxc-start".profile = ''
168 include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start
169 '';
170 "lxc-containers".profile = ''
171 include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers
172 '';
173 };
174 };
175
176 # TODO: remove once LXD gets proper support for cgroupsv2
177 # (currently most of the e.g. CPU accounting stuff doesn't work)
178 systemd.enableUnifiedCgroupHierarchy = false;
179
180 systemd.sockets.lxd = {
181 description = "LXD UNIX socket";
182 wantedBy = [ "sockets.target" ];
183
184 socketConfig = {
185 ListenStream = "/var/lib/lxd/unix.socket";
186 SocketMode = "0660";
187 SocketGroup = "lxd";
188 Service = "lxd.service";
189 };
190 };
191
192 systemd.services.lxd = {
193 description = "LXD Container Management Daemon";
194
195 wantedBy = [ "multi-user.target" ];
196 after = [
197 "network-online.target"
198 (lib.mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service")
199 ];
200 requires = [
201 "network-online.target"
202 "lxd.socket"
203 (lib.mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service")
204 ];
205 documentation = [ "man:lxd(1)" ];
206
207 path = [ pkgs.util-linux ]
208 ++ lib.optional cfg.zfsSupport config.boot.zfs.package;
209
210 environment = lib.mkIf (cfg.ui.enable) {
211 "LXD_UI" = cfg.ui.package;
212 };
213
214 serviceConfig = {
215 ExecStart = "@${cfg.package}/bin/lxd lxd --group lxd";
216 ExecStartPost = "${cfg.package}/bin/lxd waitready --timeout=${cfg.startTimeout}";
217 ExecStop = "${cfg.package}/bin/lxd shutdown";
218
219 KillMode = "process"; # when stopping, leave the containers alone
220 LimitMEMLOCK = "infinity";
221 LimitNOFILE = "1048576";
222 LimitNPROC = "infinity";
223 TasksMax = "infinity";
224
225 Restart = "on-failure";
226 TimeoutStartSec = "${cfg.startTimeout}s";
227 TimeoutStopSec = "30s";
228
229 # By default, `lxd` loads configuration files from hard-coded
230 # `/usr/share/lxc/config` - since this is a no-go for us, we have to
231 # explicitly tell it where the actual configuration files are
232 Environment = lib.mkIf (config.virtualisation.lxc.lxcfs.enable)
233 "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config";
234 };
235 };
236
237 systemd.services.lxd-preseed = lib.mkIf (cfg.preseed != null) {
238 description = "LXD initialization with preseed file";
239 wantedBy = ["multi-user.target"];
240 requires = ["lxd.service"];
241 after = ["lxd.service"];
242
243 script = ''
244 ${pkgs.coreutils}/bin/cat ${preseedFormat.generate "lxd-preseed.yaml" cfg.preseed} | ${cfg.package}/bin/lxd init --preseed
245 '';
246
247 serviceConfig = {
248 Type = "oneshot";
249 };
250 };
251
252 users.groups.lxd = {};
253
254 users.users.root = {
255 subUidRanges = [ { startUid = 1000000; count = 65536; } ];
256 subGidRanges = [ { startGid = 1000000; count = 65536; } ];
257 };
258
259 boot.kernel.sysctl = lib.mkIf cfg.recommendedSysctlSettings {
260 "fs.inotify.max_queued_events" = 1048576;
261 "fs.inotify.max_user_instances" = 1048576;
262 "fs.inotify.max_user_watches" = 1048576;
263 "vm.max_map_count" = 262144; # TODO: Default vm.max_map_count has been increased system-wide
264 "kernel.dmesg_restrict" = 1;
265 "net.ipv4.neigh.default.gc_thresh3" = 8192;
266 "net.ipv6.neigh.default.gc_thresh3" = 8192;
267 "kernel.keys.maxkeys" = 2000;
268 };
269
270 boot.kernelModules = [ "veth" "xt_comment" "xt_CHECKSUM" "xt_MASQUERADE" "vhost_vsock" ]
271 ++ lib.optionals (!config.networking.nftables.enable) [ "iptable_mangle" ];
272 };
273}