pyrox.dev
1# Systemd services for lxd.
2
3{ config, lib, pkgs, ... }:
4
5let
6 cfg = config.virtualisation.lxd;
7 preseedFormat = pkgs.formats.yaml {};
8in {
9 meta = {
10 maintainers = lib.teams.lxc.members;
11 };
12
13 imports = [
14 (lib.mkRemovedOptionModule [ "virtualisation" "lxd" "zfsPackage" ] "Override zfs in an overlay instead to override it globally")
15 ];
16
17 options = {
18 virtualisation.lxd = {
19 enable = lib.mkOption {
20 type = lib.types.bool;
21 default = false;
22 description = ''
23 This option enables lxd, a daemon that manages
24 containers. Users in the "lxd" group can interact with
25 the daemon (e.g. to start or stop containers) using the
26 {command}`lxc` command line tool, among others.
27
28 Most of the time, you'll also want to start lxcfs, so
29 that containers can "see" the limits:
30 ```
31 virtualisation.lxc.lxcfs.enable = true;
32 ```
33 '';
34 };
35
36 package = lib.mkPackageOption pkgs "lxd-lts" { };
37
38 lxcPackage = lib.mkOption {
39 type = lib.types.package;
40 default = config.virtualisation.lxc.package;
41 defaultText = lib.literalExpression "config.virtualisation.lxc.package";
42 description = "The lxc package to use.";
43 };
44
45 zfsSupport = lib.mkOption {
46 type = lib.types.bool;
47 default = config.boot.zfs.enabled;
48 defaultText = lib.literalExpression "config.boot.zfs.enabled";
49 description = ''
50 Enables lxd to use zfs as a storage for containers.
51
52 This option is enabled by default if a zfs pool is configured
53 with nixos.
54 '';
55 };
56
57 recommendedSysctlSettings = lib.mkOption {
58 type = lib.types.bool;
59 default = false;
60 description = ''
61 Enables various settings to avoid common pitfalls when
62 running containers requiring many file operations.
63 Fixes errors like "Too many open files" or
64 "neighbour: ndisc_cache: neighbor table overflow!".
65 See https://lxd.readthedocs.io/en/latest/production-setup/
66 for details.
67 '';
68 };
69
70 preseed = lib.mkOption {
71 type = lib.types.nullOr (lib.types.submodule {
72 freeformType = preseedFormat.type;
73 });
74
75 default = null;
76
77 description = ''
78 Configuration for LXD preseed, see
79 <https://documentation.ubuntu.com/lxd/en/latest/howto/initialize/#initialize-preseed>
80 for supported values.
81
82 Changes to this will be re-applied to LXD which will overwrite existing entities or create missing ones,
83 but entities will *not* be removed by preseed.
84 '';
85
86 example = lib.literalExpression ''
87 {
88 networks = [
89 {
90 name = "lxdbr0";
91 type = "bridge";
92 config = {
93 "ipv4.address" = "10.0.100.1/24";
94 "ipv4.nat" = "true";
95 };
96 }
97 ];
98 profiles = [
99 {
100 name = "default";
101 devices = {
102 eth0 = {
103 name = "eth0";
104 network = "lxdbr0";
105 type = "nic";
106 };
107 root = {
108 path = "/";
109 pool = "default";
110 size = "35GiB";
111 type = "disk";
112 };
113 };
114 }
115 ];
116 storage_pools = [
117 {
118 name = "default";
119 driver = "dir";
120 config = {
121 source = "/var/lib/lxd/storage-pools/default";
122 };
123 }
124 ];
125 }
126 '';
127 };
128
129 startTimeout = lib.mkOption {
130 type = lib.types.int;
131 default = 600;
132 apply = toString;
133 description = ''
134 Time to wait (in seconds) for LXD to become ready to process requests.
135 If LXD does not reply within the configured time, lxd.service will be
136 considered failed and systemd will attempt to restart it.
137 '';
138 };
139
140 ui = {
141 enable = lib.mkEnableOption "(experimental) LXD UI";
142
143 package = lib.mkPackageOption pkgs [ "lxd-ui" ] { };
144 };
145 };
146 };
147
148 ###### implementation
149 config = lib.mkIf cfg.enable {
150 environment.systemPackages = [ cfg.package ];
151
152 # Note: the following options are also declared in virtualisation.lxc, but
153 # the latter can't be simply enabled to reuse the formers, because it
154 # does a bunch of unrelated things.
155 systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ];
156
157 security.apparmor = {
158 packages = [ cfg.lxcPackage ];
159 policies = {
160 "bin.lxc-start".profile = ''
161 include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start
162 '';
163 "lxc-containers".profile = ''
164 include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers
165 '';
166 };
167 };
168
169 # TODO: remove once LXD gets proper support for cgroupsv2
170 # (currently most of the e.g. CPU accounting stuff doesn't work)
171 systemd.enableUnifiedCgroupHierarchy = false;
172
173 systemd.sockets.lxd = {
174 description = "LXD UNIX socket";
175 wantedBy = [ "sockets.target" ];
176
177 socketConfig = {
178 ListenStream = "/var/lib/lxd/unix.socket";
179 SocketMode = "0660";
180 SocketGroup = "lxd";
181 Service = "lxd.service";
182 };
183 };
184
185 systemd.services.lxd = {
186 description = "LXD Container Management Daemon";
187
188 wantedBy = [ "multi-user.target" ];
189 after = [
190 "network-online.target"
191 (lib.mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service")
192 ];
193 requires = [
194 "network-online.target"
195 "lxd.socket"
196 (lib.mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service")
197 ];
198 documentation = [ "man:lxd(1)" ];
199
200 path = [ pkgs.util-linux ]
201 ++ lib.optional cfg.zfsSupport config.boot.zfs.package;
202
203 environment = lib.mkIf (cfg.ui.enable) {
204 "LXD_UI" = cfg.ui.package;
205 };
206
207 serviceConfig = {
208 ExecStart = "@${cfg.package}/bin/lxd lxd --group lxd";
209 ExecStartPost = "${cfg.package}/bin/lxd waitready --timeout=${cfg.startTimeout}";
210 ExecStop = "${cfg.package}/bin/lxd shutdown";
211
212 KillMode = "process"; # when stopping, leave the containers alone
213 LimitMEMLOCK = "infinity";
214 LimitNOFILE = "1048576";
215 LimitNPROC = "infinity";
216 TasksMax = "infinity";
217
218 # By default, `lxd` loads configuration files from hard-coded
219 # `/usr/share/lxc/config` - since this is a no-go for us, we have to
220 # explicitly tell it where the actual configuration files are
221 Environment = lib.mkIf (config.virtualisation.lxc.lxcfs.enable)
222 "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config";
223 };
224
225 unitConfig.ConditionPathExists = "!/var/lib/incus/.migrated-from-lxd";
226 };
227
228 systemd.services.lxd-preseed = lib.mkIf (cfg.preseed != null) {
229 description = "LXD initialization with preseed file";
230 wantedBy = ["multi-user.target"];
231 requires = ["lxd.service"];
232 after = ["lxd.service"];
233
234 script = ''
235 ${pkgs.coreutils}/bin/cat ${preseedFormat.generate "lxd-preseed.yaml" cfg.preseed} | ${cfg.package}/bin/lxd init --preseed
236 '';
237
238 serviceConfig = {
239 Type = "oneshot";
240 };
241 };
242
243 users.groups.lxd = {};
244
245 users.users.root = {
246 subUidRanges = [ { startUid = 1000000; count = 65536; } ];
247 subGidRanges = [ { startGid = 1000000; count = 65536; } ];
248 };
249
250 boot.kernel.sysctl = lib.mkIf cfg.recommendedSysctlSettings {
251 "fs.inotify.max_queued_events" = 1048576;
252 "fs.inotify.max_user_instances" = 1048576;
253 "fs.inotify.max_user_watches" = 1048576;
254 "vm.max_map_count" = 262144; # TODO: Default vm.max_map_count has been increased system-wide
255 "kernel.dmesg_restrict" = 1;
256 "net.ipv4.neigh.default.gc_thresh3" = 8192;
257 "net.ipv6.neigh.default.gc_thresh3" = 8192;
258 "kernel.keys.maxkeys" = 2000;
259 };
260
261 boot.kernelModules = [ "veth" "xt_comment" "xt_CHECKSUM" "xt_MASQUERADE" "vhost_vsock" ]
262 ++ lib.optionals (!config.networking.nftables.enable) [ "iptable_mangle" ];
263 };
264}