pyrox.dev
1# Systemd services for lxd.
2
3{
4 config,
5 lib,
6 pkgs,
7 ...
8}:
9
10let
11 cfg = config.virtualisation.lxd;
12 preseedFormat = pkgs.formats.yaml { };
13in
14{
15 imports = [
16 (lib.mkRemovedOptionModule [
17 "virtualisation"
18 "lxd"
19 "zfsPackage"
20 ] "Override zfs in an overlay instead to override it globally")
21 ];
22
23 options = {
24 virtualisation.lxd = {
25 enable = lib.mkOption {
26 type = lib.types.bool;
27 default = false;
28 description = ''
29 This option enables lxd, a daemon that manages
30 containers. Users in the "lxd" group can interact with
31 the daemon (e.g. to start or stop containers) using the
32 {command}`lxc` command line tool, among others.
33
34 Most of the time, you'll also want to start lxcfs, so
35 that containers can "see" the limits:
36 ```
37 virtualisation.lxc.lxcfs.enable = true;
38 ```
39 '';
40 };
41
42 package = lib.mkPackageOption pkgs "lxd-lts" { };
43
44 lxcPackage = lib.mkOption {
45 type = lib.types.package;
46 default = config.virtualisation.lxc.package;
47 defaultText = lib.literalExpression "config.virtualisation.lxc.package";
48 description = "The lxc package to use.";
49 };
50
51 zfsSupport = lib.mkOption {
52 type = lib.types.bool;
53 default = config.boot.zfs.enabled;
54 defaultText = lib.literalExpression "config.boot.zfs.enabled";
55 description = ''
56 Enables lxd to use zfs as a storage for containers.
57
58 This option is enabled by default if a zfs pool is configured
59 with nixos.
60 '';
61 };
62
63 recommendedSysctlSettings = lib.mkOption {
64 type = lib.types.bool;
65 default = false;
66 description = ''
67 Enables various settings to avoid common pitfalls when
68 running containers requiring many file operations.
69 Fixes errors like "Too many open files" or
70 "neighbour: ndisc_cache: neighbor table overflow!".
71 See https://lxd.readthedocs.io/en/latest/production-setup/
72 for details.
73 '';
74 };
75
76 preseed = lib.mkOption {
77 type = lib.types.nullOr (
78 lib.types.submodule {
79 freeformType = preseedFormat.type;
80 }
81 );
82
83 default = null;
84
85 description = ''
86 Configuration for LXD preseed, see
87 <https://documentation.ubuntu.com/lxd/en/latest/howto/initialize/#initialize-preseed>
88 for supported values.
89
90 Changes to this will be re-applied to LXD which will overwrite existing entities or create missing ones,
91 but entities will *not* be removed by preseed.
92 '';
93
94 example = lib.literalExpression ''
95 {
96 networks = [
97 {
98 name = "lxdbr0";
99 type = "bridge";
100 config = {
101 "ipv4.address" = "10.0.100.1/24";
102 "ipv4.nat" = "true";
103 };
104 }
105 ];
106 profiles = [
107 {
108 name = "default";
109 devices = {
110 eth0 = {
111 name = "eth0";
112 network = "lxdbr0";
113 type = "nic";
114 };
115 root = {
116 path = "/";
117 pool = "default";
118 size = "35GiB";
119 type = "disk";
120 };
121 };
122 }
123 ];
124 storage_pools = [
125 {
126 name = "default";
127 driver = "dir";
128 config = {
129 source = "/var/lib/lxd/storage-pools/default";
130 };
131 }
132 ];
133 }
134 '';
135 };
136
137 startTimeout = lib.mkOption {
138 type = lib.types.int;
139 default = 600;
140 apply = toString;
141 description = ''
142 Time to wait (in seconds) for LXD to become ready to process requests.
143 If LXD does not reply within the configured time, lxd.service will be
144 considered failed and systemd will attempt to restart it.
145 '';
146 };
147
148 ui = {
149 enable = lib.mkEnableOption "(experimental) LXD UI";
150
151 package = lib.mkPackageOption pkgs [ "lxd-ui" ] { };
152 };
153 };
154 };
155
156 ###### implementation
157 config = lib.mkIf cfg.enable {
158 environment.systemPackages = [ cfg.package ];
159
160 # Note: the following options are also declared in virtualisation.lxc, but
161 # the latter can't be simply enabled to reuse the formers, because it
162 # does a bunch of unrelated things.
163 systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ];
164
165 security.apparmor = {
166 packages = [ cfg.lxcPackage ];
167 policies = {
168 "bin.lxc-start".profile = ''
169 include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start
170 '';
171 "lxc-containers".profile = ''
172 include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers
173 '';
174 };
175 };
176
177 systemd.sockets.lxd = {
178 description = "LXD UNIX socket";
179 wantedBy = [ "sockets.target" ];
180
181 socketConfig = {
182 ListenStream = "/var/lib/lxd/unix.socket";
183 SocketMode = "0660";
184 SocketGroup = "lxd";
185 Service = "lxd.service";
186 };
187 };
188
189 systemd.services.lxd = {
190 description = "LXD Container Management Daemon";
191
192 wantedBy = [ "multi-user.target" ];
193 after = [
194 "network-online.target"
195 (lib.mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service")
196 ];
197 requires = [
198 "network-online.target"
199 "lxd.socket"
200 (lib.mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service")
201 ];
202 documentation = [ "man:lxd(1)" ];
203
204 path = [ pkgs.util-linux ] ++ lib.optional cfg.zfsSupport config.boot.zfs.package;
205
206 environment = lib.mkIf (cfg.ui.enable) {
207 "LXD_UI" = cfg.ui.package;
208 };
209
210 serviceConfig = {
211 ExecStart = "@${cfg.package}/bin/lxd lxd --group lxd";
212 ExecStartPost = "${cfg.package}/bin/lxd waitready --timeout=${cfg.startTimeout}";
213 ExecStop = "${cfg.package}/bin/lxd shutdown";
214
215 KillMode = "process"; # when stopping, leave the containers alone
216 LimitMEMLOCK = "infinity";
217 LimitNOFILE = "1048576";
218 LimitNPROC = "infinity";
219 TasksMax = "infinity";
220 Delegate = true; # LXD needs to manage cgroups in its subtree
221
222 # By default, `lxd` loads configuration files from hard-coded
223 # `/usr/share/lxc/config` - since this is a no-go for us, we have to
224 # explicitly tell it where the actual configuration files are
225 Environment = lib.mkIf (config.virtualisation.lxc.lxcfs.enable) "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config";
226 };
227
228 unitConfig.ConditionPathExists = "!/var/lib/incus/.migrated-from-lxd";
229 };
230
231 systemd.services.lxd-preseed = lib.mkIf (cfg.preseed != null) {
232 description = "LXD initialization with preseed file";
233 wantedBy = [ "multi-user.target" ];
234 requires = [ "lxd.service" ];
235 after = [ "lxd.service" ];
236
237 script = ''
238 ${pkgs.coreutils}/bin/cat ${preseedFormat.generate "lxd-preseed.yaml" cfg.preseed} | ${cfg.package}/bin/lxd init --preseed
239 '';
240
241 serviceConfig = {
242 Type = "oneshot";
243 };
244 };
245
246 users.groups.lxd = { };
247
248 users.users.root = {
249 subUidRanges = [
250 {
251 startUid = 1000000;
252 count = 65536;
253 }
254 ];
255 subGidRanges = [
256 {
257 startGid = 1000000;
258 count = 65536;
259 }
260 ];
261 };
262
263 boot.kernel.sysctl = lib.mkIf cfg.recommendedSysctlSettings {
264 "fs.inotify.max_queued_events" = 1048576;
265 "fs.inotify.max_user_instances" = 1048576;
266 "fs.inotify.max_user_watches" = 1048576;
267 "vm.max_map_count" = 262144; # TODO: Default vm.max_map_count has been increased system-wide
268 "kernel.dmesg_restrict" = 1;
269 "net.ipv4.neigh.default.gc_thresh3" = 8192;
270 "net.ipv6.neigh.default.gc_thresh3" = 8192;
271 "kernel.keys.maxkeys" = 2000;
272 };
273
274 boot.kernelModules = [
275 "veth"
276 "xt_comment"
277 "xt_CHECKSUM"
278 "xt_MASQUERADE"
279 "vhost_vsock"
280 ] ++ lib.optionals (!config.networking.nftables.enable) [ "iptable_mangle" ];
281 };
282}