pyrox.dev
1# Systemd services for lxd.
2
3{ config, lib, pkgs, ... }:
4
5with lib;
6
7let
8 cfg = config.virtualisation.lxd;
9in {
10 imports = [
11 (mkRemovedOptionModule [ "virtualisation" "lxd" "zfsPackage" ] "Override zfs in an overlay instead to override it globally")
12 ];
13
14 ###### interface
15
16 options = {
17 virtualisation.lxd = {
18 enable = mkOption {
19 type = types.bool;
20 default = false;
21 description = lib.mdDoc ''
22 This option enables lxd, a daemon that manages
23 containers. Users in the "lxd" group can interact with
24 the daemon (e.g. to start or stop containers) using the
25 {command}`lxc` command line tool, among others.
26
27 Most of the time, you'll also want to start lxcfs, so
28 that containers can "see" the limits:
29 ```
30 virtualisation.lxc.lxcfs.enable = true;
31 ```
32 '';
33 };
34
35 package = mkOption {
36 type = types.package;
37 default = pkgs.lxd;
38 defaultText = literalExpression "pkgs.lxd";
39 description = lib.mdDoc ''
40 The LXD package to use.
41 '';
42 };
43
44 lxcPackage = mkOption {
45 type = types.package;
46 default = pkgs.lxc;
47 defaultText = literalExpression "pkgs.lxc";
48 description = lib.mdDoc ''
49 The LXC package to use with LXD (required for AppArmor profiles).
50 '';
51 };
52
53 zfsSupport = mkOption {
54 type = types.bool;
55 default = config.boot.zfs.enabled;
56 defaultText = literalExpression "config.boot.zfs.enabled";
57 description = lib.mdDoc ''
58 Enables lxd to use zfs as a storage for containers.
59
60 This option is enabled by default if a zfs pool is configured
61 with nixos.
62 '';
63 };
64
65 recommendedSysctlSettings = mkOption {
66 type = types.bool;
67 default = false;
68 description = lib.mdDoc ''
69 Enables various settings to avoid common pitfalls when
70 running containers requiring many file operations.
71 Fixes errors like "Too many open files" or
72 "neighbour: ndisc_cache: neighbor table overflow!".
73 See https://lxd.readthedocs.io/en/latest/production-setup/
74 for details.
75 '';
76 };
77
78 startTimeout = mkOption {
79 type = types.int;
80 default = 600;
81 apply = toString;
82 description = lib.mdDoc ''
83 Time to wait (in seconds) for LXD to become ready to process requests.
84 If LXD does not reply within the configured time, lxd.service will be
85 considered failed and systemd will attempt to restart it.
86 '';
87 };
88 };
89 };
90
91 ###### implementation
92 config = mkIf cfg.enable {
93 environment.systemPackages = [ cfg.package ];
94
95 # Note: the following options are also declared in virtualisation.lxc, but
96 # the latter can't be simply enabled to reuse the formers, because it
97 # does a bunch of unrelated things.
98 systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ];
99
100 security.apparmor = {
101 packages = [ cfg.lxcPackage ];
102 policies = {
103 "bin.lxc-start".profile = ''
104 include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start
105 '';
106 "lxc-containers".profile = ''
107 include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers
108 '';
109 };
110 };
111
112 # TODO: remove once LXD gets proper support for cgroupsv2
113 # (currently most of the e.g. CPU accounting stuff doesn't work)
114 systemd.enableUnifiedCgroupHierarchy = false;
115
116 systemd.sockets.lxd = {
117 description = "LXD UNIX socket";
118 wantedBy = [ "sockets.target" ];
119
120 socketConfig = {
121 ListenStream = "/var/lib/lxd/unix.socket";
122 SocketMode = "0660";
123 SocketGroup = "lxd";
124 Service = "lxd.service";
125 };
126 };
127
128 systemd.services.lxd = {
129 description = "LXD Container Management Daemon";
130
131 wantedBy = [ "multi-user.target" ];
132 after = [
133 "network-online.target"
134 (mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service")
135 ];
136 requires = [
137 "network-online.target"
138 "lxd.socket"
139 (mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service")
140 ];
141 documentation = [ "man:lxd(1)" ];
142
143 path = [ pkgs.util-linux ]
144 ++ optional cfg.zfsSupport config.boot.zfs.package;
145
146 serviceConfig = {
147 ExecStart = "@${cfg.package}/bin/lxd lxd --group lxd";
148 ExecStartPost = "${cfg.package}/bin/lxd waitready --timeout=${cfg.startTimeout}";
149 ExecStop = "${cfg.package}/bin/lxd shutdown";
150
151 KillMode = "process"; # when stopping, leave the containers alone
152 LimitMEMLOCK = "infinity";
153 LimitNOFILE = "1048576";
154 LimitNPROC = "infinity";
155 TasksMax = "infinity";
156
157 Restart = "on-failure";
158 TimeoutStartSec = "${cfg.startTimeout}s";
159 TimeoutStopSec = "30s";
160
161 # By default, `lxd` loads configuration files from hard-coded
162 # `/usr/share/lxc/config` - since this is a no-go for us, we have to
163 # explicitly tell it where the actual configuration files are
164 Environment = mkIf (config.virtualisation.lxc.lxcfs.enable)
165 "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config";
166 };
167 };
168
169 users.groups.lxd = {};
170
171 users.users.root = {
172 subUidRanges = [ { startUid = 1000000; count = 65536; } ];
173 subGidRanges = [ { startGid = 1000000; count = 65536; } ];
174 };
175
176 boot.kernel.sysctl = mkIf cfg.recommendedSysctlSettings {
177 "fs.inotify.max_queued_events" = 1048576;
178 "fs.inotify.max_user_instances" = 1048576;
179 "fs.inotify.max_user_watches" = 1048576;
180 "vm.max_map_count" = 262144;
181 "kernel.dmesg_restrict" = 1;
182 "net.ipv4.neigh.default.gc_thresh3" = 8192;
183 "net.ipv6.neigh.default.gc_thresh3" = 8192;
184 "kernel.keys.maxkeys" = 2000;
185 };
186
187 boot.kernelModules = [ "veth" "xt_comment" "xt_CHECKSUM" "xt_MASQUERADE" ]
188 ++ optionals (!config.networking.nftables.enable) [ "iptable_mangle" ];
189 };
190}