pyrox.dev
1{ pkgs, ... }:
2let
3
4 container =
5 { config, ... }:
6 {
7 # We re-use the NixOS container option ...
8 boot.isContainer = true;
9 # ... and revert unwanted defaults
10 networking.useHostResolvConf = false;
11
12 # use networkd to obtain systemd network setup
13 networking.useNetworkd = true;
14 networking.useDHCP = false;
15
16 # systemd-nspawn expects /sbin/init
17 boot.loader.initScript.enable = true;
18
19 imports = [ ../modules/profiles/minimal.nix ];
20
21 system.stateVersion = config.system.nixos.release;
22
23 nixpkgs.pkgs = pkgs;
24 };
25
26 containerSystem =
27 (import ../lib/eval-config.nix {
28 system = null;
29 modules = [ container ];
30 }).config.system.build.toplevel;
31
32 containerName = "container";
33 containerRoot = "/var/lib/machines/${containerName}";
34
35 containerTarball = pkgs.callPackage ../lib/make-system-tarball.nix {
36 storeContents = [
37 {
38 object = containerSystem;
39 symlink = "/nix/var/nix/profiles/system";
40 }
41 ];
42
43 contents = [
44 {
45 source = containerSystem + "/etc/os-release";
46 target = "/etc/os-release";
47 }
48 {
49 source = containerSystem + "/init";
50 target = "/sbin/init";
51 }
52 ];
53 };
54in
55{
56 name = "systemd-machinectl";
57
58 nodes.machine =
59 { lib, ... }:
60 {
61 # use networkd to obtain systemd network setup
62 networking.useNetworkd = true;
63 networking.useDHCP = false;
64
65 # do not try to access cache.nixos.org
66 nix.settings.substituters = lib.mkForce [ ];
67
68 # auto-start container
69 systemd.targets.machines.wants = [ "systemd-nspawn@${containerName}.service" ];
70
71 virtualisation.additionalPaths = [
72 containerSystem
73 containerTarball
74 ];
75
76 systemd.tmpfiles.rules = [
77 "d /var/lib/machines/shared-decl 0755 root root - -"
78 ];
79 systemd.nspawn.shared-decl = {
80 execConfig = {
81 Boot = false;
82 Parameters = "${containerSystem}/init";
83 };
84 filesConfig = {
85 BindReadOnly = "/nix/store";
86 };
87 };
88
89 systemd.nspawn.${containerName} = {
90 filesConfig = {
91 # workaround to fix kernel namespaces; needed for Nix sandbox
92 # https://github.com/systemd/systemd/issues/27994#issuecomment-1704005670
93 Bind = "/proc:/run/proc";
94 };
95 };
96
97 systemd.services."systemd-nspawn@${containerName}" = {
98 serviceConfig.Environment = [
99 # Disable tmpfs for /tmp
100 "SYSTEMD_NSPAWN_TMPFS_TMP=0"
101
102 # force unified cgroup delegation, which would be the default
103 # if systemd could check the capabilities of the installed systemd.
104 # see also: https://github.com/NixOS/nixpkgs/pull/198526
105 "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1"
106 ];
107 overrideStrategy = "asDropin";
108 };
109
110 # open DHCP for container
111 networking.firewall.extraCommands = ''
112 ${pkgs.iptables}/bin/iptables -A nixos-fw -i ve-+ -p udp -m udp --dport 67 -j nixos-fw-accept
113 '';
114 };
115
116 testScript = ''
117 start_all()
118 machine.wait_for_unit("default.target");
119
120 # Test machinectl start stop of shared-decl
121 machine.succeed("machinectl start shared-decl");
122 machine.wait_until_succeeds("systemctl -M shared-decl is-active default.target");
123 machine.succeed("machinectl stop shared-decl");
124
125 # create containers root
126 machine.succeed("mkdir -p ${containerRoot}");
127
128 # start container with shared nix store by using same arguments as for systemd-nspawn@.service
129 machine.succeed("systemd-run systemd-nspawn --machine=${containerName} --network-veth -U --bind-ro=/nix/store ${containerSystem}/init")
130 machine.wait_until_succeeds("systemctl -M ${containerName} is-active default.target");
131
132 # Test machinectl stop
133 machine.succeed("machinectl stop ${containerName}");
134
135 # Install container
136 # Workaround for nixos-install
137 machine.succeed("chmod o+rx /var/lib/machines");
138 machine.succeed("nixos-install --root ${containerRoot} --system ${containerSystem} --no-channel-copy --no-root-passwd");
139
140 # Allow systemd-nspawn to apply user namespace on immutable files
141 machine.succeed("chattr -i ${containerRoot}/var/empty");
142
143 # Test machinectl start
144 machine.succeed("machinectl start ${containerName}");
145 machine.wait_until_succeeds("systemctl -M ${containerName} is-active default.target");
146
147 # Test systemd-nspawn configured unified cgroup delegation
148 # see also:
149 # https://github.com/systemd/systemd/blob/main/docs/CGROUP_DELEGATION.md#three-different-tree-setups-
150 machine.succeed('systemd-run --pty --wait -M ${containerName} /run/current-system/sw/bin/stat --format="%T" --file-system /sys/fs/cgroup > fstype')
151 machine.succeed('test $(tr -d "\\r" < fstype) = cgroup2fs')
152
153 # Test if systemd-nspawn provides a working environment for nix to build derivations
154 # https://nixos.org/guides/nix-pills/07-working-derivation
155 machine.succeed('systemd-run --pty --wait -M ${containerName} /run/current-system/sw/bin/nix-instantiate --expr \'derivation { name = "myname"; builder = "/bin/sh"; args = [ "-c" "echo foo > $out" ]; system = "${pkgs.system}"; }\' --add-root /tmp/drv')
156 machine.succeed('systemd-run --pty --wait -M ${containerName} /run/current-system/sw/bin/nix-store --option substitute false --realize /tmp/drv')
157
158 # Test nss_mymachines without nscd
159 machine.succeed('LD_LIBRARY_PATH="/run/current-system/sw/lib" getent -s hosts:mymachines hosts ${containerName}');
160
161 # Test nss_mymachines via nscd
162 machine.succeed("getent hosts ${containerName}");
163
164 # Test systemd-nspawn network configuration to container
165 machine.succeed("networkctl --json=short status ve-${containerName} | ${pkgs.jq}/bin/jq -e '.OperationalState == \"routable\"'");
166
167 # Test systemd-nspawn network configuration to host
168 machine.succeed("machinectl shell ${containerName} /run/current-system/sw/bin/networkctl --json=short status host0 | ${pkgs.jq}/bin/jq -r '.OperationalState == \"routable\"'");
169
170 # Test systemd-nspawn network configuration
171 machine.succeed("ping -n -c 1 ${containerName}");
172
173 # Test systemd-nspawn uses a user namespace
174 machine.succeed("test $(machinectl status ${containerName} | grep 'UID Shift: ' | wc -l) = 1")
175
176 # Test systemd-nspawn reboot
177 machine.succeed("machinectl shell ${containerName} /run/current-system/sw/bin/reboot");
178 machine.wait_until_succeeds("systemctl -M ${containerName} is-active default.target");
179
180 # Test machinectl reboot
181 machine.succeed("machinectl reboot ${containerName}");
182 machine.wait_until_succeeds("systemctl -M ${containerName} is-active default.target");
183
184 # Restart machine
185 machine.shutdown()
186 machine.start()
187 machine.wait_for_unit("default.target");
188
189 # Test auto-start
190 machine.succeed("machinectl show ${containerName}")
191
192 # Test machinectl stop
193 machine.succeed("machinectl stop ${containerName}");
194 machine.wait_until_succeeds("test $(systemctl is-active systemd-nspawn@${containerName}) = inactive");
195
196 # Test tmpfs for /tmp
197 machine.fail("mountpoint /tmp");
198
199 # Show to to delete the container
200 machine.succeed("chattr -i ${containerRoot}/var/empty");
201 machine.succeed("rm -rf ${containerRoot}");
202
203 # Test import tarball, start, stop and remove
204 machine.succeed("machinectl import-tar ${containerTarball}/tarball/*.tar* ${containerName}");
205 machine.succeed("machinectl start ${containerName}");
206 machine.wait_until_succeeds("systemctl -M ${containerName} is-active default.target");
207 machine.succeed("machinectl stop ${containerName}");
208 machine.wait_until_succeeds("test $(systemctl is-active systemd-nspawn@${containerName}) = inactive");
209 machine.succeed("machinectl remove ${containerName}");
210 '';
211}