pyrox.dev
1import ./make-test-python.nix (
2 { pkgs, lib, ... }:
3 let
4
5 # We'll need to be able to trade cert files between nodes via scp.
6 inherit (import ./ssh-keys.nix pkgs)
7 snakeOilPrivateKey
8 snakeOilPublicKey
9 ;
10
11 makeNebulaNode =
12 { config, ... }:
13 name: extraConfig:
14 lib.mkMerge [
15 {
16 # Expose nebula for doing cert signing.
17 environment.systemPackages = [
18 pkgs.dig
19 pkgs.nebula
20 ];
21 users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
22 services.openssh.enable = true;
23 networking.firewall.enable = true; # Implicitly true, but let's make sure.
24 networking.interfaces.eth1.useDHCP = false;
25
26 services.nebula.networks.smoke = {
27 # Note that these paths won't exist when the machine is first booted.
28 ca = "/etc/nebula/ca.crt";
29 cert = "/etc/nebula/${name}.crt";
30 key = "/etc/nebula/${name}.key";
31 listen = {
32 host = "0.0.0.0";
33 port =
34 if
35 (
36 config.services.nebula.networks.smoke.isLighthouse || config.services.nebula.networks.smoke.isRelay
37 )
38 then
39 4242
40 else
41 0;
42 };
43 };
44 }
45 extraConfig
46 ];
47
48 in
49 {
50 name = "nebula";
51
52 nodes = {
53
54 lighthouse =
55 { ... }@args:
56 makeNebulaNode args "lighthouse" {
57 networking.firewall.allowedUDPPorts = [ 53 ];
58 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
59 {
60 address = "192.168.1.1";
61 prefixLength = 24;
62 }
63 ];
64
65 services.nebula.networks.smoke = {
66 isLighthouse = true;
67 isRelay = true;
68 firewall = {
69 outbound = [
70 {
71 port = "any";
72 proto = "any";
73 host = "any";
74 }
75 ];
76 inbound = [
77 {
78 port = "any";
79 proto = "any";
80 host = "any";
81 }
82 ];
83 };
84 lighthouse = {
85 dns = {
86 enable = true;
87 host = "10.0.100.1"; # bind to lighthouse interface
88 port = 53; # answer on standard DNS port
89 };
90 };
91 };
92 };
93
94 allowAny =
95 { ... }@args:
96 makeNebulaNode args "allowAny" {
97 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
98 {
99 address = "192.168.1.2";
100 prefixLength = 24;
101 }
102 ];
103
104 services.nebula.networks.smoke = {
105 staticHostMap = {
106 "10.0.100.1" = [ "192.168.1.1:4242" ];
107 };
108 isLighthouse = false;
109 lighthouses = [ "10.0.100.1" ];
110 relays = [ "10.0.100.1" ];
111 firewall = {
112 outbound = [
113 {
114 port = "any";
115 proto = "any";
116 host = "any";
117 }
118 ];
119 inbound = [
120 {
121 port = "any";
122 proto = "any";
123 host = "any";
124 }
125 ];
126 };
127 };
128 };
129
130 allowFromLighthouse =
131 { ... }@args:
132 makeNebulaNode args "allowFromLighthouse" {
133 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
134 {
135 address = "192.168.1.3";
136 prefixLength = 24;
137 }
138 ];
139
140 services.nebula.networks.smoke = {
141 staticHostMap = {
142 "10.0.100.1" = [ "192.168.1.1:4242" ];
143 };
144 isLighthouse = false;
145 lighthouses = [ "10.0.100.1" ];
146 relays = [ "10.0.100.1" ];
147 firewall = {
148 outbound = [
149 {
150 port = "any";
151 proto = "any";
152 host = "any";
153 }
154 ];
155 inbound = [
156 {
157 port = "any";
158 proto = "any";
159 host = "lighthouse";
160 }
161 ];
162 };
163 };
164 };
165
166 allowToLighthouse =
167 { ... }@args:
168 makeNebulaNode args "allowToLighthouse" {
169 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
170 {
171 address = "192.168.1.4";
172 prefixLength = 24;
173 }
174 ];
175
176 services.nebula.networks.smoke = {
177 enable = true;
178 staticHostMap = {
179 "10.0.100.1" = [ "192.168.1.1:4242" ];
180 };
181 isLighthouse = false;
182 lighthouses = [ "10.0.100.1" ];
183 relays = [ "10.0.100.1" ];
184 firewall = {
185 outbound = [
186 {
187 port = "any";
188 proto = "any";
189 host = "lighthouse";
190 }
191 ];
192 inbound = [
193 {
194 port = "any";
195 proto = "any";
196 host = "any";
197 }
198 ];
199 };
200 };
201 };
202
203 disabled =
204 { ... }@args:
205 makeNebulaNode args "disabled" {
206 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
207 {
208 address = "192.168.1.5";
209 prefixLength = 24;
210 }
211 ];
212
213 services.nebula.networks.smoke = {
214 enable = false;
215 staticHostMap = {
216 "10.0.100.1" = [ "192.168.1.1:4242" ];
217 };
218 isLighthouse = false;
219 lighthouses = [ "10.0.100.1" ];
220 relays = [ "10.0.100.1" ];
221 firewall = {
222 outbound = [
223 {
224 port = "any";
225 proto = "any";
226 host = "lighthouse";
227 }
228 ];
229 inbound = [
230 {
231 port = "any";
232 proto = "any";
233 host = "any";
234 }
235 ];
236 };
237 };
238 };
239
240 };
241
242 testScript =
243 let
244
245 setUpPrivateKey = name: ''
246 ${name}.start()
247 ${name}.succeed(
248 "mkdir -p /root/.ssh",
249 "chmod 700 /root/.ssh",
250 "cat '${snakeOilPrivateKey}' > /root/.ssh/id_snakeoil",
251 "chmod 600 /root/.ssh/id_snakeoil",
252 "mkdir -p /root"
253 )
254 '';
255
256 # From what I can tell, StrictHostKeyChecking=no is necessary for ssh to work between machines.
257 sshOpts = "-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oIdentityFile=/root/.ssh/id_snakeoil";
258
259 restartAndCheckNebula = name: ip: ''
260 ${name}.systemctl("restart nebula@smoke.service")
261 ${name}.succeed("ping -c5 ${ip}")
262 '';
263
264 # Create a keypair on the client node, then use the public key to sign a cert on the lighthouse.
265 signKeysFor = name: ip: ''
266 lighthouse.wait_for_unit("sshd.service")
267 ${name}.wait_for_unit("sshd.service")
268 ${name}.succeed(
269 "mkdir -p /etc/nebula",
270 "nebula-cert keygen -out-key /etc/nebula/${name}.key -out-pub /etc/nebula/${name}.pub",
271 "scp ${sshOpts} /etc/nebula/${name}.pub root@192.168.1.1:/root/${name}.pub",
272 )
273 lighthouse.succeed(
274 'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -ip "${ip}" -in-pub /root/${name}.pub -out-crt /root/${name}.crt'
275 )
276 ${name}.succeed(
277 "scp ${sshOpts} root@192.168.1.1:/root/${name}.crt /etc/nebula/${name}.crt",
278 "scp ${sshOpts} root@192.168.1.1:/etc/nebula/ca.crt /etc/nebula/ca.crt",
279 '(id nebula-smoke >/dev/null && chown -R nebula-smoke:nebula-smoke /etc/nebula) || true'
280 )
281 '';
282
283 getPublicIp = node: ''
284 ${node}.succeed("ip --brief addr show eth1 | awk '{print $3}' | tail -n1 | cut -d/ -f1").strip()
285 '';
286
287 # Never do this for anything security critical! (Thankfully it's just a test.)
288 # Restart Nebula right after the mutual block and/or restore so the state is fresh.
289 blockTrafficBetween = nodeA: nodeB: ''
290 node_a = ${getPublicIp nodeA}
291 node_b = ${getPublicIp nodeB}
292 ${nodeA}.succeed("iptables -I INPUT -s " + node_b + " -j DROP")
293 ${nodeB}.succeed("iptables -I INPUT -s " + node_a + " -j DROP")
294 ${nodeA}.systemctl("restart nebula@smoke.service")
295 ${nodeB}.systemctl("restart nebula@smoke.service")
296 '';
297 allowTrafficBetween = nodeA: nodeB: ''
298 node_a = ${getPublicIp nodeA}
299 node_b = ${getPublicIp nodeB}
300 ${nodeA}.succeed("iptables -D INPUT -s " + node_b + " -j DROP")
301 ${nodeB}.succeed("iptables -D INPUT -s " + node_a + " -j DROP")
302 ${nodeA}.systemctl("restart nebula@smoke.service")
303 ${nodeB}.systemctl("restart nebula@smoke.service")
304 '';
305 in
306 ''
307 # Create the certificate and sign the lighthouse's keys.
308 ${setUpPrivateKey "lighthouse"}
309 lighthouse.succeed(
310 "mkdir -p /etc/nebula",
311 'nebula-cert ca -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key',
312 'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
313 'chown -R nebula-smoke:nebula-smoke /etc/nebula'
314 )
315
316 # Reboot the lighthouse and verify that the nebula service comes up on boot.
317 # Since rebooting takes a while, we'll just restart the service on the other nodes.
318 lighthouse.shutdown()
319 lighthouse.start()
320 lighthouse.wait_for_unit("nebula@smoke.service")
321 lighthouse.succeed("ping -c5 10.0.100.1")
322
323 # Create keys for allowAny's nebula service and test that it comes up.
324 ${setUpPrivateKey "allowAny"}
325 ${signKeysFor "allowAny" "10.0.100.2/24"}
326 ${restartAndCheckNebula "allowAny" "10.0.100.2"}
327
328 # Create keys for allowFromLighthouse's nebula service and test that it comes up.
329 ${setUpPrivateKey "allowFromLighthouse"}
330 ${signKeysFor "allowFromLighthouse" "10.0.100.3/24"}
331 ${restartAndCheckNebula "allowFromLighthouse" "10.0.100.3"}
332
333 # Create keys for allowToLighthouse's nebula service and test that it comes up.
334 ${setUpPrivateKey "allowToLighthouse"}
335 ${signKeysFor "allowToLighthouse" "10.0.100.4/24"}
336 ${restartAndCheckNebula "allowToLighthouse" "10.0.100.4"}
337
338 # Create keys for disabled's nebula service and test that it does not come up.
339 ${setUpPrivateKey "disabled"}
340 ${signKeysFor "disabled" "10.0.100.5/24"}
341 disabled.fail("systemctl status nebula@smoke.service")
342 disabled.fail("ping -c5 10.0.100.5")
343
344 # The lighthouse can ping allowAny and allowFromLighthouse but not disabled
345 lighthouse.succeed("ping -c3 10.0.100.2")
346 lighthouse.succeed("ping -c3 10.0.100.3")
347 lighthouse.fail("ping -c3 10.0.100.5")
348
349 # allowAny can ping the lighthouse, but not allowFromLighthouse because of its inbound firewall
350 allowAny.succeed("ping -c3 10.0.100.1")
351 allowAny.fail("ping -c3 10.0.100.3")
352 # allowAny can also resolve DNS on lighthouse
353 allowAny.succeed("dig @10.0.100.1 allowToLighthouse | grep -E 'allowToLighthouse\.\s+[0-9]+\s+IN\s+A\s+10\.0\.100\.4'")
354
355 # allowFromLighthouse can ping the lighthouse and allowAny
356 allowFromLighthouse.succeed("ping -c3 10.0.100.1")
357 allowFromLighthouse.succeed("ping -c3 10.0.100.2")
358
359 # block allowFromLighthouse <-> allowAny, and allowFromLighthouse -> allowAny should still work.
360 ${blockTrafficBetween "allowFromLighthouse" "allowAny"}
361 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
362 ${allowTrafficBetween "allowFromLighthouse" "allowAny"}
363 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
364
365 # allowToLighthouse can ping the lighthouse but not allowAny or allowFromLighthouse
366 allowToLighthouse.succeed("ping -c3 10.0.100.1")
367 allowToLighthouse.fail("ping -c3 10.0.100.2")
368 allowToLighthouse.fail("ping -c3 10.0.100.3")
369
370 # allowAny can ping allowFromLighthouse now that allowFromLighthouse pinged it first
371 allowAny.succeed("ping -c3 10.0.100.3")
372
373 # block allowAny <-> allowFromLighthouse, and allowAny -> allowFromLighthouse should still work.
374 ${blockTrafficBetween "allowAny" "allowFromLighthouse"}
375 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
376 allowAny.succeed("ping -c10 10.0.100.3")
377 ${allowTrafficBetween "allowAny" "allowFromLighthouse"}
378 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
379 allowAny.succeed("ping -c10 10.0.100.3")
380
381 # allowToLighthouse can ping allowAny if allowAny pings it first
382 allowAny.succeed("ping -c3 10.0.100.4")
383 allowToLighthouse.succeed("ping -c3 10.0.100.2")
384
385 # block allowToLighthouse <-> allowAny, and allowAny <-> allowToLighthouse should still work.
386 ${blockTrafficBetween "allowAny" "allowToLighthouse"}
387 allowAny.succeed("ping -c10 10.0.100.4")
388 allowToLighthouse.succeed("ping -c10 10.0.100.2")
389 ${allowTrafficBetween "allowAny" "allowToLighthouse"}
390 allowAny.succeed("ping -c10 10.0.100.4")
391 allowToLighthouse.succeed("ping -c10 10.0.100.2")
392
393 # block lighthouse <-> allowFromLighthouse and allowAny <-> allowFromLighthouse; allowFromLighthouse won't get to allowAny
394 ${blockTrafficBetween "allowFromLighthouse" "lighthouse"}
395 ${blockTrafficBetween "allowFromLighthouse" "allowAny"}
396 allowFromLighthouse.fail("ping -c3 10.0.100.2")
397 ${allowTrafficBetween "allowFromLighthouse" "lighthouse"}
398 ${allowTrafficBetween "allowFromLighthouse" "allowAny"}
399 allowFromLighthouse.succeed("ping -c3 10.0.100.2")
400
401 # block lighthouse <-> allowAny, allowAny <-> allowFromLighthouse, and allowAny <-> allowToLighthouse; it won't get to allowFromLighthouse or allowToLighthouse
402 ${blockTrafficBetween "allowAny" "lighthouse"}
403 ${blockTrafficBetween "allowAny" "allowFromLighthouse"}
404 ${blockTrafficBetween "allowAny" "allowToLighthouse"}
405 allowFromLighthouse.fail("ping -c3 10.0.100.2")
406 allowAny.fail("ping -c3 10.0.100.3")
407 allowAny.fail("ping -c3 10.0.100.4")
408 ${allowTrafficBetween "allowAny" "lighthouse"}
409 ${allowTrafficBetween "allowAny" "allowFromLighthouse"}
410 ${allowTrafficBetween "allowAny" "allowToLighthouse"}
411 allowFromLighthouse.succeed("ping -c3 10.0.100.2")
412 allowAny.succeed("ping -c3 10.0.100.3")
413 allowAny.succeed("ping -c3 10.0.100.4")
414
415 # block lighthouse <-> allowToLighthouse and allowToLighthouse <-> allowAny; it won't get to allowAny
416 ${blockTrafficBetween "allowToLighthouse" "lighthouse"}
417 ${blockTrafficBetween "allowToLighthouse" "allowAny"}
418 allowAny.fail("ping -c3 10.0.100.4")
419 allowToLighthouse.fail("ping -c3 10.0.100.2")
420 ${allowTrafficBetween "allowToLighthouse" "lighthouse"}
421 ${allowTrafficBetween "allowToLighthouse" "allowAny"}
422 allowAny.succeed("ping -c3 10.0.100.4")
423 allowToLighthouse.succeed("ping -c3 10.0.100.2")
424 '';
425 }
426)