pyrox.dev
1{ pkgs, lib, ... }:
2let
3
4 # We'll need to be able to trade cert files between nodes via scp.
5 inherit (import ./ssh-keys.nix pkgs)
6 snakeOilPrivateKey
7 snakeOilPublicKey
8 ;
9
10 makeNebulaNode =
11 { config, ... }:
12 name: extraConfig:
13 lib.mkMerge [
14 {
15 # Expose nebula for doing cert signing.
16 environment.systemPackages = [
17 pkgs.dig
18 pkgs.nebula
19 ];
20 users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
21 services.openssh.enable = true;
22 networking.firewall.enable = true; # Implicitly true, but let's make sure.
23 networking.interfaces.eth1.useDHCP = false;
24
25 services.nebula.networks.smoke = {
26 # Note that these paths won't exist when the machine is first booted.
27 ca = "/etc/nebula/ca.crt";
28 cert = "/etc/nebula/${name}.crt";
29 key = "/etc/nebula/${name}.key";
30 listen = {
31 host = "0.0.0.0";
32 port =
33 if
34 (
35 config.services.nebula.networks.smoke.isLighthouse || config.services.nebula.networks.smoke.isRelay
36 )
37 then
38 4242
39 else
40 0;
41 };
42 };
43 }
44 extraConfig
45 ];
46
47in
48{
49 name = "nebula";
50
51 nodes = {
52
53 lighthouse =
54 { ... }@args:
55 makeNebulaNode args "lighthouse" {
56 networking.firewall.allowedUDPPorts = [ 53 ];
57 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
58 {
59 address = "192.168.1.1";
60 prefixLength = 24;
61 }
62 ];
63
64 services.nebula.networks.smoke = {
65 isLighthouse = true;
66 isRelay = true;
67 firewall = {
68 outbound = [
69 {
70 port = "any";
71 proto = "any";
72 host = "any";
73 }
74 ];
75 inbound = [
76 {
77 port = "any";
78 proto = "any";
79 host = "any";
80 }
81 ];
82 };
83 lighthouse = {
84 dns = {
85 enable = true;
86 host = "10.0.100.1"; # bind to lighthouse interface
87 port = 53; # answer on standard DNS port
88 };
89 };
90 };
91 };
92
93 allowAny =
94 { ... }@args:
95 makeNebulaNode args "allowAny" {
96 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
97 {
98 address = "192.168.1.2";
99 prefixLength = 24;
100 }
101 ];
102
103 services.nebula.networks.smoke = {
104 staticHostMap = {
105 "10.0.100.1" = [ "192.168.1.1:4242" ];
106 };
107 isLighthouse = false;
108 lighthouses = [ "10.0.100.1" ];
109 relays = [ "10.0.100.1" ];
110 firewall = {
111 outbound = [
112 {
113 port = "any";
114 proto = "any";
115 host = "any";
116 }
117 ];
118 inbound = [
119 {
120 port = "any";
121 proto = "any";
122 host = "any";
123 }
124 ];
125 };
126 };
127 };
128
129 allowFromLighthouse =
130 { ... }@args:
131 makeNebulaNode args "allowFromLighthouse" {
132 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
133 {
134 address = "192.168.1.3";
135 prefixLength = 24;
136 }
137 ];
138
139 services.nebula.networks.smoke = {
140 staticHostMap = {
141 "10.0.100.1" = [ "192.168.1.1:4242" ];
142 };
143 isLighthouse = false;
144 lighthouses = [ "10.0.100.1" ];
145 relays = [ "10.0.100.1" ];
146 firewall = {
147 outbound = [
148 {
149 port = "any";
150 proto = "any";
151 host = "any";
152 }
153 ];
154 inbound = [
155 {
156 port = "any";
157 proto = "any";
158 host = "lighthouse";
159 }
160 ];
161 };
162 };
163 };
164
165 allowToLighthouse =
166 { ... }@args:
167 makeNebulaNode args "allowToLighthouse" {
168 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
169 {
170 address = "192.168.1.4";
171 prefixLength = 24;
172 }
173 ];
174
175 services.nebula.networks.smoke = {
176 enable = true;
177 staticHostMap = {
178 "10.0.100.1" = [ "192.168.1.1:4242" ];
179 };
180 isLighthouse = false;
181 lighthouses = [ "10.0.100.1" ];
182 relays = [ "10.0.100.1" ];
183 firewall = {
184 outbound = [
185 {
186 port = "any";
187 proto = "any";
188 host = "lighthouse";
189 }
190 ];
191 inbound = [
192 {
193 port = "any";
194 proto = "any";
195 host = "any";
196 }
197 ];
198 };
199 };
200 };
201
202 disabled =
203 { ... }@args:
204 makeNebulaNode args "disabled" {
205 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
206 {
207 address = "192.168.1.5";
208 prefixLength = 24;
209 }
210 ];
211
212 services.nebula.networks.smoke = {
213 enable = false;
214 staticHostMap = {
215 "10.0.100.1" = [ "192.168.1.1:4242" ];
216 };
217 isLighthouse = false;
218 lighthouses = [ "10.0.100.1" ];
219 relays = [ "10.0.100.1" ];
220 firewall = {
221 outbound = [
222 {
223 port = "any";
224 proto = "any";
225 host = "lighthouse";
226 }
227 ];
228 inbound = [
229 {
230 port = "any";
231 proto = "any";
232 host = "any";
233 }
234 ];
235 };
236 };
237 };
238
239 };
240
241 testScript =
242 let
243
244 setUpPrivateKey = name: ''
245 ${name}.start()
246 ${name}.succeed(
247 "mkdir -p /root/.ssh",
248 "chmod 700 /root/.ssh",
249 "cat '${snakeOilPrivateKey}' > /root/.ssh/id_snakeoil",
250 "chmod 600 /root/.ssh/id_snakeoil",
251 "mkdir -p /root"
252 )
253 '';
254
255 # From what I can tell, StrictHostKeyChecking=no is necessary for ssh to work between machines.
256 sshOpts = "-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oIdentityFile=/root/.ssh/id_snakeoil";
257
258 restartAndCheckNebula = name: ip: ''
259 ${name}.systemctl("restart nebula@smoke.service")
260 ${name}.succeed("ping -c5 ${ip}")
261 '';
262
263 # Create a keypair on the client node, then use the public key to sign a cert on the lighthouse.
264 signKeysFor = name: ip: ''
265 lighthouse.wait_for_unit("sshd.service")
266 ${name}.wait_for_unit("sshd.service")
267 ${name}.succeed(
268 "mkdir -p /etc/nebula",
269 "nebula-cert keygen -out-key /etc/nebula/${name}.key -out-pub /etc/nebula/${name}.pub",
270 "scp ${sshOpts} /etc/nebula/${name}.pub root@192.168.1.1:/root/${name}.pub",
271 )
272 lighthouse.succeed(
273 'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -ip "${ip}" -in-pub /root/${name}.pub -out-crt /root/${name}.crt'
274 )
275 ${name}.succeed(
276 "scp ${sshOpts} root@192.168.1.1:/root/${name}.crt /etc/nebula/${name}.crt",
277 "scp ${sshOpts} root@192.168.1.1:/etc/nebula/ca.crt /etc/nebula/ca.crt",
278 '(id nebula-smoke >/dev/null && chown -R nebula-smoke:nebula-smoke /etc/nebula) || true'
279 )
280 '';
281
282 getPublicIp = node: ''
283 ${node}.succeed("ip --brief addr show eth1 | awk '{print $3}' | tail -n1 | cut -d/ -f1").strip()
284 '';
285
286 # Never do this for anything security critical! (Thankfully it's just a test.)
287 # Restart Nebula right after the mutual block and/or restore so the state is fresh.
288 blockTrafficBetween = nodeA: nodeB: ''
289 node_a = ${getPublicIp nodeA}
290 node_b = ${getPublicIp nodeB}
291 ${nodeA}.succeed("iptables -I INPUT -s " + node_b + " -j DROP")
292 ${nodeB}.succeed("iptables -I INPUT -s " + node_a + " -j DROP")
293 ${nodeA}.systemctl("restart nebula@smoke.service")
294 ${nodeB}.systemctl("restart nebula@smoke.service")
295 '';
296 allowTrafficBetween = nodeA: nodeB: ''
297 node_a = ${getPublicIp nodeA}
298 node_b = ${getPublicIp nodeB}
299 ${nodeA}.succeed("iptables -D INPUT -s " + node_b + " -j DROP")
300 ${nodeB}.succeed("iptables -D INPUT -s " + node_a + " -j DROP")
301 ${nodeA}.systemctl("restart nebula@smoke.service")
302 ${nodeB}.systemctl("restart nebula@smoke.service")
303 '';
304 in
305 ''
306 # Create the certificate and sign the lighthouse's keys.
307 ${setUpPrivateKey "lighthouse"}
308 lighthouse.succeed(
309 "mkdir -p /etc/nebula",
310 'nebula-cert ca -duration $((10*365*24*60))m -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key',
311 'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
312 'chown -R nebula-smoke:nebula-smoke /etc/nebula'
313 )
314
315 # Reboot the lighthouse and verify that the nebula service comes up on boot.
316 # Since rebooting takes a while, we'll just restart the service on the other nodes.
317 lighthouse.shutdown()
318 lighthouse.start()
319 lighthouse.wait_for_unit("nebula@smoke.service")
320 lighthouse.succeed("ping -c5 10.0.100.1")
321
322 # Create keys for allowAny's nebula service and test that it comes up.
323 ${setUpPrivateKey "allowAny"}
324 ${signKeysFor "allowAny" "10.0.100.2/24"}
325 ${restartAndCheckNebula "allowAny" "10.0.100.2"}
326
327 # Create keys for allowFromLighthouse's nebula service and test that it comes up.
328 ${setUpPrivateKey "allowFromLighthouse"}
329 ${signKeysFor "allowFromLighthouse" "10.0.100.3/24"}
330 ${restartAndCheckNebula "allowFromLighthouse" "10.0.100.3"}
331
332 # Create keys for allowToLighthouse's nebula service and test that it comes up.
333 ${setUpPrivateKey "allowToLighthouse"}
334 ${signKeysFor "allowToLighthouse" "10.0.100.4/24"}
335 ${restartAndCheckNebula "allowToLighthouse" "10.0.100.4"}
336
337 # Create keys for disabled's nebula service and test that it does not come up.
338 ${setUpPrivateKey "disabled"}
339 ${signKeysFor "disabled" "10.0.100.5/24"}
340 disabled.fail("systemctl status nebula@smoke.service")
341 disabled.fail("ping -c5 10.0.100.5")
342
343 # The lighthouse can ping allowAny and allowFromLighthouse but not disabled
344 lighthouse.succeed("ping -c3 10.0.100.2")
345 lighthouse.succeed("ping -c3 10.0.100.3")
346 lighthouse.fail("ping -c3 10.0.100.5")
347
348 # allowAny can ping the lighthouse, but not allowFromLighthouse because of its inbound firewall
349 allowAny.succeed("ping -c3 10.0.100.1")
350 allowAny.fail("ping -c3 10.0.100.3")
351 # allowAny can also resolve DNS on lighthouse
352 allowAny.succeed("dig @10.0.100.1 allowToLighthouse | grep -E 'allowToLighthouse\.\s+[0-9]+\s+IN\s+A\s+10\.0\.100\.4'")
353
354 # allowFromLighthouse can ping the lighthouse and allowAny
355 allowFromLighthouse.succeed("ping -c3 10.0.100.1")
356 allowFromLighthouse.succeed("ping -c3 10.0.100.2")
357
358 # block allowFromLighthouse <-> allowAny, and allowFromLighthouse -> allowAny should still work.
359 ${blockTrafficBetween "allowFromLighthouse" "allowAny"}
360 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
361 ${allowTrafficBetween "allowFromLighthouse" "allowAny"}
362 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
363
364 # allowToLighthouse can ping the lighthouse but not allowAny or allowFromLighthouse
365 allowToLighthouse.succeed("ping -c3 10.0.100.1")
366 allowToLighthouse.fail("ping -c3 10.0.100.2")
367 allowToLighthouse.fail("ping -c3 10.0.100.3")
368
369 # allowAny can ping allowFromLighthouse now that allowFromLighthouse pinged it first
370 allowAny.succeed("ping -c3 10.0.100.3")
371
372 # block allowAny <-> allowFromLighthouse, and allowAny -> allowFromLighthouse should still work.
373 ${blockTrafficBetween "allowAny" "allowFromLighthouse"}
374 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
375 allowAny.succeed("ping -c10 10.0.100.3")
376 ${allowTrafficBetween "allowAny" "allowFromLighthouse"}
377 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
378 allowAny.succeed("ping -c10 10.0.100.3")
379
380 # allowToLighthouse can ping allowAny if allowAny pings it first
381 allowAny.succeed("ping -c3 10.0.100.4")
382 allowToLighthouse.succeed("ping -c3 10.0.100.2")
383
384 # block allowToLighthouse <-> allowAny, and allowAny <-> allowToLighthouse should still work.
385 ${blockTrafficBetween "allowAny" "allowToLighthouse"}
386 allowAny.succeed("ping -c10 10.0.100.4")
387 allowToLighthouse.succeed("ping -c10 10.0.100.2")
388 ${allowTrafficBetween "allowAny" "allowToLighthouse"}
389 allowAny.succeed("ping -c10 10.0.100.4")
390 allowToLighthouse.succeed("ping -c10 10.0.100.2")
391
392 # block lighthouse <-> allowFromLighthouse and allowAny <-> allowFromLighthouse; allowFromLighthouse won't get to allowAny
393 ${blockTrafficBetween "allowFromLighthouse" "lighthouse"}
394 ${blockTrafficBetween "allowFromLighthouse" "allowAny"}
395 allowFromLighthouse.fail("ping -c3 10.0.100.2")
396 ${allowTrafficBetween "allowFromLighthouse" "lighthouse"}
397 ${allowTrafficBetween "allowFromLighthouse" "allowAny"}
398 allowFromLighthouse.succeed("ping -c3 10.0.100.2")
399
400 # block lighthouse <-> allowAny, allowAny <-> allowFromLighthouse, and allowAny <-> allowToLighthouse; it won't get to allowFromLighthouse or allowToLighthouse
401 ${blockTrafficBetween "allowAny" "lighthouse"}
402 ${blockTrafficBetween "allowAny" "allowFromLighthouse"}
403 ${blockTrafficBetween "allowAny" "allowToLighthouse"}
404 allowFromLighthouse.fail("ping -c3 10.0.100.2")
405 allowAny.fail("ping -c3 10.0.100.3")
406 allowAny.fail("ping -c3 10.0.100.4")
407 ${allowTrafficBetween "allowAny" "lighthouse"}
408 ${allowTrafficBetween "allowAny" "allowFromLighthouse"}
409 ${allowTrafficBetween "allowAny" "allowToLighthouse"}
410 allowFromLighthouse.succeed("ping -c3 10.0.100.2")
411 allowAny.succeed("ping -c3 10.0.100.3")
412 allowAny.succeed("ping -c3 10.0.100.4")
413
414 # block lighthouse <-> allowToLighthouse and allowToLighthouse <-> allowAny; it won't get to allowAny
415 ${blockTrafficBetween "allowToLighthouse" "lighthouse"}
416 ${blockTrafficBetween "allowToLighthouse" "allowAny"}
417 allowAny.fail("ping -c3 10.0.100.4")
418 allowToLighthouse.fail("ping -c3 10.0.100.2")
419 ${allowTrafficBetween "allowToLighthouse" "lighthouse"}
420 ${allowTrafficBetween "allowToLighthouse" "allowAny"}
421 allowAny.succeed("ping -c3 10.0.100.4")
422 allowToLighthouse.succeed("ping -c3 10.0.100.2")
423 '';
424}