pyrox.dev
1import ./make-test-python.nix ({ pkgs, lib, ... }: let
2
3 # We'll need to be able to trade cert files between nodes via scp.
4 inherit (import ./ssh-keys.nix pkgs)
5 snakeOilPrivateKey snakeOilPublicKey;
6
7 makeNebulaNode = { config, ... }: name: extraConfig: lib.mkMerge [
8 {
9 # Expose nebula for doing cert signing.
10 environment.systemPackages = [ pkgs.nebula ];
11 users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
12 services.openssh.enable = true;
13 networking.interfaces.eth1.useDHCP = false;
14
15 services.nebula.networks.smoke = {
16 # Note that these paths won't exist when the machine is first booted.
17 ca = "/etc/nebula/ca.crt";
18 cert = "/etc/nebula/${name}.crt";
19 key = "/etc/nebula/${name}.key";
20 listen = { host = "0.0.0.0"; port = 4242; };
21 };
22 }
23 extraConfig
24 ];
25
26in
27{
28 name = "nebula";
29
30 nodes = {
31
32 lighthouse = { ... } @ args:
33 makeNebulaNode args "lighthouse" {
34 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
35 address = "192.168.1.1";
36 prefixLength = 24;
37 }];
38
39 services.nebula.networks.smoke = {
40 isLighthouse = true;
41 isRelay = true;
42 firewall = {
43 outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
44 inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
45 };
46 };
47 };
48
49 allowAny = { ... } @ args:
50 makeNebulaNode args "allowAny" {
51 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
52 address = "192.168.1.2";
53 prefixLength = 24;
54 }];
55
56 services.nebula.networks.smoke = {
57 staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
58 isLighthouse = false;
59 lighthouses = [ "10.0.100.1" ];
60 relays = [ "10.0.100.1" ];
61 firewall = {
62 outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
63 inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
64 };
65 };
66 };
67
68 allowFromLighthouse = { ... } @ args:
69 makeNebulaNode args "allowFromLighthouse" {
70 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
71 address = "192.168.1.3";
72 prefixLength = 24;
73 }];
74
75 services.nebula.networks.smoke = {
76 staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
77 isLighthouse = false;
78 lighthouses = [ "10.0.100.1" ];
79 relays = [ "10.0.100.1" ];
80 firewall = {
81 outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
82 inbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
83 };
84 };
85 };
86
87 allowToLighthouse = { ... } @ args:
88 makeNebulaNode args "allowToLighthouse" {
89 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
90 address = "192.168.1.4";
91 prefixLength = 24;
92 }];
93
94 services.nebula.networks.smoke = {
95 enable = true;
96 staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
97 isLighthouse = false;
98 lighthouses = [ "10.0.100.1" ];
99 relays = [ "10.0.100.1" ];
100 firewall = {
101 outbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
102 inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
103 };
104 };
105 };
106
107 disabled = { ... } @ args:
108 makeNebulaNode args "disabled" {
109 networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
110 address = "192.168.1.5";
111 prefixLength = 24;
112 }];
113
114 services.nebula.networks.smoke = {
115 enable = false;
116 staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
117 isLighthouse = false;
118 lighthouses = [ "10.0.100.1" ];
119 relays = [ "10.0.100.1" ];
120 firewall = {
121 outbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
122 inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
123 };
124 };
125 };
126
127 };
128
129 testScript = let
130
131 setUpPrivateKey = name: ''
132 ${name}.start()
133 ${name}.succeed(
134 "mkdir -p /root/.ssh",
135 "chown 700 /root/.ssh",
136 "cat '${snakeOilPrivateKey}' > /root/.ssh/id_snakeoil",
137 "chown 600 /root/.ssh/id_snakeoil",
138 "mkdir -p /root"
139 )
140 '';
141
142 # From what I can tell, StrictHostKeyChecking=no is necessary for ssh to work between machines.
143 sshOpts = "-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oIdentityFile=/root/.ssh/id_snakeoil";
144
145 restartAndCheckNebula = name: ip: ''
146 ${name}.systemctl("restart nebula@smoke.service")
147 ${name}.succeed("ping -c5 ${ip}")
148 '';
149
150 # Create a keypair on the client node, then use the public key to sign a cert on the lighthouse.
151 signKeysFor = name: ip: ''
152 lighthouse.wait_for_unit("sshd.service")
153 ${name}.wait_for_unit("sshd.service")
154 ${name}.succeed(
155 "mkdir -p /etc/nebula",
156 "nebula-cert keygen -out-key /etc/nebula/${name}.key -out-pub /etc/nebula/${name}.pub",
157 "scp ${sshOpts} /etc/nebula/${name}.pub root@192.168.1.1:/root/${name}.pub",
158 )
159 lighthouse.succeed(
160 'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -ip "${ip}" -in-pub /root/${name}.pub -out-crt /root/${name}.crt'
161 )
162 ${name}.succeed(
163 "scp ${sshOpts} root@192.168.1.1:/root/${name}.crt /etc/nebula/${name}.crt",
164 "scp ${sshOpts} root@192.168.1.1:/etc/nebula/ca.crt /etc/nebula/ca.crt",
165 '(id nebula-smoke >/dev/null && chown -R nebula-smoke:nebula-smoke /etc/nebula) || true'
166 )
167 '';
168
169 getPublicIp = node: ''
170 ${node}.succeed("ip --brief addr show eth1 | awk '{print $3}' | tail -n1 | cut -d/ -f1").strip()
171 '';
172
173 # Never do this for anything security critical! (Thankfully it's just a test.)
174 # Restart Nebula right after the mutual block and/or restore so the state is fresh.
175 blockTrafficBetween = nodeA: nodeB: ''
176 node_a = ${getPublicIp nodeA}
177 node_b = ${getPublicIp nodeB}
178 ${nodeA}.succeed("iptables -I INPUT -s " + node_b + " -j DROP")
179 ${nodeB}.succeed("iptables -I INPUT -s " + node_a + " -j DROP")
180 ${nodeA}.systemctl("restart nebula@smoke.service")
181 ${nodeB}.systemctl("restart nebula@smoke.service")
182 '';
183 allowTrafficBetween = nodeA: nodeB: ''
184 node_a = ${getPublicIp nodeA}
185 node_b = ${getPublicIp nodeB}
186 ${nodeA}.succeed("iptables -D INPUT -s " + node_b + " -j DROP")
187 ${nodeB}.succeed("iptables -D INPUT -s " + node_a + " -j DROP")
188 ${nodeA}.systemctl("restart nebula@smoke.service")
189 ${nodeB}.systemctl("restart nebula@smoke.service")
190 '';
191 in ''
192 # Create the certificate and sign the lighthouse's keys.
193 ${setUpPrivateKey "lighthouse"}
194 lighthouse.succeed(
195 "mkdir -p /etc/nebula",
196 'nebula-cert ca -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key',
197 'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
198 'chown -R nebula-smoke:nebula-smoke /etc/nebula'
199 )
200
201 # Reboot the lighthouse and verify that the nebula service comes up on boot.
202 # Since rebooting takes a while, we'll just restart the service on the other nodes.
203 lighthouse.shutdown()
204 lighthouse.start()
205 lighthouse.wait_for_unit("nebula@smoke.service")
206 lighthouse.succeed("ping -c5 10.0.100.1")
207
208 # Create keys for allowAny's nebula service and test that it comes up.
209 ${setUpPrivateKey "allowAny"}
210 ${signKeysFor "allowAny" "10.0.100.2/24"}
211 ${restartAndCheckNebula "allowAny" "10.0.100.2"}
212
213 # Create keys for allowFromLighthouse's nebula service and test that it comes up.
214 ${setUpPrivateKey "allowFromLighthouse"}
215 ${signKeysFor "allowFromLighthouse" "10.0.100.3/24"}
216 ${restartAndCheckNebula "allowFromLighthouse" "10.0.100.3"}
217
218 # Create keys for allowToLighthouse's nebula service and test that it comes up.
219 ${setUpPrivateKey "allowToLighthouse"}
220 ${signKeysFor "allowToLighthouse" "10.0.100.4/24"}
221 ${restartAndCheckNebula "allowToLighthouse" "10.0.100.4"}
222
223 # Create keys for disabled's nebula service and test that it does not come up.
224 ${setUpPrivateKey "disabled"}
225 ${signKeysFor "disabled" "10.0.100.5/24"}
226 disabled.fail("systemctl status nebula@smoke.service")
227 disabled.fail("ping -c5 10.0.100.5")
228
229 # The lighthouse can ping allowAny and allowFromLighthouse but not disabled
230 lighthouse.succeed("ping -c3 10.0.100.2")
231 lighthouse.succeed("ping -c3 10.0.100.3")
232 lighthouse.fail("ping -c3 10.0.100.5")
233
234 # allowAny can ping the lighthouse, but not allowFromLighthouse because of its inbound firewall
235 allowAny.succeed("ping -c3 10.0.100.1")
236 allowAny.fail("ping -c3 10.0.100.3")
237
238 # allowFromLighthouse can ping the lighthouse and allowAny
239 allowFromLighthouse.succeed("ping -c3 10.0.100.1")
240 allowFromLighthouse.succeed("ping -c3 10.0.100.2")
241
242 # block allowFromLighthouse <-> allowAny, and allowFromLighthouse -> allowAny should still work.
243 ${blockTrafficBetween "allowFromLighthouse" "allowAny"}
244 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
245 ${allowTrafficBetween "allowFromLighthouse" "allowAny"}
246 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
247
248 # allowToLighthouse can ping the lighthouse but not allowAny or allowFromLighthouse
249 allowToLighthouse.succeed("ping -c3 10.0.100.1")
250 allowToLighthouse.fail("ping -c3 10.0.100.2")
251 allowToLighthouse.fail("ping -c3 10.0.100.3")
252
253 # allowAny can ping allowFromLighthouse now that allowFromLighthouse pinged it first
254 allowAny.succeed("ping -c3 10.0.100.3")
255
256 # block allowAny <-> allowFromLighthouse, and allowAny -> allowFromLighthouse should still work.
257 ${blockTrafficBetween "allowAny" "allowFromLighthouse"}
258 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
259 allowAny.succeed("ping -c10 10.0.100.3")
260 ${allowTrafficBetween "allowAny" "allowFromLighthouse"}
261 allowFromLighthouse.succeed("ping -c10 10.0.100.2")
262 allowAny.succeed("ping -c10 10.0.100.3")
263
264 # allowToLighthouse can ping allowAny if allowAny pings it first
265 allowAny.succeed("ping -c3 10.0.100.4")
266 allowToLighthouse.succeed("ping -c3 10.0.100.2")
267
268 # block allowToLighthouse <-> allowAny, and allowAny <-> allowToLighthouse should still work.
269 ${blockTrafficBetween "allowAny" "allowToLighthouse"}
270 allowAny.succeed("ping -c10 10.0.100.4")
271 allowToLighthouse.succeed("ping -c10 10.0.100.2")
272 ${allowTrafficBetween "allowAny" "allowToLighthouse"}
273 allowAny.succeed("ping -c10 10.0.100.4")
274 allowToLighthouse.succeed("ping -c10 10.0.100.2")
275
276 # block lighthouse <-> allowFromLighthouse and allowAny <-> allowFromLighthouse; allowFromLighthouse won't get to allowAny
277 ${blockTrafficBetween "allowFromLighthouse" "lighthouse"}
278 ${blockTrafficBetween "allowFromLighthouse" "allowAny"}
279 allowFromLighthouse.fail("ping -c3 10.0.100.2")
280 ${allowTrafficBetween "allowFromLighthouse" "lighthouse"}
281 ${allowTrafficBetween "allowFromLighthouse" "allowAny"}
282 allowFromLighthouse.succeed("ping -c3 10.0.100.2")
283
284 # block lighthouse <-> allowAny, allowAny <-> allowFromLighthouse, and allowAny <-> allowToLighthouse; it won't get to allowFromLighthouse or allowToLighthouse
285 ${blockTrafficBetween "allowAny" "lighthouse"}
286 ${blockTrafficBetween "allowAny" "allowFromLighthouse"}
287 ${blockTrafficBetween "allowAny" "allowToLighthouse"}
288 allowFromLighthouse.fail("ping -c3 10.0.100.2")
289 allowAny.fail("ping -c3 10.0.100.3")
290 allowAny.fail("ping -c3 10.0.100.4")
291 ${allowTrafficBetween "allowAny" "lighthouse"}
292 ${allowTrafficBetween "allowAny" "allowFromLighthouse"}
293 ${allowTrafficBetween "allowAny" "allowToLighthouse"}
294 allowFromLighthouse.succeed("ping -c3 10.0.100.2")
295 allowAny.succeed("ping -c3 10.0.100.3")
296 allowAny.succeed("ping -c3 10.0.100.4")
297
298 # block lighthouse <-> allowToLighthouse and allowToLighthouse <-> allowAny; it won't get to allowAny
299 ${blockTrafficBetween "allowToLighthouse" "lighthouse"}
300 ${blockTrafficBetween "allowToLighthouse" "allowAny"}
301 allowAny.fail("ping -c3 10.0.100.4")
302 allowToLighthouse.fail("ping -c3 10.0.100.2")
303 ${allowTrafficBetween "allowToLighthouse" "lighthouse"}
304 ${allowTrafficBetween "allowToLighthouse" "allowAny"}
305 allowAny.succeed("ping -c3 10.0.100.4")
306 allowToLighthouse.succeed("ping -c3 10.0.100.2")
307 '';
308})