pyrox.dev
1import ./make-test-python.nix ({ pkgs, lib, ... }: let
2
3 # We'll need to be able to trade cert files between nodes via scp.
4 inherit (import ./ssh-keys.nix pkgs)
5 snakeOilPrivateKey snakeOilPublicKey;
6
7 makeNebulaNode = { config, ... }: name: extraConfig: lib.mkMerge [
8 {
9 # Expose nebula for doing cert signing.
10 environment.systemPackages = [ pkgs.nebula ];
11 users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
12 services.openssh.enable = true;
13
14 services.nebula.networks.smoke = {
15 # Note that these paths won't exist when the machine is first booted.
16 ca = "/etc/nebula/ca.crt";
17 cert = "/etc/nebula/${name}.crt";
18 key = "/etc/nebula/${name}.key";
19 listen = { host = "0.0.0.0"; port = 4242; };
20 };
21 }
22 extraConfig
23 ];
24
25in
26{
27 name = "nebula";
28
29 nodes = {
30
31 lighthouse = { ... } @ args:
32 makeNebulaNode args "lighthouse" {
33 networking.interfaces.eth1.ipv4.addresses = [{
34 address = "192.168.1.1";
35 prefixLength = 24;
36 }];
37
38 services.nebula.networks.smoke = {
39 isLighthouse = true;
40 firewall = {
41 outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
42 inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
43 };
44 };
45 };
46
47 node2 = { ... } @ args:
48 makeNebulaNode args "node2" {
49 networking.interfaces.eth1.ipv4.addresses = [{
50 address = "192.168.1.2";
51 prefixLength = 24;
52 }];
53
54 services.nebula.networks.smoke = {
55 staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
56 isLighthouse = false;
57 lighthouses = [ "10.0.100.1" ];
58 firewall = {
59 outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
60 inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
61 };
62 };
63 };
64
65 node3 = { ... } @ args:
66 makeNebulaNode args "node3" {
67 networking.interfaces.eth1.ipv4.addresses = [{
68 address = "192.168.1.3";
69 prefixLength = 24;
70 }];
71
72 services.nebula.networks.smoke = {
73 staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
74 isLighthouse = false;
75 lighthouses = [ "10.0.100.1" ];
76 firewall = {
77 outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
78 inbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
79 };
80 };
81 };
82
83 node4 = { ... } @ args:
84 makeNebulaNode args "node4" {
85 networking.interfaces.eth1.ipv4.addresses = [{
86 address = "192.168.1.4";
87 prefixLength = 24;
88 }];
89
90 services.nebula.networks.smoke = {
91 enable = true;
92 staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
93 isLighthouse = false;
94 lighthouses = [ "10.0.100.1" ];
95 firewall = {
96 outbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
97 inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
98 };
99 };
100 };
101
102 node5 = { ... } @ args:
103 makeNebulaNode args "node5" {
104 networking.interfaces.eth1.ipv4.addresses = [{
105 address = "192.168.1.5";
106 prefixLength = 24;
107 }];
108
109 services.nebula.networks.smoke = {
110 enable = false;
111 staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
112 isLighthouse = false;
113 lighthouses = [ "10.0.100.1" ];
114 firewall = {
115 outbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
116 inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
117 };
118 };
119 };
120
121 };
122
123 testScript = let
124
125 setUpPrivateKey = name: ''
126 ${name}.succeed(
127 "mkdir -p /root/.ssh",
128 "chown 700 /root/.ssh",
129 "cat '${snakeOilPrivateKey}' > /root/.ssh/id_snakeoil",
130 "chown 600 /root/.ssh/id_snakeoil",
131 )
132 '';
133
134 # From what I can tell, StrictHostKeyChecking=no is necessary for ssh to work between machines.
135 sshOpts = "-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oIdentityFile=/root/.ssh/id_snakeoil";
136
137 restartAndCheckNebula = name: ip: ''
138 ${name}.systemctl("restart nebula@smoke.service")
139 ${name}.succeed("ping -c5 ${ip}")
140 '';
141
142 # Create a keypair on the client node, then use the public key to sign a cert on the lighthouse.
143 signKeysFor = name: ip: ''
144 lighthouse.wait_for_unit("sshd.service")
145 ${name}.wait_for_unit("sshd.service")
146 ${name}.succeed(
147 "mkdir -p /etc/nebula",
148 "nebula-cert keygen -out-key /etc/nebula/${name}.key -out-pub /etc/nebula/${name}.pub",
149 "scp ${sshOpts} /etc/nebula/${name}.pub 192.168.1.1:/tmp/${name}.pub",
150 )
151 lighthouse.succeed(
152 'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -ip "${ip}" -in-pub /tmp/${name}.pub -out-crt /tmp/${name}.crt',
153 )
154 ${name}.succeed(
155 "scp ${sshOpts} 192.168.1.1:/tmp/${name}.crt /etc/nebula/${name}.crt",
156 "scp ${sshOpts} 192.168.1.1:/etc/nebula/ca.crt /etc/nebula/ca.crt",
157 )
158 '';
159
160 in ''
161 start_all()
162
163 # Create the certificate and sign the lighthouse's keys.
164 ${setUpPrivateKey "lighthouse"}
165 lighthouse.succeed(
166 "mkdir -p /etc/nebula",
167 'nebula-cert ca -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key',
168 'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
169 )
170
171 # Reboot the lighthouse and verify that the nebula service comes up on boot.
172 # Since rebooting takes a while, we'll just restart the service on the other nodes.
173 lighthouse.shutdown()
174 lighthouse.start()
175 lighthouse.wait_for_unit("nebula@smoke.service")
176 lighthouse.succeed("ping -c5 10.0.100.1")
177
178 # Create keys for node2's nebula service and test that it comes up.
179 ${setUpPrivateKey "node2"}
180 ${signKeysFor "node2" "10.0.100.2/24"}
181 ${restartAndCheckNebula "node2" "10.0.100.2"}
182
183 # Create keys for node3's nebula service and test that it comes up.
184 ${setUpPrivateKey "node3"}
185 ${signKeysFor "node3" "10.0.100.3/24"}
186 ${restartAndCheckNebula "node3" "10.0.100.3"}
187
188 # Create keys for node4's nebula service and test that it comes up.
189 ${setUpPrivateKey "node4"}
190 ${signKeysFor "node4" "10.0.100.4/24"}
191 ${restartAndCheckNebula "node4" "10.0.100.4"}
192
193 # Create keys for node4's nebula service and test that it does not come up.
194 ${setUpPrivateKey "node5"}
195 ${signKeysFor "node5" "10.0.100.5/24"}
196 node5.fail("systemctl status nebula@smoke.service")
197 node5.fail("ping -c5 10.0.100.5")
198
199 # The lighthouse can ping node2 and node3 but not node5
200 lighthouse.succeed("ping -c3 10.0.100.2")
201 lighthouse.succeed("ping -c3 10.0.100.3")
202 lighthouse.fail("ping -c3 10.0.100.5")
203
204 # node2 can ping the lighthouse, but not node3 because of its inbound firewall
205 node2.succeed("ping -c3 10.0.100.1")
206 node2.fail("ping -c3 10.0.100.3")
207
208 # node3 can ping the lighthouse and node2
209 node3.succeed("ping -c3 10.0.100.1")
210 node3.succeed("ping -c3 10.0.100.2")
211
212 # node4 can ping the lighthouse but not node2 or node3
213 node4.succeed("ping -c3 10.0.100.1")
214 node4.fail("ping -c3 10.0.100.2")
215 node4.fail("ping -c3 10.0.100.3")
216
217 # node2 can ping node3 now that node3 pinged it first
218 node2.succeed("ping -c3 10.0.100.3")
219 # node4 can ping node2 if node2 pings it first
220 node2.succeed("ping -c3 10.0.100.4")
221 node4.succeed("ping -c3 10.0.100.2")
222 '';
223})