pyrox.dev
1# This is a distributed test of the Network Address Translation involving a topology
2# with a router inbetween three separate virtual networks:
3# - "external" -- i.e. the internet,
4# - "internal" -- i.e. an office LAN,
5#
6# This test puts one server on each of those networks and its primary goal is to ensure that:
7# - server (named client in the code) in internal network can reach server (named server in the code) on the external network,
8# - server in external network can not reach server in internal network (skipped in some cases),
9# - when using externalIP, only the specified IP is used for NAT,
10# - port forwarding functionality behaves correctly
11#
12# The client is behind the nat (read: protected by the nat) and the server is on the external network, attempting to access services behind the NAT.
13
14import ./make-test-python.nix (
15 {
16 pkgs,
17 lib,
18 withFirewall ? false,
19 nftables ? false,
20 ...
21 }:
22 let
23 unit = if nftables then "nftables" else (if withFirewall then "firewall" else "nat");
24
25 routerAlternativeExternalIp = "192.168.2.234";
26
27 makeNginxConfig = hostname: {
28 enable = true;
29 virtualHosts."${hostname}" = {
30 root = "/etc";
31 locations."/".index = "hostname";
32 listen = [
33 {
34 addr = "0.0.0.0";
35 port = 80;
36 }
37 {
38 addr = "0.0.0.0";
39 port = 8080;
40 }
41 ];
42 };
43 };
44
45 makeCommonConfig = hostname: {
46 services.nginx = makeNginxConfig hostname;
47 services.vsftpd = {
48 enable = true;
49 anonymousUser = true;
50 localRoot = "/etc/";
51 extraConfig = ''
52 pasv_min_port=51000
53 pasv_max_port=51999
54 '';
55 };
56
57 # Disable eth0 autoconfiguration
58 networking.useDHCP = false;
59
60 environment.systemPackages = [
61 (pkgs.writeScriptBin "check-connection" ''
62 #!/usr/bin/env bash
63
64 set -e
65
66 if [[ "$2" == "" || "$3" == "" || "$1" == "--help" || "$1" == "-h" ]];
67 then
68 echo "check-connection <target-address> <target-hostname> <[expect-success|expect-failure]>"
69 exit 1
70 fi
71
72 ADDRESS="$1"
73 HOSTNAME="$2"
74
75 function test_icmp() { timeout 3 ping -c 1 $ADDRESS; }
76 function test_http() { [[ `timeout 3 curl $ADDRESS` == "$HOSTNAME" ]]; }
77 function test_ftp() { timeout 3 curl ftp://$ADDRESS; }
78
79 if [[ "$3" == "expect-success" ]];
80 then
81 test_icmp; test_http; test_ftp
82 else
83 ! test_icmp; ! test_http; ! test_ftp
84 fi
85 '')
86 (pkgs.writeScriptBin "check-last-clients-ip" ''
87 #!/usr/bin/env bash
88 set -e
89
90 [[ `cat /var/log/nginx/access.log | tail -n1 | awk '{print $1}'` == "$1" ]]
91 '')
92 ];
93 };
94
95 in
96 # VLANS:
97 # 1 -- simulates the internal network
98 # 2 -- simulates the external network
99 {
100 name =
101 "nat"
102 + (lib.optionalString nftables "Nftables")
103 + (if withFirewall then "WithFirewall" else "Standalone");
104 meta = with pkgs.lib.maintainers; {
105 maintainers = [
106 tne
107 rob
108 ];
109 };
110
111 nodes = {
112 client =
113 { pkgs, nodes, ... }:
114 lib.mkMerge [
115 (makeCommonConfig "client")
116 {
117 virtualisation.vlans = [ 1 ];
118 networking.defaultGateway =
119 (pkgs.lib.head nodes.router.networking.interfaces.eth1.ipv4.addresses).address;
120 networking.nftables.enable = nftables;
121 networking.firewall.enable = false;
122 }
123 ];
124
125 router =
126 { nodes, ... }:
127 lib.mkMerge [
128 (makeCommonConfig "router")
129 {
130 virtualisation.vlans = [
131 1
132 2
133 ];
134 networking.firewall = {
135 enable = withFirewall;
136 filterForward = nftables;
137 allowedTCPPorts = [
138 21
139 80
140 8080
141 ];
142 # For FTP passive mode
143 allowedTCPPortRanges = [
144 {
145 from = 51000;
146 to = 51999;
147 }
148 ];
149 };
150 networking.nftables.enable = nftables;
151 networking.nat =
152 let
153 clientIp = (pkgs.lib.head nodes.client.networking.interfaces.eth1.ipv4.addresses).address;
154 serverIp = (pkgs.lib.head nodes.router.networking.interfaces.eth2.ipv4.addresses).address;
155 in
156 {
157 enable = true;
158 internalIPs = [ "${clientIp}/24" ];
159 # internalInterfaces = [ "eth1" ];
160 externalInterface = "eth2";
161 externalIP = serverIp;
162
163 forwardPorts = [
164 {
165 destination = "${clientIp}:8080";
166 proto = "tcp";
167 sourcePort = 8080;
168
169 loopbackIPs = [ serverIp ];
170 }
171 ];
172 };
173
174 networking.interfaces.eth2.ipv4.addresses = lib.mkOrder 10000 [
175 {
176 address = routerAlternativeExternalIp;
177 prefixLength = 24;
178 }
179 ];
180
181 services.nginx.virtualHosts.router.listen = lib.mkOrder (-1) [
182 {
183 addr = routerAlternativeExternalIp;
184 port = 8080;
185 }
186 ];
187
188 specialisation.no-nat.configuration = {
189 networking.nat.enable = lib.mkForce false;
190 };
191 }
192 ];
193
194 server =
195 { nodes, ... }:
196 lib.mkMerge [
197 (makeCommonConfig "server")
198 {
199 virtualisation.vlans = [ 2 ];
200 networking.firewall.enable = false;
201
202 networking.defaultGateway =
203 (pkgs.lib.head nodes.router.networking.interfaces.eth2.ipv4.addresses).address;
204 }
205 ];
206 };
207
208 testScript =
209 { nodes, ... }:
210 let
211 clientIp = (pkgs.lib.head nodes.client.networking.interfaces.eth1.ipv4.addresses).address;
212 serverIp = (pkgs.lib.head nodes.server.networking.interfaces.eth1.ipv4.addresses).address;
213 routerIp = (pkgs.lib.head nodes.router.networking.interfaces.eth2.ipv4.addresses).address;
214 in
215 ''
216 def wait_for_machine(m):
217 m.wait_for_unit("network.target")
218 m.wait_for_unit("nginx.service")
219
220 client.start()
221 router.start()
222 server.start()
223
224 wait_for_machine(router)
225 wait_for_machine(client)
226 wait_for_machine(server)
227
228 # We assume we are isolated from layer 2 attacks or are securely configured (like disabling forwarding by default)
229 # Relevant moby issue describing the problem allowing bypassing of NAT: https://github.com/moby/moby/issues/14041
230 ${lib.optionalString (!nftables) ''
231 router.succeed("iptables -P FORWARD DROP")
232 ''}
233
234 # Sanity checks.
235 ## The router should have direct access to the server
236 router.succeed("check-connection ${serverIp} server expect-success")
237 ## The server should have direct access to the router
238 server.succeed("check-connection ${routerIp} router expect-success")
239
240 # The client should be also able to connect via the NAT router...
241 client.succeed("check-connection ${serverIp} server expect-success")
242 # ... but its IP should be rewritten to be that of the router.
243 server.succeed("check-last-clients-ip ${routerIp}")
244
245 # Active FTP (where the FTP server connects back to us via a random port) should work directly...
246 router.succeed("timeout 3 curl -P eth2:51000-51999 ftp://${serverIp}")
247 # ... but not from behind NAT.
248 client.fail("timeout 3 curl -P eth1:51000-51999 ftp://${serverIp};")
249
250 # If using nftables without firewall, filterForward can't be used and L2 security can't easily be simulated like with iptables, skipping.
251 # See moby github issue mentioned above.
252 ${lib.optionalString (nftables && withFirewall) ''
253 # The server should not be able to reach the client directly...
254 server.succeed("check-connection ${clientIp} client expect-failure")
255 ''}
256 # ... but the server should be able to reach a port forwarded address of the client
257 server.succeed('[[ `timeout 3 curl http://${routerIp}:8080` == "client" ]]')
258 # The IP address the client sees should not be rewritten to be that of the router (#277016)
259 client.succeed("check-last-clients-ip ${serverIp}")
260
261 # But this forwarded port shouldn't intercept communication with
262 # other IPs than externalIp.
263 server.succeed('[[ `timeout 3 curl http://${routerAlternativeExternalIp}:8080` == "router" ]]')
264
265 # The loopback should allow the router itself to access the forwarded port
266 # Note: The reason we use routerIp here is because only routerIp is listed for reflection in networking.nat.forwardPorts.loopbackIPs
267 # The purpose of loopbackIPs is to allow things inside of the NAT to for example access their own public domain when a service has to make a request
268 # to itself/another service on the same NAT through a public address
269 router.succeed('[[ `timeout 3 curl http://${routerIp}:8080` == "client" ]]')
270 # The loopback should also allow the client to access its own forwarded port
271 client.succeed('[[ `timeout 3 curl http://${routerIp}:8080` == "client" ]]')
272
273 # If we turn off NAT, nothing should work
274 router.succeed(
275 "systemctl stop ${unit}.service"
276 )
277
278 # If using nftables and firewall, this makes no sense. We deactivated the firewall after all,
279 # so we are once again affected by the same issue as the moby github issue mentioned above.
280 # If using nftables without firewall, filterForward can't be used and L2 security can't easily be simulated like with iptables, skipping.
281 # See moby github issue mentioned above.
282 ${lib.optionalString (!nftables) ''
283 client.succeed("check-connection ${serverIp} server expect-failure")
284 server.succeed("check-connection ${clientIp} client expect-failure")
285 ''}
286 # These should revert to their pre-NATed versions
287 server.succeed('[[ `timeout 3 curl http://${routerIp}:8080` == "router" ]]')
288 router.succeed('[[ `timeout 3 curl http://${routerIp}:8080` == "router" ]]')
289
290 # Reverse the effect of nat stop
291 router.succeed(
292 "systemctl start ${unit}.service"
293 )
294
295 # Switch to a config without NAT at all, again nothing should work
296 router.succeed(
297 "/run/booted-system/specialisation/no-nat/bin/switch-to-configuration test 2>&1"
298 )
299
300 # If using nftables without firewall, filterForward can't be used and L2 security can't easily be simulated like with iptables, skipping.
301 # See moby github issue mentioned above.
302 ${lib.optionalString (nftables && withFirewall) ''
303 client.succeed("check-connection ${serverIp} server expect-failure")
304 server.succeed("check-connection ${clientIp} client expect-failure")
305 ''}
306
307 # These should revert to their pre-NATed versions
308 server.succeed('[[ `timeout 3 curl http://${routerIp}:8080` == "router" ]]')
309 router.succeed('[[ `timeout 3 curl http://${routerIp}:8080` == "router" ]]')
310 '';
311 }
312)