pyrox.dev
1let
2 commonConfig = ./common/acme/client;
3
4 dnsServerIP = nodes: nodes.dnsserver.config.networking.primaryIPAddress;
5
6 dnsScript = {pkgs, nodes}: let
7 dnsAddress = dnsServerIP nodes;
8 in pkgs.writeShellScript "dns-hook.sh" ''
9 set -euo pipefail
10 echo '[INFO]' "[$2]" 'dns-hook.sh' $*
11 if [ "$1" = "present" ]; then
12 ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'", "value": "'"$3"'"}' http://${dnsAddress}:8055/set-txt
13 else
14 ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'"}' http://${dnsAddress}:8055/clear-txt
15 fi
16 '';
17
18 documentRoot = pkgs: pkgs.runCommand "docroot" {} ''
19 mkdir -p "$out"
20 echo hello world > "$out/index.html"
21 '';
22
23 vhostBase = pkgs: {
24 forceSSL = true;
25 locations."/".root = documentRoot pkgs;
26 };
27
28in import ./make-test-python.nix ({ lib, ... }: {
29 name = "acme";
30 meta.maintainers = lib.teams.acme.members;
31
32 nodes = {
33 # The fake ACME server which will respond to client requests
34 acme = { nodes, lib, ... }: {
35 imports = [ ./common/acme/server ];
36 networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
37 };
38
39 # A fake DNS server which can be configured with records as desired
40 # Used to test DNS-01 challenge
41 dnsserver = { nodes, pkgs, ... }: {
42 networking.firewall.allowedTCPPorts = [ 8055 53 ];
43 networking.firewall.allowedUDPPorts = [ 53 ];
44 systemd.services.pebble-challtestsrv = {
45 enable = true;
46 description = "Pebble ACME challenge test server";
47 wantedBy = [ "network.target" ];
48 serviceConfig = {
49 ExecStart = "${pkgs.pebble}/bin/pebble-challtestsrv -dns01 ':53' -defaultIPv6 '' -defaultIPv4 '${nodes.webserver.config.networking.primaryIPAddress}'";
50 # Required to bind on privileged ports.
51 AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
52 };
53 };
54 };
55
56 # A web server which will be the node requesting certs
57 webserver = { pkgs, nodes, lib, config, ... }: {
58 imports = [ commonConfig ];
59 networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
60 networking.firewall.allowedTCPPorts = [ 80 443 ];
61
62 # OpenSSL will be used for more thorough certificate validation
63 environment.systemPackages = [ pkgs.openssl ];
64
65 # Set log level to info so that we can see when the service is reloaded
66 services.nginx.enable = true;
67 services.nginx.logError = "stderr info";
68
69 # First tests configure a basic cert and run a bunch of openssl checks
70 services.nginx.virtualHosts."a.example.test" = (vhostBase pkgs) // {
71 enableACME = true;
72 };
73
74 # Used to determine if service reload was triggered
75 systemd.targets.test-renew-nginx = {
76 wants = [ "acme-a.example.test.service" ];
77 after = [ "acme-a.example.test.service" "nginx-config-reload.service" ];
78 };
79
80 # Test that account creation is collated into one service
81 specialisation.account-creation.configuration = { nodes, pkgs, lib, ... }: let
82 email = "newhostmaster@example.test";
83 caDomain = nodes.acme.config.test-support.acme.caDomain;
84 # Exit 99 to make it easier to track if this is the reason a renew failed
85 testScript = ''
86 test -e accounts/${caDomain}/${email}/account.json || exit 99
87 '';
88 in {
89 security.acme.email = lib.mkForce email;
90 systemd.services."b.example.test".preStart = testScript;
91 systemd.services."c.example.test".preStart = testScript;
92
93 services.nginx.virtualHosts."b.example.test" = (vhostBase pkgs) // {
94 enableACME = true;
95 };
96 services.nginx.virtualHosts."c.example.test" = (vhostBase pkgs) // {
97 enableACME = true;
98 };
99 };
100
101 # Cert config changes will not cause the nginx configuration to change.
102 # This tests that the reload service is correctly triggered.
103 # It also tests that postRun is exec'd as root
104 specialisation.cert-change.configuration = { pkgs, ... }: {
105 security.acme.certs."a.example.test".keyType = "ec384";
106 security.acme.certs."a.example.test".postRun = ''
107 set -euo pipefail
108 touch /home/test
109 chown root:root /home/test
110 echo testing > /home/test
111 '';
112 };
113
114 # Now adding an alias to ensure that the certs are updated
115 specialisation.nginx-aliases.configuration = { pkgs, ... }: {
116 services.nginx.virtualHosts."a.example.test" = {
117 serverAliases = [ "b.example.test" ];
118 };
119 };
120
121 # Test OCSP Stapling
122 specialisation.ocsp-stapling.configuration = { pkgs, ... }: {
123 security.acme.certs."a.example.test" = {
124 ocspMustStaple = true;
125 };
126 services.nginx.virtualHosts."a.example.com" = {
127 extraConfig = ''
128 ssl_stapling on;
129 ssl_stapling_verify on;
130 '';
131 };
132 };
133
134 # Test using Apache HTTPD
135 specialisation.httpd-aliases.configuration = { pkgs, config, lib, ... }: {
136 services.nginx.enable = lib.mkForce false;
137 services.httpd.enable = true;
138 services.httpd.adminAddr = config.security.acme.email;
139 services.httpd.virtualHosts."c.example.test" = {
140 serverAliases = [ "d.example.test" ];
141 forceSSL = true;
142 enableACME = true;
143 documentRoot = documentRoot pkgs;
144 };
145
146 # Used to determine if service reload was triggered
147 systemd.targets.test-renew-httpd = {
148 wants = [ "acme-c.example.test.service" ];
149 after = [ "acme-c.example.test.service" "httpd-config-reload.service" ];
150 };
151 };
152
153 # Validation via DNS-01 challenge
154 specialisation.dns-01.configuration = { pkgs, config, nodes, ... }: {
155 security.acme.certs."example.test" = {
156 domain = "*.example.test";
157 group = config.services.nginx.group;
158 dnsProvider = "exec";
159 dnsPropagationCheck = false;
160 credentialsFile = pkgs.writeText "wildcard.env" ''
161 EXEC_PATH=${dnsScript { inherit pkgs nodes; }}
162 '';
163 };
164
165 services.nginx.virtualHosts."dns.example.test" = (vhostBase pkgs) // {
166 useACMEHost = "example.test";
167 };
168 };
169
170 # Validate service relationships by adding a slow start service to nginx' wants.
171 # Reproducer for https://github.com/NixOS/nixpkgs/issues/81842
172 specialisation.slow-startup.configuration = { pkgs, config, nodes, lib, ... }: {
173 systemd.services.my-slow-service = {
174 wantedBy = [ "multi-user.target" "nginx.service" ];
175 before = [ "nginx.service" ];
176 preStart = "sleep 5";
177 script = "${pkgs.python3}/bin/python -m http.server";
178 };
179
180 services.nginx.virtualHosts."slow.example.com" = {
181 forceSSL = true;
182 enableACME = true;
183 locations."/".proxyPass = "http://localhost:8000";
184 };
185 };
186 };
187
188 # The client will be used to curl the webserver to validate configuration
189 client = {nodes, lib, pkgs, ...}: {
190 imports = [ commonConfig ];
191 networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
192
193 # OpenSSL will be used for more thorough certificate validation
194 environment.systemPackages = [ pkgs.openssl ];
195 };
196 };
197
198 testScript = {nodes, ...}:
199 let
200 caDomain = nodes.acme.config.test-support.acme.caDomain;
201 newServerSystem = nodes.webserver.config.system.build.toplevel;
202 switchToNewServer = "${newServerSystem}/bin/switch-to-configuration test";
203 in
204 # Note, wait_for_unit does not work for oneshot services that do not have RemainAfterExit=true,
205 # this is because a oneshot goes from inactive => activating => inactive, and never
206 # reaches the active state. Targets do not have this issue.
207
208 ''
209 import time
210
211
212 has_switched = False
213
214
215 def switch_to(node, name):
216 global has_switched
217 if has_switched:
218 node.succeed(
219 "${switchToNewServer}"
220 )
221 has_switched = True
222 node.succeed(
223 f"/run/current-system/specialisation/{name}/bin/switch-to-configuration test"
224 )
225
226
227 # Ensures the issuer of our cert matches the chain
228 # and matches the issuer we expect it to be.
229 # It's a good validation to ensure the cert.pem and fullchain.pem
230 # are not still selfsigned afer verification
231 def check_issuer(node, cert_name, issuer):
232 for fname in ("cert.pem", "fullchain.pem"):
233 actual_issuer = node.succeed(
234 f"openssl x509 -noout -issuer -in /var/lib/acme/{cert_name}/{fname}"
235 ).partition("=")[2]
236 print(f"{fname} issuer: {actual_issuer}")
237 assert issuer.lower() in actual_issuer.lower()
238
239
240 # Ensure cert comes before chain in fullchain.pem
241 def check_fullchain(node, cert_name):
242 subject_data = node.succeed(
243 f"openssl crl2pkcs7 -nocrl -certfile /var/lib/acme/{cert_name}/fullchain.pem"
244 " | openssl pkcs7 -print_certs -noout"
245 )
246 for line in subject_data.lower().split("\n"):
247 if "subject" in line:
248 print(f"First subject in fullchain.pem: {line}")
249 assert cert_name.lower() in line
250 return
251
252 assert False
253
254
255 def check_connection(node, domain, retries=3):
256 assert retries >= 0, f"Failed to connect to https://{domain}"
257
258 result = node.succeed(
259 "openssl s_client -brief -verify 2 -CAfile /tmp/ca.crt"
260 f" -servername {domain} -connect {domain}:443 < /dev/null 2>&1"
261 )
262
263 for line in result.lower().split("\n"):
264 if "verification" in line and "error" in line:
265 time.sleep(3)
266 return check_connection(node, domain, retries - 1)
267
268
269 def check_connection_key_bits(node, domain, bits, retries=3):
270 assert retries >= 0, f"Did not find expected number of bits ({bits}) in key"
271
272 result = node.succeed(
273 "openssl s_client -CAfile /tmp/ca.crt"
274 f" -servername {domain} -connect {domain}:443 < /dev/null"
275 " | openssl x509 -noout -text | grep -i Public-Key"
276 )
277 print("Key type:", result)
278
279 if bits not in result:
280 time.sleep(3)
281 return check_connection_key_bits(node, domain, bits, retries - 1)
282
283
284 def check_stapling(node, domain, retries=3):
285 assert retries >= 0, "OCSP Stapling check failed"
286
287 # Pebble doesn't provide a full OCSP responder, so just check the URL
288 result = node.succeed(
289 "openssl s_client -CAfile /tmp/ca.crt"
290 f" -servername {domain} -connect {domain}:443 < /dev/null"
291 " | openssl x509 -noout -ocsp_uri"
292 )
293 print("OCSP Responder URL:", result)
294
295 if "${caDomain}:4002" not in result.lower():
296 time.sleep(3)
297 return check_stapling(node, domain, retries - 1)
298
299
300 def download_ca_certs(node, retries=5):
301 assert retries >= 0, "Failed to connect to pebble to download root CA certs"
302
303 exit_code, _ = node.execute("curl https://${caDomain}:15000/roots/0 > /tmp/ca.crt")
304 exit_code_2, _ = node.execute(
305 "curl https://${caDomain}:15000/intermediate-keys/0 >> /tmp/ca.crt"
306 )
307
308 if exit_code + exit_code_2 > 0:
309 time.sleep(3)
310 return download_ca_certs(node, retries - 1)
311
312
313 client.start()
314 dnsserver.start()
315
316 dnsserver.wait_for_unit("pebble-challtestsrv.service")
317 client.wait_for_unit("default.target")
318
319 client.succeed(
320 'curl --data \'{"host": "${caDomain}", "addresses": ["${nodes.acme.config.networking.primaryIPAddress}"]}\' http://${dnsServerIP nodes}:8055/add-a'
321 )
322
323 acme.start()
324 webserver.start()
325
326 acme.wait_for_unit("network-online.target")
327 acme.wait_for_unit("pebble.service")
328
329 download_ca_certs(client)
330
331 with subtest("Can request certificate with HTTPS-01 challenge"):
332 webserver.wait_for_unit("acme-finished-a.example.test.target")
333
334 with subtest("Certificates and accounts have safe + valid permissions"):
335 group = "${nodes.webserver.config.security.acme.certs."a.example.test".group}"
336 webserver.succeed(
337 f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
338 )
339 webserver.succeed(
340 f"test $(stat -L -c '%a %U %G' /var/lib/acme/.lego/a.example.test/**/a.example.test* | tee /dev/stderr | grep '600 acme {group}' | wc -l) -eq 4"
341 )
342 webserver.succeed(
343 f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test | tee /dev/stderr | grep '750 acme {group}' | wc -l) -eq 1"
344 )
345 webserver.succeed(
346 f"test $(find /var/lib/acme/accounts -type f -exec stat -L -c '%a %U %G' {{}} \\; | tee /dev/stderr | grep -v '600 acme {group}' | wc -l) -eq 0"
347 )
348
349 with subtest("Certs are accepted by web server"):
350 webserver.succeed("systemctl start nginx.service")
351 check_fullchain(webserver, "a.example.test")
352 check_issuer(webserver, "a.example.test", "pebble")
353 check_connection(client, "a.example.test")
354
355 # Selfsigned certs tests happen late so we aren't fighting the system init triggering cert renewal
356 with subtest("Can generate valid selfsigned certs"):
357 webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
358 webserver.succeed("systemctl start acme-selfsigned-a.example.test.service")
359 check_fullchain(webserver, "a.example.test")
360 check_issuer(webserver, "a.example.test", "minica")
361 # Check selfsigned permissions
362 webserver.succeed(
363 f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
364 )
365 # Will succeed if nginx can load the certs
366 webserver.succeed("systemctl start nginx-config-reload.service")
367
368 with subtest("Can reload nginx when timer triggers renewal"):
369 webserver.succeed("systemctl start test-renew-nginx.target")
370 check_issuer(webserver, "a.example.test", "pebble")
371 check_connection(client, "a.example.test")
372
373 with subtest("Runs 1 cert for account creation before others"):
374 switch_to(webserver, "account-creation")
375 webserver.wait_for_unit("acme-finished-a.example.test.target")
376 check_connection(client, "a.example.test")
377 webserver.wait_for_unit("acme-finished-b.example.test.target")
378 webserver.wait_for_unit("acme-finished-c.example.test.target")
379 check_connection(client, "b.example.test")
380 check_connection(client, "c.example.test")
381
382 with subtest("Can reload web server when cert configuration changes"):
383 switch_to(webserver, "cert-change")
384 webserver.wait_for_unit("acme-finished-a.example.test.target")
385 check_connection_key_bits(client, "a.example.test", "384")
386 webserver.succeed("grep testing /home/test")
387 # Clean to remove the testing file (and anything else messy we did)
388 webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
389
390 with subtest("Correctly implements OCSP stapling"):
391 switch_to(webserver, "ocsp-stapling")
392 webserver.wait_for_unit("acme-finished-a.example.test.target")
393 check_stapling(client, "a.example.test")
394
395 with subtest("Can request certificate with HTTPS-01 when nginx startup is delayed"):
396 switch_to(webserver, "slow-startup")
397 webserver.wait_for_unit("acme-finished-slow.example.com.target")
398 check_issuer(webserver, "slow.example.com", "pebble")
399 check_connection(client, "slow.example.com")
400
401 with subtest("Can request certificate for vhost + aliases (nginx)"):
402 # Check the key hash before and after adding an alias. It should not change.
403 # The previous test reverts the ed384 change
404 webserver.wait_for_unit("acme-finished-a.example.test.target")
405 switch_to(webserver, "nginx-aliases")
406 webserver.wait_for_unit("acme-finished-a.example.test.target")
407 check_issuer(webserver, "a.example.test", "pebble")
408 check_connection(client, "a.example.test")
409 check_connection(client, "b.example.test")
410
411 with subtest("Can request certificates for vhost + aliases (apache-httpd)"):
412 try:
413 switch_to(webserver, "httpd-aliases")
414 webserver.wait_for_unit("acme-finished-c.example.test.target")
415 except Exception as err:
416 _, output = webserver.execute(
417 "cat /var/log/httpd/*.log && ls -al /var/lib/acme/acme-challenge"
418 )
419 print(output)
420 raise err
421 check_issuer(webserver, "c.example.test", "pebble")
422 check_connection(client, "c.example.test")
423 check_connection(client, "d.example.test")
424
425 with subtest("Can reload httpd when timer triggers renewal"):
426 # Switch to selfsigned first
427 webserver.succeed("systemctl clean acme-c.example.test.service --what=state")
428 webserver.succeed("systemctl start acme-selfsigned-c.example.test.service")
429 check_issuer(webserver, "c.example.test", "minica")
430 webserver.succeed("systemctl start httpd-config-reload.service")
431 webserver.succeed("systemctl start test-renew-httpd.target")
432 check_issuer(webserver, "c.example.test", "pebble")
433 check_connection(client, "c.example.test")
434
435 with subtest("Can request wildcard certificates using DNS-01 challenge"):
436 switch_to(webserver, "dns-01")
437 webserver.wait_for_unit("acme-finished-example.test.target")
438 check_issuer(webserver, "example.test", "pebble")
439 check_connection(client, "dns.example.test")
440 '';
441})