pyrox.dev
1{ system ? builtins.currentSystem
2, config ? { }
3, pkgs ? import ../.. { inherit system config; }
4}:
5
6let
7 inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
8 inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge
9 removeSuffix replaceChars singleton splitString;
10
11 /*
12 * The attrset `exporterTests` contains one attribute
13 * for each exporter test. Each of these attributes
14 * is expected to be an attrset containing:
15 *
16 * `exporterConfig`:
17 * this attribute set contains config for the exporter itself
18 *
19 * `exporterTest`
20 * this attribute set contains test instructions
21 *
22 * `metricProvider` (optional)
23 * this attribute contains additional machine config
24 *
25 * `nodeName` (optional)
26 * override an incompatible testnode name
27 *
28 * Example:
29 * exporterTests.<exporterName> = {
30 * exporterConfig = {
31 * enable = true;
32 * };
33 * metricProvider = {
34 * services.<metricProvider>.enable = true;
35 * };
36 * exporterTest = ''
37 * wait_for_unit("prometheus-<exporterName>-exporter.service")
38 * wait_for_open_port(1234)
39 * succeed("curl -sSf 'localhost:1234/metrics'")
40 * '';
41 * };
42 *
43 * # this would generate the following test config:
44 *
45 * nodes.<exporterName> = {
46 * services.prometheus.<exporterName> = {
47 * enable = true;
48 * };
49 * services.<metricProvider>.enable = true;
50 * };
51 *
52 * testScript = ''
53 * <exporterName>.start()
54 * <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service")
55 * <exporterName>.wait_for_open_port(1234)
56 * <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'")
57 * <exporterName>.shutdown()
58 * '';
59 */
60
61 exporterTests = {
62 apcupsd = {
63 exporterConfig = {
64 enable = true;
65 };
66 metricProvider = {
67 services.apcupsd.enable = true;
68 };
69 exporterTest = ''
70 wait_for_unit("apcupsd.service")
71 wait_for_open_port(3551)
72 wait_for_unit("prometheus-apcupsd-exporter.service")
73 wait_for_open_port(9162)
74 succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'")
75 '';
76 };
77
78 artifactory = {
79 exporterConfig = {
80 enable = true;
81 artiUsername = "artifactory-username";
82 artiPassword = "artifactory-password";
83 };
84 exporterTest = ''
85 wait_for_unit("prometheus-artifactory-exporter.service")
86 wait_for_open_port(9531)
87 succeed(
88 "curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'"
89 )
90 '';
91 };
92
93 bind = {
94 exporterConfig = {
95 enable = true;
96 };
97 metricProvider = {
98 services.bind.enable = true;
99 services.bind.extraConfig = ''
100 statistics-channels {
101 inet 127.0.0.1 port 8053 allow { localhost; };
102 };
103 '';
104 };
105 exporterTest = ''
106 wait_for_unit("prometheus-bind-exporter.service")
107 wait_for_open_port(9119)
108 succeed(
109 "curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'"
110 )
111 '';
112 };
113
114 bird = {
115 exporterConfig = {
116 enable = true;
117 };
118 metricProvider = {
119 services.bird2.enable = true;
120 services.bird2.config = ''
121 router id 127.0.0.1;
122
123 protocol kernel MyObviousTestString {
124 ipv4 {
125 import all;
126 export none;
127 };
128 }
129
130 protocol device {
131 }
132 '';
133 };
134 exporterTest = ''
135 wait_for_unit("prometheus-bird-exporter.service")
136 wait_for_open_port(9324)
137 wait_until_succeeds(
138 "curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'"
139 )
140 '';
141 };
142
143 bitcoin = {
144 exporterConfig = {
145 enable = true;
146 rpcUser = "bitcoinrpc";
147 rpcPasswordFile = pkgs.writeText "password" "hunter2";
148 };
149 metricProvider = {
150 services.bitcoind.default.enable = true;
151 services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC = "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7";
152 };
153 exporterTest = ''
154 wait_for_unit("prometheus-bitcoin-exporter.service")
155 wait_for_unit("bitcoind-default.service")
156 wait_for_open_port(9332)
157 succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '")
158 '';
159 };
160
161 blackbox = {
162 exporterConfig = {
163 enable = true;
164 configFile = pkgs.writeText "config.yml" (builtins.toJSON {
165 modules.icmp_v6 = {
166 prober = "icmp";
167 icmp.preferred_ip_protocol = "ip6";
168 };
169 });
170 };
171 exporterTest = ''
172 wait_for_unit("prometheus-blackbox-exporter.service")
173 wait_for_open_port(9115)
174 succeed(
175 "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'"
176 )
177 '';
178 };
179
180 collectd = {
181 exporterConfig = {
182 enable = true;
183 extraFlags = [ "--web.collectd-push-path /collectd" ];
184 };
185 exporterTest = let postData = replaceChars [ "\n" ] [ "" ] ''
186 [{
187 "values":[23],
188 "dstypes":["gauge"],
189 "type":"gauge",
190 "interval":1000,
191 "host":"testhost",
192 "plugin":"testplugin",
193 "time":DATE
194 }]
195 ''; in
196 ''
197 wait_for_unit("prometheus-collectd-exporter.service")
198 wait_for_open_port(9103)
199 succeed(
200 'echo \'${postData}\'> /tmp/data.json'
201 )
202 succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json')
203 succeed(
204 "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd"
205 )
206 succeed(
207 "curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"
208 )
209 '';
210 };
211
212 dnsmasq = {
213 exporterConfig = {
214 enable = true;
215 leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
216 };
217 metricProvider = {
218 services.dnsmasq.enable = true;
219 };
220 exporterTest = ''
221 wait_for_unit("prometheus-dnsmasq-exporter.service")
222 wait_for_open_port(9153)
223 succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'")
224 '';
225 };
226
227 # Access to WHOIS server is required to properly test this exporter, so
228 # just perform basic sanity check that the exporter is running and returns
229 # a failure.
230 domain = {
231 exporterConfig = {
232 enable = true;
233 };
234 exporterTest = ''
235 wait_for_unit("prometheus-domain-exporter.service")
236 wait_for_open_port(9222)
237 succeed(
238 "curl -sSf 'http://localhost:9222/probe?target=nixos.org' | grep 'domain_probe_success 0'"
239 )
240 '';
241 };
242
243 dovecot = {
244 exporterConfig = {
245 enable = true;
246 scopes = [ "global" ];
247 socketPath = "/var/run/dovecot2/old-stats";
248 user = "root"; # <- don't use user root in production
249 };
250 metricProvider = {
251 services.dovecot2.enable = true;
252 };
253 exporterTest = ''
254 wait_for_unit("prometheus-dovecot-exporter.service")
255 wait_for_open_port(9166)
256 succeed(
257 "curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'"
258 )
259 '';
260 };
261
262 fastly = {
263 exporterConfig = {
264 enable = true;
265 tokenPath = pkgs.writeText "token" "abc123";
266 };
267
268 # noop: fastly's exporter can't start without first talking to fastly
269 # see: https://github.com/peterbourgon/fastly-exporter/issues/87
270 exporterTest = ''
271 succeed("true");
272 '';
273 };
274
275 fritzbox = {
276 # TODO add proper test case
277 exporterConfig = {
278 enable = true;
279 };
280 exporterTest = ''
281 wait_for_unit("prometheus-fritzbox-exporter.service")
282 wait_for_open_port(9133)
283 succeed(
284 "curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'"
285 )
286 '';
287 };
288
289 influxdb = {
290 exporterConfig = {
291 enable = true;
292 sampleExpiry = "3s";
293 };
294 exporterTest = ''
295 wait_for_unit("prometheus-influxdb-exporter.service")
296 wait_for_open_port(9122)
297 succeed(
298 "curl -XPOST http://localhost:9122/write --data-binary 'influxdb_exporter,distro=nixos,added_in=21.09 value=1'"
299 )
300 succeed(
301 "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
302 )
303 execute("sleep 5")
304 fail(
305 "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
306 )
307 '';
308 };
309
310 ipmi = {
311 exporterConfig = {
312 enable = true;
313 };
314 exporterTest = ''
315 wait_for_unit("prometheus-ipmi-exporter.service")
316 wait_for_open_port(9290)
317 succeed(
318 "curl -sSf http://localhost:9290/metrics | grep 'ipmi_scrape_duration_seconds'"
319 )
320 '';
321 };
322
323 jitsi = {
324 exporterConfig = {
325 enable = true;
326 };
327 metricProvider = {
328 systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ];
329 services.jitsi-videobridge = {
330 enable = true;
331 apis = [ "colibri" "rest" ];
332 };
333 };
334 exporterTest = ''
335 wait_for_unit("jitsi-videobridge2.service")
336 wait_for_open_port(8080)
337 wait_for_unit("prometheus-jitsi-exporter.service")
338 wait_for_open_port(9700)
339 wait_until_succeeds(
340 'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"'
341 )
342 succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'")
343 '';
344 };
345
346 json = {
347 exporterConfig = {
348 enable = true;
349 url = "http://localhost";
350 configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON {
351 metrics = [
352 { name = "json_test_metric"; path = "{ .test }"; }
353 ];
354 });
355 };
356 metricProvider = {
357 systemd.services.prometheus-json-exporter.after = [ "nginx.service" ];
358 services.nginx = {
359 enable = true;
360 virtualHosts.localhost.locations."/".extraConfig = ''
361 return 200 "{\"test\":1}";
362 '';
363 };
364 };
365 exporterTest = ''
366 wait_for_unit("nginx.service")
367 wait_for_open_port(80)
368 wait_for_unit("prometheus-json-exporter.service")
369 wait_for_open_port(7979)
370 succeed(
371 "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'"
372 )
373 '';
374 };
375
376 kea = let
377 controlSocketPathV4 = "/run/kea/dhcp4.sock";
378 controlSocketPathV6 = "/run/kea/dhcp6.sock";
379 in
380 {
381 exporterConfig = {
382 enable = true;
383 controlSocketPaths = [
384 controlSocketPathV4
385 controlSocketPathV6
386 ];
387 };
388 metricProvider = {
389 services.kea = {
390 dhcp4 = {
391 enable = true;
392 settings = {
393 control-socket = {
394 socket-type = "unix";
395 socket-name = controlSocketPathV4;
396 };
397 };
398 };
399 dhcp6 = {
400 enable = true;
401 settings = {
402 control-socket = {
403 socket-type = "unix";
404 socket-name = controlSocketPathV6;
405 };
406 };
407 };
408 };
409 };
410
411 exporterTest = ''
412 wait_for_unit("kea-dhcp4-server.service")
413 wait_for_unit("kea-dhcp6-server.service")
414 wait_for_file("${controlSocketPathV4}")
415 wait_for_file("${controlSocketPathV6}")
416 wait_for_unit("prometheus-kea-exporter.service")
417 wait_for_open_port(9547)
418 succeed(
419 "curl --fail localhost:9547/metrics | grep 'packets_received_total'"
420 )
421 '';
422 };
423
424 knot = {
425 exporterConfig = {
426 enable = true;
427 };
428 metricProvider = {
429 services.knot = {
430 enable = true;
431 extraArgs = [ "-v" ];
432 extraConfig = ''
433 server:
434 listen: 127.0.0.1@53
435
436 template:
437 - id: default
438 global-module: mod-stats
439 dnssec-signing: off
440 zonefile-sync: -1
441 journal-db: /var/lib/knot/journal
442 kasp-db: /var/lib/knot/kasp
443 timer-db: /var/lib/knot/timer
444 zonefile-load: difference
445 storage: ${pkgs.buildEnv {
446 name = "foo";
447 paths = [
448 (pkgs.writeTextDir "test.zone" ''
449 @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
450 @ NS ns1
451 @ NS ns2
452 ns1 A 192.168.0.1
453 '')
454 ];
455 }}
456
457 mod-stats:
458 - id: custom
459 edns-presence: on
460 query-type: on
461
462 zone:
463 - domain: test
464 file: test.zone
465 module: mod-stats/custom
466 '';
467 };
468 };
469 exporterTest = ''
470 wait_for_unit("knot.service")
471 wait_for_unit("prometheus-knot-exporter.service")
472 wait_for_open_port(9433)
473 succeed("curl -sSf 'localhost:9433' | grep 'knot_server_zone_count 1.0'")
474 '';
475 };
476
477 keylight = {
478 # A hardware device is required to properly test this exporter, so just
479 # perform a couple of basic sanity checks that the exporter is running
480 # and requires a target, but cannot reach a specified target.
481 exporterConfig = {
482 enable = true;
483 };
484 exporterTest = ''
485 wait_for_unit("prometheus-keylight-exporter.service")
486 wait_for_open_port(9288)
487 succeed(
488 "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'"
489 )
490 succeed(
491 "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'"
492 )
493 '';
494 };
495
496 lnd = {
497 exporterConfig = {
498 enable = true;
499 lndTlsPath = "/var/lib/lnd/tls.cert";
500 lndMacaroonDir = "/var/lib/lnd";
501 extraFlags = [ "--lnd.network=regtest" ];
502 };
503 metricProvider = {
504 systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15;
505 systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ];
506 services.bitcoind.regtest = {
507 enable = true;
508 extraConfig = ''
509 rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7
510 zmqpubrawblock=tcp://127.0.0.1:28332
511 zmqpubrawtx=tcp://127.0.0.1:28333
512 '';
513 extraCmdlineOptions = [ "-regtest" ];
514 };
515 systemd.services.lnd = {
516 serviceConfig.ExecStart = ''
517 ${pkgs.lnd}/bin/lnd \
518 --datadir=/var/lib/lnd \
519 --tlscertpath=/var/lib/lnd/tls.cert \
520 --tlskeypath=/var/lib/lnd/tls.key \
521 --logdir=/var/log/lnd \
522 --bitcoin.active \
523 --bitcoin.regtest \
524 --bitcoin.node=bitcoind \
525 --bitcoind.rpcuser=bitcoinrpc \
526 --bitcoind.rpcpass=hunter2 \
527 --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \
528 --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \
529 --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon
530 '';
531 serviceConfig.StateDirectory = "lnd";
532 wantedBy = [ "multi-user.target" ];
533 after = [ "network.target" ];
534 };
535 # initialize wallet, creates macaroon needed by exporter
536 systemd.services.lnd.postStart = ''
537 ${pkgs.curl}/bin/curl \
538 --retry 20 \
539 --retry-delay 1 \
540 --retry-connrefused \
541 --cacert /var/lib/lnd/tls.cert \
542 -X GET \
543 https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed
544 ${pkgs.curl}/bin/curl \
545 --retry 20 \
546 --retry-delay 1 \
547 --retry-connrefused \
548 --cacert /var/lib/lnd/tls.cert \
549 -X POST \
550 -d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \
551 https://localhost:8080/v1/initwallet
552 '';
553 };
554 exporterTest = ''
555 wait_for_unit("lnd.service")
556 wait_for_open_port(10009)
557 wait_for_unit("prometheus-lnd-exporter.service")
558 wait_for_open_port(9092)
559 succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'")
560 '';
561 };
562
563 mail = {
564 exporterConfig = {
565 enable = true;
566 configuration = {
567 monitoringInterval = "2s";
568 mailCheckTimeout = "10s";
569 servers = [{
570 name = "testserver";
571 server = "localhost";
572 port = 25;
573 from = "mail-exporter@localhost";
574 to = "mail-exporter@localhost";
575 detectionDir = "/var/spool/mail/mail-exporter/new";
576 }];
577 };
578 };
579 metricProvider = {
580 services.postfix.enable = true;
581 systemd.services.prometheus-mail-exporter = {
582 after = [ "postfix.service" ];
583 requires = [ "postfix.service" ];
584 serviceConfig = {
585 ExecStartPre = [
586 "${pkgs.writeShellScript "create-maildir" ''
587 mkdir -p -m 0700 mail-exporter/new
588 ''}"
589 ];
590 ProtectHome = true;
591 ReadOnlyPaths = "/";
592 ReadWritePaths = "/var/spool/mail";
593 WorkingDirectory = "/var/spool/mail";
594 };
595 };
596 users.users.mailexporter = {
597 isSystemUser = true;
598 group = "mailexporter";
599 };
600 users.groups.mailexporter = {};
601 };
602 exporterTest = ''
603 wait_for_unit("postfix.service")
604 wait_for_unit("prometheus-mail-exporter.service")
605 wait_for_open_port(9225)
606 wait_until_succeeds(
607 "curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'"
608 )
609 '';
610 };
611
612 mikrotik = {
613 exporterConfig = {
614 enable = true;
615 extraFlags = [ "-timeout=1s" ];
616 configuration = {
617 devices = [
618 {
619 name = "router";
620 address = "192.168.42.48";
621 user = "prometheus";
622 password = "shh";
623 }
624 ];
625 features = {
626 bgp = true;
627 dhcp = true;
628 dhcpl = true;
629 dhcpv6 = true;
630 health = true;
631 routes = true;
632 poe = true;
633 pools = true;
634 optics = true;
635 w60g = true;
636 wlansta = true;
637 wlanif = true;
638 monitor = true;
639 ipsec = true;
640 };
641 };
642 };
643 exporterTest = ''
644 wait_for_unit("prometheus-mikrotik-exporter.service")
645 wait_for_open_port(9436)
646 succeed(
647 "curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'"
648 )
649 '';
650 };
651
652 modemmanager = {
653 exporterConfig = {
654 enable = true;
655 refreshRate = "10s";
656 };
657 metricProvider = {
658 # ModemManager is installed when NetworkManager is enabled. Ensure it is
659 # started and is wanted by NM and the exporter to start everything up
660 # in the right order.
661 networking.networkmanager.enable = true;
662 systemd.services.ModemManager = {
663 enable = true;
664 wantedBy = [ "NetworkManager.service" "prometheus-modemmanager-exporter.service" ];
665 };
666 };
667 exporterTest = ''
668 wait_for_unit("ModemManager.service")
669 wait_for_unit("prometheus-modemmanager-exporter.service")
670 wait_for_open_port(9539)
671 succeed(
672 "curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'"
673 )
674 '';
675 };
676
677 nextcloud = {
678 exporterConfig = {
679 enable = true;
680 passwordFile = "/var/nextcloud-pwfile";
681 url = "http://localhost";
682 };
683 metricProvider = {
684 systemd.services.nc-pwfile =
685 let
686 passfile = (pkgs.writeText "pwfile" "snakeoilpw");
687 in
688 {
689 requiredBy = [ "prometheus-nextcloud-exporter.service" ];
690 before = [ "prometheus-nextcloud-exporter.service" ];
691 serviceConfig.ExecStart = ''
692 ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile
693 '';
694 };
695 services.nginx = {
696 enable = true;
697 virtualHosts."localhost" = {
698 basicAuth.nextcloud-exporter = "snakeoilpw";
699 locations."/" = {
700 root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
701 tryFiles = "/negative-space.json =404";
702 };
703 };
704 };
705 };
706 exporterTest = ''
707 wait_for_unit("nginx.service")
708 wait_for_unit("prometheus-nextcloud-exporter.service")
709 wait_for_open_port(9205)
710 succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'")
711 '';
712 };
713
714 nginx = {
715 exporterConfig = {
716 enable = true;
717 };
718 metricProvider = {
719 services.nginx = {
720 enable = true;
721 statusPage = true;
722 virtualHosts."test".extraConfig = "return 204;";
723 };
724 };
725 exporterTest = ''
726 wait_for_unit("nginx.service")
727 wait_for_unit("prometheus-nginx-exporter.service")
728 wait_for_open_port(9113)
729 succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up 1'")
730 '';
731 };
732
733 nginxlog = {
734 exporterConfig = {
735 enable = true;
736 group = "nginx";
737 settings = {
738 namespaces = [
739 {
740 name = "filelogger";
741 source = {
742 files = [ "/var/log/nginx/filelogger.access.log" ];
743 };
744 }
745 {
746 name = "syslogger";
747 source = {
748 syslog = {
749 listen_address = "udp://127.0.0.1:10000";
750 format = "rfc3164";
751 tags = [ "nginx" ];
752 };
753 };
754 }
755 ];
756 };
757 };
758 metricProvider = {
759 services.nginx = {
760 enable = true;
761 httpConfig = ''
762 server {
763 listen 80;
764 server_name filelogger.local;
765 access_log /var/log/nginx/filelogger.access.log;
766 }
767 server {
768 listen 81;
769 server_name syslogger.local;
770 access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
771 }
772 '';
773 };
774 };
775 exporterTest = ''
776 wait_for_unit("nginx.service")
777 wait_for_unit("prometheus-nginxlog-exporter.service")
778 wait_for_open_port(9117)
779 wait_for_open_port(80)
780 wait_for_open_port(81)
781 succeed("curl http://localhost")
782 execute("sleep 1")
783 succeed(
784 "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1"
785 )
786 succeed("curl http://localhost:81")
787 execute("sleep 1")
788 succeed(
789 "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1"
790 )
791 '';
792 };
793
794 node = {
795 exporterConfig = {
796 enable = true;
797 };
798 exporterTest = ''
799 wait_for_unit("prometheus-node-exporter.service")
800 wait_for_open_port(9100)
801 succeed(
802 "curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'"
803 )
804 '';
805 };
806
807 openldap = {
808 exporterConfig = {
809 enable = true;
810 ldapCredentialFile = "${pkgs.writeText "exporter.yml" ''
811 ldapUser: "cn=root,dc=example"
812 ldapPass: "notapassword"
813 ''}";
814 };
815 metricProvider = {
816 services.openldap = {
817 enable = true;
818 settings.children = {
819 "cn=schema".includes = [
820 "${pkgs.openldap}/etc/schema/core.ldif"
821 "${pkgs.openldap}/etc/schema/cosine.ldif"
822 "${pkgs.openldap}/etc/schema/inetorgperson.ldif"
823 "${pkgs.openldap}/etc/schema/nis.ldif"
824 ];
825 "olcDatabase={1}mdb" = {
826 attrs = {
827 objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ];
828 olcDatabase = "{1}mdb";
829 olcDbDirectory = "/var/db/openldap";
830 olcSuffix = "dc=example";
831 olcRootDN = {
832 # cn=root,dc=example
833 base64 = "Y249cm9vdCxkYz1leGFtcGxl";
834 };
835 olcRootPW = {
836 path = "${pkgs.writeText "rootpw" "notapassword"}";
837 };
838 };
839 };
840 "olcDatabase={2}monitor".attrs = {
841 objectClass = [ "olcDatabaseConfig" ];
842 olcDatabase = "{2}monitor";
843 olcAccess = [ "to dn.subtree=cn=monitor by users read" ];
844 };
845 };
846 declarativeContents."dc=example" = ''
847 dn: dc=example
848 objectClass: domain
849 dc: example
850
851 dn: ou=users,dc=example
852 objectClass: organizationalUnit
853 ou: users
854 '';
855 };
856 };
857 exporterTest = ''
858 wait_for_unit("prometheus-openldap-exporter.service")
859 wait_for_open_port(389)
860 wait_for_open_port(9330)
861 wait_until_succeeds(
862 "curl -sSf http://localhost:9330/metrics | grep 'openldap_scrape{result=\"ok\"} 1'"
863 )
864 '';
865 };
866
867 openvpn = {
868 exporterConfig = {
869 enable = true;
870 group = "openvpn";
871 statusPaths = [ "/run/openvpn-test" ];
872 };
873 metricProvider = {
874 users.groups.openvpn = { };
875 services.openvpn.servers.test = {
876 config = ''
877 dev tun
878 status /run/openvpn-test
879 status-version 3
880 '';
881 up = "chmod g+r /run/openvpn-test";
882 };
883 systemd.services."openvpn-test".serviceConfig.Group = "openvpn";
884 };
885 exporterTest = ''
886 wait_for_unit("openvpn-test.service")
887 wait_for_unit("prometheus-openvpn-exporter.service")
888 succeed("curl -sSf http://localhost:9176/metrics | grep 'openvpn_up{.*} 1'")
889 '';
890 };
891
892 postfix = {
893 exporterConfig = {
894 enable = true;
895 };
896 metricProvider = {
897 services.postfix.enable = true;
898 };
899 exporterTest = ''
900 wait_for_unit("prometheus-postfix-exporter.service")
901 wait_for_file("/var/lib/postfix/queue/public/showq")
902 wait_for_open_port(9154)
903 wait_until_succeeds(
904 "curl -sSf http://localhost:9154/metrics | grep 'postfix_up{path=\"/var/lib/postfix/queue/public/showq\"} 1'"
905 )
906 succeed(
907 "curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'"
908 )
909 succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'")
910 '';
911 };
912
913 postgres = {
914 exporterConfig = {
915 enable = true;
916 runAsLocalSuperUser = true;
917 };
918 metricProvider = {
919 services.postgresql.enable = true;
920 };
921 exporterTest = ''
922 wait_for_unit("prometheus-postgres-exporter.service")
923 wait_for_open_port(9187)
924 wait_for_unit("postgresql.service")
925 succeed(
926 "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
927 )
928 succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
929 systemctl("stop postgresql.service")
930 succeed(
931 "curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'"
932 )
933 succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'")
934 systemctl("start postgresql.service")
935 wait_for_unit("postgresql.service")
936 succeed(
937 "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
938 )
939 succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
940 '';
941 };
942
943 process = {
944 exporterConfig = {
945 enable = true;
946 settings.process_names = [
947 # Remove nix store path from process name
948 { name = "{{.Matches.Wrapped}} {{ .Matches.Args }}"; cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ]; }
949 ];
950 };
951 exporterTest = ''
952 wait_for_unit("prometheus-process-exporter.service")
953 wait_for_open_port(9256)
954 wait_until_succeeds(
955 "curl -sSf localhost:9256/metrics | grep -q '{}'".format(
956 'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter '
957 )
958 )
959 '';
960 };
961
962 pve = let
963 pveExporterEnvFile = pkgs.writeTextFile {
964 name = "pve.env";
965 text = ''
966 PVE_USER="test_user@pam"
967 PVE_PASSWORD="hunter3"
968 PVE_VERIFY_SSL="false"
969 '';
970 };
971 in {
972 exporterConfig = {
973 enable = true;
974 environmentFile = pveExporterEnvFile;
975 };
976 exporterTest = ''
977 wait_for_unit("prometheus-pve-exporter.service")
978 wait_for_open_port(9221)
979 wait_until_succeeds("curl localhost:9221")
980 '';
981 };
982
983 py-air-control = {
984 nodeName = "py_air_control";
985 exporterConfig = {
986 enable = true;
987 deviceHostname = "127.0.0.1";
988 };
989 exporterTest = ''
990 wait_for_unit("prometheus-py-air-control-exporter.service")
991 wait_for_open_port(9896)
992 succeed(
993 "curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'"
994 )
995 '';
996 };
997
998 redis = {
999 exporterConfig = {
1000 enable = true;
1001 };
1002 metricProvider.services.redis.servers."".enable = true;
1003 exporterTest = ''
1004 wait_for_unit("redis.service")
1005 wait_for_unit("prometheus-redis-exporter.service")
1006 wait_for_open_port(6379)
1007 wait_for_open_port(9121)
1008 wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'")
1009 '';
1010 };
1011
1012 rspamd = {
1013 exporterConfig = {
1014 enable = true;
1015 };
1016 metricProvider = {
1017 services.rspamd.enable = true;
1018 };
1019 exporterTest = ''
1020 wait_for_unit("rspamd.service")
1021 wait_for_unit("prometheus-rspamd-exporter.service")
1022 wait_for_open_port(11334)
1023 wait_for_open_port(7980)
1024 wait_until_succeeds(
1025 "curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep 'rspamd_scanned{host=\"rspamd\"} 0'"
1026 )
1027 '';
1028 };
1029
1030 rtl_433 = {
1031 exporterConfig = {
1032 enable = true;
1033 };
1034 metricProvider = {
1035 # Mock rtl_433 binary to return a dummy metric stream.
1036 nixpkgs.overlays = [
1037 (self: super: {
1038 rtl_433 = self.runCommand "rtl_433" { } ''
1039 mkdir -p "$out/bin"
1040 cat <<EOF > "$out/bin/rtl_433"
1041 #!/bin/sh
1042 while true; do
1043 printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n'
1044 sleep 4
1045 done
1046 EOF
1047 chmod +x "$out/bin/rtl_433"
1048 '';
1049 })
1050 ];
1051 };
1052 exporterTest = ''
1053 wait_for_unit("prometheus-rtl_433-exporter.service")
1054 wait_for_open_port(9550)
1055 wait_until_succeeds(
1056 "curl -sSf localhost:9550/metrics | grep '{}'".format(
1057 'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18'
1058 )
1059 )
1060 '';
1061 };
1062
1063 script = {
1064 exporterConfig = {
1065 enable = true;
1066 settings.scripts = [
1067 { name = "success"; script = "sleep 1"; }
1068 ];
1069 };
1070 exporterTest = ''
1071 wait_for_unit("prometheus-script-exporter.service")
1072 wait_for_open_port(9172)
1073 wait_until_succeeds(
1074 "curl -sSf 'localhost:9172/probe?name=success' | grep -q '{}'".format(
1075 'script_success{script="success"} 1'
1076 )
1077 )
1078 '';
1079 };
1080
1081 smartctl = {
1082 exporterConfig = {
1083 enable = true;
1084 devices = [
1085 "/dev/vda"
1086 ];
1087 };
1088 exporterTest = ''
1089 wait_for_unit("prometheus-smartctl-exporter.service")
1090 wait_for_open_port(9633)
1091 wait_until_succeeds(
1092 "curl -sSf 'localhost:9633/metrics'"
1093 )
1094 wait_until_succeeds(
1095 'journalctl -eu prometheus-smartctl-exporter.service -o cat | grep "/dev/vda: Unable to detect device type"'
1096 )
1097 '';
1098 };
1099
1100 smokeping = {
1101 exporterConfig = {
1102 enable = true;
1103 hosts = [ "127.0.0.1" ];
1104 };
1105 exporterTest = ''
1106 wait_for_unit("prometheus-smokeping-exporter.service")
1107 wait_for_open_port(9374)
1108 wait_until_succeeds(
1109 "curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format(
1110 'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1"} '
1111 )
1112 )
1113 wait_until_succeeds(
1114 "curl -sSf localhost:9374/metrics | grep '{}'".format(
1115 'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1"}'
1116 )
1117 )
1118 '';
1119 };
1120
1121 snmp = {
1122 exporterConfig = {
1123 enable = true;
1124 configuration.default = {
1125 version = 2;
1126 auth.community = "public";
1127 };
1128 };
1129 exporterTest = ''
1130 wait_for_unit("prometheus-snmp-exporter.service")
1131 wait_for_open_port(9116)
1132 succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'")
1133 '';
1134 };
1135
1136 sql = {
1137 exporterConfig = {
1138 configuration.jobs.points = {
1139 interval = "1m";
1140 connections = [
1141 "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable"
1142 ];
1143 queries = {
1144 points = {
1145 labels = [ "name" ];
1146 help = "Amount of points accumulated per person";
1147 values = [ "amount" ];
1148 query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name";
1149 };
1150 };
1151 };
1152 enable = true;
1153 user = "prometheus-sql-exporter";
1154 };
1155 metricProvider = {
1156 services.postgresql = {
1157 enable = true;
1158 initialScript = builtins.toFile "init.sql" ''
1159 CREATE DATABASE data;
1160 \c data;
1161 CREATE TABLE points (amount INT, name TEXT);
1162 INSERT INTO points(amount, name) VALUES (1, 'jack');
1163 INSERT INTO points(amount, name) VALUES (2, 'jill');
1164 INSERT INTO points(amount, name) VALUES (3, 'jack');
1165
1166 CREATE USER "prometheus-sql-exporter";
1167 GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter";
1168 GRANT SELECT ON points TO "prometheus-sql-exporter";
1169 '';
1170 };
1171 systemd.services.prometheus-sql-exporter.after = [ "postgresql.service" ];
1172 };
1173 exporterTest = ''
1174 wait_for_unit("prometheus-sql-exporter.service")
1175 wait_for_open_port(9237)
1176 succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2")
1177 '';
1178 };
1179
1180 surfboard = {
1181 exporterConfig = {
1182 enable = true;
1183 modemAddress = "localhost";
1184 };
1185 metricProvider = {
1186 systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ];
1187 services.nginx = {
1188 enable = true;
1189 virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = ''
1190 return 204;
1191 '';
1192 };
1193 };
1194 exporterTest = ''
1195 wait_for_unit("nginx.service")
1196 wait_for_open_port(80)
1197 wait_for_unit("prometheus-surfboard-exporter.service")
1198 wait_for_open_port(9239)
1199 succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'")
1200 '';
1201 };
1202
1203 systemd = {
1204 exporterConfig = {
1205 enable = true;
1206
1207 extraFlags = [
1208 "--systemd.collector.enable-restart-count"
1209 ];
1210 };
1211 metricProvider = { };
1212 exporterTest = ''
1213 wait_for_unit("prometheus-systemd-exporter.service")
1214 wait_for_open_port(9558)
1215 wait_until_succeeds(
1216 "curl -sSf localhost:9558/metrics | grep '{}'".format(
1217 'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
1218 )
1219 )
1220 succeed(
1221 "curl -sSf localhost:9558/metrics | grep '{}'".format(
1222 'systemd_service_restart_total{name="prometheus-systemd-exporter.service"} 0'
1223 )
1224 )
1225 '';
1226 };
1227
1228 tor = {
1229 exporterConfig = {
1230 enable = true;
1231 };
1232 metricProvider = {
1233 # Note: this does not connect the test environment to the Tor network.
1234 # Client, relay, bridge or exit connectivity are disabled by default.
1235 services.tor.enable = true;
1236 services.tor.settings.ControlPort = 9051;
1237 };
1238 exporterTest = ''
1239 wait_for_unit("tor.service")
1240 wait_for_open_port(9051)
1241 wait_for_unit("prometheus-tor-exporter.service")
1242 wait_for_open_port(9130)
1243 succeed("curl -sSf localhost:9130/metrics | grep 'tor_version{.\\+} 1'")
1244 '';
1245 };
1246
1247 unifi-poller = {
1248 nodeName = "unifi_poller";
1249 exporterConfig.enable = true;
1250 exporterConfig.controllers = [{ }];
1251 exporterTest = ''
1252 wait_for_unit("prometheus-unifi-poller-exporter.service")
1253 wait_for_open_port(9130)
1254 succeed(
1255 "curl -sSf localhost:9130/metrics | grep 'unifipoller_build_info{.\\+} 1'"
1256 )
1257 '';
1258 };
1259
1260 unbound = {
1261 exporterConfig = {
1262 enable = true;
1263 fetchType = "uds";
1264 controlInterface = "/run/unbound/unbound.ctl";
1265 };
1266 metricProvider = {
1267 services.unbound = {
1268 enable = true;
1269 localControlSocketPath = "/run/unbound/unbound.ctl";
1270 };
1271 systemd.services.prometheus-unbound-exporter.serviceConfig = {
1272 SupplementaryGroups = [ "unbound" ];
1273 };
1274 };
1275 exporterTest = ''
1276 wait_for_unit("unbound.service")
1277 wait_for_unit("prometheus-unbound-exporter.service")
1278 wait_for_open_port(9167)
1279 succeed("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'")
1280 '';
1281 };
1282
1283 v2ray = {
1284 exporterConfig = {
1285 enable = true;
1286 };
1287
1288 metricProvider = {
1289 systemd.services.prometheus-nginx-exporter.after = [ "v2ray.service" ];
1290 services.v2ray = {
1291 enable = true;
1292 config = {
1293 stats = {};
1294 api = {
1295 tag = "api";
1296 services = [ "StatsService" ];
1297 };
1298 inbounds = [
1299 {
1300 port = 1080;
1301 listen = "127.0.0.1";
1302 protocol = "http";
1303 }
1304 {
1305 listen = "127.0.0.1";
1306 port = 54321;
1307 protocol = "dokodemo-door";
1308 settings = { address = "127.0.0.1"; };
1309 tag = "api";
1310 }
1311 ];
1312 outbounds = [
1313 {
1314 protocol = "freedom";
1315 }
1316 {
1317 protocol = "freedom";
1318 settings = {};
1319 tag = "api";
1320 }
1321 ];
1322 routing = {
1323 strategy = "rules";
1324 settings = {
1325 rules = [
1326 {
1327 inboundTag = [ "api" ];
1328 outboundTag = "api";
1329 type = "field";
1330 }
1331 ];
1332 };
1333 };
1334 };
1335 };
1336 };
1337 exporterTest = ''
1338 wait_for_unit("prometheus-v2ray-exporter.service")
1339 wait_for_open_port(9299)
1340 succeed("curl -sSf localhost:9299/scrape | grep 'v2ray_up 1'")
1341 '';
1342 };
1343
1344 varnish = {
1345 exporterConfig = {
1346 enable = true;
1347 instance = "/var/spool/varnish/varnish";
1348 group = "varnish";
1349 };
1350 metricProvider = {
1351 systemd.services.prometheus-varnish-exporter.after = [
1352 "varnish.service"
1353 ];
1354 services.varnish = {
1355 enable = true;
1356 config = ''
1357 vcl 4.0;
1358 backend default {
1359 .host = "127.0.0.1";
1360 .port = "80";
1361 }
1362 '';
1363 };
1364 };
1365 exporterTest = ''
1366 wait_for_unit("prometheus-varnish-exporter.service")
1367 wait_for_open_port(6081)
1368 wait_for_open_port(9131)
1369 succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'")
1370 '';
1371 };
1372
1373 wireguard = let snakeoil = import ./wireguard/snakeoil-keys.nix; in
1374 {
1375 exporterConfig.enable = true;
1376 metricProvider = {
1377 networking.wireguard.interfaces.wg0 = {
1378 ips = [ "10.23.42.1/32" "fc00::1/128" ];
1379 listenPort = 23542;
1380
1381 inherit (snakeoil.peer0) privateKey;
1382
1383 peers = singleton {
1384 allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ];
1385
1386 inherit (snakeoil.peer1) publicKey;
1387 };
1388 };
1389 systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ];
1390 };
1391 exporterTest = ''
1392 wait_for_unit("prometheus-wireguard-exporter.service")
1393 wait_for_open_port(9586)
1394 wait_until_succeeds(
1395 "curl -sSf http://localhost:9586/metrics | grep '${snakeoil.peer1.publicKey}'"
1396 )
1397 '';
1398 };
1399
1400 zfs = {
1401 exporterConfig = {
1402 enable = true;
1403 };
1404 metricProvider = {
1405 boot.supportedFilesystems = [ "zfs" ];
1406 networking.hostId = "7327ded7";
1407 };
1408 exporterTest = ''
1409 wait_for_unit("prometheus-zfs-exporter.service")
1410 wait_for_unit("zfs.target")
1411 wait_for_open_port(9134)
1412 wait_until_succeeds("curl -f localhost:9134/metrics | grep 'zfs_scrape_collector_success{.*} 1'")
1413 '';
1414 };
1415 };
1416in
1417mapAttrs
1418 (exporter: testConfig: (makeTest (
1419 let
1420 nodeName = testConfig.nodeName or exporter;
1421
1422 in
1423 {
1424 name = "prometheus-${exporter}-exporter";
1425
1426 nodes.${nodeName} = mkMerge [{
1427 services.prometheus.exporters.${exporter} = testConfig.exporterConfig;
1428 } testConfig.metricProvider or { }];
1429
1430 testScript = ''
1431 ${nodeName}.start()
1432 ${concatStringsSep "\n" (map (line:
1433 if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")")
1434 then line
1435 else "${nodeName}.${line}"
1436 ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))}
1437 ${nodeName}.shutdown()
1438 '';
1439
1440 meta = with maintainers; {
1441 maintainers = [ willibutz ];
1442 };
1443 }
1444 )))
1445 exporterTests