pyrox.dev
1{ system ? builtins.currentSystem
2, config ? { }
3, pkgs ? import ../.. { inherit system config; }
4}:
5
6let
7 inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
8 inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge
9 removeSuffix replaceStrings singleton splitString;
10
11 /*
12 * The attrset `exporterTests` contains one attribute
13 * for each exporter test. Each of these attributes
14 * is expected to be an attrset containing:
15 *
16 * `exporterConfig`:
17 * this attribute set contains config for the exporter itself
18 *
19 * `exporterTest`
20 * this attribute set contains test instructions
21 *
22 * `metricProvider` (optional)
23 * this attribute contains additional machine config
24 *
25 * `nodeName` (optional)
26 * override an incompatible testnode name
27 *
28 * Example:
29 * exporterTests.<exporterName> = {
30 * exporterConfig = {
31 * enable = true;
32 * };
33 * metricProvider = {
34 * services.<metricProvider>.enable = true;
35 * };
36 * exporterTest = ''
37 * wait_for_unit("prometheus-<exporterName>-exporter.service")
38 * wait_for_open_port(1234)
39 * succeed("curl -sSf 'localhost:1234/metrics'")
40 * '';
41 * };
42 *
43 * # this would generate the following test config:
44 *
45 * nodes.<exporterName> = {
46 * services.prometheus.<exporterName> = {
47 * enable = true;
48 * };
49 * services.<metricProvider>.enable = true;
50 * };
51 *
52 * testScript = ''
53 * <exporterName>.start()
54 * <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service")
55 * <exporterName>.wait_for_open_port(1234)
56 * <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'")
57 * <exporterName>.shutdown()
58 * '';
59 */
60
61 exporterTests = {
62 apcupsd = {
63 exporterConfig = {
64 enable = true;
65 };
66 metricProvider = {
67 services.apcupsd.enable = true;
68 };
69 exporterTest = ''
70 wait_for_unit("apcupsd.service")
71 wait_for_open_port(3551)
72 wait_for_unit("prometheus-apcupsd-exporter.service")
73 wait_for_open_port(9162)
74 succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'")
75 '';
76 };
77
78 artifactory = {
79 exporterConfig = {
80 enable = true;
81 artiUsername = "artifactory-username";
82 artiPassword = "artifactory-password";
83 };
84 exporterTest = ''
85 wait_for_unit("prometheus-artifactory-exporter.service")
86 wait_for_open_port(9531)
87 succeed(
88 "curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'"
89 )
90 '';
91 };
92
93 bind = {
94 exporterConfig = {
95 enable = true;
96 };
97 metricProvider = {
98 services.bind.enable = true;
99 services.bind.extraConfig = ''
100 statistics-channels {
101 inet 127.0.0.1 port 8053 allow { localhost; };
102 };
103 '';
104 };
105 exporterTest = ''
106 wait_for_unit("prometheus-bind-exporter.service")
107 wait_for_open_port(9119)
108 succeed(
109 "curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'"
110 )
111 '';
112 };
113
114 bird = {
115 exporterConfig = {
116 enable = true;
117 };
118 metricProvider = {
119 services.bird2.enable = true;
120 services.bird2.config = ''
121 router id 127.0.0.1;
122
123 protocol kernel MyObviousTestString {
124 ipv4 {
125 import all;
126 export none;
127 };
128 }
129
130 protocol device {
131 }
132 '';
133 };
134 exporterTest = ''
135 wait_for_unit("prometheus-bird-exporter.service")
136 wait_for_open_port(9324)
137 wait_until_succeeds(
138 "curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'"
139 )
140 '';
141 };
142
143 bitcoin = {
144 exporterConfig = {
145 enable = true;
146 rpcUser = "bitcoinrpc";
147 rpcPasswordFile = pkgs.writeText "password" "hunter2";
148 };
149 metricProvider = {
150 services.bitcoind.default.enable = true;
151 services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC = "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7";
152 };
153 exporterTest = ''
154 wait_for_unit("prometheus-bitcoin-exporter.service")
155 wait_for_unit("bitcoind-default.service")
156 wait_for_open_port(9332)
157 succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '")
158 '';
159 };
160
161 blackbox = {
162 exporterConfig = {
163 enable = true;
164 configFile = pkgs.writeText "config.yml" (builtins.toJSON {
165 modules.icmp_v6 = {
166 prober = "icmp";
167 icmp.preferred_ip_protocol = "ip6";
168 };
169 });
170 };
171 exporterTest = ''
172 wait_for_unit("prometheus-blackbox-exporter.service")
173 wait_for_open_port(9115)
174 succeed(
175 "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'"
176 )
177 '';
178 };
179
180 collectd = {
181 exporterConfig = {
182 enable = true;
183 extraFlags = [ "--web.collectd-push-path /collectd" ];
184 };
185 exporterTest = let postData = replaceStrings [ "\n" ] [ "" ] ''
186 [{
187 "values":[23],
188 "dstypes":["gauge"],
189 "type":"gauge",
190 "interval":1000,
191 "host":"testhost",
192 "plugin":"testplugin",
193 "time":DATE
194 }]
195 ''; in
196 ''
197 wait_for_unit("prometheus-collectd-exporter.service")
198 wait_for_open_port(9103)
199 succeed(
200 'echo \'${postData}\'> /tmp/data.json'
201 )
202 succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json')
203 succeed(
204 "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd"
205 )
206 succeed(
207 "curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"
208 )
209 '';
210 };
211
212 dnsmasq = {
213 exporterConfig = {
214 enable = true;
215 leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
216 };
217 metricProvider = {
218 services.dnsmasq.enable = true;
219 };
220 exporterTest = ''
221 wait_for_unit("prometheus-dnsmasq-exporter.service")
222 wait_for_open_port(9153)
223 succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'")
224 '';
225 };
226
227 # Access to WHOIS server is required to properly test this exporter, so
228 # just perform basic sanity check that the exporter is running and returns
229 # a failure.
230 domain = {
231 exporterConfig = {
232 enable = true;
233 };
234 exporterTest = ''
235 wait_for_unit("prometheus-domain-exporter.service")
236 wait_for_open_port(9222)
237 succeed("curl -sSf 'http://localhost:9222/probe?target=nixos.org'")
238 '';
239 };
240
241 dovecot = {
242 exporterConfig = {
243 enable = true;
244 scopes = [ "global" ];
245 socketPath = "/var/run/dovecot2/old-stats";
246 user = "root"; # <- don't use user root in production
247 };
248 metricProvider = {
249 services.dovecot2.enable = true;
250 };
251 exporterTest = ''
252 wait_for_unit("prometheus-dovecot-exporter.service")
253 wait_for_open_port(9166)
254 succeed(
255 "curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'"
256 )
257 '';
258 };
259
260 fastly = {
261 exporterConfig = {
262 enable = true;
263 tokenPath = pkgs.writeText "token" "abc123";
264 };
265
266 # noop: fastly's exporter can't start without first talking to fastly
267 # see: https://github.com/peterbourgon/fastly-exporter/issues/87
268 exporterTest = ''
269 succeed("true");
270 '';
271 };
272
273 fritzbox = {
274 # TODO add proper test case
275 exporterConfig = {
276 enable = true;
277 };
278 exporterTest = ''
279 wait_for_unit("prometheus-fritzbox-exporter.service")
280 wait_for_open_port(9133)
281 succeed(
282 "curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'"
283 )
284 '';
285 };
286
287 influxdb = {
288 exporterConfig = {
289 enable = true;
290 sampleExpiry = "3s";
291 };
292 exporterTest = ''
293 wait_for_unit("prometheus-influxdb-exporter.service")
294 wait_for_open_port(9122)
295 succeed(
296 "curl -XPOST http://localhost:9122/write --data-binary 'influxdb_exporter,distro=nixos,added_in=21.09 value=1'"
297 )
298 succeed(
299 "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
300 )
301 execute("sleep 5")
302 fail(
303 "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
304 )
305 '';
306 };
307
308 ipmi = {
309 exporterConfig = {
310 enable = true;
311 };
312 exporterTest = ''
313 wait_for_unit("prometheus-ipmi-exporter.service")
314 wait_for_open_port(9290)
315 succeed(
316 "curl -sSf http://localhost:9290/metrics | grep 'ipmi_scrape_duration_seconds'"
317 )
318 '';
319 };
320
321 jitsi = {
322 exporterConfig = {
323 enable = true;
324 };
325 metricProvider = {
326 systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ];
327 services.jitsi-videobridge = {
328 enable = true;
329 colibriRestApi = true;
330 };
331 };
332 exporterTest = ''
333 wait_for_unit("jitsi-videobridge2.service")
334 wait_for_open_port(8080)
335 wait_for_unit("prometheus-jitsi-exporter.service")
336 wait_for_open_port(9700)
337 wait_until_succeeds(
338 'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"'
339 )
340 succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'")
341 '';
342 };
343
344 json = {
345 exporterConfig = {
346 enable = true;
347 url = "http://localhost";
348 configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON {
349 modules = {
350 default = {
351 metrics = [
352 { name = "json_test_metric"; path = "{ .test }"; }
353 ];
354 };
355 };
356 });
357 };
358 metricProvider = {
359 systemd.services.prometheus-json-exporter.after = [ "nginx.service" ];
360 services.nginx = {
361 enable = true;
362 virtualHosts.localhost.locations."/".extraConfig = ''
363 return 200 "{\"test\":1}";
364 '';
365 };
366 };
367 exporterTest = ''
368 wait_for_unit("nginx.service")
369 wait_for_open_port(80)
370 wait_for_unit("prometheus-json-exporter.service")
371 wait_for_open_port(7979)
372 succeed(
373 "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'"
374 )
375 '';
376 };
377
378 kea = let
379 controlSocketPathV4 = "/run/kea/dhcp4.sock";
380 controlSocketPathV6 = "/run/kea/dhcp6.sock";
381 in
382 {
383 exporterConfig = {
384 enable = true;
385 controlSocketPaths = [
386 controlSocketPathV4
387 controlSocketPathV6
388 ];
389 };
390 metricProvider = {
391 services.kea = {
392 dhcp4 = {
393 enable = true;
394 settings = {
395 control-socket = {
396 socket-type = "unix";
397 socket-name = controlSocketPathV4;
398 };
399 };
400 };
401 dhcp6 = {
402 enable = true;
403 settings = {
404 control-socket = {
405 socket-type = "unix";
406 socket-name = controlSocketPathV6;
407 };
408 };
409 };
410 };
411 };
412
413 exporterTest = ''
414 wait_for_unit("kea-dhcp4-server.service")
415 wait_for_unit("kea-dhcp6-server.service")
416 wait_for_file("${controlSocketPathV4}")
417 wait_for_file("${controlSocketPathV6}")
418 wait_for_unit("prometheus-kea-exporter.service")
419 wait_for_open_port(9547)
420 succeed(
421 "curl --fail localhost:9547/metrics | grep 'packets_received_total'"
422 )
423 '';
424 };
425
426 knot = {
427 exporterConfig = {
428 enable = true;
429 };
430 metricProvider = {
431 services.knot = {
432 enable = true;
433 extraArgs = [ "-v" ];
434 extraConfig = ''
435 server:
436 listen: 127.0.0.1@53
437
438 template:
439 - id: default
440 global-module: mod-stats
441 dnssec-signing: off
442 zonefile-sync: -1
443 journal-db: /var/lib/knot/journal
444 kasp-db: /var/lib/knot/kasp
445 timer-db: /var/lib/knot/timer
446 zonefile-load: difference
447 storage: ${pkgs.buildEnv {
448 name = "foo";
449 paths = [
450 (pkgs.writeTextDir "test.zone" ''
451 @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
452 @ NS ns1
453 @ NS ns2
454 ns1 A 192.168.0.1
455 '')
456 ];
457 }}
458
459 mod-stats:
460 - id: custom
461 edns-presence: on
462 query-type: on
463
464 zone:
465 - domain: test
466 file: test.zone
467 module: mod-stats/custom
468 '';
469 };
470 };
471 exporterTest = ''
472 wait_for_unit("knot.service")
473 wait_for_unit("prometheus-knot-exporter.service")
474 wait_for_open_port(9433)
475 succeed("curl -sSf 'localhost:9433' | grep 'knot_server_zone_count 1.0'")
476 '';
477 };
478
479 keylight = {
480 # A hardware device is required to properly test this exporter, so just
481 # perform a couple of basic sanity checks that the exporter is running
482 # and requires a target, but cannot reach a specified target.
483 exporterConfig = {
484 enable = true;
485 };
486 exporterTest = ''
487 wait_for_unit("prometheus-keylight-exporter.service")
488 wait_for_open_port(9288)
489 succeed(
490 "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'"
491 )
492 succeed(
493 "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'"
494 )
495 '';
496 };
497
498 lnd = {
499 exporterConfig = {
500 enable = true;
501 lndTlsPath = "/var/lib/lnd/tls.cert";
502 lndMacaroonDir = "/var/lib/lnd";
503 extraFlags = [ "--lnd.network=regtest" ];
504 };
505 metricProvider = {
506 systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15;
507 systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ];
508 services.bitcoind.regtest = {
509 enable = true;
510 extraConfig = ''
511 rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7
512 zmqpubrawblock=tcp://127.0.0.1:28332
513 zmqpubrawtx=tcp://127.0.0.1:28333
514 '';
515 extraCmdlineOptions = [ "-regtest" ];
516 };
517 systemd.services.lnd = {
518 serviceConfig.ExecStart = ''
519 ${pkgs.lnd}/bin/lnd \
520 --datadir=/var/lib/lnd \
521 --tlscertpath=/var/lib/lnd/tls.cert \
522 --tlskeypath=/var/lib/lnd/tls.key \
523 --logdir=/var/log/lnd \
524 --bitcoin.active \
525 --bitcoin.regtest \
526 --bitcoin.node=bitcoind \
527 --bitcoind.rpcuser=bitcoinrpc \
528 --bitcoind.rpcpass=hunter2 \
529 --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \
530 --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \
531 --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon
532 '';
533 serviceConfig.StateDirectory = "lnd";
534 wantedBy = [ "multi-user.target" ];
535 after = [ "network.target" ];
536 };
537 # initialize wallet, creates macaroon needed by exporter
538 systemd.services.lnd.postStart = ''
539 ${pkgs.curl}/bin/curl \
540 --retry 20 \
541 --retry-delay 1 \
542 --retry-connrefused \
543 --cacert /var/lib/lnd/tls.cert \
544 -X GET \
545 https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed
546 ${pkgs.curl}/bin/curl \
547 --retry 20 \
548 --retry-delay 1 \
549 --retry-connrefused \
550 --cacert /var/lib/lnd/tls.cert \
551 -X POST \
552 -d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \
553 https://localhost:8080/v1/initwallet
554 '';
555 };
556 exporterTest = ''
557 wait_for_unit("lnd.service")
558 wait_for_open_port(10009)
559 wait_for_unit("prometheus-lnd-exporter.service")
560 wait_for_open_port(9092)
561 succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'")
562 '';
563 };
564
565 mail = {
566 exporterConfig = {
567 enable = true;
568 configuration = {
569 monitoringInterval = "2s";
570 mailCheckTimeout = "10s";
571 servers = [{
572 name = "testserver";
573 server = "localhost";
574 port = 25;
575 from = "mail-exporter@localhost";
576 to = "mail-exporter@localhost";
577 detectionDir = "/var/spool/mail/mail-exporter/new";
578 }];
579 };
580 };
581 metricProvider = {
582 services.postfix.enable = true;
583 systemd.services.prometheus-mail-exporter = {
584 after = [ "postfix.service" ];
585 requires = [ "postfix.service" ];
586 serviceConfig = {
587 ExecStartPre = [
588 "${pkgs.writeShellScript "create-maildir" ''
589 mkdir -p -m 0700 mail-exporter/new
590 ''}"
591 ];
592 ProtectHome = true;
593 ReadOnlyPaths = "/";
594 ReadWritePaths = "/var/spool/mail";
595 WorkingDirectory = "/var/spool/mail";
596 };
597 };
598 users.users.mailexporter = {
599 isSystemUser = true;
600 group = "mailexporter";
601 };
602 users.groups.mailexporter = {};
603 };
604 exporterTest = ''
605 wait_for_unit("postfix.service")
606 wait_for_unit("prometheus-mail-exporter.service")
607 wait_for_open_port(9225)
608 wait_until_succeeds(
609 "curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'"
610 )
611 '';
612 };
613
614 mikrotik = {
615 exporterConfig = {
616 enable = true;
617 extraFlags = [ "-timeout=1s" ];
618 configuration = {
619 devices = [
620 {
621 name = "router";
622 address = "192.168.42.48";
623 user = "prometheus";
624 password = "shh";
625 }
626 ];
627 features = {
628 bgp = true;
629 dhcp = true;
630 dhcpl = true;
631 dhcpv6 = true;
632 health = true;
633 routes = true;
634 poe = true;
635 pools = true;
636 optics = true;
637 w60g = true;
638 wlansta = true;
639 wlanif = true;
640 monitor = true;
641 ipsec = true;
642 };
643 };
644 };
645 exporterTest = ''
646 wait_for_unit("prometheus-mikrotik-exporter.service")
647 wait_for_open_port(9436)
648 succeed(
649 "curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'"
650 )
651 '';
652 };
653
654 modemmanager = {
655 exporterConfig = {
656 enable = true;
657 refreshRate = "10s";
658 };
659 metricProvider = {
660 # ModemManager is installed when NetworkManager is enabled. Ensure it is
661 # started and is wanted by NM and the exporter to start everything up
662 # in the right order.
663 networking.networkmanager.enable = true;
664 systemd.services.ModemManager = {
665 enable = true;
666 wantedBy = [ "NetworkManager.service" "prometheus-modemmanager-exporter.service" ];
667 };
668 };
669 exporterTest = ''
670 wait_for_unit("ModemManager.service")
671 wait_for_unit("prometheus-modemmanager-exporter.service")
672 wait_for_open_port(9539)
673 succeed(
674 "curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'"
675 )
676 '';
677 };
678
679 nextcloud = {
680 exporterConfig = {
681 enable = true;
682 passwordFile = "/var/nextcloud-pwfile";
683 url = "http://localhost";
684 };
685 metricProvider = {
686 systemd.services.nc-pwfile =
687 let
688 passfile = (pkgs.writeText "pwfile" "snakeoilpw");
689 in
690 {
691 requiredBy = [ "prometheus-nextcloud-exporter.service" ];
692 before = [ "prometheus-nextcloud-exporter.service" ];
693 serviceConfig.ExecStart = ''
694 ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile
695 '';
696 };
697 services.nginx = {
698 enable = true;
699 virtualHosts."localhost" = {
700 basicAuth.nextcloud-exporter = "snakeoilpw";
701 locations."/" = {
702 root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
703 tryFiles = "/negative-space.json =404";
704 };
705 };
706 };
707 };
708 exporterTest = ''
709 wait_for_unit("nginx.service")
710 wait_for_unit("prometheus-nextcloud-exporter.service")
711 wait_for_open_port(9205)
712 succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'")
713 '';
714 };
715
716 nginx = {
717 exporterConfig = {
718 enable = true;
719 };
720 metricProvider = {
721 services.nginx = {
722 enable = true;
723 statusPage = true;
724 virtualHosts."test".extraConfig = "return 204;";
725 };
726 };
727 exporterTest = ''
728 wait_for_unit("nginx.service")
729 wait_for_unit("prometheus-nginx-exporter.service")
730 wait_for_open_port(9113)
731 succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up 1'")
732 '';
733 };
734
735 nginxlog = {
736 exporterConfig = {
737 enable = true;
738 group = "nginx";
739 settings = {
740 namespaces = [
741 {
742 name = "filelogger";
743 source = {
744 files = [ "/var/log/nginx/filelogger.access.log" ];
745 };
746 }
747 {
748 name = "syslogger";
749 source = {
750 syslog = {
751 listen_address = "udp://127.0.0.1:10000";
752 format = "rfc3164";
753 tags = [ "nginx" ];
754 };
755 };
756 }
757 ];
758 };
759 };
760 metricProvider = {
761 services.nginx = {
762 enable = true;
763 httpConfig = ''
764 server {
765 listen 80;
766 server_name filelogger.local;
767 access_log /var/log/nginx/filelogger.access.log;
768 }
769 server {
770 listen 81;
771 server_name syslogger.local;
772 access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
773 }
774 '';
775 };
776 };
777 exporterTest = ''
778 wait_for_unit("nginx.service")
779 wait_for_unit("prometheus-nginxlog-exporter.service")
780 wait_for_open_port(9117)
781 wait_for_open_port(80)
782 wait_for_open_port(81)
783 succeed("curl http://localhost")
784 execute("sleep 1")
785 succeed(
786 "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1"
787 )
788 succeed("curl http://localhost:81")
789 execute("sleep 1")
790 succeed(
791 "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1"
792 )
793 '';
794 };
795
796 node = {
797 exporterConfig = {
798 enable = true;
799 };
800 exporterTest = ''
801 wait_for_unit("prometheus-node-exporter.service")
802 wait_for_open_port(9100)
803 succeed(
804 "curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'"
805 )
806 '';
807 };
808
809 openldap = {
810 exporterConfig = {
811 enable = true;
812 ldapCredentialFile = "${pkgs.writeText "exporter.yml" ''
813 ldapUser: "cn=root,dc=example"
814 ldapPass: "notapassword"
815 ''}";
816 };
817 metricProvider = {
818 services.openldap = {
819 enable = true;
820 settings.children = {
821 "cn=schema".includes = [
822 "${pkgs.openldap}/etc/schema/core.ldif"
823 "${pkgs.openldap}/etc/schema/cosine.ldif"
824 "${pkgs.openldap}/etc/schema/inetorgperson.ldif"
825 "${pkgs.openldap}/etc/schema/nis.ldif"
826 ];
827 "olcDatabase={1}mdb" = {
828 attrs = {
829 objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ];
830 olcDatabase = "{1}mdb";
831 olcDbDirectory = "/var/db/openldap";
832 olcSuffix = "dc=example";
833 olcRootDN = {
834 # cn=root,dc=example
835 base64 = "Y249cm9vdCxkYz1leGFtcGxl";
836 };
837 olcRootPW = {
838 path = "${pkgs.writeText "rootpw" "notapassword"}";
839 };
840 };
841 };
842 "olcDatabase={2}monitor".attrs = {
843 objectClass = [ "olcDatabaseConfig" ];
844 olcDatabase = "{2}monitor";
845 olcAccess = [ "to dn.subtree=cn=monitor by users read" ];
846 };
847 };
848 declarativeContents."dc=example" = ''
849 dn: dc=example
850 objectClass: domain
851 dc: example
852
853 dn: ou=users,dc=example
854 objectClass: organizationalUnit
855 ou: users
856 '';
857 };
858 };
859 exporterTest = ''
860 wait_for_unit("prometheus-openldap-exporter.service")
861 wait_for_open_port(389)
862 wait_for_open_port(9330)
863 wait_until_succeeds(
864 "curl -sSf http://localhost:9330/metrics | grep 'openldap_scrape{result=\"ok\"} 1'"
865 )
866 '';
867 };
868
869 openvpn = {
870 exporterConfig = {
871 enable = true;
872 group = "openvpn";
873 statusPaths = [ "/run/openvpn-test" ];
874 };
875 metricProvider = {
876 users.groups.openvpn = { };
877 services.openvpn.servers.test = {
878 config = ''
879 dev tun
880 status /run/openvpn-test
881 status-version 3
882 '';
883 up = "chmod g+r /run/openvpn-test";
884 };
885 systemd.services."openvpn-test".serviceConfig.Group = "openvpn";
886 };
887 exporterTest = ''
888 wait_for_unit("openvpn-test.service")
889 wait_for_unit("prometheus-openvpn-exporter.service")
890 succeed("curl -sSf http://localhost:9176/metrics | grep 'openvpn_up{.*} 1'")
891 '';
892 };
893
894 postfix = {
895 exporterConfig = {
896 enable = true;
897 };
898 metricProvider = {
899 services.postfix.enable = true;
900 };
901 exporterTest = ''
902 wait_for_unit("prometheus-postfix-exporter.service")
903 wait_for_file("/var/lib/postfix/queue/public/showq")
904 wait_for_open_port(9154)
905 wait_until_succeeds(
906 "curl -sSf http://localhost:9154/metrics | grep 'postfix_up{path=\"/var/lib/postfix/queue/public/showq\"} 1'"
907 )
908 succeed(
909 "curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'"
910 )
911 succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'")
912 '';
913 };
914
915 postgres = {
916 exporterConfig = {
917 enable = true;
918 runAsLocalSuperUser = true;
919 };
920 metricProvider = {
921 services.postgresql.enable = true;
922 };
923 exporterTest = ''
924 wait_for_unit("prometheus-postgres-exporter.service")
925 wait_for_open_port(9187)
926 wait_for_unit("postgresql.service")
927 succeed(
928 "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
929 )
930 succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
931 systemctl("stop postgresql.service")
932 succeed(
933 "curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'"
934 )
935 succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'")
936 systemctl("start postgresql.service")
937 wait_for_unit("postgresql.service")
938 succeed(
939 "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
940 )
941 succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
942 '';
943 };
944
945 process = {
946 exporterConfig = {
947 enable = true;
948 settings.process_names = [
949 # Remove nix store path from process name
950 { name = "{{.Matches.Wrapped}} {{ .Matches.Args }}"; cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ]; }
951 ];
952 };
953 exporterTest = ''
954 wait_for_unit("prometheus-process-exporter.service")
955 wait_for_open_port(9256)
956 wait_until_succeeds(
957 "curl -sSf localhost:9256/metrics | grep -q '{}'".format(
958 'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter '
959 )
960 )
961 '';
962 };
963
964 pve = let
965 pveExporterEnvFile = pkgs.writeTextFile {
966 name = "pve.env";
967 text = ''
968 PVE_USER="test_user@pam"
969 PVE_PASSWORD="hunter3"
970 PVE_VERIFY_SSL="false"
971 '';
972 };
973 in {
974 exporterConfig = {
975 enable = true;
976 environmentFile = pveExporterEnvFile;
977 };
978 exporterTest = ''
979 wait_for_unit("prometheus-pve-exporter.service")
980 wait_for_open_port(9221)
981 wait_until_succeeds("curl localhost:9221")
982 '';
983 };
984
985 py-air-control = {
986 nodeName = "py_air_control";
987 exporterConfig = {
988 enable = true;
989 deviceHostname = "127.0.0.1";
990 };
991 exporterTest = ''
992 wait_for_unit("prometheus-py-air-control-exporter.service")
993 wait_for_open_port(9896)
994 succeed(
995 "curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'"
996 )
997 '';
998 };
999
1000 redis = {
1001 exporterConfig = {
1002 enable = true;
1003 };
1004 metricProvider.services.redis.servers."".enable = true;
1005 exporterTest = ''
1006 wait_for_unit("redis.service")
1007 wait_for_unit("prometheus-redis-exporter.service")
1008 wait_for_open_port(6379)
1009 wait_for_open_port(9121)
1010 wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'")
1011 '';
1012 };
1013
1014 rspamd = {
1015 exporterConfig = {
1016 enable = true;
1017 };
1018 metricProvider = {
1019 services.rspamd.enable = true;
1020 };
1021 exporterTest = ''
1022 wait_for_unit("rspamd.service")
1023 wait_for_unit("prometheus-rspamd-exporter.service")
1024 wait_for_open_port(11334)
1025 wait_for_open_port(7980)
1026 wait_until_succeeds(
1027 "curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep 'rspamd_scanned{host=\"rspamd\"} 0'"
1028 )
1029 '';
1030 };
1031
1032 rtl_433 = {
1033 exporterConfig = {
1034 enable = true;
1035 };
1036 metricProvider = {
1037 # Mock rtl_433 binary to return a dummy metric stream.
1038 nixpkgs.overlays = [
1039 (self: super: {
1040 rtl_433 = self.runCommand "rtl_433" { } ''
1041 mkdir -p "$out/bin"
1042 cat <<EOF > "$out/bin/rtl_433"
1043 #!/bin/sh
1044 while true; do
1045 printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n'
1046 sleep 4
1047 done
1048 EOF
1049 chmod +x "$out/bin/rtl_433"
1050 '';
1051 })
1052 ];
1053 };
1054 exporterTest = ''
1055 wait_for_unit("prometheus-rtl_433-exporter.service")
1056 wait_for_open_port(9550)
1057 wait_until_succeeds(
1058 "curl -sSf localhost:9550/metrics | grep '{}'".format(
1059 'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18'
1060 )
1061 )
1062 '';
1063 };
1064
1065 shelly = {
1066 exporterConfig = {
1067 enable = true;
1068 metrics-file = "${pkgs.writeText "test.json" ''{}''}";
1069 };
1070 exporterTest = ''
1071 wait_for_unit("prometheus-shelly-exporter.service")
1072 wait_for_open_port(9784)
1073 wait_until_succeeds(
1074 "curl -sSf 'localhost:9784/metrics'"
1075 )
1076 '';
1077 };
1078
1079 script = {
1080 exporterConfig = {
1081 enable = true;
1082 settings.scripts = [
1083 { name = "success"; script = "sleep 1"; }
1084 ];
1085 };
1086 exporterTest = ''
1087 wait_for_unit("prometheus-script-exporter.service")
1088 wait_for_open_port(9172)
1089 wait_until_succeeds(
1090 "curl -sSf 'localhost:9172/probe?name=success' | grep -q '{}'".format(
1091 'script_success{script="success"} 1'
1092 )
1093 )
1094 '';
1095 };
1096
1097 smartctl = {
1098 exporterConfig = {
1099 enable = true;
1100 devices = [
1101 "/dev/vda"
1102 ];
1103 };
1104 exporterTest = ''
1105 wait_until_succeeds(
1106 'journalctl -eu prometheus-smartctl-exporter.service -o cat | grep "Device unavailable"'
1107 )
1108 '';
1109 };
1110
1111 smokeping = {
1112 exporterConfig = {
1113 enable = true;
1114 hosts = [ "127.0.0.1" ];
1115 };
1116 exporterTest = ''
1117 wait_for_unit("prometheus-smokeping-exporter.service")
1118 wait_for_open_port(9374)
1119 wait_until_succeeds(
1120 "curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format(
1121 'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1"} '
1122 )
1123 )
1124 wait_until_succeeds(
1125 "curl -sSf localhost:9374/metrics | grep '{}'".format(
1126 'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1"}'
1127 )
1128 )
1129 '';
1130 };
1131
1132 snmp = {
1133 exporterConfig = {
1134 enable = true;
1135 configuration.default = {
1136 version = 2;
1137 auth.community = "public";
1138 };
1139 };
1140 exporterTest = ''
1141 wait_for_unit("prometheus-snmp-exporter.service")
1142 wait_for_open_port(9116)
1143 succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'")
1144 '';
1145 };
1146
1147 sql = {
1148 exporterConfig = {
1149 configuration.jobs.points = {
1150 interval = "1m";
1151 connections = [
1152 "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable"
1153 ];
1154 queries = {
1155 points = {
1156 labels = [ "name" ];
1157 help = "Amount of points accumulated per person";
1158 values = [ "amount" ];
1159 query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name";
1160 };
1161 };
1162 };
1163 enable = true;
1164 user = "prometheus-sql-exporter";
1165 };
1166 metricProvider = {
1167 services.postgresql = {
1168 enable = true;
1169 initialScript = builtins.toFile "init.sql" ''
1170 CREATE DATABASE data;
1171 \c data;
1172 CREATE TABLE points (amount INT, name TEXT);
1173 INSERT INTO points(amount, name) VALUES (1, 'jack');
1174 INSERT INTO points(amount, name) VALUES (2, 'jill');
1175 INSERT INTO points(amount, name) VALUES (3, 'jack');
1176
1177 CREATE USER "prometheus-sql-exporter";
1178 GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter";
1179 GRANT SELECT ON points TO "prometheus-sql-exporter";
1180 '';
1181 };
1182 systemd.services.prometheus-sql-exporter.after = [ "postgresql.service" ];
1183 };
1184 exporterTest = ''
1185 wait_for_unit("prometheus-sql-exporter.service")
1186 wait_for_open_port(9237)
1187 succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2")
1188 '';
1189 };
1190
1191 statsd = {
1192 exporterConfig = {
1193 enable = true;
1194 };
1195 exporterTest = ''
1196 wait_for_unit("prometheus-statsd-exporter.service")
1197 wait_for_open_port(9102)
1198 succeed("curl http://localhost:9102/metrics | grep 'statsd_exporter_build_info{'")
1199 wait_until_succeeds(
1200 "echo 'test.udp:1|c' > /dev/udp/localhost/9125 && \
1201 curl http://localhost:9102/metrics | grep 'test_udp 1'",
1202 timeout=10
1203 )
1204 wait_until_succeeds(
1205 "echo 'test.tcp:1|c' > /dev/tcp/localhost/9125 && \
1206 curl http://localhost:9102/metrics | grep 'test_tcp 1'",
1207 timeout=10
1208 )
1209 '';
1210 };
1211
1212 surfboard = {
1213 exporterConfig = {
1214 enable = true;
1215 modemAddress = "localhost";
1216 };
1217 metricProvider = {
1218 systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ];
1219 services.nginx = {
1220 enable = true;
1221 virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = ''
1222 return 204;
1223 '';
1224 };
1225 };
1226 exporterTest = ''
1227 wait_for_unit("nginx.service")
1228 wait_for_open_port(80)
1229 wait_for_unit("prometheus-surfboard-exporter.service")
1230 wait_for_open_port(9239)
1231 succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'")
1232 '';
1233 };
1234
1235 systemd = {
1236 exporterConfig = {
1237 enable = true;
1238
1239 extraFlags = [
1240 "--systemd.collector.enable-restart-count"
1241 ];
1242 };
1243 metricProvider = { };
1244 exporterTest = ''
1245 wait_for_unit("prometheus-systemd-exporter.service")
1246 wait_for_open_port(9558)
1247 wait_until_succeeds(
1248 "curl -sSf localhost:9558/metrics | grep '{}'".format(
1249 'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
1250 )
1251 )
1252 succeed(
1253 "curl -sSf localhost:9558/metrics | grep '{}'".format(
1254 'systemd_service_restart_total{name="prometheus-systemd-exporter.service"} 0'
1255 )
1256 )
1257 '';
1258 };
1259
1260 tor = {
1261 exporterConfig = {
1262 enable = true;
1263 };
1264 metricProvider = {
1265 # Note: this does not connect the test environment to the Tor network.
1266 # Client, relay, bridge or exit connectivity are disabled by default.
1267 services.tor.enable = true;
1268 services.tor.settings.ControlPort = 9051;
1269 };
1270 exporterTest = ''
1271 wait_for_unit("tor.service")
1272 wait_for_open_port(9051)
1273 wait_for_unit("prometheus-tor-exporter.service")
1274 wait_for_open_port(9130)
1275 succeed("curl -sSf localhost:9130/metrics | grep 'tor_version{.\\+} 1'")
1276 '';
1277 };
1278
1279 unpoller = {
1280 nodeName = "unpoller";
1281 exporterConfig.enable = true;
1282 exporterConfig.controllers = [{ }];
1283 exporterTest = ''
1284 wait_until_succeeds(
1285 'journalctl -eu prometheus-unpoller-exporter.service -o cat | grep "Connection Error"'
1286 )
1287 '';
1288 };
1289
1290 unbound = {
1291 exporterConfig = {
1292 enable = true;
1293 fetchType = "uds";
1294 controlInterface = "/run/unbound/unbound.ctl";
1295 };
1296 metricProvider = {
1297 services.unbound = {
1298 enable = true;
1299 localControlSocketPath = "/run/unbound/unbound.ctl";
1300 };
1301 systemd.services.prometheus-unbound-exporter.serviceConfig = {
1302 SupplementaryGroups = [ "unbound" ];
1303 };
1304 };
1305 exporterTest = ''
1306 wait_for_unit("unbound.service")
1307 wait_for_unit("prometheus-unbound-exporter.service")
1308 wait_for_open_port(9167)
1309 succeed("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'")
1310 '';
1311 };
1312
1313 v2ray = {
1314 exporterConfig = {
1315 enable = true;
1316 };
1317
1318 metricProvider = {
1319 systemd.services.prometheus-nginx-exporter.after = [ "v2ray.service" ];
1320 services.v2ray = {
1321 enable = true;
1322 config = {
1323 stats = {};
1324 api = {
1325 tag = "api";
1326 services = [ "StatsService" ];
1327 };
1328 inbounds = [
1329 {
1330 port = 1080;
1331 listen = "127.0.0.1";
1332 protocol = "http";
1333 }
1334 {
1335 listen = "127.0.0.1";
1336 port = 54321;
1337 protocol = "dokodemo-door";
1338 settings = { address = "127.0.0.1"; };
1339 tag = "api";
1340 }
1341 ];
1342 outbounds = [
1343 {
1344 protocol = "freedom";
1345 }
1346 {
1347 protocol = "freedom";
1348 settings = {};
1349 tag = "api";
1350 }
1351 ];
1352 routing = {
1353 strategy = "rules";
1354 settings = {
1355 rules = [
1356 {
1357 inboundTag = [ "api" ];
1358 outboundTag = "api";
1359 type = "field";
1360 }
1361 ];
1362 };
1363 };
1364 };
1365 };
1366 };
1367 exporterTest = ''
1368 wait_for_unit("prometheus-v2ray-exporter.service")
1369 wait_for_open_port(9299)
1370 succeed("curl -sSf localhost:9299/scrape | grep 'v2ray_up 1'")
1371 '';
1372 };
1373
1374 varnish = {
1375 exporterConfig = {
1376 enable = true;
1377 instance = "/var/spool/varnish/varnish";
1378 group = "varnish";
1379 };
1380 metricProvider = {
1381 systemd.services.prometheus-varnish-exporter.after = [
1382 "varnish.service"
1383 ];
1384 services.varnish = {
1385 enable = true;
1386 config = ''
1387 vcl 4.0;
1388 backend default {
1389 .host = "127.0.0.1";
1390 .port = "80";
1391 }
1392 '';
1393 };
1394 };
1395 exporterTest = ''
1396 wait_for_unit("prometheus-varnish-exporter.service")
1397 wait_for_open_port(6081)
1398 wait_for_open_port(9131)
1399 succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'")
1400 '';
1401 };
1402
1403 wireguard = let
1404 snakeoil = import ./wireguard/snakeoil-keys.nix;
1405 publicKeyWithoutNewlines = replaceStrings [ "\n" ] [ "" ] snakeoil.peer1.publicKey;
1406 in
1407 {
1408 exporterConfig.enable = true;
1409 metricProvider = {
1410 networking.wireguard.interfaces.wg0 = {
1411 ips = [ "10.23.42.1/32" "fc00::1/128" ];
1412 listenPort = 23542;
1413
1414 inherit (snakeoil.peer0) privateKey;
1415
1416 peers = singleton {
1417 allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ];
1418
1419 inherit (snakeoil.peer1) publicKey;
1420 };
1421 };
1422 systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ];
1423 };
1424 exporterTest = ''
1425 wait_for_unit("prometheus-wireguard-exporter.service")
1426 wait_for_open_port(9586)
1427 wait_until_succeeds(
1428 "curl -sSf http://localhost:9586/metrics | grep '${publicKeyWithoutNewlines}'"
1429 )
1430 '';
1431 };
1432
1433 zfs = {
1434 exporterConfig = {
1435 enable = true;
1436 };
1437 metricProvider = {
1438 boot.supportedFilesystems = [ "zfs" ];
1439 networking.hostId = "7327ded7";
1440 };
1441 exporterTest = ''
1442 wait_for_unit("prometheus-zfs-exporter.service")
1443 wait_for_unit("zfs.target")
1444 wait_for_open_port(9134)
1445 wait_until_succeeds("curl -f localhost:9134/metrics | grep 'zfs_scrape_collector_success{.*} 1'")
1446 '';
1447 };
1448 };
1449in
1450mapAttrs
1451 (exporter: testConfig: (makeTest (
1452 let
1453 nodeName = testConfig.nodeName or exporter;
1454
1455 in
1456 {
1457 name = "prometheus-${exporter}-exporter";
1458
1459 nodes.${nodeName} = mkMerge [{
1460 services.prometheus.exporters.${exporter} = testConfig.exporterConfig;
1461 } testConfig.metricProvider or { }];
1462
1463 testScript = ''
1464 ${nodeName}.start()
1465 ${concatStringsSep "\n" (map (line:
1466 if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")")
1467 then line
1468 else "${nodeName}.${line}"
1469 ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))}
1470 ${nodeName}.shutdown()
1471 '';
1472
1473 meta = with maintainers; {
1474 maintainers = [ willibutz ];
1475 };
1476 }
1477 )))
1478 exporterTests