pyrox.dev
1{
2 system ? builtins.currentSystem,
3 config ? { },
4 pkgs ? import ../.. { inherit system config; },
5}:
6
7let
8 inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
9 inherit (pkgs.lib)
10 concatStringsSep
11 maintainers
12 mapAttrs
13 mkMerge
14 removeSuffix
15 replaceStrings
16 singleton
17 splitString
18 makeBinPath
19 ;
20
21 /*
22 * The attrset `exporterTests` contains one attribute
23 * for each exporter test. Each of these attributes
24 * is expected to be an attrset containing:
25 *
26 * `exporterConfig`:
27 * this attribute set contains config for the exporter itself
28 *
29 * `exporterTest`
30 * this attribute set contains test instructions
31 *
32 * `metricProvider` (optional)
33 * this attribute contains additional machine config
34 *
35 * `nodeName` (optional)
36 * override an incompatible testnode name
37 *
38 * Example:
39 * exporterTests.<exporterName> = {
40 * exporterConfig = {
41 * enable = true;
42 * };
43 * metricProvider = {
44 * services.<metricProvider>.enable = true;
45 * };
46 * exporterTest = ''
47 * wait_for_unit("prometheus-<exporterName>-exporter.service")
48 * wait_for_open_port(1234)
49 * succeed("curl -sSf 'localhost:1234/metrics'")
50 * '';
51 * };
52 *
53 * # this would generate the following test config:
54 *
55 * nodes.<exporterName> = {
56 * services.prometheus.<exporterName> = {
57 * enable = true;
58 * };
59 * services.<metricProvider>.enable = true;
60 * };
61 *
62 * testScript = ''
63 * <exporterName>.start()
64 * <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service")
65 * <exporterName>.wait_for_open_port(1234)
66 * <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'")
67 * <exporterName>.shutdown()
68 * '';
69 */
70
71 exporterTests = {
72 apcupsd = {
73 exporterConfig = {
74 enable = true;
75 };
76 metricProvider = {
77 services.apcupsd.enable = true;
78 };
79 exporterTest = ''
80 wait_for_unit("apcupsd.service")
81 wait_for_open_port(3551)
82 wait_for_unit("prometheus-apcupsd-exporter.service")
83 wait_for_open_port(9162)
84 succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'")
85 '';
86 };
87
88 artifactory = {
89 exporterConfig = {
90 enable = true;
91 artiUsername = "artifactory-username";
92 artiPassword = "artifactory-password";
93 };
94 exporterTest = ''
95 wait_for_unit("prometheus-artifactory-exporter.service")
96 wait_for_open_port(9531)
97 succeed(
98 "curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'"
99 )
100 '';
101 };
102
103 bind = {
104 exporterConfig = {
105 enable = true;
106 };
107 metricProvider = {
108 services.bind.enable = true;
109 services.bind.extraConfig = ''
110 statistics-channels {
111 inet 127.0.0.1 port 8053 allow { localhost; };
112 };
113 '';
114 };
115 exporterTest = ''
116 wait_for_unit("prometheus-bind-exporter.service")
117 wait_for_open_port(9119)
118 succeed(
119 "curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'"
120 )
121 '';
122 };
123
124 bird = {
125 exporterConfig = {
126 enable = true;
127 };
128 metricProvider = {
129 services.bird.enable = true;
130 services.bird.config = ''
131 router id 127.0.0.1;
132
133 protocol kernel MyObviousTestString {
134 ipv4 {
135 import all;
136 export none;
137 };
138 }
139
140 protocol device {
141 }
142 '';
143 };
144 exporterTest = ''
145 wait_for_unit("prometheus-bird-exporter.service")
146 wait_for_open_port(9324)
147 wait_until_succeeds(
148 "curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'"
149 )
150 '';
151 };
152
153 bitcoin = {
154 exporterConfig = {
155 enable = true;
156 rpcUser = "bitcoinrpc";
157 rpcPasswordFile = pkgs.writeText "password" "hunter2";
158 };
159 metricProvider = {
160 services.bitcoind.default.enable = true;
161 services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC =
162 "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7";
163 };
164 exporterTest = ''
165 wait_for_unit("prometheus-bitcoin-exporter.service")
166 wait_for_unit("bitcoind-default.service")
167 wait_for_open_port(9332)
168 succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '")
169 '';
170 };
171
172 blackbox = {
173 exporterConfig = {
174 enable = true;
175 configFile = pkgs.writeText "config.yml" (
176 builtins.toJSON {
177 modules.icmp_v6 = {
178 prober = "icmp";
179 icmp.preferred_ip_protocol = "ip6";
180 };
181 }
182 );
183 };
184 exporterTest = ''
185 wait_for_unit("prometheus-blackbox-exporter.service")
186 wait_for_open_port(9115)
187 succeed(
188 "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'"
189 )
190 '';
191 };
192
193 borgmatic = {
194 exporterConfig = {
195 enable = true;
196 user = "root";
197 };
198 metricProvider = {
199 services.borgmatic.enable = true;
200 services.borgmatic.settings.source_directories = [ "/home" ];
201 services.borgmatic.settings.repositories = [
202 {
203 label = "local";
204 path = "/var/backup";
205 }
206 ];
207 services.borgmatic.settings.keep_daily = 10;
208 };
209 exporterTest = ''
210 succeed("borgmatic rcreate -e none")
211 succeed("borgmatic")
212 wait_for_unit("prometheus-borgmatic-exporter.service")
213 wait_for_open_port(9996)
214 succeed("curl -sSf localhost:9996/metrics | grep 'borg_total_backups{repository=\"/var/backup\"} 1'")
215 '';
216 };
217
218 collectd = {
219 exporterConfig = {
220 enable = true;
221 extraFlags = [ "--web.collectd-push-path /collectd" ];
222 };
223 exporterTest =
224 let
225 postData = replaceStrings [ "\n" ] [ "" ] ''
226 [{
227 "values":[23],
228 "dstypes":["gauge"],
229 "type":"gauge",
230 "interval":1000,
231 "host":"testhost",
232 "plugin":"testplugin",
233 "time":DATE
234 }]
235 '';
236 in
237 ''
238 wait_for_unit("prometheus-collectd-exporter.service")
239 wait_for_open_port(9103)
240 succeed(
241 'echo \'${postData}\'> /tmp/data.json'
242 )
243 succeed('sed -i -e "s DATE $(date +%s) " /tmp/data.json')
244 succeed(
245 "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd"
246 )
247 succeed(
248 "curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"
249 )
250 '';
251 };
252
253 deluge = {
254 exporterConfig = {
255 enable = true;
256 port = 1234;
257 listenAddress = "127.0.0.1";
258
259 delugeUser = "user";
260 delugePort = 2345;
261 delugePasswordFile = pkgs.writeText "password" "weak_password";
262 };
263 metricProvider = {
264 services.deluge.enable = true;
265 services.deluge.declarative = true;
266 services.deluge.config.daemon_port = 2345;
267 services.deluge.authFile = pkgs.writeText "authFile" ''
268 localclient:abcdef:10
269 user:weak_password:10
270 '';
271 };
272 exporterTest = ''
273 wait_for_unit("deluged.service")
274 wait_for_open_port(2345)
275 wait_for_unit("prometheus-deluge-exporter.service")
276 wait_for_open_port(1234)
277 succeed("curl -sSf http://localhost:1234 | grep 'deluge_torrents'")
278 '';
279 };
280
281 dnsmasq = {
282 exporterConfig = {
283 enable = true;
284 leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
285 };
286 metricProvider = {
287 services.dnsmasq.enable = true;
288 };
289 exporterTest = ''
290 wait_for_unit("dnsmasq.service")
291 wait_for_open_port(53)
292 wait_for_file("/var/lib/dnsmasq/dnsmasq.leases")
293 wait_for_unit("prometheus-dnsmasq-exporter.service")
294 wait_for_open_port(9153)
295 succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'")
296 '';
297 };
298
299 dnssec = {
300 exporterConfig = {
301 enable = true;
302 configuration = {
303 records = [
304 {
305 zone = "example.com";
306 record = "@";
307 type = "SOA";
308 }
309 ];
310 };
311 resolvers = [ "127.0.0.1:53" ];
312 };
313 metricProvider = {
314 services.knot = {
315 enable = true;
316 settingsFile = pkgs.writeText "knot.conf" ''
317 server:
318 listen: 127.0.0.1@53
319 template:
320 - id: default
321 storage: ${
322 pkgs.buildEnv {
323 name = "zones";
324 paths = [
325 (pkgs.writeTextDir "example.com.zone" ''
326 @ SOA ns1.example.com. noc.example.com. 2024032401 86400 7200 3600000 172800
327 @ NS ns1
328 ns1 A 192.168.0.1
329 '')
330 ];
331 }
332 }
333 zonefile-load: difference
334 zonefile-sync: -1
335 zone:
336 - domain: example.com
337 file: example.com.zone
338 dnssec-signing: on
339 '';
340 };
341 };
342 exporterTest = ''
343 wait_for_unit("knot.service")
344 wait_for_open_port(53)
345 wait_for_unit("prometheus-dnssec-exporter.service")
346 wait_for_open_port(9204)
347 succeed("curl -sSf http://localhost:9204/metrics | grep 'example.com'")
348 '';
349 };
350
351 # Access to WHOIS server is required to properly test this exporter, so
352 # just perform basic sanity check that the exporter is running and returns
353 # a failure.
354 domain = {
355 exporterConfig = {
356 enable = true;
357 };
358 exporterTest = ''
359 wait_for_unit("prometheus-domain-exporter.service")
360 wait_for_open_port(9222)
361 succeed("curl -sSf 'http://localhost:9222/probe?target=nixos.org'")
362 '';
363 };
364
365 dovecot = {
366 exporterConfig = {
367 enable = true;
368 scopes = [ "global" ];
369 socketPath = "/var/run/dovecot2/old-stats";
370 user = "root"; # <- don't use user root in production
371 };
372 metricProvider = {
373 services.dovecot2.enable = true;
374 };
375 exporterTest = ''
376 wait_for_unit("prometheus-dovecot-exporter.service")
377 wait_for_open_port(9166)
378 succeed(
379 "curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'"
380 )
381 '';
382 };
383
384 exportarr-sonarr =
385 let
386 apikey = "eccff6a992bc2e4b88e46d064b26bb4e";
387 in
388 {
389 nodeName = "exportarr_sonarr";
390 exporterConfig = {
391 enable = true;
392 url = "http://127.0.0.1:8989";
393 apiKeyFile = pkgs.writeText "dummy-api-key" apikey;
394 };
395 metricProvider = {
396 services.sonarr = {
397 enable = true;
398 environmentFiles = [ (pkgs.writeText "sonarr-env" "SONARR__AUTH__APIKEY=${apikey}") ];
399 };
400 };
401 exporterTest = ''
402 wait_for_unit("sonarr.service")
403 wait_for_open_port(8989)
404 wait_for_unit("prometheus-exportarr-sonarr-exporter.service")
405 wait_for_open_port(9708)
406 succeed("curl -sSf http://localhost:9708/metrics | grep sonarr_series_total")
407 '';
408 };
409
410 ebpf = {
411 exporterConfig = {
412 enable = true;
413 names = [ "timers" ];
414 };
415 exporterTest = ''
416 wait_for_unit("prometheus-ebpf-exporter.service")
417 wait_for_open_port(9435)
418 succeed(
419 "curl -sSf http://localhost:9435/metrics | grep 'ebpf_exporter_enabled_configs{name=\"timers\"} 1'"
420 )
421 '';
422 };
423
424 fastly = {
425 exporterConfig = {
426 enable = true;
427 environmentFile = pkgs.writeText "fastly-exporter-env" "FASTLY_API_TOKEN=abc123";
428 };
429
430 exporterTest = ''
431 wait_for_unit("prometheus-fastly-exporter.service")
432 wait_for_open_port(9118)
433 '';
434 };
435
436 fritzbox = {
437 # TODO add proper test case
438 exporterConfig = {
439 enable = true;
440 };
441 exporterTest = ''
442 wait_for_unit("prometheus-fritzbox-exporter.service")
443 wait_for_open_port(9133)
444 succeed(
445 "curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'"
446 )
447 '';
448 };
449
450 graphite = {
451 exporterConfig = {
452 enable = true;
453 port = 9108;
454 graphitePort = 9109;
455 mappingSettings.mappings = [
456 {
457 match = "test.*.*";
458 name = "testing";
459 labels = {
460 protocol = "$1";
461 author = "$2";
462 };
463 }
464 ];
465 };
466 exporterTest = ''
467 wait_for_unit("prometheus-graphite-exporter.service")
468 wait_for_open_port(9108)
469 wait_for_open_port(9109)
470 succeed("echo test.tcp.foo-bar 1234 $(date +%s) | nc -w1 localhost 9109")
471 succeed("curl -sSf http://localhost:9108/metrics | grep 'testing{author=\"foo-bar\",protocol=\"tcp\"} 1234'")
472 '';
473 };
474
475 idrac = {
476 exporterConfig = {
477 enable = true;
478 port = 9348;
479 configuration = {
480 hosts = {
481 default = {
482 username = "username";
483 password = "password";
484 };
485 };
486 };
487 };
488 exporterTest = ''
489 wait_for_unit("prometheus-idrac-exporter.service")
490 wait_for_open_port(9348)
491 wait_until_succeeds("curl localhost:9348")
492 '';
493 };
494
495 influxdb = {
496 exporterConfig = {
497 enable = true;
498 sampleExpiry = "3s";
499 };
500 exporterTest = ''
501 wait_for_unit("prometheus-influxdb-exporter.service")
502 wait_for_open_port(9122)
503 succeed(
504 "curl -XPOST http://localhost:9122/write --data-binary 'influxdb_exporter,distro=nixos,added_in=21.09 value=1'"
505 )
506 succeed(
507 "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
508 )
509 execute("sleep 5")
510 fail(
511 "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
512 )
513 '';
514 };
515
516 ipmi = {
517 exporterConfig = {
518 enable = true;
519 };
520 exporterTest = ''
521 wait_for_unit("prometheus-ipmi-exporter.service")
522 wait_for_open_port(9290)
523 succeed(
524 "curl -sSf http://localhost:9290/metrics | grep 'ipmi_scrape_duration_seconds'"
525 )
526 '';
527 };
528
529 jitsi = {
530 exporterConfig = {
531 enable = true;
532 };
533 metricProvider = {
534 systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ];
535 services.jitsi-videobridge = {
536 enable = true;
537 colibriRestApi = true;
538 };
539 };
540 exporterTest = ''
541 wait_for_unit("jitsi-videobridge2.service")
542 wait_for_open_port(8080)
543 wait_for_unit("prometheus-jitsi-exporter.service")
544 wait_for_open_port(9700)
545 wait_until_succeeds(
546 'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"'
547 )
548 succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'")
549 '';
550 };
551
552 json = {
553 exporterConfig = {
554 enable = true;
555 configFile = pkgs.writeText "json-exporter-conf.json" (
556 builtins.toJSON {
557 modules = {
558 default = {
559 metrics = [
560 {
561 name = "json_test_metric";
562 path = "{ .test }";
563 }
564 ];
565 };
566 };
567 }
568 );
569 };
570 metricProvider = {
571 systemd.services.prometheus-json-exporter.after = [ "nginx.service" ];
572 services.nginx = {
573 enable = true;
574 virtualHosts.localhost.locations."/".extraConfig = ''
575 return 200 "{\"test\":1}";
576 '';
577 };
578 };
579 exporterTest = ''
580 wait_for_unit("nginx.service")
581 wait_for_open_port(80)
582 wait_for_unit("prometheus-json-exporter.service")
583 wait_for_open_port(7979)
584 succeed(
585 "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'"
586 )
587 '';
588 };
589
590 kafka = {
591 exporterConfig = {
592 enable = true;
593 environmentFile = pkgs.writeTextFile {
594 name = "/tmp/prometheus-kafka-exporter.env";
595 text = ''
596 KAFKA_BROKERS="localhost:9092"
597 '';
598 };
599 };
600 metricProvider = {
601 services.apache-kafka = {
602 enable = true;
603
604 clusterId = "pHG8aWuXSfWAibHFDCnsCQ";
605
606 formatLogDirs = true;
607
608 settings = {
609 "node.id" = 1;
610 "process.roles" = [
611 "broker"
612 "controller"
613 ];
614 "listeners" = [
615 "PLAINTEXT://:9092"
616 "CONTROLLER://:9093"
617 ];
618 "listener.security.protocol.map" = [
619 "PLAINTEXT:PLAINTEXT"
620 "CONTROLLER:PLAINTEXT"
621 ];
622 "controller.quorum.voters" = [
623 "1@localhost:9093"
624 ];
625 "controller.listener.names" = [ "CONTROLLER" ];
626 "log.dirs" = [ "/var/lib/apache-kafka" ];
627 };
628 };
629
630 systemd.services.apache-kafka.serviceConfig.StateDirectory = "apache-kafka";
631 };
632 exporterTest = ''
633 wait_for_unit("apache-kafka")
634 wait_for_open_port(9092)
635 wait_for_open_port(9093)
636 wait_for_unit("prometheus-kafka-exporter.service")
637 wait_for_open_port(8080)
638 wait_until_succeeds(
639 "journalctl -o cat -u prometheus-kafka-exporter.service | grep '\"version\":\"${pkgs.kminion.version}\"'"
640 )
641 succeed(
642 "curl -sSf http://localhost:8080/metrics | grep 'kminion_exporter_up'"
643 )
644 '';
645 };
646
647 knot = {
648 exporterConfig = {
649 enable = true;
650 };
651 metricProvider = {
652 services.knot = {
653 enable = true;
654 extraArgs = [ "-v" ];
655 settingsFile = pkgs.writeText "knot.conf" ''
656 server:
657 listen: 127.0.0.1@53
658
659 template:
660 - id: default
661 global-module: mod-stats
662 dnssec-signing: off
663 zonefile-sync: -1
664 zonefile-load: difference
665 storage: ${
666 pkgs.buildEnv {
667 name = "foo";
668 paths = [
669 (pkgs.writeTextDir "test.zone" ''
670 @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
671 @ NS ns1
672 @ NS ns2
673 ns1 A 192.168.0.1
674 '')
675 ];
676 }
677 }
678
679 mod-stats:
680 - id: custom
681 edns-presence: on
682 query-type: on
683
684 zone:
685 - domain: test
686 file: test.zone
687 module: mod-stats/custom
688 '';
689 };
690 };
691 exporterTest = ''
692 wait_for_unit("knot.service")
693 wait_for_unit("prometheus-knot-exporter.service")
694 wait_for_open_port(9433)
695 succeed("curl -sSf 'localhost:9433' | grep '2\.019031301'")
696 '';
697 };
698
699 keylight = {
700 # A hardware device is required to properly test this exporter, so just
701 # perform a couple of basic sanity checks that the exporter is running
702 # and requires a target, but cannot reach a specified target.
703 exporterConfig = {
704 enable = true;
705 };
706 exporterTest = ''
707 wait_for_unit("prometheus-keylight-exporter.service")
708 wait_for_open_port(9288)
709 succeed(
710 "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'"
711 )
712 succeed(
713 "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'"
714 )
715 '';
716 };
717
718 lnd = {
719 exporterConfig = {
720 enable = true;
721 lndTlsPath = "/var/lib/lnd/tls.cert";
722 lndMacaroonDir = "/var/lib/lnd";
723 extraFlags = [ "--lnd.network=regtest" ];
724 };
725 metricProvider = {
726 systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15;
727 systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ];
728 services.bitcoind.regtest = {
729 enable = true;
730 extraConfig = ''
731 rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7
732 zmqpubrawblock=tcp://127.0.0.1:28332
733 zmqpubrawtx=tcp://127.0.0.1:28333
734 # https://github.com/lightningnetwork/lnd/issues/9163
735 deprecatedrpc=warnings
736 '';
737 extraCmdlineOptions = [ "-regtest" ];
738 };
739 systemd.services.lnd = {
740 serviceConfig.ExecStart = ''
741 ${pkgs.lnd}/bin/lnd \
742 --datadir=/var/lib/lnd \
743 --tlscertpath=/var/lib/lnd/tls.cert \
744 --tlskeypath=/var/lib/lnd/tls.key \
745 --logdir=/var/log/lnd \
746 --bitcoin.active \
747 --bitcoin.regtest \
748 --bitcoin.node=bitcoind \
749 --bitcoind.rpcuser=bitcoinrpc \
750 --bitcoind.rpcpass=hunter2 \
751 --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \
752 --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \
753 --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon
754 '';
755 serviceConfig.StateDirectory = "lnd";
756 wantedBy = [ "multi-user.target" ];
757 after = [ "network.target" ];
758 };
759 # initialize wallet, creates macaroon needed by exporter
760 systemd.services.lnd.postStart = ''
761 ${pkgs.curl}/bin/curl \
762 --retry 20 \
763 --retry-delay 1 \
764 --retry-connrefused \
765 --cacert /var/lib/lnd/tls.cert \
766 -X GET \
767 https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed
768 ${pkgs.curl}/bin/curl \
769 --retry 20 \
770 --retry-delay 1 \
771 --retry-connrefused \
772 --cacert /var/lib/lnd/tls.cert \
773 -X POST \
774 -d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \
775 https://localhost:8080/v1/initwallet
776 '';
777 };
778 exporterTest = ''
779 wait_for_unit("lnd.service")
780 wait_for_open_port(10009)
781 wait_for_unit("prometheus-lnd-exporter.service")
782 wait_for_open_port(9092)
783 succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'")
784 '';
785 };
786
787 mail = {
788 exporterConfig = {
789 enable = true;
790 configuration = {
791 monitoringInterval = "2s";
792 mailCheckTimeout = "10s";
793 servers = [
794 {
795 name = "testserver";
796 server = "localhost";
797 port = 25;
798 from = "mail-exporter@localhost";
799 to = "mail-exporter@localhost";
800 detectionDir = "/var/spool/mail/mail-exporter/new";
801 }
802 ];
803 };
804 };
805 metricProvider = {
806 services.postfix.enable = true;
807 systemd.services.prometheus-mail-exporter = {
808 after = [ "postfix.service" ];
809 requires = [ "postfix.service" ];
810 serviceConfig = {
811 ExecStartPre = [
812 "${pkgs.writeShellScript "create-maildir" ''
813 mkdir -p -m 0700 mail-exporter/new
814 ''}"
815 ];
816 ProtectHome = true;
817 ReadOnlyPaths = "/";
818 ReadWritePaths = "/var/spool/mail";
819 WorkingDirectory = "/var/spool/mail";
820 };
821 };
822 users.users.mailexporter = {
823 isSystemUser = true;
824 group = "mailexporter";
825 };
826 users.groups.mailexporter = { };
827 };
828 exporterTest = ''
829 wait_for_unit("postfix.service")
830 wait_for_unit("prometheus-mail-exporter.service")
831 wait_for_open_port(9225)
832 wait_until_succeeds(
833 "curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'"
834 )
835 '';
836 };
837
838 mikrotik = {
839 exporterConfig = {
840 enable = true;
841 extraFlags = [ "-timeout=1s" ];
842 configuration = {
843 devices = [
844 {
845 name = "router";
846 address = "192.168.42.48";
847 user = "prometheus";
848 password = "shh";
849 }
850 ];
851 features = {
852 bgp = true;
853 dhcp = true;
854 dhcpl = true;
855 dhcpv6 = true;
856 health = true;
857 routes = true;
858 poe = true;
859 pools = true;
860 optics = true;
861 w60g = true;
862 wlansta = true;
863 wlanif = true;
864 monitor = true;
865 ipsec = true;
866 };
867 };
868 };
869 exporterTest = ''
870 wait_for_unit("prometheus-mikrotik-exporter.service")
871 wait_for_open_port(9436)
872 succeed(
873 "curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'"
874 )
875 '';
876 };
877
878 modemmanager = {
879 exporterConfig = {
880 enable = true;
881 refreshRate = "10s";
882 };
883 metricProvider = {
884 # ModemManager is installed when NetworkManager is enabled. Ensure it is
885 # started and is wanted by NM and the exporter to start everything up
886 # in the right order.
887 networking.networkmanager.enable = true;
888 systemd.services.ModemManager = {
889 enable = true;
890 wantedBy = [
891 "NetworkManager.service"
892 "prometheus-modemmanager-exporter.service"
893 ];
894 };
895 };
896 exporterTest = ''
897 wait_for_unit("ModemManager.service")
898 wait_for_unit("prometheus-modemmanager-exporter.service")
899 wait_for_open_port(9539)
900 succeed(
901 "curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'"
902 )
903 '';
904 };
905
906 mqtt = {
907 exporterConfig = {
908 enable = true;
909 environmentFile = pkgs.writeText "mqtt-exporter-envfile" ''
910 MQTT_PASSWORD=testpassword
911 '';
912 };
913 metricProvider = {
914 services.mosquitto = {
915 enable = true;
916 listeners = [
917 {
918 users.exporter = {
919 acl = [ "read #" ];
920 passwordFile = pkgs.writeText "mosquitto-password" "testpassword";
921 };
922 }
923 ];
924 };
925 systemd.services.prometheus-mqtt-exporter = {
926 wants = [ "mosquitto.service" ];
927 after = [ "mosquitto.service" ];
928 };
929 };
930 exporterTest = ''
931 wait_for_unit("mosquitto.service")
932 wait_for_unit("prometheus-mqtt-exporter.service")
933 wait_for_open_port(9000)
934 succeed(
935 "curl -sSf http://localhost:9000/metrics | grep '^python_info'"
936 )
937 '';
938 };
939
940 mysqld = {
941 exporterConfig = {
942 enable = true;
943 runAsLocalSuperUser = true;
944 configFile = pkgs.writeText "test-prometheus-exporter-mysqld-config.my-cnf" ''
945 [client]
946 user = exporter
947 password = snakeoilpassword
948 '';
949 };
950 metricProvider = {
951 services.mysql = {
952 enable = true;
953 package = pkgs.mariadb;
954 initialScript = pkgs.writeText "mysql-init-script.sql" ''
955 CREATE USER 'exporter'@'localhost'
956 IDENTIFIED BY 'snakeoilpassword'
957 WITH MAX_USER_CONNECTIONS 3;
958 GRANT PROCESS, REPLICATION CLIENT, SLAVE MONITOR, SELECT ON *.* TO 'exporter'@'localhost';
959 '';
960 };
961 };
962 exporterTest = ''
963 wait_for_unit("prometheus-mysqld-exporter.service")
964 wait_for_open_port(9104)
965 wait_for_unit("mysql.service")
966 succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
967 systemctl("stop mysql.service")
968 succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 0'")
969 systemctl("start mysql.service")
970 wait_for_unit("mysql.service")
971 succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
972 '';
973 };
974
975 nextcloud = {
976 exporterConfig = {
977 enable = true;
978 passwordFile = "/var/nextcloud-pwfile";
979 url = "http://localhost";
980 };
981 metricProvider = {
982 systemd.services.nc-pwfile =
983 let
984 passfile = (pkgs.writeText "pwfile" "snakeoilpw");
985 in
986 {
987 requiredBy = [ "prometheus-nextcloud-exporter.service" ];
988 before = [ "prometheus-nextcloud-exporter.service" ];
989 serviceConfig.ExecStart = ''
990 ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile
991 '';
992 };
993 services.nginx = {
994 enable = true;
995 virtualHosts."localhost" = {
996 basicAuth.nextcloud-exporter = "snakeoilpw";
997 locations."/" = {
998 root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
999 tryFiles = "/negative-space.json =404";
1000 };
1001 };
1002 };
1003 };
1004 exporterTest = ''
1005 wait_for_unit("nginx.service")
1006 wait_for_unit("prometheus-nextcloud-exporter.service")
1007 wait_for_open_port(9205)
1008 succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'")
1009 '';
1010 };
1011
1012 nginx = {
1013 exporterConfig = {
1014 enable = true;
1015 constLabels = [ "foo=bar" ];
1016 };
1017 metricProvider = {
1018 services.nginx = {
1019 enable = true;
1020 statusPage = true;
1021 virtualHosts."test".extraConfig = "return 204;";
1022 };
1023 };
1024 exporterTest = ''
1025 wait_for_unit("nginx.service")
1026 wait_for_unit("prometheus-nginx-exporter.service")
1027 wait_for_open_port(9113)
1028 succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up{foo=\"bar\"} 1'")
1029 '';
1030 };
1031
1032 nginxlog = {
1033 exporterConfig = {
1034 enable = true;
1035 group = "nginx";
1036 settings = {
1037 namespaces = [
1038 {
1039 name = "filelogger";
1040 source = {
1041 files = [ "/var/log/nginx/filelogger.access.log" ];
1042 };
1043 }
1044 {
1045 name = "syslogger";
1046 source = {
1047 syslog = {
1048 listen_address = "udp://127.0.0.1:10000";
1049 format = "rfc3164";
1050 tags = [ "nginx" ];
1051 };
1052 };
1053 }
1054 ];
1055 };
1056 };
1057 metricProvider = {
1058 services.nginx = {
1059 enable = true;
1060 httpConfig = ''
1061 server {
1062 listen 80;
1063 server_name filelogger.local;
1064 access_log /var/log/nginx/filelogger.access.log;
1065 }
1066 server {
1067 listen 81;
1068 server_name syslogger.local;
1069 access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
1070 }
1071 '';
1072 };
1073 };
1074 exporterTest = ''
1075 wait_for_unit("nginx.service")
1076 wait_for_unit("prometheus-nginxlog-exporter.service")
1077 wait_for_open_port(9117)
1078 wait_for_open_port(80)
1079 wait_for_open_port(81)
1080 succeed("curl http://localhost")
1081 execute("sleep 1")
1082 succeed(
1083 "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1"
1084 )
1085 succeed("curl http://localhost:81")
1086 execute("sleep 1")
1087 succeed(
1088 "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1"
1089 )
1090 '';
1091 };
1092
1093 node = {
1094 exporterConfig = {
1095 enable = true;
1096 };
1097 exporterTest = ''
1098 wait_for_unit("prometheus-node-exporter.service")
1099 wait_for_open_port(9100)
1100 succeed(
1101 "curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'"
1102 )
1103 '';
1104 };
1105
1106 node-cert = {
1107 nodeName = "node_cert";
1108 exporterConfig = {
1109 enable = true;
1110 paths = [ "/run/certs" ];
1111 };
1112 exporterTest = ''
1113 wait_for_unit("prometheus-node-cert-exporter.service")
1114 wait_for_open_port(9141)
1115 wait_until_succeeds(
1116 "curl -sSf http://localhost:9141/metrics | grep 'ssl_certificate_expiry_seconds{.\\+path=\"/run/certs/node-cert\\.cert\".\\+}'"
1117 )
1118 '';
1119
1120 metricProvider = {
1121 system.activationScripts.cert.text = ''
1122 mkdir -p /run/certs
1123 cd /run/certs
1124
1125 cat >ca.template <<EOF
1126 organization = "prometheus-node-cert-exporter"
1127 cn = "prometheus-node-cert-exporter"
1128 expiration_days = 365
1129 ca
1130 cert_signing_key
1131 crl_signing_key
1132 EOF
1133
1134 ${pkgs.gnutls}/bin/certtool \
1135 --generate-privkey \
1136 --key-type rsa \
1137 --sec-param High \
1138 --outfile node-cert.key
1139
1140 ${pkgs.gnutls}/bin/certtool \
1141 --generate-self-signed \
1142 --load-privkey node-cert.key \
1143 --template ca.template \
1144 --outfile node-cert.cert
1145 '';
1146 };
1147 };
1148
1149 pgbouncer = {
1150 exporterConfig = {
1151 enable = true;
1152 connectionEnvFile = "${pkgs.writeText "connstr-env" ''
1153 PGBOUNCER_EXPORTER_CONNECTION_STRING=postgres://admin@localhost:6432/pgbouncer?sslmode=disable
1154 ''}";
1155 };
1156
1157 metricProvider = {
1158 services.postgresql.enable = true;
1159 services.pgbouncer = {
1160 enable = true;
1161 settings = {
1162 pgbouncer = {
1163 listen_addr = "*";
1164 auth_type = "any";
1165 max_client_conn = 99;
1166 # https://github.com/prometheus-community/pgbouncer_exporter#pgbouncer-configuration
1167 ignore_startup_parameters = "extra_float_digits";
1168 };
1169 databases = {
1170 postgres = concatStringsSep " " [
1171 "host=/run/postgresql"
1172 "port=5432"
1173 "auth_user=postgres"
1174 "dbname=postgres"
1175 ];
1176 };
1177 };
1178 };
1179 };
1180 exporterTest = ''
1181 wait_for_unit("postgresql.target")
1182 wait_for_unit("pgbouncer.service")
1183 wait_for_unit("prometheus-pgbouncer-exporter.service")
1184 wait_for_open_port(9127)
1185 succeed("curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_up 1'")
1186 succeed(
1187 "curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_config_max_client_connections 99'"
1188 )
1189 '';
1190 };
1191
1192 php-fpm = {
1193 nodeName = "php_fpm";
1194 exporterConfig = {
1195 enable = true;
1196 environmentFile = pkgs.writeTextFile {
1197 name = "/tmp/prometheus-php-fpm-exporter.env";
1198 text = ''
1199 PHP_FPM_SCRAPE_URI="tcp://127.0.0.1:9000/status"
1200 '';
1201 };
1202 };
1203 metricProvider = {
1204 users.users."php-fpm-exporter" = {
1205 isSystemUser = true;
1206 group = "php-fpm-exporter";
1207 };
1208 users.groups."php-fpm-exporter" = { };
1209 services.phpfpm.pools."php-fpm-exporter" = {
1210 user = "php-fpm-exporter";
1211 group = "php-fpm-exporter";
1212 settings = {
1213 "pm" = "dynamic";
1214 "pm.max_children" = 32;
1215 "pm.max_requests" = 500;
1216 "pm.start_servers" = 2;
1217 "pm.min_spare_servers" = 2;
1218 "pm.max_spare_servers" = 5;
1219 "pm.status_path" = "/status";
1220 "listen" = "127.0.0.1:9000";
1221 "listen.allowed_clients" = "127.0.0.1";
1222 };
1223 phpEnv."PATH" = makeBinPath [ pkgs.php ];
1224 };
1225 };
1226 exporterTest = ''
1227 wait_for_unit("phpfpm-php-fpm-exporter.service")
1228 wait_for_unit("prometheus-php-fpm-exporter.service")
1229 succeed("curl -sSf http://localhost:9253/metrics | grep 'phpfpm_up{.*} 1'")
1230 '';
1231 };
1232
1233 ping = {
1234 exporterConfig = {
1235 enable = true;
1236
1237 settings = {
1238 targets = [
1239 {
1240 "localhost" = {
1241 alias = "local machine";
1242 env = "prod";
1243 type = "domain";
1244 };
1245 }
1246 {
1247 "127.0.0.1" = {
1248 alias = "local machine";
1249 type = "v4";
1250 };
1251 }
1252 {
1253 "::1" = {
1254 alias = "local machine";
1255 type = "v6";
1256 };
1257 }
1258 {
1259 "google.com" = { };
1260 }
1261 ];
1262 dns = { };
1263 ping = {
1264 interval = "2s";
1265 timeout = "3s";
1266 history-size = 42;
1267 payload-size = 56;
1268 };
1269 log = {
1270 level = "warn";
1271 };
1272 };
1273 };
1274
1275 exporterTest = ''
1276 wait_for_unit("prometheus-ping-exporter.service")
1277 wait_for_open_port(9427)
1278 succeed("curl -sSf http://localhost:9427/metrics | grep 'ping_up{.*} 1'")
1279 '';
1280 };
1281
1282 postfix = {
1283 exporterConfig = {
1284 enable = true;
1285 };
1286 metricProvider = {
1287 services.postfix.enable = true;
1288 };
1289 exporterTest = ''
1290 wait_for_unit("prometheus-postfix-exporter.service")
1291 wait_for_file("/var/lib/postfix/queue/public/showq")
1292 wait_for_open_port(9154)
1293 wait_until_succeeds(
1294 "curl -sSf http://localhost:9154/metrics | grep 'postfix_up{path=\"unix:///var/lib/postfix/queue/public/showq\"} 1'"
1295 )
1296 succeed(
1297 "curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'"
1298 )
1299 succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'")
1300 '';
1301 };
1302
1303 postgres = {
1304 exporterConfig = {
1305 enable = true;
1306 runAsLocalSuperUser = true;
1307 };
1308 metricProvider = {
1309 services.postgresql.enable = true;
1310 };
1311 exporterTest = ''
1312 wait_for_unit("prometheus-postgres-exporter.service")
1313 wait_for_open_port(9187)
1314 wait_for_unit("postgresql.target")
1315 succeed(
1316 "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
1317 )
1318 succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
1319 systemctl("stop postgresql")
1320 succeed(
1321 "curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'"
1322 )
1323 succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'")
1324 systemctl("start postgresql")
1325 wait_for_unit("postgresql.target")
1326 succeed(
1327 "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
1328 )
1329 succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
1330 '';
1331 };
1332
1333 process = {
1334 exporterConfig = {
1335 enable = true;
1336 settings.process_names = [
1337 # Remove nix store path from process name
1338 {
1339 name = "{{.Matches.Wrapped}} {{ .Matches.Args }}";
1340 cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ];
1341 }
1342 ];
1343 };
1344 exporterTest = ''
1345 wait_for_unit("prometheus-process-exporter.service")
1346 wait_for_open_port(9256)
1347 wait_until_succeeds(
1348 "curl -sSf localhost:9256/metrics | grep -q '{}'".format(
1349 'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter '
1350 )
1351 )
1352 '';
1353 };
1354
1355 pve =
1356 let
1357 pveExporterEnvFile = pkgs.writeTextFile {
1358 name = "pve.env";
1359 text = ''
1360 PVE_USER="test_user@pam"
1361 PVE_PASSWORD="hunter3"
1362 PVE_VERIFY_SSL="false"
1363 '';
1364 };
1365 in
1366 {
1367 exporterConfig = {
1368 enable = true;
1369 environmentFile = pveExporterEnvFile;
1370 };
1371 exporterTest = ''
1372 wait_for_unit("prometheus-pve-exporter.service")
1373 wait_for_open_port(9221)
1374 wait_until_succeeds("curl localhost:9221")
1375 '';
1376 };
1377
1378 py-air-control = {
1379 nodeName = "py_air_control";
1380 exporterConfig = {
1381 enable = true;
1382 deviceHostname = "127.0.0.1";
1383 };
1384 exporterTest = ''
1385 wait_for_unit("prometheus-py-air-control-exporter.service")
1386 wait_for_open_port(9896)
1387 succeed(
1388 "curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'"
1389 )
1390 '';
1391 };
1392
1393 redis = {
1394 exporterConfig = {
1395 enable = true;
1396 };
1397 metricProvider.services.redis.servers."".enable = true;
1398 exporterTest = ''
1399 wait_for_unit("redis.service")
1400 wait_for_unit("prometheus-redis-exporter.service")
1401 wait_for_open_port(6379)
1402 wait_for_open_port(9121)
1403 wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'")
1404 '';
1405 };
1406
1407 restic =
1408 let
1409 repository = "rest:http://127.0.0.1:8000";
1410 passwordFile = pkgs.writeText "restic-test-password" "test-password";
1411 in
1412 {
1413 exporterConfig = {
1414 enable = true;
1415 inherit repository passwordFile;
1416 };
1417 metricProvider = {
1418 services.restic.server = {
1419 enable = true;
1420 extraFlags = [ "--no-auth" ];
1421 };
1422 environment.systemPackages = [ pkgs.restic ];
1423 };
1424 exporterTest = ''
1425 # prometheus-restic-exporter.service fails without initialised repository
1426 systemctl("stop prometheus-restic-exporter.service")
1427
1428 # Initialise the repository
1429 wait_for_unit("restic-rest-server.service")
1430 wait_for_open_port(8000)
1431 succeed("restic init --repo ${repository} --password-file ${passwordFile}")
1432
1433 systemctl("start prometheus-restic-exporter.service")
1434 wait_for_unit("prometheus-restic-exporter.service")
1435 wait_for_open_port(9753)
1436 wait_until_succeeds("curl -sSf localhost:9753/metrics | grep 'restic_check_success 1.0'")
1437 '';
1438 };
1439
1440 rspamd = {
1441 exporterConfig = {
1442 enable = true;
1443 };
1444 metricProvider = {
1445 services.rspamd.enable = true;
1446 };
1447 exporterTest = ''
1448 wait_for_unit("rspamd.service")
1449 wait_for_unit("prometheus-rspamd-exporter.service")
1450 wait_for_open_port(11334)
1451 wait_for_open_port(7980)
1452 wait_until_succeeds(
1453 "curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep 'rspamd_scanned{host=\"rspamd\"} 0'"
1454 )
1455 '';
1456 };
1457
1458 rtl_433 = {
1459 exporterConfig = {
1460 enable = true;
1461 };
1462 metricProvider = {
1463 # Mock rtl_433 binary to return a dummy metric stream.
1464 nixpkgs.overlays = [
1465 (self: super: {
1466 rtl_433 = self.runCommand "rtl_433" { } ''
1467 mkdir -p "$out/bin"
1468 cat <<EOF > "$out/bin/rtl_433"
1469 #!/bin/sh
1470 while true; do
1471 printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n'
1472 sleep 4
1473 done
1474 EOF
1475 chmod +x "$out/bin/rtl_433"
1476 '';
1477 })
1478 ];
1479 };
1480 exporterTest = ''
1481 wait_for_unit("prometheus-rtl_433-exporter.service")
1482 wait_for_open_port(9550)
1483 wait_until_succeeds(
1484 "curl -sSf localhost:9550/metrics | grep '{}'".format(
1485 'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18'
1486 )
1487 )
1488 '';
1489 };
1490
1491 sabnzbd = {
1492 exporterConfig = {
1493 enable = true;
1494 servers = [
1495 {
1496 baseUrl = "http://localhost:8080";
1497 apiKeyFile = "/var/sabnzbd-apikey";
1498 }
1499 ];
1500 };
1501
1502 metricProvider = {
1503 services.sabnzbd.enable = true;
1504
1505 # unrar is required for sabnzbd
1506 nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ "unrar" ];
1507
1508 # extract the generated api key before starting
1509 systemd.services.sabnzbd-apikey = {
1510 requires = [ "sabnzbd.service" ];
1511 after = [ "sabnzbd.service" ];
1512 requiredBy = [ "prometheus-sabnzbd-exporter.service" ];
1513 before = [ "prometheus-sabnzbd-exporter.service" ];
1514 script = ''
1515 grep -Po '^api_key = \K.+' /var/lib/sabnzbd/sabnzbd.ini > /var/sabnzbd-apikey
1516 '';
1517 };
1518 };
1519
1520 exporterTest = ''
1521 wait_for_unit("sabnzbd.service")
1522 wait_for_unit("prometheus-sabnzbd-exporter.service")
1523 wait_for_open_port(8080)
1524 wait_for_open_port(9387)
1525 wait_until_succeeds(
1526 "curl -sSf 'localhost:9387/metrics' | grep 'sabnzbd_queue_size{sabnzbd_instance=\"http://localhost:8080\"} 0.0'"
1527 )
1528 '';
1529 };
1530
1531 scaphandre = {
1532 exporterConfig = {
1533 enable = true;
1534 };
1535 metricProvider = {
1536 boot.kernelModules = [ "intel_rapl_common" ];
1537 };
1538 exporterTest = ''
1539 wait_for_unit("prometheus-scaphandre-exporter.service")
1540 wait_for_open_port(8080)
1541 wait_until_succeeds(
1542 "curl -sSf 'localhost:8080/metrics'"
1543 )
1544 '';
1545 };
1546
1547 shelly = {
1548 exporterConfig = {
1549 enable = true;
1550 metrics-file = "${pkgs.writeText "test.json" ''{}''}";
1551 };
1552 exporterTest = ''
1553 wait_for_unit("prometheus-shelly-exporter.service")
1554 wait_for_open_port(9784)
1555 wait_until_succeeds(
1556 "curl -sSf 'localhost:9784/metrics'"
1557 )
1558 '';
1559 };
1560
1561 script = {
1562 exporterConfig = {
1563 enable = true;
1564 settings.scripts = [
1565 {
1566 name = "success";
1567 command = [ "sleep" ];
1568 args = [ "1" ];
1569 }
1570 ];
1571 };
1572 exporterTest = ''
1573 wait_for_unit("prometheus-script-exporter.service")
1574 wait_for_open_port(9172)
1575 wait_until_succeeds(
1576 "curl -sSf 'localhost:9172/probe?script=success' | grep -q '{}'".format(
1577 'script_success{script="success"} 1'
1578 )
1579 )
1580 '';
1581 };
1582
1583 smartctl = {
1584 exporterConfig = {
1585 enable = true;
1586 devices = [
1587 "/dev/vda"
1588 ];
1589 };
1590 exporterTest = ''
1591 wait_until_succeeds(
1592 'journalctl -eu prometheus-smartctl-exporter.service -o cat | grep "Unable to detect device type"'
1593 )
1594 '';
1595 };
1596
1597 smokeping = {
1598 exporterConfig = {
1599 enable = true;
1600 hosts = [ "127.0.0.1" ];
1601 };
1602 exporterTest = ''
1603 wait_for_unit("prometheus-smokeping-exporter.service")
1604 wait_for_open_port(9374)
1605 wait_until_succeeds(
1606 "curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format(
1607 'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1",source="",tos="0"} '
1608 )
1609 )
1610 wait_until_succeeds(
1611 "curl -sSf localhost:9374/metrics | grep '{}'".format(
1612 'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1",source="",tos="0"}'
1613 )
1614 )
1615 '';
1616 };
1617
1618 storagebox = {
1619 exporterConfig = {
1620 enable = true;
1621 tokenFile = "/tmp/faketoken";
1622 };
1623 exporterTest = ''
1624 succeed(
1625 'echo faketoken > /tmp/faketoken'
1626 )
1627 wait_for_unit("prometheus-storagebox-exporter.service")
1628 wait_for_open_port(9509)
1629 succeed("curl -sSf localhost:9509/metrics | grep 'process_open_fds'")
1630 '';
1631 };
1632
1633 snmp = {
1634 exporterConfig = {
1635 enable = true;
1636 configuration = {
1637 auths.public_v2 = {
1638 community = "public";
1639 version = 2;
1640 };
1641 };
1642 };
1643 exporterTest = ''
1644 wait_for_unit("prometheus-snmp-exporter.service")
1645 wait_for_open_port(9116)
1646 succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'")
1647 '';
1648 };
1649
1650 sql = {
1651 exporterConfig = {
1652 configuration.jobs.points = {
1653 interval = "1m";
1654 connections = [
1655 "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable"
1656 ];
1657 queries = {
1658 points = {
1659 labels = [ "name" ];
1660 help = "Amount of points accumulated per person";
1661 values = [ "amount" ];
1662 query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name";
1663 };
1664 };
1665 };
1666 enable = true;
1667 user = "prometheus-sql-exporter";
1668 };
1669 metricProvider = {
1670 services.postgresql = {
1671 enable = true;
1672 initialScript = builtins.toFile "init.sql" ''
1673 CREATE DATABASE data;
1674 \c data;
1675 CREATE TABLE points (amount INT, name TEXT);
1676 INSERT INTO points(amount, name) VALUES (1, 'jack');
1677 INSERT INTO points(amount, name) VALUES (2, 'jill');
1678 INSERT INTO points(amount, name) VALUES (3, 'jack');
1679
1680 CREATE USER "prometheus-sql-exporter";
1681 GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter";
1682 GRANT SELECT ON points TO "prometheus-sql-exporter";
1683 '';
1684 };
1685 systemd.services.prometheus-sql-exporter.after = [ "postgresql.target" ];
1686 };
1687 exporterTest = ''
1688 wait_for_unit("prometheus-sql-exporter.service")
1689 wait_for_open_port(9237)
1690 succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2")
1691 '';
1692 };
1693
1694 statsd = {
1695 exporterConfig = {
1696 enable = true;
1697 };
1698 exporterTest = ''
1699 wait_for_unit("prometheus-statsd-exporter.service")
1700 wait_for_open_port(9102)
1701 succeed("curl http://localhost:9102/metrics | grep 'statsd_exporter_build_info{'")
1702 wait_until_succeeds(
1703 "echo 'test.udp:1|c' > /dev/udp/localhost/9125 && \
1704 curl http://localhost:9102/metrics | grep 'test_udp 1'",
1705 timeout=10
1706 )
1707 wait_until_succeeds(
1708 "echo 'test.tcp:1|c' > /dev/tcp/localhost/9125 && \
1709 curl http://localhost:9102/metrics | grep 'test_tcp 1'",
1710 timeout=10
1711 )
1712 '';
1713 };
1714
1715 surfboard = {
1716 exporterConfig = {
1717 enable = true;
1718 modemAddress = "localhost";
1719 };
1720 metricProvider = {
1721 systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ];
1722 services.nginx = {
1723 enable = true;
1724 virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = ''
1725 return 204;
1726 '';
1727 };
1728 };
1729 exporterTest = ''
1730 wait_for_unit("nginx.service")
1731 wait_for_open_port(80)
1732 wait_for_unit("prometheus-surfboard-exporter.service")
1733 wait_for_open_port(9239)
1734 succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'")
1735 '';
1736 };
1737
1738 systemd = {
1739 exporterConfig = {
1740 enable = true;
1741
1742 extraFlags = [
1743 "--systemd.collector.enable-restart-count"
1744 ];
1745 };
1746 metricProvider = { };
1747 exporterTest = ''
1748 wait_for_unit("prometheus-systemd-exporter.service")
1749 wait_for_open_port(9558)
1750 wait_until_succeeds(
1751 "curl -sSf localhost:9558/metrics | grep '{}'".format(
1752 'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
1753 )
1754 )
1755 succeed(
1756 "curl -sSf localhost:9558/metrics | grep '{}'".format(
1757 'systemd_service_restart_total{name="prometheus-systemd-exporter.service"} 0'
1758 )
1759 )
1760 '';
1761 };
1762
1763 unpoller = {
1764 nodeName = "unpoller";
1765 exporterConfig.enable = true;
1766 exporterConfig.controllers = [ { } ];
1767 exporterTest = ''
1768 wait_until_succeeds(
1769 'journalctl -eu prometheus-unpoller-exporter.service -o cat | grep "Connection Error"'
1770 )
1771 '';
1772 };
1773
1774 unbound = {
1775 exporterConfig = {
1776 enable = true;
1777 unbound.host = "unix:///run/unbound/unbound.ctl";
1778 };
1779 metricProvider = {
1780 services.unbound = {
1781 enable = true;
1782 localControlSocketPath = "/run/unbound/unbound.ctl";
1783 };
1784 systemd.services.prometheus-unbound-exporter.serviceConfig = {
1785 SupplementaryGroups = [ "unbound" ];
1786 };
1787 };
1788 exporterTest = ''
1789 wait_for_unit("unbound.service")
1790 wait_for_unit("prometheus-unbound-exporter.service")
1791 wait_for_open_port(9167)
1792 wait_until_succeeds("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'")
1793 '';
1794 };
1795
1796 v2ray = {
1797 exporterConfig = {
1798 enable = true;
1799 };
1800
1801 metricProvider = {
1802 systemd.services.prometheus-nginx-exporter.after = [ "v2ray.service" ];
1803 services.v2ray = {
1804 enable = true;
1805 config = {
1806 stats = { };
1807 api = {
1808 tag = "api";
1809 services = [ "StatsService" ];
1810 };
1811 inbounds = [
1812 {
1813 port = 1080;
1814 listen = "127.0.0.1";
1815 protocol = "http";
1816 }
1817 {
1818 listen = "127.0.0.1";
1819 port = 54321;
1820 protocol = "dokodemo-door";
1821 settings = {
1822 address = "127.0.0.1";
1823 };
1824 tag = "api";
1825 }
1826 ];
1827 outbounds = [
1828 {
1829 protocol = "freedom";
1830 }
1831 {
1832 protocol = "freedom";
1833 settings = { };
1834 tag = "api";
1835 }
1836 ];
1837 routing = {
1838 strategy = "rules";
1839 settings = {
1840 rules = [
1841 {
1842 inboundTag = [ "api" ];
1843 outboundTag = "api";
1844 type = "field";
1845 }
1846 ];
1847 };
1848 };
1849 };
1850 };
1851 };
1852 exporterTest = ''
1853 wait_for_unit("prometheus-v2ray-exporter.service")
1854 wait_for_open_port(9299)
1855 succeed("curl -sSf localhost:9299/scrape | grep 'v2ray_up 1'")
1856 '';
1857 };
1858
1859 varnish = {
1860 exporterConfig = {
1861 enable = true;
1862 instance = "/run/varnish/varnish";
1863 group = "varnish";
1864 };
1865 metricProvider = {
1866 systemd.services.prometheus-varnish-exporter.after = [
1867 "varnish.service"
1868 ];
1869 services.varnish = {
1870 enable = true;
1871 stateDir = "/run/varnish/varnish";
1872 config = ''
1873 vcl 4.0;
1874 backend default {
1875 .host = "127.0.0.1";
1876 .port = "80";
1877 }
1878 '';
1879 };
1880 };
1881 exporterTest = ''
1882 wait_for_unit("prometheus-varnish-exporter.service")
1883 wait_for_open_port(6081)
1884 wait_for_open_port(9131)
1885 succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'")
1886 '';
1887 };
1888
1889 wireguard =
1890 let
1891 snakeoil = import ./wireguard/snakeoil-keys.nix;
1892 publicKeyWithoutNewlines = replaceStrings [ "\n" ] [ "" ] snakeoil.peer1.publicKey;
1893 in
1894 {
1895 exporterConfig.enable = true;
1896 metricProvider = {
1897 networking.wireguard.interfaces.wg0 = {
1898 ips = [
1899 "10.23.42.1/32"
1900 "fc00::1/128"
1901 ];
1902 listenPort = 23542;
1903
1904 inherit (snakeoil.peer0) privateKey;
1905
1906 peers = singleton {
1907 allowedIPs = [
1908 "10.23.42.2/32"
1909 "fc00::2/128"
1910 ];
1911
1912 inherit (snakeoil.peer1) publicKey;
1913 };
1914 };
1915 systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ];
1916 };
1917 exporterTest = ''
1918 wait_for_unit("prometheus-wireguard-exporter.service")
1919 wait_for_open_port(9586)
1920 wait_until_succeeds(
1921 "curl -sSf http://localhost:9586/metrics | grep '${publicKeyWithoutNewlines}'"
1922 )
1923 '';
1924 };
1925
1926 zfs = {
1927 exporterConfig = {
1928 enable = true;
1929 };
1930 metricProvider = {
1931 boot.supportedFilesystems = [ "zfs" ];
1932 networking.hostId = "7327ded7";
1933 };
1934 exporterTest = ''
1935 wait_for_unit("prometheus-zfs-exporter.service")
1936 wait_for_unit("zfs.target")
1937 wait_for_open_port(9134)
1938 wait_until_succeeds("curl -f localhost:9134/metrics | grep 'zfs_scrape_collector_success{.*} 1'")
1939 '';
1940 };
1941 };
1942in
1943mapAttrs (
1944 exporter: testConfig:
1945 (makeTest (
1946 let
1947 nodeName = testConfig.nodeName or exporter;
1948
1949 in
1950 {
1951 name = "prometheus-${exporter}-exporter";
1952
1953 nodes.${nodeName} = mkMerge [
1954 {
1955 services.prometheus.exporters.${exporter} = testConfig.exporterConfig;
1956 }
1957 testConfig.metricProvider or { }
1958 ];
1959
1960 testScript = ''
1961 ${nodeName}.start()
1962 ${concatStringsSep "\n" (
1963 map (
1964 line:
1965 if
1966 builtins.any (b: b) [
1967 (builtins.match "^[[:space:]]*$" line != null)
1968 (builtins.substring 0 1 line == "#")
1969 (builtins.substring 0 1 line == " ")
1970 (builtins.substring 0 1 line == ")")
1971 ]
1972 then
1973 line
1974 else
1975 "${nodeName}.${line}"
1976 ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest))
1977 )}
1978 ${nodeName}.shutdown()
1979 '';
1980
1981 meta.maintainers = [ ];
1982 }
1983 ))
1984) exporterTests