pyrox.dev
1{
2 name = "prometheus-federation";
3
4 nodes = {
5 global1 =
6 { config, pkgs, ... }:
7 {
8 environment.systemPackages = [ pkgs.jq ];
9
10 networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
11
12 services.prometheus = {
13 enable = true;
14 globalConfig.scrape_interval = "2s";
15
16 scrapeConfigs = [
17 {
18 job_name = "federate";
19 honor_labels = true;
20 metrics_path = "/federate";
21
22 params = {
23 "match[]" = [
24 "{job=\"node\"}"
25 "{job=\"prometheus\"}"
26 ];
27 };
28
29 static_configs = [
30 {
31 targets = [
32 "prometheus1:${toString config.services.prometheus.port}"
33 "prometheus2:${toString config.services.prometheus.port}"
34 ];
35 }
36 ];
37 }
38 {
39 job_name = "prometheus";
40 static_configs = [
41 {
42 targets = [
43 "global1:${toString config.services.prometheus.port}"
44 "global2:${toString config.services.prometheus.port}"
45 ];
46 }
47 ];
48 }
49 ];
50 };
51 };
52
53 global2 =
54 { config, pkgs, ... }:
55 {
56 environment.systemPackages = [ pkgs.jq ];
57
58 networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
59
60 services.prometheus = {
61 enable = true;
62 globalConfig.scrape_interval = "2s";
63
64 scrapeConfigs = [
65 {
66 job_name = "federate";
67 honor_labels = true;
68 metrics_path = "/federate";
69
70 params = {
71 "match[]" = [
72 "{job=\"node\"}"
73 "{job=\"prometheus\"}"
74 ];
75 };
76
77 static_configs = [
78 {
79 targets = [
80 "prometheus1:${toString config.services.prometheus.port}"
81 "prometheus2:${toString config.services.prometheus.port}"
82 ];
83 }
84 ];
85 }
86 {
87 job_name = "prometheus";
88 static_configs = [
89 {
90 targets = [
91 "global1:${toString config.services.prometheus.port}"
92 "global2:${toString config.services.prometheus.port}"
93 ];
94 }
95 ];
96 }
97 ];
98 };
99 };
100
101 prometheus1 =
102 { config, pkgs, ... }:
103 {
104 environment.systemPackages = [ pkgs.jq ];
105
106 networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
107
108 services.prometheus = {
109 enable = true;
110 globalConfig.scrape_interval = "2s";
111
112 scrapeConfigs = [
113 {
114 job_name = "node";
115 static_configs = [
116 { targets = [ "node1:${toString config.services.prometheus.exporters.node.port}" ]; }
117 ];
118 }
119 {
120 job_name = "prometheus";
121 static_configs = [ { targets = [ "prometheus1:${toString config.services.prometheus.port}" ]; } ];
122 }
123 ];
124 };
125 };
126
127 prometheus2 =
128 { config, pkgs, ... }:
129 {
130 environment.systemPackages = [ pkgs.jq ];
131
132 networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
133
134 services.prometheus = {
135 enable = true;
136 globalConfig.scrape_interval = "2s";
137
138 scrapeConfigs = [
139 {
140 job_name = "node";
141 static_configs = [
142 { targets = [ "node2:${toString config.services.prometheus.exporters.node.port}" ]; }
143 ];
144 }
145 {
146 job_name = "prometheus";
147 static_configs = [ { targets = [ "prometheus2:${toString config.services.prometheus.port}" ]; } ];
148 }
149 ];
150 };
151 };
152
153 node1 =
154 { config, pkgs, ... }:
155 {
156 services.prometheus.exporters.node = {
157 enable = true;
158 openFirewall = true;
159 };
160 };
161
162 node2 =
163 { config, pkgs, ... }:
164 {
165 services.prometheus.exporters.node = {
166 enable = true;
167 openFirewall = true;
168 };
169 };
170 };
171
172 testScript = ''
173 for machine in node1, node2:
174 machine.wait_for_unit("prometheus-node-exporter")
175 machine.wait_for_open_port(9100)
176
177 for machine in prometheus1, prometheus2, global1, global2:
178 machine.wait_for_unit("prometheus")
179 machine.wait_for_open_port(9090)
180
181 # Verify both servers got the same data from the exporter
182 for machine in prometheus1, prometheus2:
183 machine.wait_until_succeeds(
184 "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
185 + "jq '.data.result[0].value[1]' | grep '\"1\"'"
186 )
187 machine.wait_until_succeeds(
188 "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
189 + "jq '.data.result[0].value[1]' | grep '\"1\"'"
190 )
191
192 for machine in global1, global2:
193 machine.wait_until_succeeds(
194 "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
195 + "jq '.data.result[0].value[1]' | grep '\"2\"'"
196 )
197
198 machine.wait_until_succeeds(
199 "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
200 + "jq '.data.result[0].value[1]' | grep '\"4\"'"
201 )
202 '';
203}