pyrox.dev
1{
2 config,
3 lib,
4 options,
5 pkgs,
6 ...
7}:
8let
9 cfg = config.services.graphite;
10 opt = options.services.graphite;
11 writeTextOrNull = f: t: lib.mapNullable (pkgs.writeTextDir f) t;
12
13 dataDir = cfg.dataDir;
14 staticDir = cfg.dataDir + "/static";
15
16 graphiteLocalSettingsDir =
17 pkgs.runCommand "graphite_local_settings"
18 {
19 inherit graphiteLocalSettings;
20 preferLocalBuild = true;
21 }
22 ''
23 mkdir -p $out
24 ln -s $graphiteLocalSettings $out/graphite_local_settings.py
25 '';
26
27 graphiteLocalSettings = pkgs.writeText "graphite_local_settings.py" (
28 "STATIC_ROOT = '${staticDir}'\n"
29 + lib.optionalString (config.time.timeZone != null) "TIME_ZONE = '${config.time.timeZone}'\n"
30 + cfg.web.extraConfig
31 );
32
33 seyrenConfig = {
34 SEYREN_URL = cfg.seyren.seyrenUrl;
35 MONGO_URL = cfg.seyren.mongoUrl;
36 GRAPHITE_URL = cfg.seyren.graphiteUrl;
37 }
38 // cfg.seyren.extraConfig;
39
40 configDir = pkgs.buildEnv {
41 name = "graphite-config";
42 paths = lib.lists.filter (el: el != null) [
43 (writeTextOrNull "carbon.conf" cfg.carbon.config)
44 (writeTextOrNull "storage-aggregation.conf" cfg.carbon.storageAggregation)
45 (writeTextOrNull "storage-schemas.conf" cfg.carbon.storageSchemas)
46 (writeTextOrNull "blacklist.conf" cfg.carbon.blacklist)
47 (writeTextOrNull "whitelist.conf" cfg.carbon.whitelist)
48 (writeTextOrNull "rewrite-rules.conf" cfg.carbon.rewriteRules)
49 (writeTextOrNull "relay-rules.conf" cfg.carbon.relayRules)
50 (writeTextOrNull "aggregation-rules.conf" cfg.carbon.aggregationRules)
51 ];
52 };
53
54 carbonOpts =
55 name: with config.ids; ''
56 --nodaemon --syslog --prefix=${name} --pidfile /run/${name}/${name}.pid ${name}
57 '';
58
59 carbonEnv = {
60 PYTHONPATH =
61 let
62 cenv = pkgs.python3.buildEnv.override {
63 extraLibs = [ pkgs.python3Packages.carbon ];
64 };
65 in
66 "${cenv}/${pkgs.python3.sitePackages}";
67 GRAPHITE_ROOT = dataDir;
68 GRAPHITE_CONF_DIR = configDir;
69 GRAPHITE_STORAGE_DIR = dataDir;
70 };
71
72in
73{
74
75 imports = [
76 (lib.mkRemovedOptionModule [ "services" "graphite" "api" ] "")
77 (lib.mkRemovedOptionModule [ "services" "graphite" "beacon" ] "")
78 (lib.mkRemovedOptionModule [ "services" "graphite" "pager" ] "")
79 ];
80
81 ###### interface
82
83 options.services.graphite = {
84 dataDir = lib.mkOption {
85 type = lib.types.path;
86 default = "/var/db/graphite";
87 description = ''
88 Data directory for graphite.
89 '';
90 };
91
92 web = {
93 enable = lib.mkOption {
94 description = "Whether to enable graphite web frontend.";
95 default = false;
96 type = lib.types.bool;
97 };
98
99 listenAddress = lib.mkOption {
100 description = "Graphite web frontend listen address.";
101 default = "127.0.0.1";
102 type = lib.types.str;
103 };
104
105 port = lib.mkOption {
106 description = "Graphite web frontend port.";
107 default = 8080;
108 type = lib.types.port;
109 };
110
111 extraConfig = lib.mkOption {
112 type = lib.types.str;
113 default = "";
114 description = ''
115 Graphite webapp settings. See:
116 <https://graphite.readthedocs.io/en/latest/config-local-settings.html>
117 '';
118 };
119 };
120
121 carbon = {
122 config = lib.mkOption {
123 description = "Content of carbon configuration file.";
124 default = ''
125 [cache]
126 # Listen on localhost by default for security reasons
127 UDP_RECEIVER_INTERFACE = 127.0.0.1
128 PICKLE_RECEIVER_INTERFACE = 127.0.0.1
129 LINE_RECEIVER_INTERFACE = 127.0.0.1
130 CACHE_QUERY_INTERFACE = 127.0.0.1
131 # Do not log every update
132 LOG_UPDATES = False
133 LOG_CACHE_HITS = False
134 '';
135 type = lib.types.str;
136 };
137
138 enableCache = lib.mkOption {
139 description = "Whether to enable carbon cache, the graphite storage daemon.";
140 default = false;
141 type = lib.types.bool;
142 };
143
144 storageAggregation = lib.mkOption {
145 description = "Defines how to aggregate data to lower-precision retentions.";
146 default = null;
147 type = lib.types.nullOr lib.types.str;
148 example = ''
149 [all_min]
150 pattern = \.min$
151 xFilesFactor = 0.1
152 aggregationMethod = min
153 '';
154 };
155
156 storageSchemas = lib.mkOption {
157 description = "Defines retention rates for storing metrics.";
158 default = "";
159 type = lib.types.nullOr lib.types.str;
160 example = ''
161 [apache_busyWorkers]
162 pattern = ^servers\.www.*\.workers\.busyWorkers$
163 retentions = 15s:7d,1m:21d,15m:5y
164 '';
165 };
166
167 blacklist = lib.mkOption {
168 description = "Any metrics received which match one of the expressions will be dropped.";
169 default = null;
170 type = lib.types.nullOr lib.types.str;
171 example = "^some\\.noisy\\.metric\\.prefix\\..*";
172 };
173
174 whitelist = lib.mkOption {
175 description = "Only metrics received which match one of the expressions will be persisted.";
176 default = null;
177 type = lib.types.nullOr lib.types.str;
178 example = ".*";
179 };
180
181 rewriteRules = lib.mkOption {
182 description = ''
183 Regular expression patterns that can be used to rewrite metric names
184 in a search and replace fashion.
185 '';
186 default = null;
187 type = lib.types.nullOr lib.types.str;
188 example = ''
189 [post]
190 _sum$ =
191 _avg$ =
192 '';
193 };
194
195 enableRelay = lib.mkOption {
196 description = "Whether to enable carbon relay, the carbon replication and sharding service.";
197 default = false;
198 type = lib.types.bool;
199 };
200
201 relayRules = lib.mkOption {
202 description = "Relay rules are used to send certain metrics to a certain backend.";
203 default = null;
204 type = lib.types.nullOr lib.types.str;
205 example = ''
206 [example]
207 pattern = ^mydata\.foo\..+
208 servers = 10.1.2.3, 10.1.2.4:2004, myserver.mydomain.com
209 '';
210 };
211
212 enableAggregator = lib.mkOption {
213 description = "Whether to enable carbon aggregator, the carbon buffering service.";
214 default = false;
215 type = lib.types.bool;
216 };
217
218 aggregationRules = lib.mkOption {
219 description = "Defines if and how received metrics will be aggregated.";
220 default = null;
221 type = lib.types.nullOr lib.types.str;
222 example = ''
223 <env>.applications.<app>.all.requests (60) = sum <env>.applications.<app>.*.requests
224 <env>.applications.<app>.all.latency (60) = avg <env>.applications.<app>.*.latency
225 '';
226 };
227 };
228
229 seyren = {
230 enable = lib.mkOption {
231 description = "Whether to enable seyren service.";
232 default = false;
233 type = lib.types.bool;
234 };
235
236 port = lib.mkOption {
237 description = "Seyren listening port.";
238 default = 8081;
239 type = lib.types.port;
240 };
241
242 seyrenUrl = lib.mkOption {
243 default = "http://localhost:${toString cfg.seyren.port}/";
244 defaultText = lib.literalExpression ''"http://localhost:''${toString config.${opt.seyren.port}}/"'';
245 description = "Host where seyren is accessible.";
246 type = lib.types.str;
247 };
248
249 graphiteUrl = lib.mkOption {
250 default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
251 defaultText = lib.literalExpression ''"http://''${config.${opt.web.listenAddress}}:''${toString config.${opt.web.port}}"'';
252 description = "Host where graphite service runs.";
253 type = lib.types.str;
254 };
255
256 mongoUrl = lib.mkOption {
257 default = "mongodb://${config.services.mongodb.bind_ip}:27017/seyren";
258 defaultText = lib.literalExpression ''"mongodb://''${config.services.mongodb.bind_ip}:27017/seyren"'';
259 description = "Mongodb connection string.";
260 type = lib.types.str;
261 };
262
263 extraConfig = lib.mkOption {
264 default = { };
265 description = ''
266 Extra seyren configuration. See
267 <https://github.com/scobal/seyren#config>
268 '';
269 type = lib.types.attrsOf lib.types.str;
270 example = lib.literalExpression ''
271 {
272 GRAPHITE_USERNAME = "user";
273 GRAPHITE_PASSWORD = "pass";
274 }
275 '';
276 };
277 };
278 };
279
280 ###### implementation
281
282 config = lib.mkMerge [
283 (lib.mkIf cfg.carbon.enableCache {
284 systemd.services.carbonCache =
285 let
286 name = "carbon-cache";
287 in
288 {
289 description = "Graphite Data Storage Backend";
290 wantedBy = [ "multi-user.target" ];
291 after = [ "network.target" ];
292 environment = carbonEnv;
293 serviceConfig = {
294 Slice = "system-graphite.slice";
295 RuntimeDirectory = name;
296 ExecStart = "${lib.getExe' pkgs.python3Packages.twisted "twistd"} ${carbonOpts name}";
297 User = "graphite";
298 Group = "graphite";
299 PermissionsStartOnly = true;
300 PIDFile = "/run/${name}/${name}.pid";
301 };
302 preStart = ''
303 install -dm0700 -o graphite -g graphite ${cfg.dataDir}
304 install -dm0700 -o graphite -g graphite ${cfg.dataDir}/whisper
305 '';
306 };
307 })
308
309 (lib.mkIf cfg.carbon.enableAggregator {
310 systemd.services.carbonAggregator =
311 let
312 name = "carbon-aggregator";
313 in
314 {
315 enable = cfg.carbon.enableAggregator;
316 description = "Carbon Data Aggregator";
317 wantedBy = [ "multi-user.target" ];
318 after = [ "network.target" ];
319 environment = carbonEnv;
320 serviceConfig = {
321 Slice = "system-graphite.slice";
322 RuntimeDirectory = name;
323 ExecStart = "${lib.getExe' pkgs.python3Packages.twisted "twistd"} ${carbonOpts name}";
324 User = "graphite";
325 Group = "graphite";
326 PIDFile = "/run/${name}/${name}.pid";
327 };
328 };
329 })
330
331 (lib.mkIf cfg.carbon.enableRelay {
332 systemd.services.carbonRelay =
333 let
334 name = "carbon-relay";
335 in
336 {
337 description = "Carbon Data Relay";
338 wantedBy = [ "multi-user.target" ];
339 after = [ "network.target" ];
340 environment = carbonEnv;
341 serviceConfig = {
342 Slice = "system-graphite.slice";
343 RuntimeDirectory = name;
344 ExecStart = "${lib.getExe' pkgs.python3Packages.twisted "twistd"} ${carbonOpts name}";
345 User = "graphite";
346 Group = "graphite";
347 PIDFile = "/run/${name}/${name}.pid";
348 };
349 };
350 })
351
352 (lib.mkIf (cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay) {
353 environment.systemPackages = [
354 pkgs.python3Packages.carbon
355 ];
356 })
357
358 (lib.mkIf cfg.web.enable ({
359 systemd.services.graphiteWeb = {
360 description = "Graphite Web Interface";
361 wantedBy = [ "multi-user.target" ];
362 after = [ "network.target" ];
363 path = [ pkgs.perl ];
364 environment = {
365 PYTHONPATH =
366 let
367 penv = pkgs.python3.buildEnv.override {
368 extraLibs = [
369 pkgs.python3Packages.graphite-web
370 ];
371 };
372 penvPack = "${penv}/${pkgs.python3.sitePackages}";
373 in
374 lib.concatStringsSep ":" [
375 "${graphiteLocalSettingsDir}"
376 "${penvPack}"
377 # explicitly adding pycairo in path because it cannot be imported via buildEnv
378 "${pkgs.python3Packages.pycairo}/${pkgs.python3.sitePackages}"
379 ];
380 DJANGO_SETTINGS_MODULE = "graphite.settings";
381 GRAPHITE_SETTINGS_MODULE = "graphite_local_settings";
382 GRAPHITE_CONF_DIR = configDir;
383 GRAPHITE_STORAGE_DIR = dataDir;
384 LD_LIBRARY_PATH = "${pkgs.cairo.out}/lib";
385 };
386 serviceConfig = {
387 ExecStart = ''
388 ${lib.getExe pkgs.python3Packages.waitress-django} \
389 --host=${cfg.web.listenAddress} --port=${toString cfg.web.port}
390 '';
391 User = "graphite";
392 Group = "graphite";
393 PermissionsStartOnly = true;
394 Slice = "system-graphite.slice";
395 };
396 preStart = ''
397 if ! test -e ${dataDir}/db-created; then
398 mkdir -p ${dataDir}/{whisper/,log/webapp/}
399 chmod 0700 ${dataDir}/{whisper/,log/webapp/}
400
401 ${lib.getExe' pkgs.python3Packages.django "django-admin"} migrate --noinput
402
403 chown -R graphite:graphite ${dataDir}
404
405 touch ${dataDir}/db-created
406 fi
407
408 # Only collect static files when graphite_web changes.
409 if ! [ "${dataDir}/current_graphite_web" -ef "${pkgs.python3Packages.graphite-web}" ]; then
410 mkdir -p ${staticDir}
411 ${lib.getExe' pkgs.python3Packages.django "django-admin"} collectstatic --noinput --clear
412 chown -R graphite:graphite ${staticDir}
413 ln -sfT "${pkgs.python3Packages.graphite-web}" "${dataDir}/current_graphite_web"
414 fi
415 '';
416 };
417
418 environment.systemPackages = [ pkgs.python3Packages.graphite-web ];
419 }))
420
421 (lib.mkIf cfg.seyren.enable {
422 systemd.services.seyren = {
423 description = "Graphite Alerting Dashboard";
424 wantedBy = [ "multi-user.target" ];
425 after = [
426 "network.target"
427 "mongodb.service"
428 ];
429 environment = seyrenConfig;
430 serviceConfig = {
431 ExecStart = "${lib.getExe pkgs.seyren} -httpPort ${toString cfg.seyren.port}";
432 WorkingDirectory = dataDir;
433 User = "graphite";
434 Group = "graphite";
435 Slice = "system-graphite.slice";
436 };
437 preStart = ''
438 if ! test -e ${dataDir}/db-created; then
439 mkdir -p ${dataDir}
440 chown graphite:graphite ${dataDir}
441 fi
442 '';
443 };
444
445 services.mongodb.enable = lib.mkDefault true;
446 })
447
448 (lib.mkIf
449 (
450 cfg.carbon.enableCache
451 || cfg.carbon.enableAggregator
452 || cfg.carbon.enableRelay
453 || cfg.web.enable
454 || cfg.seyren.enable
455 )
456 {
457 systemd.slices.system-graphite = {
458 description = "Graphite Graphing System Slice";
459 documentation = [ "https://graphite.readthedocs.io/en/latest/overview.html" ];
460 };
461
462 users.users.graphite = {
463 uid = config.ids.uids.graphite;
464 group = "graphite";
465 description = "Graphite daemon user";
466 home = dataDir;
467 };
468 users.groups.graphite.gid = config.ids.gids.graphite;
469 }
470 )
471 ];
472}