pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.graphite;
7 writeTextOrNull = f: t: mapNullable (pkgs.writeTextDir f) t;
8
9 dataDir = cfg.dataDir;
10 staticDir = cfg.dataDir + "/static";
11
12 graphiteLocalSettingsDir = pkgs.runCommand "graphite_local_settings" {
13 inherit graphiteLocalSettings;
14 preferLocalBuild = true;
15 } ''
16 mkdir -p $out
17 ln -s $graphiteLocalSettings $out/graphite_local_settings.py
18 '';
19
20 graphiteLocalSettings = pkgs.writeText "graphite_local_settings.py" (
21 "STATIC_ROOT = '${staticDir}'\n" +
22 optionalString (config.time.timeZone != null) "TIME_ZONE = '${config.time.timeZone}'\n"
23 + cfg.web.extraConfig
24 );
25
26 graphiteApiConfig = pkgs.writeText "graphite-api.yaml" ''
27 search_index: ${dataDir}/index
28 ${optionalString (config.time.timeZone != null) "time_zone: ${config.time.timeZone}"}
29 ${optionalString (cfg.api.finders != []) "finders:"}
30 ${concatMapStringsSep "\n" (f: " - " + f.moduleName) cfg.api.finders}
31 ${optionalString (cfg.api.functions != []) "functions:"}
32 ${concatMapStringsSep "\n" (f: " - " + f) cfg.api.functions}
33 ${cfg.api.extraConfig}
34 '';
35
36 seyrenConfig = {
37 SEYREN_URL = cfg.seyren.seyrenUrl;
38 MONGO_URL = cfg.seyren.mongoUrl;
39 GRAPHITE_URL = cfg.seyren.graphiteUrl;
40 } // cfg.seyren.extraConfig;
41
42 configDir = pkgs.buildEnv {
43 name = "graphite-config";
44 paths = lists.filter (el: el != null) [
45 (writeTextOrNull "carbon.conf" cfg.carbon.config)
46 (writeTextOrNull "storage-aggregation.conf" cfg.carbon.storageAggregation)
47 (writeTextOrNull "storage-schemas.conf" cfg.carbon.storageSchemas)
48 (writeTextOrNull "blacklist.conf" cfg.carbon.blacklist)
49 (writeTextOrNull "whitelist.conf" cfg.carbon.whitelist)
50 (writeTextOrNull "rewrite-rules.conf" cfg.carbon.rewriteRules)
51 (writeTextOrNull "relay-rules.conf" cfg.carbon.relayRules)
52 (writeTextOrNull "aggregation-rules.conf" cfg.carbon.aggregationRules)
53 ];
54 };
55
56 carbonOpts = name: with config.ids; ''
57 --nodaemon --syslog --prefix=${name} --pidfile /run/${name}/${name}.pid ${name}
58 '';
59
60 carbonEnv = {
61 PYTHONPATH = let
62 cenv = pkgs.python3.buildEnv.override {
63 extraLibs = [ pkgs.python3Packages.carbon ];
64 };
65 in "${cenv}/${pkgs.python3.sitePackages}";
66 GRAPHITE_ROOT = dataDir;
67 GRAPHITE_CONF_DIR = configDir;
68 GRAPHITE_STORAGE_DIR = dataDir;
69 };
70
71in {
72
73 imports = [
74 (mkRemovedOptionModule ["services" "graphite" "pager"] "")
75 ];
76
77 ###### interface
78
79 options.services.graphite = {
80 dataDir = mkOption {
81 type = types.path;
82 default = "/var/db/graphite";
83 description = ''
84 Data directory for graphite.
85 '';
86 };
87
88 web = {
89 enable = mkOption {
90 description = "Whether to enable graphite web frontend.";
91 default = false;
92 type = types.bool;
93 };
94
95 listenAddress = mkOption {
96 description = "Graphite web frontend listen address.";
97 default = "127.0.0.1";
98 type = types.str;
99 };
100
101 port = mkOption {
102 description = "Graphite web frontend port.";
103 default = 8080;
104 type = types.int;
105 };
106
107 extraConfig = mkOption {
108 type = types.str;
109 default = "";
110 description = ''
111 Graphite webapp settings. See:
112 <link xlink:href="http://graphite.readthedocs.io/en/latest/config-local-settings.html"/>
113 '';
114 };
115 };
116
117 api = {
118 enable = mkOption {
119 description = ''
120 Whether to enable graphite api. Graphite api is lightweight alternative
121 to graphite web, with api and without dashboard. It's advised to use
122 grafana as alternative dashboard and influxdb as alternative to
123 graphite carbon.
124
125 For more information visit
126 <link xlink:href="https://graphite-api.readthedocs.org/en/latest/"/>
127 '';
128 default = false;
129 type = types.bool;
130 };
131
132 finders = mkOption {
133 description = "List of finder plugins to load.";
134 default = [];
135 example = literalExpression "[ pkgs.python3Packages.influxgraph ]";
136 type = types.listOf types.package;
137 };
138
139 functions = mkOption {
140 description = "List of functions to load.";
141 default = [
142 "graphite_api.functions.SeriesFunctions"
143 "graphite_api.functions.PieFunctions"
144 ];
145 type = types.listOf types.str;
146 };
147
148 listenAddress = mkOption {
149 description = "Graphite web service listen address.";
150 default = "127.0.0.1";
151 type = types.str;
152 };
153
154 port = mkOption {
155 description = "Graphite api service port.";
156 default = 8080;
157 type = types.int;
158 };
159
160 package = mkOption {
161 description = "Package to use for graphite api.";
162 default = pkgs.python3Packages.graphite_api;
163 defaultText = literalExpression "pkgs.python3Packages.graphite_api";
164 type = types.package;
165 };
166
167 extraConfig = mkOption {
168 description = "Extra configuration for graphite api.";
169 default = ''
170 whisper:
171 directories:
172 - ${dataDir}/whisper
173 '';
174 example = ''
175 allowed_origins:
176 - dashboard.example.com
177 cheat_times: true
178 influxdb:
179 host: localhost
180 port: 8086
181 user: influxdb
182 pass: influxdb
183 db: metrics
184 cache:
185 CACHE_TYPE: 'filesystem'
186 CACHE_DIR: '/tmp/graphite-api-cache'
187 '';
188 type = types.lines;
189 };
190 };
191
192 carbon = {
193 config = mkOption {
194 description = "Content of carbon configuration file.";
195 default = ''
196 [cache]
197 # Listen on localhost by default for security reasons
198 UDP_RECEIVER_INTERFACE = 127.0.0.1
199 PICKLE_RECEIVER_INTERFACE = 127.0.0.1
200 LINE_RECEIVER_INTERFACE = 127.0.0.1
201 CACHE_QUERY_INTERFACE = 127.0.0.1
202 # Do not log every update
203 LOG_UPDATES = False
204 LOG_CACHE_HITS = False
205 '';
206 type = types.str;
207 };
208
209 enableCache = mkOption {
210 description = "Whether to enable carbon cache, the graphite storage daemon.";
211 default = false;
212 type = types.bool;
213 };
214
215 storageAggregation = mkOption {
216 description = "Defines how to aggregate data to lower-precision retentions.";
217 default = null;
218 type = types.nullOr types.str;
219 example = ''
220 [all_min]
221 pattern = \.min$
222 xFilesFactor = 0.1
223 aggregationMethod = min
224 '';
225 };
226
227 storageSchemas = mkOption {
228 description = "Defines retention rates for storing metrics.";
229 default = "";
230 type = types.nullOr types.str;
231 example = ''
232 [apache_busyWorkers]
233 pattern = ^servers\.www.*\.workers\.busyWorkers$
234 retentions = 15s:7d,1m:21d,15m:5y
235 '';
236 };
237
238 blacklist = mkOption {
239 description = "Any metrics received which match one of the experssions will be dropped.";
240 default = null;
241 type = types.nullOr types.str;
242 example = "^some\\.noisy\\.metric\\.prefix\\..*";
243 };
244
245 whitelist = mkOption {
246 description = "Only metrics received which match one of the experssions will be persisted.";
247 default = null;
248 type = types.nullOr types.str;
249 example = ".*";
250 };
251
252 rewriteRules = mkOption {
253 description = ''
254 Regular expression patterns that can be used to rewrite metric names
255 in a search and replace fashion.
256 '';
257 default = null;
258 type = types.nullOr types.str;
259 example = ''
260 [post]
261 _sum$ =
262 _avg$ =
263 '';
264 };
265
266 enableRelay = mkOption {
267 description = "Whether to enable carbon relay, the carbon replication and sharding service.";
268 default = false;
269 type = types.bool;
270 };
271
272 relayRules = mkOption {
273 description = "Relay rules are used to send certain metrics to a certain backend.";
274 default = null;
275 type = types.nullOr types.str;
276 example = ''
277 [example]
278 pattern = ^mydata\.foo\..+
279 servers = 10.1.2.3, 10.1.2.4:2004, myserver.mydomain.com
280 '';
281 };
282
283 enableAggregator = mkOption {
284 description = "Whether to enable carbon aggregator, the carbon buffering service.";
285 default = false;
286 type = types.bool;
287 };
288
289 aggregationRules = mkOption {
290 description = "Defines if and how received metrics will be aggregated.";
291 default = null;
292 type = types.nullOr types.str;
293 example = ''
294 <env>.applications.<app>.all.requests (60) = sum <env>.applications.<app>.*.requests
295 <env>.applications.<app>.all.latency (60) = avg <env>.applications.<app>.*.latency
296 '';
297 };
298 };
299
300 seyren = {
301 enable = mkOption {
302 description = "Whether to enable seyren service.";
303 default = false;
304 type = types.bool;
305 };
306
307 port = mkOption {
308 description = "Seyren listening port.";
309 default = 8081;
310 type = types.int;
311 };
312
313 seyrenUrl = mkOption {
314 default = "http://localhost:${toString cfg.seyren.port}/";
315 description = "Host where seyren is accessible.";
316 type = types.str;
317 };
318
319 graphiteUrl = mkOption {
320 default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
321 description = "Host where graphite service runs.";
322 type = types.str;
323 };
324
325 mongoUrl = mkOption {
326 default = "mongodb://${config.services.mongodb.bind_ip}:27017/seyren";
327 description = "Mongodb connection string.";
328 type = types.str;
329 };
330
331 extraConfig = mkOption {
332 default = {};
333 description = ''
334 Extra seyren configuration. See
335 <link xlink:href='https://github.com/scobal/seyren#config' />
336 '';
337 type = types.attrsOf types.str;
338 example = literalExpression ''
339 {
340 GRAPHITE_USERNAME = "user";
341 GRAPHITE_PASSWORD = "pass";
342 }
343 '';
344 };
345 };
346
347 beacon = {
348 enable = mkEnableOption "graphite beacon";
349
350 config = mkOption {
351 description = "Graphite beacon configuration.";
352 default = {};
353 type = types.attrs;
354 };
355 };
356 };
357
358 ###### implementation
359
360 config = mkMerge [
361 (mkIf cfg.carbon.enableCache {
362 systemd.services.carbonCache = let name = "carbon-cache"; in {
363 description = "Graphite Data Storage Backend";
364 wantedBy = [ "multi-user.target" ];
365 after = [ "network.target" ];
366 environment = carbonEnv;
367 serviceConfig = {
368 RuntimeDirectory = name;
369 ExecStart = "${pkgs.python3Packages.twisted}/bin/twistd ${carbonOpts name}";
370 User = "graphite";
371 Group = "graphite";
372 PermissionsStartOnly = true;
373 PIDFile="/run/${name}/${name}.pid";
374 };
375 preStart = ''
376 install -dm0700 -o graphite -g graphite ${cfg.dataDir}
377 install -dm0700 -o graphite -g graphite ${cfg.dataDir}/whisper
378 '';
379 };
380 })
381
382 (mkIf cfg.carbon.enableAggregator {
383 systemd.services.carbonAggregator = let name = "carbon-aggregator"; in {
384 enable = cfg.carbon.enableAggregator;
385 description = "Carbon Data Aggregator";
386 wantedBy = [ "multi-user.target" ];
387 after = [ "network.target" ];
388 environment = carbonEnv;
389 serviceConfig = {
390 RuntimeDirectory = name;
391 ExecStart = "${pkgs.python3Packages.twisted}/bin/twistd ${carbonOpts name}";
392 User = "graphite";
393 Group = "graphite";
394 PIDFile="/run/${name}/${name}.pid";
395 };
396 };
397 })
398
399 (mkIf cfg.carbon.enableRelay {
400 systemd.services.carbonRelay = let name = "carbon-relay"; in {
401 description = "Carbon Data Relay";
402 wantedBy = [ "multi-user.target" ];
403 after = [ "network.target" ];
404 environment = carbonEnv;
405 serviceConfig = {
406 RuntimeDirectory = name;
407 ExecStart = "${pkgs.python3Packages.twisted}/bin/twistd ${carbonOpts name}";
408 User = "graphite";
409 Group = "graphite";
410 PIDFile="/run/${name}/${name}.pid";
411 };
412 };
413 })
414
415 (mkIf (cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay) {
416 environment.systemPackages = [
417 pkgs.python3Packages.carbon
418 ];
419 })
420
421 (mkIf cfg.web.enable ({
422 systemd.services.graphiteWeb = {
423 description = "Graphite Web Interface";
424 wantedBy = [ "multi-user.target" ];
425 after = [ "network.target" ];
426 path = [ pkgs.perl ];
427 environment = {
428 PYTHONPATH = let
429 penv = pkgs.python3.buildEnv.override {
430 extraLibs = [
431 pkgs.python3Packages.graphite-web
432 ];
433 };
434 penvPack = "${penv}/${pkgs.python3.sitePackages}";
435 in concatStringsSep ":" [
436 "${graphiteLocalSettingsDir}"
437 "${penvPack}"
438 # explicitly adding pycairo in path because it cannot be imported via buildEnv
439 "${pkgs.python3Packages.pycairo}/${pkgs.python3.sitePackages}"
440 ];
441 DJANGO_SETTINGS_MODULE = "graphite.settings";
442 GRAPHITE_SETTINGS_MODULE = "graphite_local_settings";
443 GRAPHITE_CONF_DIR = configDir;
444 GRAPHITE_STORAGE_DIR = dataDir;
445 LD_LIBRARY_PATH = "${pkgs.cairo.out}/lib";
446 };
447 serviceConfig = {
448 ExecStart = ''
449 ${pkgs.python3Packages.waitress-django}/bin/waitress-serve-django \
450 --host=${cfg.web.listenAddress} --port=${toString cfg.web.port}
451 '';
452 User = "graphite";
453 Group = "graphite";
454 PermissionsStartOnly = true;
455 };
456 preStart = ''
457 if ! test -e ${dataDir}/db-created; then
458 mkdir -p ${dataDir}/{whisper/,log/webapp/}
459 chmod 0700 ${dataDir}/{whisper/,log/webapp/}
460
461 ${pkgs.python3Packages.django}/bin/django-admin.py migrate --noinput
462
463 chown -R graphite:graphite ${dataDir}
464
465 touch ${dataDir}/db-created
466 fi
467
468 # Only collect static files when graphite_web changes.
469 if ! [ "${dataDir}/current_graphite_web" -ef "${pkgs.python3Packages.graphite-web}" ]; then
470 mkdir -p ${staticDir}
471 ${pkgs.python3Packages.django}/bin/django-admin.py collectstatic --noinput --clear
472 chown -R graphite:graphite ${staticDir}
473 ln -sfT "${pkgs.python3Packages.graphite-web}" "${dataDir}/current_graphite_web"
474 fi
475 '';
476 };
477
478 environment.systemPackages = [ pkgs.python3Packages.graphite-web ];
479 }))
480
481 (mkIf cfg.api.enable {
482 systemd.services.graphiteApi = {
483 description = "Graphite Api Interface";
484 wantedBy = [ "multi-user.target" ];
485 after = [ "network.target" ];
486 environment = {
487 PYTHONPATH = let
488 aenv = pkgs.python3.buildEnv.override {
489 extraLibs = [ cfg.api.package pkgs.cairo pkgs.python3Packages.cffi ] ++ cfg.api.finders;
490 };
491 in "${aenv}/${pkgs.python3.sitePackages}";
492 GRAPHITE_API_CONFIG = graphiteApiConfig;
493 LD_LIBRARY_PATH = "${pkgs.cairo.out}/lib";
494 };
495 serviceConfig = {
496 ExecStart = ''
497 ${pkgs.python3Packages.waitress}/bin/waitress-serve \
498 --host=${cfg.api.listenAddress} --port=${toString cfg.api.port} \
499 graphite_api.app:app
500 '';
501 User = "graphite";
502 Group = "graphite";
503 PermissionsStartOnly = true;
504 };
505 preStart = ''
506 if ! test -e ${dataDir}/db-created; then
507 mkdir -p ${dataDir}/cache/
508 chmod 0700 ${dataDir}/cache/
509
510 chown graphite:graphite ${cfg.dataDir}
511 chown -R graphite:graphite ${cfg.dataDir}/cache
512
513 touch ${dataDir}/db-created
514 fi
515 '';
516 };
517 })
518
519 (mkIf cfg.seyren.enable {
520 systemd.services.seyren = {
521 description = "Graphite Alerting Dashboard";
522 wantedBy = [ "multi-user.target" ];
523 after = [ "network.target" "mongodb.service" ];
524 environment = seyrenConfig;
525 serviceConfig = {
526 ExecStart = "${pkgs.seyren}/bin/seyren -httpPort ${toString cfg.seyren.port}";
527 WorkingDirectory = dataDir;
528 User = "graphite";
529 Group = "graphite";
530 };
531 preStart = ''
532 if ! test -e ${dataDir}/db-created; then
533 mkdir -p ${dataDir}
534 chown graphite:graphite ${dataDir}
535 fi
536 '';
537 };
538
539 services.mongodb.enable = mkDefault true;
540 })
541
542 (mkIf cfg.beacon.enable {
543 systemd.services.graphite-beacon = {
544 description = "Grpahite Beacon Alerting Daemon";
545 wantedBy = [ "multi-user.target" ];
546 serviceConfig = {
547 ExecStart = ''
548 ${pkgs.python3Packages.graphite_beacon}/bin/graphite-beacon \
549 --config=${pkgs.writeText "graphite-beacon.json" (builtins.toJSON cfg.beacon.config)}
550 '';
551 User = "graphite";
552 Group = "graphite";
553 };
554 };
555 })
556
557 (mkIf (
558 cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay ||
559 cfg.web.enable || cfg.api.enable ||
560 cfg.seyren.enable || cfg.beacon.enable
561 ) {
562 users.users.graphite = {
563 uid = config.ids.uids.graphite;
564 group = "graphite";
565 description = "Graphite daemon user";
566 home = dataDir;
567 };
568 users.groups.graphite.gid = config.ids.gids.graphite;
569 })
570 ];
571}