pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.graphite;
7 writeTextOrNull = f: t: mapNullable (pkgs.writeTextDir f) t;
8
9 dataDir = cfg.dataDir;
10
11 graphiteApiConfig = pkgs.writeText "graphite-api.yaml" ''
12 time_zone: ${config.time.timeZone}
13 search_index: ${dataDir}/index
14 ${optionalString (cfg.api.finders != []) ''finders:''}
15 ${concatMapStringsSep "\n" (f: " - " + f.moduleName) cfg.api.finders}
16 ${optionalString (cfg.api.functions != []) ''functions:''}
17 ${concatMapStringsSep "\n" (f: " - " + f) cfg.api.functions}
18 ${cfg.api.extraConfig}
19 '';
20
21 seyrenConfig = {
22 SEYREN_URL = cfg.seyren.seyrenUrl;
23 MONGO_URL = cfg.seyren.mongoUrl;
24 GRAPHITE_URL = cfg.seyren.graphiteUrl;
25 } // cfg.seyren.extraConfig;
26
27 pagerConfig = pkgs.writeText "alarms.yaml" cfg.pager.alerts;
28
29 configDir = pkgs.buildEnv {
30 name = "graphite-config";
31 paths = lists.filter (el: el != null) [
32 (writeTextOrNull "carbon.conf" cfg.carbon.config)
33 (writeTextOrNull "storage-aggregation.conf" cfg.carbon.storageAggregation)
34 (writeTextOrNull "storage-schemas.conf" cfg.carbon.storageSchemas)
35 (writeTextOrNull "blacklist.conf" cfg.carbon.blacklist)
36 (writeTextOrNull "whitelist.conf" cfg.carbon.whitelist)
37 (writeTextOrNull "rewrite-rules.conf" cfg.carbon.rewriteRules)
38 (writeTextOrNull "relay-rules.conf" cfg.carbon.relayRules)
39 (writeTextOrNull "aggregation-rules.conf" cfg.carbon.aggregationRules)
40 ];
41 };
42
43 carbonOpts = name: with config.ids; ''
44 --nodaemon --syslog --prefix=${name} --pidfile /run/${name}/${name}.pid ${name}
45 '';
46
47 mkPidFileDir = name: ''
48 mkdir -p /run/${name}
49 chmod 0700 /run/${name}
50 chown -R graphite:graphite /run/${name}
51 '';
52
53 carbonEnv = {
54 PYTHONPATH = let
55 cenv = pkgs.python.buildEnv.override {
56 extraLibs = [ pkgs.python27Packages.carbon ];
57 };
58 cenvPack = "${cenv}/${pkgs.python.sitePackages}";
59 # opt/graphite/lib contains twisted.plugins.carbon-cache
60 in "${cenvPack}/opt/graphite/lib:${cenvPack}";
61 GRAPHITE_ROOT = dataDir;
62 GRAPHITE_CONF_DIR = configDir;
63 GRAPHITE_STORAGE_DIR = dataDir;
64 };
65
66in {
67
68 ###### interface
69
70 options.services.graphite = {
71 dataDir = mkOption {
72 type = types.path;
73 default = "/var/db/graphite";
74 description = ''
75 Data directory for graphite.
76 '';
77 };
78
79 web = {
80 enable = mkOption {
81 description = "Whether to enable graphite web frontend.";
82 default = false;
83 type = types.bool;
84 };
85
86 listenAddress = mkOption {
87 description = "Graphite web frontend listen address.";
88 default = "127.0.0.1";
89 type = types.str;
90 };
91
92 port = mkOption {
93 description = "Graphite web frontend port.";
94 default = 8080;
95 type = types.int;
96 };
97 };
98
99 api = {
100 enable = mkOption {
101 description = ''
102 Whether to enable graphite api. Graphite api is lightweight alternative
103 to graphite web, with api and without dashboard. It's advised to use
104 grafana as alternative dashboard and influxdb as alternative to
105 graphite carbon.
106
107 For more information visit
108 <link xlink:href="http://graphite-api.readthedocs.org/en/latest/"/>
109 '';
110 default = false;
111 type = types.bool;
112 };
113
114 finders = mkOption {
115 description = "List of finder plugins to load.";
116 default = [];
117 example = literalExample "[ pkgs.python27Packages.graphite_influxdb ]";
118 type = types.listOf types.package;
119 };
120
121 functions = mkOption {
122 description = "List of functions to load.";
123 default = [
124 "graphite_api.functions.SeriesFunctions"
125 "graphite_api.functions.PieFunctions"
126 ];
127 type = types.listOf types.str;
128 };
129
130 listenAddress = mkOption {
131 description = "Graphite web service listen address.";
132 default = "127.0.0.1";
133 type = types.str;
134 };
135
136 port = mkOption {
137 description = "Graphite api service port.";
138 default = 8080;
139 type = types.int;
140 };
141
142 package = mkOption {
143 description = "Package to use for graphite api.";
144 default = pkgs.python27Packages.graphite_api;
145 defaultText = "pkgs.python27Packages.graphite_api";
146 type = types.package;
147 };
148
149 extraConfig = mkOption {
150 description = "Extra configuration for graphite api.";
151 default = ''
152 whisper:
153 directories:
154 - ${dataDir}/whisper
155 '';
156 example = ''
157 allowed_origins:
158 - dashboard.example.com
159 cheat_times: true
160 influxdb:
161 host: localhost
162 port: 8086
163 user: influxdb
164 pass: influxdb
165 db: metrics
166 cache:
167 CACHE_TYPE: 'filesystem'
168 CACHE_DIR: '/tmp/graphite-api-cache'
169 '';
170 type = types.lines;
171 };
172 };
173
174 carbon = {
175 config = mkOption {
176 description = "Content of carbon configuration file.";
177 default = ''
178 [cache]
179 # Listen on localhost by default for security reasons
180 UDP_RECEIVER_INTERFACE = 127.0.0.1
181 PICKLE_RECEIVER_INTERFACE = 127.0.0.1
182 LINE_RECEIVER_INTERFACE = 127.0.0.1
183 CACHE_QUERY_INTERFACE = 127.0.0.1
184 # Do not log every update
185 LOG_UPDATES = False
186 LOG_CACHE_HITS = False
187 '';
188 type = types.str;
189 };
190
191 enableCache = mkOption {
192 description = "Whether to enable carbon cache, the graphite storage daemon.";
193 default = false;
194 type = types.bool;
195 };
196
197 storageAggregation = mkOption {
198 description = "Defines how to aggregate data to lower-precision retentions.";
199 default = null;
200 type = types.uniq (types.nullOr types.string);
201 example = ''
202 [all_min]
203 pattern = \.min$
204 xFilesFactor = 0.1
205 aggregationMethod = min
206 '';
207 };
208
209 storageSchemas = mkOption {
210 description = "Defines retention rates for storing metrics.";
211 default = "";
212 type = types.uniq (types.nullOr types.string);
213 example = ''
214 [apache_busyWorkers]
215 pattern = ^servers\.www.*\.workers\.busyWorkers$
216 retentions = 15s:7d,1m:21d,15m:5y
217 '';
218 };
219
220 blacklist = mkOption {
221 description = "Any metrics received which match one of the experssions will be dropped.";
222 default = null;
223 type = types.uniq (types.nullOr types.string);
224 example = "^some\.noisy\.metric\.prefix\..*";
225 };
226
227 whitelist = mkOption {
228 description = "Only metrics received which match one of the experssions will be persisted.";
229 default = null;
230 type = types.uniq (types.nullOr types.string);
231 example = ".*";
232 };
233
234 rewriteRules = mkOption {
235 description = ''
236 Regular expression patterns that can be used to rewrite metric names
237 in a search and replace fashion.
238 '';
239 default = null;
240 type = types.uniq (types.nullOr types.string);
241 example = ''
242 [post]
243 _sum$ =
244 _avg$ =
245 '';
246 };
247
248 enableRelay = mkOption {
249 description = "Whether to enable carbon relay, the carbon replication and sharding service.";
250 default = false;
251 type = types.bool;
252 };
253
254 relayRules = mkOption {
255 description = "Relay rules are used to send certain metrics to a certain backend.";
256 default = null;
257 type = types.uniq (types.nullOr types.string);
258 example = ''
259 [example]
260 pattern = ^mydata\.foo\..+
261 servers = 10.1.2.3, 10.1.2.4:2004, myserver.mydomain.com
262 '';
263 };
264
265 enableAggregator = mkOption {
266 description = "Whether to enable carbon aggregator, the carbon buffering service.";
267 default = false;
268 type = types.bool;
269 };
270
271 aggregationRules = mkOption {
272 description = "Defines if and how received metrics will be aggregated.";
273 default = null;
274 type = types.uniq (types.nullOr types.string);
275 example = ''
276 <env>.applications.<app>.all.requests (60) = sum <env>.applications.<app>.*.requests
277 <env>.applications.<app>.all.latency (60) = avg <env>.applications.<app>.*.latency
278 '';
279 };
280 };
281
282 seyren = {
283 enable = mkOption {
284 description = "Whether to enable seyren service.";
285 default = false;
286 type = types.bool;
287 };
288
289 port = mkOption {
290 description = "Seyren listening port.";
291 default = 8081;
292 type = types.int;
293 };
294
295 seyrenUrl = mkOption {
296 default = "http://localhost:${toString cfg.seyren.port}/";
297 description = "Host where seyren is accessible.";
298 type = types.str;
299 };
300
301 graphiteUrl = mkOption {
302 default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
303 description = "Host where graphite service runs.";
304 type = types.str;
305 };
306
307 mongoUrl = mkOption {
308 default = "mongodb://${config.services.mongodb.bind_ip}:27017/seyren";
309 description = "Mongodb connection string.";
310 type = types.str;
311 };
312
313 extraConfig = mkOption {
314 default = {};
315 description = ''
316 Extra seyren configuration. See
317 <link xlink:href='https://github.com/scobal/seyren#config' />
318 '';
319 type = types.attrsOf types.str;
320 example = literalExample ''
321 {
322 GRAPHITE_USERNAME = "user";
323 GRAPHITE_PASSWORD = "pass";
324 }
325 '';
326 };
327 };
328
329 pager = {
330 enable = mkOption {
331 description = ''
332 Whether to enable graphite-pager service. For more information visit
333 <link xlink:href="https://github.com/seatgeek/graphite-pager"/>
334 '';
335 default = false;
336 type = types.bool;
337 };
338
339 redisUrl = mkOption {
340 description = "Redis connection string.";
341 default = "redis://localhost:${toString config.services.redis.port}/";
342 type = types.str;
343 };
344
345 graphiteUrl = mkOption {
346 description = "URL to your graphite service.";
347 default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
348 type = types.str;
349 };
350
351 alerts = mkOption {
352 description = "Alerts configuration for graphite-pager.";
353 default = ''
354 alerts:
355 - target: constantLine(100)
356 warning: 90
357 critical: 200
358 name: Test
359 '';
360 example = ''
361 pushbullet_key: pushbullet_api_key
362 alerts:
363 - target: stats.seatgeek.app.deal_quality.venue_info_cache.hit
364 warning: .5
365 critical: 1
366 name: Deal quality venue cache hits
367 '';
368 type = types.lines;
369 };
370 };
371
372 beacon = {
373 enable = mkEnableOption "graphite beacon";
374
375 config = mkOption {
376 description = "Graphite beacon configuration.";
377 default = {};
378 type = types.attrs;
379 };
380 };
381 };
382
383 ###### implementation
384
385 config = mkMerge [
386 (mkIf cfg.carbon.enableCache {
387 systemd.services.carbonCache = let name = "carbon-cache"; in {
388 description = "Graphite Data Storage Backend";
389 wantedBy = [ "multi-user.target" ];
390 after = [ "network.target" ];
391 environment = carbonEnv;
392 serviceConfig = {
393 ExecStart = "${pkgs.pythonPackages.twisted}/bin/twistd ${carbonOpts name}";
394 User = "graphite";
395 Group = "graphite";
396 PermissionsStartOnly = true;
397 PIDFile="/run/${name}/${name}.pid";
398 };
399 preStart = mkPidFileDir name + ''
400
401 mkdir -p ${cfg.dataDir}/whisper
402 chmod 0700 ${cfg.dataDir}/whisper
403 chown graphite:graphite ${cfg.dataDir}
404 chown graphite:graphite ${cfg.dataDir}/whisper
405 '';
406 };
407 })
408
409 (mkIf cfg.carbon.enableAggregator {
410 systemd.services.carbonAggregator = let name = "carbon-aggregator"; in {
411 enable = cfg.carbon.enableAggregator;
412 description = "Carbon Data Aggregator";
413 wantedBy = [ "multi-user.target" ];
414 after = [ "network.target" ];
415 environment = carbonEnv;
416 serviceConfig = {
417 ExecStart = "${pkgs.pythonPackages.twisted}/bin/twistd ${carbonOpts name}";
418 User = "graphite";
419 Group = "graphite";
420 PIDFile="/run/${name}/${name}.pid";
421 };
422 preStart = mkPidFileDir name;
423 };
424 })
425
426 (mkIf cfg.carbon.enableRelay {
427 systemd.services.carbonRelay = let name = "carbon-relay"; in {
428 description = "Carbon Data Relay";
429 wantedBy = [ "multi-user.target" ];
430 after = [ "network.target" ];
431 environment = carbonEnv;
432 serviceConfig = {
433 ExecStart = "${pkgs.pythonPackages.twisted}/bin/twistd ${carbonOpts name}";
434 User = "graphite";
435 Group = "graphite";
436 PIDFile="/run/${name}/${name}.pid";
437 };
438 preStart = mkPidFileDir name;
439 };
440 })
441
442 (mkIf (cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay) {
443 environment.systemPackages = [
444 pkgs.pythonPackages.carbon
445 ];
446 })
447
448 (mkIf cfg.web.enable {
449 systemd.services.graphiteWeb = {
450 description = "Graphite Web Interface";
451 wantedBy = [ "multi-user.target" ];
452 after = [ "network.target" ];
453 path = [ pkgs.perl ];
454 environment = {
455 PYTHONPATH = let
456 penv = pkgs.python.buildEnv.override {
457 extraLibs = [
458 pkgs.python27Packages.graphite_web
459 pkgs.python27Packages.pysqlite
460 ];
461 };
462 penvPack = "${penv}/${pkgs.python.sitePackages}";
463 # opt/graphite/webapp contains graphite/settings.py
464 # explicitly adding pycairo in path because it cannot be imported via buildEnv
465 in "${penvPack}/opt/graphite/webapp:${penvPack}:${pkgs.pythonPackages.pycairo}/${pkgs.python.sitePackages}";
466 DJANGO_SETTINGS_MODULE = "graphite.settings";
467 GRAPHITE_CONF_DIR = configDir;
468 GRAPHITE_STORAGE_DIR = dataDir;
469 LD_LIBRARY_PATH = "${pkgs.cairo.out}/lib";
470 };
471 serviceConfig = {
472 ExecStart = ''
473 ${pkgs.python27Packages.waitress}/bin/waitress-serve \
474 --host=${cfg.web.listenAddress} --port=${toString cfg.web.port} \
475 --call django.core.handlers.wsgi:WSGIHandler'';
476 User = "graphite";
477 Group = "graphite";
478 PermissionsStartOnly = true;
479 };
480 preStart = ''
481 if ! test -e ${dataDir}/db-created; then
482 mkdir -p ${dataDir}/{whisper/,log/webapp/}
483 chmod 0700 ${dataDir}/{whisper/,log/webapp/}
484
485 # populate database
486 ${pkgs.python27Packages.graphite_web}/bin/manage-graphite.py syncdb --noinput
487
488 # create index
489 ${pkgs.python27Packages.graphite_web}/bin/build-index.sh
490
491 chown -R graphite:graphite ${cfg.dataDir}
492
493 touch ${dataDir}/db-created
494 fi
495 '';
496 };
497
498 environment.systemPackages = [ pkgs.python27Packages.graphite_web ];
499 })
500
501 (mkIf cfg.api.enable {
502 systemd.services.graphiteApi = {
503 description = "Graphite Api Interface";
504 wantedBy = [ "multi-user.target" ];
505 after = [ "network.target" ];
506 environment = {
507 PYTHONPATH = let
508 aenv = pkgs.python.buildEnv.override {
509 extraLibs = [ cfg.api.package pkgs.cairo ] ++ cfg.api.finders;
510 };
511 in "${aenv}/${pkgs.python.sitePackages}";
512 GRAPHITE_API_CONFIG = graphiteApiConfig;
513 LD_LIBRARY_PATH = "${pkgs.cairo.out}/lib";
514 };
515 serviceConfig = {
516 ExecStart = ''
517 ${pkgs.python27Packages.waitress}/bin/waitress-serve \
518 --host=${cfg.api.listenAddress} --port=${toString cfg.api.port} \
519 graphite_api.app:app
520 '';
521 User = "graphite";
522 Group = "graphite";
523 PermissionsStartOnly = true;
524 };
525 preStart = ''
526 if ! test -e ${dataDir}/db-created; then
527 mkdir -p ${dataDir}/cache/
528 chmod 0700 ${dataDir}/cache/
529
530 chown graphite:graphite ${cfg.dataDir}
531 chown -R graphite:graphite ${cfg.dataDir}/cache
532
533 touch ${dataDir}/db-created
534 fi
535 '';
536 };
537 })
538
539 (mkIf cfg.seyren.enable {
540 systemd.services.seyren = {
541 description = "Graphite Alerting Dashboard";
542 wantedBy = [ "multi-user.target" ];
543 after = [ "network.target" "mongodb.service" ];
544 environment = seyrenConfig;
545 serviceConfig = {
546 ExecStart = "${pkgs.seyren}/bin/seyren -httpPort ${toString cfg.seyren.port}";
547 WorkingDirectory = dataDir;
548 User = "graphite";
549 Group = "graphite";
550 };
551 preStart = ''
552 if ! test -e ${dataDir}/db-created; then
553 mkdir -p ${dataDir}
554 chown graphite:graphite ${dataDir}
555 fi
556 '';
557 };
558
559 services.mongodb.enable = mkDefault true;
560 })
561
562 (mkIf cfg.pager.enable {
563 systemd.services.graphitePager = {
564 description = "Graphite Pager Alerting Daemon";
565 wantedBy = [ "multi-user.target" ];
566 after = [ "network.target" "redis.service" ];
567 environment = {
568 REDIS_URL = cfg.pager.redisUrl;
569 GRAPHITE_URL = cfg.pager.graphiteUrl;
570 };
571 serviceConfig = {
572 ExecStart = "${pkgs.pythonPackages.graphite_pager}/bin/graphite-pager --config ${pagerConfig}";
573 User = "graphite";
574 Group = "graphite";
575 };
576 };
577
578 services.redis.enable = mkDefault true;
579
580 environment.systemPackages = [ pkgs.pythonPackages.graphite_pager ];
581 })
582
583 (mkIf cfg.beacon.enable {
584 systemd.services.graphite-beacon = {
585 description = "Grpahite Beacon Alerting Daemon";
586 wantedBy = [ "multi-user.target" ];
587 serviceConfig = {
588 ExecStart = ''
589 ${pkgs.pythonPackages.graphite_beacon}/bin/graphite-beacon \
590 --config=${pkgs.writeText "graphite-beacon.json" (builtins.toJSON cfg.beacon.config)}
591 '';
592 User = "graphite";
593 Group = "graphite";
594 };
595 };
596 })
597
598 (mkIf (
599 cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay ||
600 cfg.web.enable || cfg.api.enable ||
601 cfg.seyren.enable || cfg.pager.enable || cfg.beacon.enable
602 ) {
603 users.extraUsers = singleton {
604 name = "graphite";
605 uid = config.ids.uids.graphite;
606 description = "Graphite daemon user";
607 home = dataDir;
608 };
609 users.extraGroups.graphite.gid = config.ids.gids.graphite;
610 })
611 ];
612}