pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.graphite;
7 writeTextOrNull = f: t: if t == null then null else pkgs.writeTextDir f t;
8
9 dataDir = cfg.dataDir;
10
11 graphiteApiConfig = pkgs.writeText "graphite-api.yaml" ''
12 time_zone: ${config.time.timeZone}
13 search_index: ${dataDir}/index
14 ${optionalString (cfg.api.finders != []) ''finders:''}
15 ${concatMapStringsSep "\n" (f: " - " + f.moduleName) cfg.api.finders}
16 ${optionalString (cfg.api.functions != []) ''functions:''}
17 ${concatMapStringsSep "\n" (f: " - " + f) cfg.api.functions}
18 ${cfg.api.extraConfig}
19 '';
20
21 seyrenConfig = {
22 SEYREN_URL = cfg.seyren.seyrenUrl;
23 MONGO_URL = cfg.seyren.mongoUrl;
24 GRAPHITE_URL = cfg.seyren.graphiteUrl;
25 } // cfg.seyren.extraConfig;
26
27 pagerConfig = pkgs.writeText "alarms.yaml" cfg.pager.alerts;
28
29 configDir = pkgs.buildEnv {
30 name = "graphite-config";
31 paths = lists.filter (el: el != null) [
32 (writeTextOrNull "carbon.conf" cfg.carbon.config)
33 (writeTextOrNull "storage-aggregation.conf" cfg.carbon.storageAggregation)
34 (writeTextOrNull "storage-schemas.conf" cfg.carbon.storageSchemas)
35 (writeTextOrNull "blacklist.conf" cfg.carbon.blacklist)
36 (writeTextOrNull "whitelist.conf" cfg.carbon.whitelist)
37 (writeTextOrNull "rewrite-rules.conf" cfg.carbon.rewriteRules)
38 (writeTextOrNull "relay-rules.conf" cfg.carbon.relayRules)
39 (writeTextOrNull "aggregation-rules.conf" cfg.carbon.aggregationRules)
40 ];
41 };
42
43 carbonOpts = name: with config.ids; ''
44 --nodaemon --syslog --prefix=${name} --pidfile /run/${name}/${name}.pid ${name}
45 '';
46
47 mkPidFileDir = name: ''
48 mkdir -p /run/${name}
49 chmod 0700 /run/${name}
50 chown -R graphite:graphite /run/${name}
51 '';
52
53 carbonEnv = {
54 PYTHONPATH = let
55 cenv = pkgs.python.buildEnv.override {
56 extraLibs = [ pkgs.python27Packages.carbon ];
57 };
58 cenvPack = "${cenv}/${pkgs.python.sitePackages}";
59 # opt/graphite/lib contains twisted.plugins.carbon-cache
60 in "${cenvPack}/opt/graphite/lib:${cenvPack}";
61 GRAPHITE_ROOT = dataDir;
62 GRAPHITE_CONF_DIR = configDir;
63 GRAPHITE_STORAGE_DIR = dataDir;
64 };
65
66in {
67
68 ###### interface
69
70 options.services.graphite = {
71 dataDir = mkOption {
72 type = types.path;
73 default = "/var/db/graphite";
74 description = ''
75 Data directory for graphite.
76 '';
77 };
78
79 web = {
80 enable = mkOption {
81 description = "Whether to enable graphite web frontend.";
82 default = false;
83 type = types.bool;
84 };
85
86 listenAddress = mkOption {
87 description = "Graphite web frontend listen address.";
88 default = "127.0.0.1";
89 type = types.str;
90 };
91
92 port = mkOption {
93 description = "Graphite web frontend port.";
94 default = 8080;
95 type = types.int;
96 };
97 };
98
99 api = {
100 enable = mkOption {
101 description = ''
102 Whether to enable graphite api. Graphite api is lightweight alternative
103 to graphite web, with api and without dashboard. It's advised to use
104 grafana as alternative dashboard and influxdb as alternative to
105 graphite carbon.
106
107 For more information visit
108 <link xlink:href="http://graphite-api.readthedocs.org/en/latest/"/>
109 '';
110 default = false;
111 type = types.bool;
112 };
113
114 finders = mkOption {
115 description = "List of finder plugins to load.";
116 default = [];
117 example = literalExample "[ pkgs.python27Packages.graphite_influxdb ]";
118 type = types.listOf types.package;
119 };
120
121 functions = mkOption {
122 description = "List of functions to load.";
123 default = [
124 "graphite_api.functions.SeriesFunctions"
125 "graphite_api.functions.PieFunctions"
126 ];
127 type = types.listOf types.str;
128 };
129
130 listenAddress = mkOption {
131 description = "Graphite web service listen address.";
132 default = "127.0.0.1";
133 type = types.str;
134 };
135
136 port = mkOption {
137 description = "Graphite api service port.";
138 default = 8080;
139 type = types.int;
140 };
141
142 package = mkOption {
143 description = "Package to use for graphite api.";
144 default = pkgs.python27Packages.graphite_api;
145 defaultText = "pkgs.python27Packages.graphite_api";
146 type = types.package;
147 };
148
149 extraConfig = mkOption {
150 description = "Extra configuration for graphite api.";
151 default = ''
152 whisper:
153 directories:
154 - ${dataDir}/whisper
155 '';
156 example = ''
157 allowed_origins:
158 - dashboard.example.com
159 cheat_times: true
160 influxdb:
161 host: localhost
162 port: 8086
163 user: influxdb
164 pass: influxdb
165 db: metrics
166 cache:
167 CACHE_TYPE: 'filesystem'
168 CACHE_DIR: '/tmp/graphite-api-cache'
169 '';
170 type = types.str;
171 };
172 };
173
174 carbon = {
175 config = mkOption {
176 description = "Content of carbon configuration file.";
177 default = ''
178 [cache]
179 # Listen on localhost by default for security reasons
180 UDP_RECEIVER_INTERFACE = 127.0.0.1
181 PICKLE_RECEIVER_INTERFACE = 127.0.0.1
182 LINE_RECEIVER_INTERFACE = 127.0.0.1
183 CACHE_QUERY_INTERFACE = 127.0.0.1
184 # Do not log every update
185 LOG_UPDATES = False
186 LOG_CACHE_HITS = False
187 '';
188 type = types.str;
189 };
190
191 enableCache = mkOption {
192 description = "Whether to enable carbon cache, the graphite storage daemon.";
193 default = false;
194 type = types.bool;
195 };
196
197 storageAggregation = mkOption {
198 description = "Defines how to aggregate data to lower-precision retentions.";
199 default = null;
200 type = types.uniq (types.nullOr types.string);
201 example = ''
202 [all_min]
203 pattern = \.min$
204 xFilesFactor = 0.1
205 aggregationMethod = min
206 '';
207 };
208
209 storageSchemas = mkOption {
210 description = "Defines retention rates for storing metrics.";
211 default = "";
212 type = types.uniq (types.nullOr types.string);
213 example = ''
214 [apache_busyWorkers]
215 pattern = ^servers\.www.*\.workers\.busyWorkers$
216 retentions = 15s:7d,1m:21d,15m:5y
217 '';
218 };
219
220 blacklist = mkOption {
221 description = "Any metrics received which match one of the experssions will be dropped.";
222 default = null;
223 type = types.uniq (types.nullOr types.string);
224 example = "^some\.noisy\.metric\.prefix\..*";
225 };
226
227 whitelist = mkOption {
228 description = "Only metrics received which match one of the experssions will be persisted.";
229 default = null;
230 type = types.uniq (types.nullOr types.string);
231 example = ".*";
232 };
233
234 rewriteRules = mkOption {
235 description = ''
236 Regular expression patterns that can be used to rewrite metric names
237 in a search and replace fashion.
238 '';
239 default = null;
240 type = types.uniq (types.nullOr types.string);
241 example = ''
242 [post]
243 _sum$ =
244 _avg$ =
245 '';
246 };
247
248 enableRelay = mkOption {
249 description = "Whether to enable carbon relay, the carbon replication and sharding service.";
250 default = false;
251 type = types.bool;
252 };
253
254 relayRules = mkOption {
255 description = "Relay rules are used to send certain metrics to a certain backend.";
256 default = null;
257 type = types.uniq (types.nullOr types.string);
258 example = ''
259 [example]
260 pattern = ^mydata\.foo\..+
261 servers = 10.1.2.3, 10.1.2.4:2004, myserver.mydomain.com
262 '';
263 };
264
265 enableAggregator = mkOption {
266 description = "Whether to enable carbon aggregator, the carbon buffering service.";
267 default = false;
268 type = types.bool;
269 };
270
271 aggregationRules = mkOption {
272 description = "Defines if and how received metrics will be aggregated.";
273 default = null;
274 type = types.uniq (types.nullOr types.string);
275 example = ''
276 <env>.applications.<app>.all.requests (60) = sum <env>.applications.<app>.*.requests
277 <env>.applications.<app>.all.latency (60) = avg <env>.applications.<app>.*.latency
278 '';
279 };
280 };
281
282 seyren = {
283 enable = mkOption {
284 description = "Whether to enable seyren service.";
285 default = false;
286 type = types.bool;
287 };
288
289 port = mkOption {
290 description = "Seyren listening port.";
291 default = 8081;
292 type = types.int;
293 };
294
295 seyrenUrl = mkOption {
296 default = "http://localhost:${toString cfg.seyren.port}/";
297 description = "Host where seyren is accessible.";
298 type = types.str;
299 };
300
301 graphiteUrl = mkOption {
302 default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
303 description = "Host where graphite service runs.";
304 type = types.str;
305 };
306
307 mongoUrl = mkOption {
308 default = "mongodb://${config.services.mongodb.bind_ip}:27017/seyren";
309 description = "Mongodb connection string.";
310 type = types.str;
311 };
312
313 extraConfig = mkOption {
314 default = {};
315 description = ''
316 Extra seyren configuration. See
317 <link xlink:href='https://github.com/scobal/seyren#config' />
318 '';
319 type = types.attrsOf types.str;
320 example = literalExample ''
321 {
322 GRAPHITE_USERNAME = "user";
323 GRAPHITE_PASSWORD = "pass";
324 }
325 '';
326 };
327 };
328
329 pager = {
330 enable = mkOption {
331 description = ''
332 Whether to enable graphite-pager service. For more information visit
333 <link xlink:href="https://github.com/seatgeek/graphite-pager"/>
334 '';
335 default = false;
336 type = types.bool;
337 };
338
339 redisUrl = mkOption {
340 description = "Redis connection string.";
341 default = "redis://localhost:${toString config.services.redis.port}/";
342 type = types.str;
343 };
344
345 graphiteUrl = mkOption {
346 description = "URL to your graphite service.";
347 default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
348 type = types.str;
349 };
350
351 alerts = mkOption {
352 description = "Alerts configuration for graphite-pager.";
353 default = ''
354 alerts:
355 - target: constantLine(100)
356 warning: 90
357 critical: 200
358 name: Test
359 '';
360 example = ''
361 pushbullet_key: pushbullet_api_key
362 alerts:
363 - target: stats.seatgeek.app.deal_quality.venue_info_cache.hit
364 warning: .5
365 critical: 1
366 name: Deal quality venue cache hits
367 '';
368 type = types.lines;
369 };
370 };
371
372 beacon = {
373 enable = mkEnableOption "graphite beacon";
374
375 config = mkOption {
376 description = "Graphite beacon configuration.";
377 default = {};
378 type = types.attrs;
379 };
380 };
381 };
382
383 ###### implementation
384
385 config = mkMerge [
386 (mkIf cfg.carbon.enableCache {
387 systemd.services.carbonCache = let name = "carbon-cache"; in {
388 description = "Graphite Data Storage Backend";
389 wantedBy = [ "multi-user.target" ];
390 after = [ "network-interfaces.target" ];
391 environment = carbonEnv;
392 serviceConfig = {
393 ExecStart = "${pkgs.pythonPackages.twisted}/bin/twistd ${carbonOpts name}";
394 User = "graphite";
395 Group = "graphite";
396 PermissionsStartOnly = true;
397 PIDFile="/run/${name}/${name}.pid";
398 };
399 preStart = mkPidFileDir name + ''
400
401 mkdir -p ${cfg.dataDir}/whisper
402 chmod 0700 ${cfg.dataDir}/whisper
403 chown -R graphite:graphite ${cfg.dataDir}
404 '';
405 };
406 })
407
408 (mkIf cfg.carbon.enableAggregator {
409 systemd.services.carbonAggregator = let name = "carbon-aggregator"; in {
410 enable = cfg.carbon.enableAggregator;
411 description = "Carbon Data Aggregator";
412 wantedBy = [ "multi-user.target" ];
413 after = [ "network-interfaces.target" ];
414 environment = carbonEnv;
415 serviceConfig = {
416 ExecStart = "${pkgs.pythonPackages.twisted}/bin/twistd ${carbonOpts name}";
417 User = "graphite";
418 Group = "graphite";
419 PIDFile="/run/${name}/${name}.pid";
420 };
421 preStart = mkPidFileDir name;
422 };
423 })
424
425 (mkIf cfg.carbon.enableRelay {
426 systemd.services.carbonRelay = let name = "carbon-relay"; in {
427 description = "Carbon Data Relay";
428 wantedBy = [ "multi-user.target" ];
429 after = [ "network-interfaces.target" ];
430 environment = carbonEnv;
431 serviceConfig = {
432 ExecStart = "${pkgs.pythonPackages.twisted}/bin/twistd ${carbonOpts name}";
433 User = "graphite";
434 Group = "graphite";
435 PIDFile="/run/${name}/${name}.pid";
436 };
437 preStart = mkPidFileDir name;
438 };
439 })
440
441 (mkIf (cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay) {
442 environment.systemPackages = [
443 pkgs.pythonPackages.carbon
444 ];
445 })
446
447 (mkIf cfg.web.enable {
448 systemd.services.graphiteWeb = {
449 description = "Graphite Web Interface";
450 wantedBy = [ "multi-user.target" ];
451 after = [ "network-interfaces.target" ];
452 path = [ pkgs.perl ];
453 environment = {
454 PYTHONPATH = let
455 penv = pkgs.python.buildEnv.override {
456 extraLibs = [
457 pkgs.python27Packages.graphite_web
458 pkgs.python27Packages.pysqlite
459 ];
460 };
461 penvPack = "${penv}/${pkgs.python.sitePackages}";
462 # opt/graphite/webapp contains graphite/settings.py
463 # explicitly adding pycairo in path because it cannot be imported via buildEnv
464 in "${penvPack}/opt/graphite/webapp:${penvPack}:${pkgs.pythonPackages.pycairo}/${pkgs.python.sitePackages}";
465 DJANGO_SETTINGS_MODULE = "graphite.settings";
466 GRAPHITE_CONF_DIR = configDir;
467 GRAPHITE_STORAGE_DIR = dataDir;
468 LD_LIBRARY_PATH = "${pkgs.cairo.out}/lib";
469 };
470 serviceConfig = {
471 ExecStart = ''
472 ${pkgs.python27Packages.waitress}/bin/waitress-serve \
473 --host=${cfg.web.listenAddress} --port=${toString cfg.web.port} \
474 --call django.core.handlers.wsgi:WSGIHandler'';
475 User = "graphite";
476 Group = "graphite";
477 PermissionsStartOnly = true;
478 };
479 preStart = ''
480 if ! test -e ${dataDir}/db-created; then
481 mkdir -p ${dataDir}/{whisper/,log/webapp/}
482 chmod 0700 ${dataDir}/{whisper/,log/webapp/}
483
484 # populate database
485 ${pkgs.python27Packages.graphite_web}/bin/manage-graphite.py syncdb --noinput
486
487 # create index
488 ${pkgs.python27Packages.graphite_web}/bin/build-index.sh
489
490 touch ${dataDir}/db-created
491
492 chown -R graphite:graphite ${cfg.dataDir}
493 fi
494 '';
495 };
496
497 environment.systemPackages = [ pkgs.python27Packages.graphite_web ];
498 })
499
500 (mkIf cfg.api.enable {
501 systemd.services.graphiteApi = {
502 description = "Graphite Api Interface";
503 wantedBy = [ "multi-user.target" ];
504 after = [ "network-interfaces.target" ];
505 environment = {
506 PYTHONPATH = let
507 aenv = pkgs.python.buildEnv.override {
508 extraLibs = [ cfg.api.package pkgs.cairo ] ++ cfg.api.finders;
509 };
510 in "${aenv}/${pkgs.python.sitePackages}";
511 GRAPHITE_API_CONFIG = graphiteApiConfig;
512 LD_LIBRARY_PATH = "${pkgs.cairo.out}/lib";
513 };
514 serviceConfig = {
515 ExecStart = ''
516 ${pkgs.python27Packages.waitress}/bin/waitress-serve \
517 --host=${cfg.api.listenAddress} --port=${toString cfg.api.port} \
518 graphite_api.app:app
519 '';
520 User = "graphite";
521 Group = "graphite";
522 PermissionsStartOnly = true;
523 };
524 preStart = ''
525 if ! test -e ${dataDir}/db-created; then
526 mkdir -p ${dataDir}/cache/
527 chmod 0700 ${dataDir}/cache/
528
529 touch ${dataDir}/db-created
530
531 chown -R graphite:graphite ${cfg.dataDir}
532 fi
533 '';
534 };
535 })
536
537 (mkIf cfg.seyren.enable {
538 systemd.services.seyren = {
539 description = "Graphite Alerting Dashboard";
540 wantedBy = [ "multi-user.target" ];
541 after = [ "network-interfaces.target" "mongodb.service" ];
542 environment = seyrenConfig;
543 serviceConfig = {
544 ExecStart = "${pkgs.seyren}/bin/seyren -httpPort ${toString cfg.seyren.port}";
545 WorkingDirectory = dataDir;
546 User = "graphite";
547 Group = "graphite";
548 };
549 preStart = ''
550 if ! test -e ${dataDir}/db-created; then
551 mkdir -p ${dataDir}
552 chown -R graphite:graphite ${dataDir}
553 fi
554 '';
555 };
556
557 services.mongodb.enable = mkDefault true;
558 })
559
560 (mkIf cfg.pager.enable {
561 systemd.services.graphitePager = {
562 description = "Graphite Pager Alerting Daemon";
563 wantedBy = [ "multi-user.target" ];
564 after = [ "network-interfaces.target" "redis.service" ];
565 environment = {
566 REDIS_URL = cfg.pager.redisUrl;
567 GRAPHITE_URL = cfg.pager.graphiteUrl;
568 };
569 serviceConfig = {
570 ExecStart = "${pkgs.pythonPackages.graphite_pager}/bin/graphite-pager --config ${pagerConfig}";
571 User = "graphite";
572 Group = "graphite";
573 };
574 };
575
576 services.redis.enable = mkDefault true;
577
578 environment.systemPackages = [ pkgs.pythonPackages.graphite_pager ];
579 })
580
581 (mkIf cfg.beacon.enable {
582 systemd.services.graphite-beacon = {
583 description = "Grpahite Beacon Alerting Daemon";
584 wantedBy = [ "multi-user.target" ];
585 serviceConfig = {
586 ExecStart = ''
587 ${pkgs.pythonPackages.graphite_beacon}/bin/graphite-beacon \
588 --config ${pkgs.writeText "graphite-beacon.json" (builtins.toJSON cfg.beacon.config)}
589 '';
590 User = "graphite";
591 Group = "graphite";
592 };
593 };
594 })
595
596 (mkIf (
597 cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay ||
598 cfg.web.enable || cfg.api.enable ||
599 cfg.seyren.enable || cfg.pager.enable || cfg.beacon.enable
600 ) {
601 users.extraUsers = singleton {
602 name = "graphite";
603 uid = config.ids.uids.graphite;
604 description = "Graphite daemon user";
605 home = dataDir;
606 };
607 users.extraGroups.graphite.gid = config.ids.gids.graphite;
608 })
609 ];
610}