pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.graphite;
7 writeTextOrNull = f: t: if t == null then null else pkgs.writeTextDir f t;
8
9 dataDir = cfg.dataDir;
10
11 graphiteApiConfig = pkgs.writeText "graphite-api.yaml" ''
12 time_zone: ${config.time.timeZone}
13 search_index: ${dataDir}/index
14 ${optionalString (cfg.api.finders != []) ''finders:''}
15 ${concatMapStringsSep "\n" (f: " - " + f.moduleName) cfg.api.finders}
16 ${optionalString (cfg.api.functions != []) ''functions:''}
17 ${concatMapStringsSep "\n" (f: " - " + f) cfg.api.functions}
18 ${cfg.api.extraConfig}
19 '';
20
21 seyrenConfig = {
22 SEYREN_URL = cfg.seyren.seyrenUrl;
23 MONGO_URL = cfg.seyren.mongoUrl;
24 GRAPHITE_URL = cfg.seyren.graphiteUrl;
25 } // cfg.seyren.extraConfig;
26
27 pagerConfig = pkgs.writeText "alarms.yaml" cfg.pager.alerts;
28
29 configDir = pkgs.buildEnv {
30 name = "graphite-config";
31 paths = lists.filter (el: el != null) [
32 (writeTextOrNull "carbon.conf" cfg.carbon.config)
33 (writeTextOrNull "storage-aggregation.conf" cfg.carbon.storageAggregation)
34 (writeTextOrNull "storage-schemas.conf" cfg.carbon.storageSchemas)
35 (writeTextOrNull "blacklist.conf" cfg.carbon.blacklist)
36 (writeTextOrNull "whitelist.conf" cfg.carbon.whitelist)
37 (writeTextOrNull "rewrite-rules.conf" cfg.carbon.rewriteRules)
38 (writeTextOrNull "relay-rules.conf" cfg.carbon.relayRules)
39 (writeTextOrNull "aggregation-rules.conf" cfg.carbon.aggregationRules)
40 ];
41 };
42
43 carbonOpts = name: with config.ids; ''
44 --nodaemon --syslog --prefix=${name} --pidfile /run/${name}/${name}.pid ${name}
45 '';
46
47 mkPidFileDir = name: ''
48 mkdir -p /run/${name}
49 chmod 0700 /run/${name}
50 chown -R graphite:graphite /run/${name}
51 '';
52
53 carbonEnv = {
54 PYTHONPATH = "${pkgs.python27Packages.carbon}/lib/python2.7/site-packages";
55 GRAPHITE_ROOT = dataDir;
56 GRAPHITE_CONF_DIR = configDir;
57 GRAPHITE_STORAGE_DIR = dataDir;
58 };
59
60in {
61
62 ###### interface
63
64 options.services.graphite = {
65 dataDir = mkOption {
66 type = types.path;
67 default = "/var/db/graphite";
68 description = ''
69 Data directory for graphite.
70 '';
71 };
72
73 web = {
74 enable = mkOption {
75 description = "Whether to enable graphite web frontend.";
76 default = false;
77 type = types.bool;
78 };
79
80 listenAddress = mkOption {
81 description = "Graphite web frontend listen address.";
82 default = "127.0.0.1";
83 type = types.str;
84 };
85
86 port = mkOption {
87 description = "Graphite web frontend port.";
88 default = 8080;
89 type = types.int;
90 };
91 };
92
93 api = {
94 enable = mkOption {
95 description = ''
96 Whether to enable graphite api. Graphite api is lightweight alternative
97 to graphite web, with api and without dashboard. It's advised to use
98 grafana as alternative dashboard and influxdb as alternative to
99 graphite carbon.
100
101 For more information visit
102 <link xlink:href="http://graphite-api.readthedocs.org/en/latest/"/>
103 '';
104 default = false;
105 type = types.bool;
106 };
107
108 finders = mkOption {
109 description = "List of finder plugins to load.";
110 default = [];
111 example = literalExample "[ pkgs.python27Packages.graphite_influxdb ]";
112 type = types.listOf types.package;
113 };
114
115 functions = mkOption {
116 description = "List of functions to load.";
117 default = [
118 "graphite_api.functions.SeriesFunctions"
119 "graphite_api.functions.PieFunctions"
120 ];
121 type = types.listOf types.str;
122 };
123
124 listenAddress = mkOption {
125 description = "Graphite web service listen address.";
126 default = "127.0.0.1";
127 type = types.str;
128 };
129
130 port = mkOption {
131 description = "Graphite api service port.";
132 default = 8080;
133 type = types.int;
134 };
135
136 package = mkOption {
137 description = "Package to use for graphite api.";
138 default = pkgs.python27Packages.graphite_api;
139 defaultText = "pkgs.python27Packages.graphite_api";
140 type = types.package;
141 };
142
143 extraConfig = mkOption {
144 description = "Extra configuration for graphite api.";
145 default = ''
146 whisper:
147 directories:
148 - ${dataDir}/whisper
149 '';
150 example = ''
151 allowed_origins:
152 - dashboard.example.com
153 cheat_times: true
154 influxdb:
155 host: localhost
156 port: 8086
157 user: influxdb
158 pass: influxdb
159 db: metrics
160 cache:
161 CACHE_TYPE: 'filesystem'
162 CACHE_DIR: '/tmp/graphite-api-cache'
163 '';
164 type = types.str;
165 };
166 };
167
168 carbon = {
169 config = mkOption {
170 description = "Content of carbon configuration file.";
171 default = ''
172 [cache]
173 # Listen on localhost by default for security reasons
174 UDP_RECEIVER_INTERFACE = 127.0.0.1
175 PICKLE_RECEIVER_INTERFACE = 127.0.0.1
176 LINE_RECEIVER_INTERFACE = 127.0.0.1
177 CACHE_QUERY_INTERFACE = 127.0.0.1
178 # Do not log every update
179 LOG_UPDATES = False
180 LOG_CACHE_HITS = False
181 '';
182 type = types.str;
183 };
184
185 enableCache = mkOption {
186 description = "Whether to enable carbon cache, the graphite storage daemon.";
187 default = false;
188 type = types.bool;
189 };
190
191 storageAggregation = mkOption {
192 description = "Defines how to aggregate data to lower-precision retentions.";
193 default = null;
194 type = types.uniq (types.nullOr types.string);
195 example = ''
196 [all_min]
197 pattern = \.min$
198 xFilesFactor = 0.1
199 aggregationMethod = min
200 '';
201 };
202
203 storageSchemas = mkOption {
204 description = "Defines retention rates for storing metrics.";
205 default = "";
206 type = types.uniq (types.nullOr types.string);
207 example = ''
208 [apache_busyWorkers]
209 pattern = ^servers\.www.*\.workers\.busyWorkers$
210 retentions = 15s:7d,1m:21d,15m:5y
211 '';
212 };
213
214 blacklist = mkOption {
215 description = "Any metrics received which match one of the experssions will be dropped.";
216 default = null;
217 type = types.uniq (types.nullOr types.string);
218 example = "^some\.noisy\.metric\.prefix\..*";
219 };
220
221 whitelist = mkOption {
222 description = "Only metrics received which match one of the experssions will be persisted.";
223 default = null;
224 type = types.uniq (types.nullOr types.string);
225 example = ".*";
226 };
227
228 rewriteRules = mkOption {
229 description = ''
230 Regular expression patterns that can be used to rewrite metric names
231 in a search and replace fashion.
232 '';
233 default = null;
234 type = types.uniq (types.nullOr types.string);
235 example = ''
236 [post]
237 _sum$ =
238 _avg$ =
239 '';
240 };
241
242 enableRelay = mkOption {
243 description = "Whether to enable carbon relay, the carbon replication and sharding service.";
244 default = false;
245 type = types.bool;
246 };
247
248 relayRules = mkOption {
249 description = "Relay rules are used to send certain metrics to a certain backend.";
250 default = null;
251 type = types.uniq (types.nullOr types.string);
252 example = ''
253 [example]
254 pattern = ^mydata\.foo\..+
255 servers = 10.1.2.3, 10.1.2.4:2004, myserver.mydomain.com
256 '';
257 };
258
259 enableAggregator = mkOption {
260 description = "Whether to enable carbon aggregator, the carbon buffering service.";
261 default = false;
262 type = types.bool;
263 };
264
265 aggregationRules = mkOption {
266 description = "Defines if and how received metrics will be aggregated.";
267 default = null;
268 type = types.uniq (types.nullOr types.string);
269 example = ''
270 <env>.applications.<app>.all.requests (60) = sum <env>.applications.<app>.*.requests
271 <env>.applications.<app>.all.latency (60) = avg <env>.applications.<app>.*.latency
272 '';
273 };
274 };
275
276 seyren = {
277 enable = mkOption {
278 description = "Whether to enable seyren service.";
279 default = false;
280 type = types.bool;
281 };
282
283 port = mkOption {
284 description = "Seyren listening port.";
285 default = 8081;
286 type = types.int;
287 };
288
289 seyrenUrl = mkOption {
290 default = "http://localhost:${toString cfg.seyren.port}/";
291 description = "Host where seyren is accessible.";
292 type = types.str;
293 };
294
295 graphiteUrl = mkOption {
296 default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
297 description = "Host where graphite service runs.";
298 type = types.str;
299 };
300
301 mongoUrl = mkOption {
302 default = "mongodb://${config.services.mongodb.bind_ip}:27017/seyren";
303 description = "Mongodb connection string.";
304 type = types.str;
305 };
306
307 extraConfig = mkOption {
308 default = {};
309 description = ''
310 Extra seyren configuration. See
311 <link xlink:href='https://github.com/scobal/seyren#config' />
312 '';
313 type = types.attrsOf types.str;
314 example = literalExample ''
315 {
316 GRAPHITE_USERNAME = "user";
317 GRAPHITE_PASSWORD = "pass";
318 }
319 '';
320 };
321 };
322
323 pager = {
324 enable = mkOption {
325 description = ''
326 Whether to enable graphite-pager service. For more information visit
327 <link xlink:href="https://github.com/seatgeek/graphite-pager"/>
328 '';
329 default = false;
330 type = types.bool;
331 };
332
333 redisUrl = mkOption {
334 description = "Redis connection string.";
335 default = "redis://localhost:${toString config.services.redis.port}/";
336 type = types.str;
337 };
338
339 graphiteUrl = mkOption {
340 description = "URL to your graphite service.";
341 default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
342 type = types.str;
343 };
344
345 alerts = mkOption {
346 description = "Alerts configuration for graphite-pager.";
347 default = ''
348 alerts:
349 - target: constantLine(100)
350 warning: 90
351 critical: 200
352 name: Test
353 '';
354 example = ''
355 pushbullet_key: pushbullet_api_key
356 alerts:
357 - target: stats.seatgeek.app.deal_quality.venue_info_cache.hit
358 warning: .5
359 critical: 1
360 name: Deal quality venue cache hits
361 '';
362 type = types.lines;
363 };
364 };
365
366 beacon = {
367 enable = mkEnableOption "graphite beacon";
368
369 config = mkOption {
370 description = "Graphite beacon configuration.";
371 default = {};
372 type = types.attrs;
373 };
374 };
375 };
376
377 ###### implementation
378
379 config = mkMerge [
380 (mkIf cfg.carbon.enableCache {
381 systemd.services.carbonCache = let name = "carbon-cache"; in {
382 description = "Graphite Data Storage Backend";
383 wantedBy = [ "multi-user.target" ];
384 after = [ "network-interfaces.target" ];
385 environment = carbonEnv;
386 serviceConfig = {
387 ExecStart = "${pkgs.twisted}/bin/twistd ${carbonOpts name}";
388 User = "graphite";
389 Group = "graphite";
390 PermissionsStartOnly = true;
391 PIDFile="/run/${name}/${name}.pid";
392 };
393 preStart = mkPidFileDir name + ''
394
395 mkdir -p ${cfg.dataDir}/whisper
396 chmod 0700 ${cfg.dataDir}/whisper
397 chown -R graphite:graphite ${cfg.dataDir}
398 '';
399 };
400 })
401
402 (mkIf cfg.carbon.enableAggregator {
403 systemd.services.carbonAggregator = let name = "carbon-aggregator"; in {
404 enable = cfg.carbon.enableAggregator;
405 description = "Carbon Data Aggregator";
406 wantedBy = [ "multi-user.target" ];
407 after = [ "network-interfaces.target" ];
408 environment = carbonEnv;
409 serviceConfig = {
410 ExecStart = "${pkgs.twisted}/bin/twistd ${carbonOpts name}";
411 User = "graphite";
412 Group = "graphite";
413 PIDFile="/run/${name}/${name}.pid";
414 };
415 preStart = mkPidFileDir name;
416 };
417 })
418
419 (mkIf cfg.carbon.enableRelay {
420 systemd.services.carbonRelay = let name = "carbon-relay"; in {
421 description = "Carbon Data Relay";
422 wantedBy = [ "multi-user.target" ];
423 after = [ "network-interfaces.target" ];
424 environment = carbonEnv;
425 serviceConfig = {
426 ExecStart = "${pkgs.twisted}/bin/twistd ${carbonOpts name}";
427 User = "graphite";
428 Group = "graphite";
429 PIDFile="/run/${name}/${name}.pid";
430 };
431 preStart = mkPidFileDir name;
432 };
433 })
434
435 (mkIf (cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay) {
436 environment.systemPackages = [
437 pkgs.pythonPackages.carbon
438 ];
439 })
440
441 (mkIf cfg.web.enable {
442 systemd.services.graphiteWeb = {
443 description = "Graphite Web Interface";
444 wantedBy = [ "multi-user.target" ];
445 after = [ "network-interfaces.target" ];
446 path = [ pkgs.perl ];
447 environment = {
448 PYTHONPATH = "${pkgs.python27Packages.graphite_web}/lib/python2.7/site-packages";
449 DJANGO_SETTINGS_MODULE = "graphite.settings";
450 GRAPHITE_CONF_DIR = configDir;
451 GRAPHITE_STORAGE_DIR = dataDir;
452 };
453 serviceConfig = {
454 ExecStart = ''
455 ${pkgs.python27Packages.waitress}/bin/waitress-serve \
456 --host=${cfg.web.listenAddress} --port=${toString cfg.web.port} \
457 --call django.core.handlers.wsgi:WSGIHandler'';
458 User = "graphite";
459 Group = "graphite";
460 PermissionsStartOnly = true;
461 };
462 preStart = ''
463 if ! test -e ${dataDir}/db-created; then
464 mkdir -p ${dataDir}/{whisper/,log/webapp/}
465 chmod 0700 ${dataDir}/{whisper/,log/webapp/}
466
467 # populate database
468 ${pkgs.python27Packages.graphite_web}/bin/manage-graphite.py syncdb --noinput
469
470 # create index
471 ${pkgs.python27Packages.graphite_web}/bin/build-index.sh
472
473 touch ${dataDir}/db-created
474
475 chown -R graphite:graphite ${cfg.dataDir}
476 fi
477 '';
478 };
479
480 environment.systemPackages = [ pkgs.python27Packages.graphite_web ];
481 })
482
483 (mkIf cfg.api.enable {
484 systemd.services.graphiteApi = {
485 description = "Graphite Api Interface";
486 wantedBy = [ "multi-user.target" ];
487 after = [ "network-interfaces.target" ];
488 environment = {
489 PYTHONPATH =
490 "${cfg.api.package}/lib/python2.7/site-packages:" +
491 concatMapStringsSep ":" (f: f + "/lib/python2.7/site-packages") cfg.api.finders;
492 GRAPHITE_API_CONFIG = graphiteApiConfig;
493 LD_LIBRARY_PATH = "${pkgs.cairo}/lib";
494 };
495 serviceConfig = {
496 ExecStart = ''
497 ${pkgs.python27Packages.waitress}/bin/waitress-serve \
498 --host=${cfg.api.listenAddress} --port=${toString cfg.api.port} \
499 graphite_api.app:app
500 '';
501 User = "graphite";
502 Group = "graphite";
503 PermissionsStartOnly = true;
504 };
505 preStart = ''
506 if ! test -e ${dataDir}/db-created; then
507 mkdir -p ${dataDir}/cache/
508 chmod 0700 ${dataDir}/cache/
509
510 touch ${dataDir}/db-created
511
512 chown -R graphite:graphite ${cfg.dataDir}
513 fi
514 '';
515 };
516 })
517
518 (mkIf cfg.seyren.enable {
519 systemd.services.seyren = {
520 description = "Graphite Alerting Dashboard";
521 wantedBy = [ "multi-user.target" ];
522 after = [ "network-interfaces.target" "mongodb.service" ];
523 environment = seyrenConfig;
524 serviceConfig = {
525 ExecStart = "${pkgs.seyren}/bin/seyren -httpPort ${toString cfg.seyren.port}";
526 WorkingDirectory = dataDir;
527 User = "graphite";
528 Group = "graphite";
529 };
530 preStart = ''
531 if ! test -e ${dataDir}/db-created; then
532 mkdir -p ${dataDir}
533 chown -R graphite:graphite ${dataDir}
534 fi
535 '';
536 };
537
538 services.mongodb.enable = mkDefault true;
539 })
540
541 (mkIf cfg.pager.enable {
542 systemd.services.graphitePager = {
543 description = "Graphite Pager Alerting Daemon";
544 wantedBy = [ "multi-user.target" ];
545 after = [ "network-interfaces.target" "redis.service" ];
546 environment = {
547 REDIS_URL = cfg.pager.redisUrl;
548 GRAPHITE_URL = cfg.pager.graphiteUrl;
549 };
550 serviceConfig = {
551 ExecStart = "${pkgs.pythonPackages.graphite_pager}/bin/graphite-pager --config ${pagerConfig}";
552 User = "graphite";
553 Group = "graphite";
554 };
555 };
556
557 services.redis.enable = mkDefault true;
558
559 environment.systemPackages = [ pkgs.pythonPackages.graphite_pager ];
560 })
561
562 (mkIf cfg.beacon.enable {
563 systemd.services.graphite-beacon = {
564 description = "Grpahite Beacon Alerting Daemon";
565 wantedBy = [ "multi-user.target" ];
566 serviceConfig = {
567 ExecStart = ''
568 ${pkgs.pythonPackages.graphite_beacon}/bin/graphite-beacon \
569 --config ${pkgs.writeText "graphite-beacon.json" (builtins.toJSON cfg.beacon.config)}
570 '';
571 User = "graphite";
572 Group = "graphite";
573 };
574 };
575 })
576
577 (mkIf (
578 cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay ||
579 cfg.web.enable || cfg.api.enable ||
580 cfg.seyren.enable || cfg.pager.enable || cfg.beacon.enable
581 ) {
582 users.extraUsers = singleton {
583 name = "graphite";
584 uid = config.ids.uids.graphite;
585 description = "Graphite daemon user";
586 home = dataDir;
587 };
588 users.extraGroups.graphite.gid = config.ids.gids.graphite;
589 })
590 ];
591}