pyrox.dev
1{ system ? builtins.currentSystem, enableUnfree ? false }:
2with import ../lib/testing.nix { inherit system; };
3with pkgs.lib;
4let
5 esUrl = "http://localhost:9200";
6
7 mkElkTest = name : elk : makeTest {
8 inherit name;
9 meta = with pkgs.stdenv.lib.maintainers; {
10 maintainers = [ eelco chaoflow offline basvandijk ];
11 };
12 nodes = {
13 one =
14 { pkgs, ... }: {
15 # Not giving the machine at least 2060MB results in elasticsearch failing with the following error:
16 #
17 # OpenJDK 64-Bit Server VM warning:
18 # INFO: os::commit_memory(0x0000000085330000, 2060255232, 0)
19 # failed; error='Cannot allocate memory' (errno=12)
20 #
21 # There is insufficient memory for the Java Runtime Environment to continue.
22 # Native memory allocation (mmap) failed to map 2060255232 bytes for committing reserved memory.
23 #
24 # When setting this to 2500 I got "Kernel panic - not syncing: Out of
25 # memory: compulsory panic_on_oom is enabled" so lets give it even a
26 # bit more room:
27 virtualisation.memorySize = 3000;
28
29 # For querying JSON objects returned from elasticsearch and kibana.
30 environment.systemPackages = [ pkgs.jq ];
31
32 services = {
33 logstash = {
34 enable = true;
35 package = elk.logstash;
36 inputConfig = ''
37 exec { command => "echo -n flowers" interval => 1 type => "test" }
38 exec { command => "echo -n dragons" interval => 1 type => "test" }
39 '';
40 filterConfig = ''
41 if [message] =~ /dragons/ {
42 drop {}
43 }
44 '';
45 outputConfig = ''
46 file {
47 path => "/tmp/logstash.out"
48 codec => line { format => "%{message}" }
49 }
50 elasticsearch {
51 hosts => [ "${esUrl}" ]
52 }
53 '';
54 };
55
56 elasticsearch = {
57 enable = true;
58 package = elk.elasticsearch;
59 };
60
61 kibana = {
62 enable = true;
63 package = elk.kibana;
64 elasticsearch.url = esUrl;
65 };
66
67 elasticsearch-curator = {
68 enable = true;
69 actionYAML = ''
70 ---
71 actions:
72 1:
73 action: delete_indices
74 description: >-
75 Delete indices older than 1 second (based on index name), for logstash-
76 prefixed indices. Ignore the error if the filter does not result in an
77 actionable list of indices (ignore_empty_list) and exit cleanly.
78 options:
79 ignore_empty_list: True
80 disable_action: False
81 filters:
82 - filtertype: pattern
83 kind: prefix
84 value: logstash-
85 - filtertype: age
86 source: name
87 direction: older
88 timestring: '%Y.%m.%d'
89 unit: seconds
90 unit_count: 1
91 '';
92 };
93 };
94 };
95 };
96
97 testScript = ''
98 startAll;
99
100 $one->waitForUnit("elasticsearch.service");
101
102 # Continue as long as the status is not "red". The status is probably
103 # "yellow" instead of "green" because we are using a single elasticsearch
104 # node which elasticsearch considers risky.
105 #
106 # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green".
107 $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red");
108
109 # Perform some simple logstash tests.
110 $one->waitForUnit("logstash.service");
111 $one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers");
112 $one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons");
113
114 # See if kibana is healthy.
115 $one->waitForUnit("kibana.service");
116 $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green");
117
118 # See if logstash messages arive in elasticsearch.
119 $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
120 $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
121
122 # Test elasticsearch-curator.
123 $one->systemctl("stop logstash");
124 $one->systemctl("start elasticsearch-curator");
125 $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1");
126 '';
127 };
128in mapAttrs mkElkTest {
129 "ELK-5" = {
130 elasticsearch = pkgs.elasticsearch5;
131 logstash = pkgs.logstash5;
132 kibana = pkgs.kibana5;
133 };
134 "ELK-6" =
135 if enableUnfree
136 then {
137 elasticsearch = pkgs.elasticsearch6;
138 logstash = pkgs.logstash6;
139 kibana = pkgs.kibana6;
140 }
141 else {
142 elasticsearch = pkgs.elasticsearch6-oss;
143 logstash = pkgs.logstash6-oss;
144 kibana = pkgs.kibana6-oss;
145 };
146}