pyrox.dev
1{ system ? builtins.currentSystem,
2 config ? {},
3 pkgs ? import ../.. { inherit system config; },
4 enableUnfree ? false
5 # To run the test on the unfree ELK use the folllowing command:
6 # NIXPKGS_ALLOW_UNFREE=1 nix-build nixos/tests/elk.nix -A ELK-6 --arg enableUnfree true
7}:
8
9let
10 esUrl = "http://localhost:9200";
11
12 mkElkTest = name : elk :
13 import ./make-test-python.nix ({
14 inherit name;
15 meta = with pkgs.lib.maintainers; {
16 maintainers = [ eelco offline basvandijk ];
17 };
18 nodes = {
19 one =
20 { pkgs, lib, ... }: {
21 # Not giving the machine at least 2060MB results in elasticsearch failing with the following error:
22 #
23 # OpenJDK 64-Bit Server VM warning:
24 # INFO: os::commit_memory(0x0000000085330000, 2060255232, 0)
25 # failed; error='Cannot allocate memory' (errno=12)
26 #
27 # There is insufficient memory for the Java Runtime Environment to continue.
28 # Native memory allocation (mmap) failed to map 2060255232 bytes for committing reserved memory.
29 #
30 # When setting this to 2500 I got "Kernel panic - not syncing: Out of
31 # memory: compulsory panic_on_oom is enabled" so lets give it even a
32 # bit more room:
33 virtualisation.memorySize = 3000;
34
35 # For querying JSON objects returned from elasticsearch and kibana.
36 environment.systemPackages = [ pkgs.jq ];
37
38 services = {
39
40 journalbeat = let lt6 = builtins.compareVersions
41 elk.journalbeat.version "6" < 0; in {
42 enable = true;
43 package = elk.journalbeat;
44 extraConfig = pkgs.lib.mkOptionDefault (''
45 logging:
46 to_syslog: true
47 level: warning
48 metrics.enabled: false
49 output.elasticsearch:
50 hosts: [ "127.0.0.1:9200" ]
51 ${pkgs.lib.optionalString lt6 "template.enabled: false"}
52 '' + pkgs.lib.optionalString (!lt6) ''
53 journalbeat.inputs:
54 - paths: []
55 seek: cursor
56 '');
57 };
58
59 logstash = {
60 enable = true;
61 package = elk.logstash;
62 inputConfig = ''
63 exec { command => "echo -n flowers" interval => 1 type => "test" }
64 exec { command => "echo -n dragons" interval => 1 type => "test" }
65 '';
66 filterConfig = ''
67 if [message] =~ /dragons/ {
68 drop {}
69 }
70 '';
71 outputConfig = ''
72 file {
73 path => "/tmp/logstash.out"
74 codec => line { format => "%{message}" }
75 }
76 elasticsearch {
77 hosts => [ "${esUrl}" ]
78 }
79 '';
80 };
81
82 elasticsearch = {
83 enable = true;
84 package = elk.elasticsearch;
85 };
86
87 kibana = {
88 enable = true;
89 package = elk.kibana;
90 };
91
92 elasticsearch-curator = {
93 enable = true;
94 actionYAML = ''
95 ---
96 actions:
97 1:
98 action: delete_indices
99 description: >-
100 Delete indices older than 1 second (based on index name), for logstash-
101 prefixed indices. Ignore the error if the filter does not result in an
102 actionable list of indices (ignore_empty_list) and exit cleanly.
103 options:
104 allow_ilm_indices: true
105 ignore_empty_list: True
106 disable_action: False
107 filters:
108 - filtertype: pattern
109 kind: prefix
110 value: logstash-
111 - filtertype: age
112 source: name
113 direction: older
114 timestring: '%Y.%m.%d'
115 unit: seconds
116 unit_count: 1
117 '';
118 };
119 };
120 };
121 };
122
123 passthru.elkPackages = elk;
124 testScript = ''
125 import json
126
127
128 def total_hits(message):
129 dictionary = {"query": {"match": {"message": message}}}
130 return (
131 "curl --silent --show-error '${esUrl}/_search' "
132 + "-H 'Content-Type: application/json' "
133 + "-d '{}' ".format(json.dumps(dictionary))
134 + "| jq .hits.total"
135 )
136
137
138 start_all()
139
140 one.wait_for_unit("elasticsearch.service")
141 one.wait_for_open_port(9200)
142
143 # Continue as long as the status is not "red". The status is probably
144 # "yellow" instead of "green" because we are using a single elasticsearch
145 # node which elasticsearch considers risky.
146 #
147 # TODO: extend this test with multiple elasticsearch nodes
148 # and see if the status turns "green".
149 one.wait_until_succeeds(
150 "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"
151 )
152
153 with subtest("Perform some simple logstash tests"):
154 one.wait_for_unit("logstash.service")
155 one.wait_until_succeeds("cat /tmp/logstash.out | grep flowers")
156 one.wait_until_succeeds("cat /tmp/logstash.out | grep -v dragons")
157
158 with subtest("Kibana is healthy"):
159 one.wait_for_unit("kibana.service")
160 one.wait_until_succeeds(
161 "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
162 )
163
164 with subtest("Logstash messages arive in elasticsearch"):
165 one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
166 one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
167
168 with subtest(
169 "A message logged to the journal is ingested by elasticsearch via journalbeat"
170 ):
171 one.wait_for_unit("journalbeat.service")
172 one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat")
173 one.wait_until_succeeds(
174 total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
175 )
176
177 with subtest("Elasticsearch-curator works"):
178 one.systemctl("stop logstash")
179 one.systemctl("start elasticsearch-curator")
180 one.wait_until_succeeds(
181 '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep -q ^'
182 )
183 '';
184 }) {};
185in pkgs.lib.mapAttrs mkElkTest {
186 ELK-6 =
187 if enableUnfree
188 then {
189 elasticsearch = pkgs.elasticsearch6;
190 logstash = pkgs.logstash6;
191 kibana = pkgs.kibana6;
192 journalbeat = pkgs.journalbeat6;
193 }
194 else {
195 elasticsearch = pkgs.elasticsearch6-oss;
196 logstash = pkgs.logstash6-oss;
197 kibana = pkgs.kibana6-oss;
198 journalbeat = pkgs.journalbeat6;
199 };
200 ELK-7 =
201 if enableUnfree
202 then {
203 elasticsearch = pkgs.elasticsearch7;
204 logstash = pkgs.logstash7;
205 kibana = pkgs.kibana7;
206 journalbeat = pkgs.journalbeat7;
207 }
208 else {
209 elasticsearch = pkgs.elasticsearch7-oss;
210 logstash = pkgs.logstash7-oss;
211 kibana = pkgs.kibana7-oss;
212 journalbeat = pkgs.journalbeat7;
213 };
214}