pyrox.dev
1import ../make-test-python.nix (
2 { lib, pkgs, ... }:
3
4 {
5 name = "vector-nginx-clickhouse";
6 meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
7
8 nodes = {
9 clickhouse =
10 { config, pkgs, ... }:
11 {
12 virtualisation.memorySize = 4096;
13
14 # Clickhouse module can't listen on a non-loopback IP.
15 networking.firewall.allowedTCPPorts = [ 6000 ];
16 services.clickhouse.enable = true;
17
18 # Exercise Vector sink->source for now.
19 services.vector = {
20 enable = true;
21
22 settings = {
23 sources = {
24 vector_source = {
25 type = "vector";
26 address = "[::]:6000";
27 };
28 };
29
30 sinks = {
31 clickhouse = {
32 type = "clickhouse";
33 inputs = [ "vector_source" ];
34 endpoint = "http://localhost:8123";
35 database = "nginxdb";
36 table = "access_logs";
37 skip_unknown_fields = true;
38 };
39 };
40 };
41 };
42 };
43
44 nginx =
45 { config, pkgs, ... }:
46 {
47 services.nginx = {
48 enable = true;
49 virtualHosts.localhost = { };
50 };
51
52 services.vector = {
53 enable = true;
54
55 settings = {
56 sources = {
57 nginx_logs = {
58 type = "file";
59 include = [ "/var/log/nginx/access.log" ];
60 read_from = "end";
61 };
62 };
63
64 sinks = {
65 vector_sink = {
66 type = "vector";
67 inputs = [ "nginx_logs" ];
68 address = "clickhouse:6000";
69 };
70 };
71 };
72 };
73
74 systemd.services.vector.serviceConfig = {
75 SupplementaryGroups = [ "nginx" ];
76 };
77 };
78 };
79
80 testScript =
81 let
82 # work around quote/substitution complexity by Nix, Perl, bash and SQL.
83 databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS nginxdb";
84
85 tableDDL = pkgs.writeText "table.sql" ''
86 CREATE TABLE IF NOT EXISTS nginxdb.access_logs (
87 message String
88 )
89 ENGINE = MergeTree()
90 ORDER BY tuple()
91 '';
92
93 # Graciously taken from https://clickhouse.com/docs/en/integrations/vector
94 tableView = pkgs.writeText "table-view.sql" ''
95 CREATE MATERIALIZED VIEW nginxdb.access_logs_view
96 (
97 RemoteAddr String,
98 Client String,
99 RemoteUser String,
100 TimeLocal DateTime,
101 RequestMethod String,
102 Request String,
103 HttpVersion String,
104 Status Int32,
105 BytesSent Int64,
106 UserAgent String
107 )
108 ENGINE = MergeTree()
109 ORDER BY RemoteAddr
110 POPULATE AS
111 WITH
112 splitByWhitespace(message) as split,
113 splitByRegexp('\S \d+ "([^"]*)"', message) as referer
114 SELECT
115 split[1] AS RemoteAddr,
116 split[2] AS Client,
117 split[3] AS RemoteUser,
118 parseDateTimeBestEffort(replaceOne(trim(LEADING '[' FROM split[4]), ':', ' ')) AS TimeLocal,
119 trim(LEADING '"' FROM split[6]) AS RequestMethod,
120 split[7] AS Request,
121 trim(TRAILING '"' FROM split[8]) AS HttpVersion,
122 split[9] AS Status,
123 split[10] AS BytesSent,
124 trim(BOTH '"' from referer[2]) AS UserAgent
125 FROM
126 (SELECT message FROM nginxdb.access_logs)
127 '';
128
129 selectQuery = pkgs.writeText "select.sql" "SELECT * from nginxdb.access_logs_view";
130 in
131 ''
132 clickhouse.wait_for_unit("clickhouse")
133 clickhouse.wait_for_open_port(8123)
134
135 clickhouse.wait_until_succeeds(
136 "journalctl -o cat -u clickhouse.service | grep 'Started ClickHouse server'"
137 )
138
139 clickhouse.wait_for_unit("vector")
140 clickhouse.wait_for_open_port(6000)
141
142 clickhouse.succeed(
143 "cat ${databaseDDL} | clickhouse-client"
144 )
145
146 clickhouse.succeed(
147 "cat ${tableDDL} | clickhouse-client"
148 )
149
150 clickhouse.succeed(
151 "cat ${tableView} | clickhouse-client"
152 )
153
154 nginx.wait_for_unit("nginx")
155 nginx.wait_for_open_port(80)
156 nginx.wait_for_unit("vector")
157 nginx.wait_until_succeeds(
158 "journalctl -o cat -u vector.service | grep 'Starting file server'"
159 )
160
161 nginx.succeed("curl http://localhost/")
162 nginx.succeed("curl http://localhost/")
163
164 nginx.wait_for_file("/var/log/nginx/access.log")
165 nginx.wait_until_succeeds(
166 "journalctl -o cat -u vector.service | grep 'Found new file to watch. file=/var/log/nginx/access.log'"
167 )
168
169 clickhouse.wait_until_succeeds(
170 "cat ${selectQuery} | clickhouse-client | grep 'curl'"
171 )
172 '';
173 }
174)