pyrox.dev
1# This test runs simple etcd cluster
2
3{ lib, pkgs, ... }:
4let
5 runWithOpenSSL =
6 file: cmd:
7 pkgs.runCommand file {
8 buildInputs = [ pkgs.openssl ];
9 } cmd;
10
11 ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
12 ca_pem = runWithOpenSSL "ca.pem" ''
13 openssl req \
14 -x509 -new -nodes -key ${ca_key} \
15 -days 10000 -out $out -subj "/CN=etcd-ca"
16 '';
17 etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
18 etcd_csr = runWithOpenSSL "etcd.csr" ''
19 openssl req \
20 -new -key ${etcd_key} \
21 -out $out -subj "/CN=etcd" \
22 -config ${openssl_cnf}
23 '';
24 etcd_cert = runWithOpenSSL "etcd.pem" ''
25 openssl x509 \
26 -req -in ${etcd_csr} \
27 -CA ${ca_pem} -CAkey ${ca_key} \
28 -CAcreateserial -out $out \
29 -days 365 -extensions v3_req \
30 -extfile ${openssl_cnf}
31 '';
32
33 etcd_client_key = runWithOpenSSL "etcd-client-key.pem" "openssl genrsa -out $out 2048";
34
35 etcd_client_csr = runWithOpenSSL "etcd-client-key.pem" ''
36 openssl req \
37 -new -key ${etcd_client_key} \
38 -out $out -subj "/CN=etcd-client" \
39 -config ${client_openssl_cnf}
40 '';
41
42 etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''
43 openssl x509 \
44 -req -in ${etcd_client_csr} \
45 -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
46 -out $out -days 365 -extensions v3_req \
47 -extfile ${client_openssl_cnf}
48 '';
49
50 openssl_cnf = pkgs.writeText "openssl.cnf" ''
51 ions = v3_req
52 distinguished_name = req_distinguished_name
53 [req_distinguished_name]
54 [ v3_req ]
55 basicConstraints = CA:FALSE
56 keyUsage = digitalSignature, keyEncipherment
57 extendedKeyUsage = serverAuth, clientAuth
58 subjectAltName = @alt_names
59 [alt_names]
60 DNS.1 = node1
61 DNS.2 = node2
62 DNS.3 = node3
63 IP.1 = 127.0.0.1
64 '';
65
66 client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''
67 ions = v3_req
68 distinguished_name = req_distinguished_name
69 [req_distinguished_name]
70 [ v3_req ]
71 basicConstraints = CA:FALSE
72 keyUsage = digitalSignature, keyEncipherment
73 extendedKeyUsage = clientAuth
74 '';
75
76 nodeConfig = {
77 services = {
78 etcd = {
79 enable = true;
80 keyFile = etcd_key;
81 certFile = etcd_cert;
82 trustedCaFile = ca_pem;
83 clientCertAuth = true;
84 listenClientUrls = [ "https://127.0.0.1:2379" ];
85 listenPeerUrls = [ "https://0.0.0.0:2380" ];
86 };
87 };
88
89 environment.variables = {
90 ETCD_CERT_FILE = "${etcd_client_cert}";
91 ETCD_KEY_FILE = "${etcd_client_key}";
92 ETCD_CA_FILE = "${ca_pem}";
93 ETCDCTL_ENDPOINTS = "https://127.0.0.1:2379";
94 ETCDCTL_CACERT = "${ca_pem}";
95 ETCDCTL_CERT = "${etcd_cert}";
96 ETCDCTL_KEY = "${etcd_key}";
97 };
98
99 networking.firewall.allowedTCPPorts = [ 2380 ];
100 };
101in
102{
103 name = "etcd-cluster";
104
105 meta.maintainers = with lib.maintainers; [ offline ];
106
107 nodes = {
108 node1 =
109 { ... }:
110 {
111 require = [ nodeConfig ];
112 services.etcd = {
113 initialCluster = [
114 "node1=https://node1:2380"
115 "node2=https://node2:2380"
116 ];
117 initialAdvertisePeerUrls = [ "https://node1:2380" ];
118 };
119 };
120
121 node2 =
122 { ... }:
123 {
124 require = [ nodeConfig ];
125 services.etcd = {
126 initialCluster = [
127 "node1=https://node1:2380"
128 "node2=https://node2:2380"
129 ];
130 initialAdvertisePeerUrls = [ "https://node2:2380" ];
131 };
132 };
133
134 node3 =
135 { ... }:
136 {
137 require = [ nodeConfig ];
138 services.etcd = {
139 initialCluster = [
140 "node1=https://node1:2380"
141 "node2=https://node2:2380"
142 "node3=https://node3:2380"
143 ];
144 initialAdvertisePeerUrls = [ "https://node3:2380" ];
145 initialClusterState = "existing";
146 };
147 };
148 };
149
150 testScript = ''
151 with subtest("should start etcd cluster"):
152 node1.start()
153 node2.start()
154 node1.wait_for_unit("etcd.service")
155 node2.wait_for_unit("etcd.service")
156 node2.wait_until_succeeds("etcdctl endpoint status")
157 node1.succeed("etcdctl put /foo/bar 'Hello world'")
158 node2.succeed("etcdctl get /foo/bar | grep 'Hello world'")
159
160 with subtest("should add another member"):
161 node1.wait_until_succeeds("etcdctl member add node3 --peer-urls=https://node3:2380")
162 node3.start()
163 node3.wait_for_unit("etcd.service")
164 node3.wait_until_succeeds("etcdctl member list | grep 'node3'")
165 node3.succeed("etcdctl endpoint status")
166
167 with subtest("should survive member crash"):
168 node3.crash()
169 node1.succeed("etcdctl endpoint status")
170 node1.succeed("etcdctl put /foo/bar 'Hello degraded world'")
171 node1.succeed("etcdctl get /foo/bar | grep 'Hello degraded world'")
172 '';
173}