pyrox.dev
1{
2 lib,
3 mkNode,
4 package,
5 testScriptSetup,
6 ...
7}:
8let
9 extraSettings =
10 if (lib.versionAtLeast package.version "2") then
11 {
12 replication_factor = 3;
13 consistency_mode = "consistent";
14 }
15 else
16 {
17 replication_mode = "3";
18 };
19in
20{
21 name = "garage-3node-replication";
22
23 nodes = {
24 node1 = mkNode {
25 inherit extraSettings;
26 publicV6Address = "fc00:1::1";
27 };
28 node2 = mkNode {
29 inherit extraSettings;
30 publicV6Address = "fc00:1::2";
31 };
32 node3 = mkNode {
33 inherit extraSettings;
34 publicV6Address = "fc00:1::3";
35 };
36 node4 = mkNode {
37 inherit extraSettings;
38 publicV6Address = "fc00:1::4";
39 };
40 };
41
42 testScript = # python
43 ''
44 ${testScriptSetup}
45
46 with subtest("Garage works as a multi-node S3 storage"):
47 nodes = ('node1', 'node2', 'node3', 'node4')
48 rev_machines = {m.name: m for m in machines}
49 def get_machine(key): return rev_machines[key]
50 for key in nodes:
51 node = get_machine(key)
52 node.wait_for_unit("garage.service")
53 node.wait_for_open_port(3900)
54
55 # Garage is initialized on all nodes.
56 node_ids = {key: get_node_fqn(get_machine(key)) for key in nodes}
57
58 for key in nodes:
59 for other_key in nodes:
60 if other_key != key:
61 other_id = node_ids[other_key]
62 get_machine(key).succeed(f"garage node connect {other_id.node_id}@{other_id.host}")
63
64 # Provide multiple zones for the nodes.
65 zones = ["nixcon", "nixcon", "paris_meetup", "fosdem"]
66 apply_garage_layout(node1,
67 [
68 f'{ndata.node_id} -z {zones[index]} -c 1G'
69 for index, ndata in enumerate(node_ids.values())
70 ])
71 # Now Garage is operational.
72 test_bucket_writes(node1)
73 for node in nodes:
74 test_bucket_over_http(get_machine(node))
75 '';
76}