pyrox.dev
1args@{ mkNode, ... }:
2(import ../make-test-python.nix ({ pkgs, ...} :
3{
4 name = "garage-3node-replication";
5 meta = {
6 maintainers = with pkgs.lib.maintainers; [ raitobezarius ];
7 };
8
9 nodes = {
10 node1 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::1"; };
11 node2 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::2"; };
12 node3 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::3"; };
13 node4 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::4"; };
14 };
15
16 testScript = ''
17 from typing import List
18 from dataclasses import dataclass
19 import re
20 start_all()
21
22 cur_version_regex = re.compile('Current cluster layout version: (?P<ver>\d*)')
23 key_creation_regex = re.compile('Key name: (?P<key_name>.*)\nKey ID: (?P<key_id>.*)\nSecret key: (?P<secret_key>.*)')
24
25 @dataclass
26 class S3Key:
27 key_name: str
28 key_id: str
29 secret_key: str
30
31 @dataclass
32 class GarageNode:
33 node_id: str
34 host: str
35
36 def get_node_fqn(machine: Machine) -> GarageNode:
37 node_id, host = machine.succeed("garage node id").split('@')
38 return GarageNode(node_id=node_id, host=host)
39
40 def get_node_id(machine: Machine) -> str:
41 return get_node_fqn(machine).node_id
42
43 def get_layout_version(machine: Machine) -> int:
44 version_data = machine.succeed("garage layout show")
45 m = cur_version_regex.search(version_data)
46 if m and m.group('ver') is not None:
47 return int(m.group('ver')) + 1
48 else:
49 raise ValueError('Cannot find current layout version')
50
51 def apply_garage_layout(machine: Machine, layouts: List[str]):
52 for layout in layouts:
53 machine.succeed(f"garage layout assign {layout}")
54 version = get_layout_version(machine)
55 machine.succeed(f"garage layout apply --version {version}")
56
57 def create_api_key(machine: Machine, key_name: str) -> S3Key:
58 output = machine.succeed(f"garage key new --name {key_name}")
59 m = key_creation_regex.match(output)
60 if not m or not m.group('key_id') or not m.group('secret_key'):
61 raise ValueError('Cannot parse API key data')
62 return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key'))
63
64 def get_api_key(machine: Machine, key_pattern: str) -> S3Key:
65 output = machine.succeed(f"garage key info {key_pattern}")
66 m = key_creation_regex.match(output)
67 if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'):
68 raise ValueError('Cannot parse API key data')
69 return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key'))
70
71 def test_bucket_writes(node):
72 node.succeed("garage bucket create test-bucket")
73 s3_key = create_api_key(node, "test-api-key")
74 node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")
75 other_s3_key = get_api_key(node, 'test-api-key')
76 assert other_s3_key.secret_key == other_s3_key.secret_key
77 node.succeed(
78 f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"
79 )
80 node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt")
81 assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"
82
83 def test_bucket_over_http(node, bucket='test-bucket', url=None):
84 if url is None:
85 url = f"{bucket}.web.garage"
86
87 node.succeed(f'garage bucket website --allow {bucket}')
88 node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html')
89 assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'
90
91 with subtest("Garage works as a multi-node S3 storage"):
92 nodes = ('node1', 'node2', 'node3', 'node4')
93 rev_machines = {m.name: m for m in machines}
94 def get_machine(key): return rev_machines[key]
95 for key in nodes:
96 node = get_machine(key)
97 node.wait_for_unit("garage.service")
98 node.wait_for_open_port(3900)
99
100 # Garage is initialized on all nodes.
101 node_ids = {key: get_node_fqn(get_machine(key)) for key in nodes}
102
103 for key in nodes:
104 for other_key in nodes:
105 if other_key != key:
106 other_id = node_ids[other_key]
107 get_machine(key).succeed(f"garage node connect {other_id.node_id}@{other_id.host}")
108
109 # Provide multiple zones for the nodes.
110 zones = ["nixcon", "nixcon", "paris_meetup", "fosdem"]
111 apply_garage_layout(node1,
112 [
113 f'{ndata.node_id} -z {zones[index]} -c 1'
114 for index, ndata in enumerate(node_ids.values())
115 ])
116 # Now Garage is operational.
117 test_bucket_writes(node1)
118 for node in nodes:
119 test_bucket_over_http(get_machine(node))
120 '';
121})) args