pyrox.dev
1import ./make-test-python.nix ({ pkgs, ...} :
2let
3 mkNode = { replicationMode, publicV6Address ? "::1" }: { pkgs, ... }: {
4 networking.interfaces.eth1.ipv6.addresses = [{
5 address = publicV6Address;
6 prefixLength = 64;
7 }];
8
9 networking.firewall.allowedTCPPorts = [ 3901 3902 ];
10
11 services.garage = {
12 enable = true;
13 settings = {
14 replication_mode = replicationMode;
15
16 rpc_bind_addr = "[::]:3901";
17 rpc_public_addr = "[${publicV6Address}]:3901";
18 rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c";
19
20 s3_api = {
21 s3_region = "garage";
22 api_bind_addr = "[::]:3900";
23 root_domain = ".s3.garage";
24 };
25
26 s3_web = {
27 bind_addr = "[::]:3902";
28 root_domain = ".web.garage";
29 index = "index.html";
30 };
31 };
32 };
33 environment.systemPackages = [ pkgs.minio-client ];
34
35 # Garage requires at least 1GiB of free disk space to run.
36 virtualisation.diskSize = 2 * 1024;
37 };
38
39
40in {
41 name = "garage";
42 meta = {
43 maintainers = with pkgs.lib.maintainers; [ raitobezarius ];
44 };
45
46 nodes = {
47 single_node = mkNode { replicationMode = "none"; };
48 node1 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::1"; };
49 node2 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::2"; };
50 node3 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::3"; };
51 node4 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::4"; };
52 };
53
54 testScript = ''
55 from typing import List
56 from dataclasses import dataclass
57 import re
58 start_all()
59
60 cur_version_regex = re.compile('Current cluster layout version: (?P<ver>\d*)')
61 key_creation_regex = re.compile('Key name: (?P<key_name>.*)\nKey ID: (?P<key_id>.*)\nSecret key: (?P<secret_key>.*)')
62
63 @dataclass
64 class S3Key:
65 key_name: str
66 key_id: str
67 secret_key: str
68
69 @dataclass
70 class GarageNode:
71 node_id: str
72 host: str
73
74 def get_node_fqn(machine: Machine) -> GarageNode:
75 node_id, host = machine.succeed("garage node id").split('@')
76 return GarageNode(node_id=node_id, host=host)
77
78 def get_node_id(machine: Machine) -> str:
79 return get_node_fqn(machine).node_id
80
81 def get_layout_version(machine: Machine) -> int:
82 version_data = machine.succeed("garage layout show")
83 m = cur_version_regex.search(version_data)
84 if m and m.group('ver') is not None:
85 return int(m.group('ver')) + 1
86 else:
87 raise ValueError('Cannot find current layout version')
88
89 def apply_garage_layout(machine: Machine, layouts: List[str]):
90 for layout in layouts:
91 machine.succeed(f"garage layout assign {layout}")
92 version = get_layout_version(machine)
93 machine.succeed(f"garage layout apply --version {version}")
94
95 def create_api_key(machine: Machine, key_name: str) -> S3Key:
96 output = machine.succeed(f"garage key new --name {key_name}")
97 m = key_creation_regex.match(output)
98 if not m or not m.group('key_id') or not m.group('secret_key'):
99 raise ValueError('Cannot parse API key data')
100 return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key'))
101
102 def get_api_key(machine: Machine, key_pattern: str) -> S3Key:
103 output = machine.succeed(f"garage key info {key_pattern}")
104 m = key_creation_regex.match(output)
105 if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'):
106 raise ValueError('Cannot parse API key data')
107 return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key'))
108
109 def test_bucket_writes(node):
110 node.succeed("garage bucket create test-bucket")
111 s3_key = create_api_key(node, "test-api-key")
112 node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")
113 other_s3_key = get_api_key(node, 'test-api-key')
114 assert other_s3_key.secret_key == other_s3_key.secret_key
115 node.succeed(
116 f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"
117 )
118 node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt")
119 assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"
120
121 def test_bucket_over_http(node, bucket='test-bucket', url=None):
122 if url is None:
123 url = f"{bucket}.web.garage"
124
125 node.succeed(f'garage bucket website --allow {bucket}')
126 node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html')
127 assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'
128
129 with subtest("Garage works as a single-node S3 storage"):
130 single_node.wait_for_unit("garage.service")
131 single_node.wait_for_open_port(3900)
132 # Now Garage is initialized.
133 single_node_id = get_node_id(single_node)
134 apply_garage_layout(single_node, [f'-z qemutest -c 1 "{single_node_id}"'])
135 # Now Garage is operational.
136 test_bucket_writes(single_node)
137 test_bucket_over_http(single_node)
138
139 with subtest("Garage works as a multi-node S3 storage"):
140 nodes = ('node1', 'node2', 'node3', 'node4')
141 rev_machines = {m.name: m for m in machines}
142 def get_machine(key): return rev_machines[key]
143 for key in nodes:
144 node = get_machine(key)
145 node.wait_for_unit("garage.service")
146 node.wait_for_open_port(3900)
147
148 # Garage is initialized on all nodes.
149 node_ids = {key: get_node_fqn(get_machine(key)) for key in nodes}
150
151 for key in nodes:
152 for other_key in nodes:
153 if other_key != key:
154 other_id = node_ids[other_key]
155 get_machine(key).succeed(f"garage node connect {other_id.node_id}@{other_id.host}")
156
157 # Provide multiple zones for the nodes.
158 zones = ["nixcon", "nixcon", "paris_meetup", "fosdem"]
159 apply_garage_layout(node1,
160 [
161 f'{ndata.node_id} -z {zones[index]} -c 1'
162 for index, ndata in enumerate(node_ids.values())
163 ])
164 # Now Garage is operational.
165 test_bucket_writes(node1)
166 for node in nodes:
167 test_bucket_over_http(get_machine(node))
168 '';
169})