···
client2.succeed("[ $(consul kv get testkey) == 42 ]")
148
-
def rolling_reboot_test():
148
+
def rolling_reboot_test(proper_rolling_procedure=True):
Tests that the cluster can tolearate failures of any single server,
following the recommended rolling upgrade procedure from
152
-
https://www.consul.io/docs/upgrading#standard-upgrades
152
+
https://www.consul.io/docs/upgrading#standard-upgrades.
154
+
Optionally, `proper_rolling_procedure=False` can be given
155
+
to wait only for each server to be back `Healthy`, not `Stable`
156
+
in the Raft consensus, see Consul setting `ServerStabilizationTime` and
157
+
https://github.com/hashicorp/consul/issues/8118#issuecomment-645330040.
···
# Restart crashed machine.
171
-
# Wait for recovery.
172
-
wait_for_healthy_servers()
176
+
if proper_rolling_procedure:
177
+
# Wait for recovery.
178
+
wait_for_healthy_servers()
180
+
# NOT proper rolling upgrade procedure, see above.
181
+
wait_for_all_machines_alive()
# Wait for client connections.
client1.wait_until_succeeds("consul kv get -recurse")
···
client2.succeed("consul kv delete testkey")
193
+
def all_servers_crash_simultaneously_test():
195
+
Tests that the cluster will eventually come back after all
196
+
servers crash simultaneously.
199
+
for server in servers:
202
+
for server in servers:
205
+
# Wait for recovery.
206
+
wait_for_healthy_servers()
208
+
# Wait for client connections.
209
+
client1.wait_until_succeeds("consul kv get -recurse")
210
+
client2.wait_until_succeeds("consul kv get -recurse")
212
+
# Do some consul actions with servers back up.
213
+
client1.succeed("consul kv put testkey 44")
214
+
client2.succeed("[ $(consul kv get testkey) == 44 ]")
215
+
client2.succeed("consul kv delete testkey")
220
+
print("rolling_reboot_test()")
223
+
print("all_servers_crash_simultaneously_test()")
224
+
all_servers_crash_simultaneously_test()
226
+
print("rolling_reboot_test(proper_rolling_procedure=False)")
227
+
rolling_reboot_test(proper_rolling_procedure=False)
pyrox.dev