···

       
82
       
 
       
    castopod.succeed("curl -s http://localhost/cp-install | grep 'Create your Super Admin account' > /dev/null")

     

       
83
       
 
       


     

       
84
       
 
       
    with subtest("Create superadmin and log in"):

     

       
85
       
-
       
        castopod.succeed("PYTHONUNBUFFERED=1 test-runner | systemd-cat -t test-runner")

     

       
86
       
 
       
  '';

     

       
87
       
 
       
})

     

···

       
82
       
 
       
    castopod.succeed("curl -s http://localhost/cp-install | grep 'Create your Super Admin account' > /dev/null")

     

       
83
       
 
       


     

       
84
       
 
       
    with subtest("Create superadmin and log in"):

     

       
85
       
+
       
        castopod.succeed("PYTHONUNBUFFERED=1 systemd-cat -t test-runner test-runner")

     

       
86
       
 
       
  '';

     

       
87
       
 
       
})

     

···

       
176
       
 
       
    nn2.succeed("systemctl stop hdfs-zkfc")

     

       
177
       
 
       


     

       
178
       
 
       
    # Initialize zookeeper for failover controller

     

       
179
       
-
       
    nn1.succeed("sudo -u hdfs hdfs zkfc -formatZK 2>&1 | systemd-cat")

     

       
180
       
 
       


     

       
181
       
 
       
    # Format NN1 and start it

     

       
182
       
-
       
    nn1.succeed("sudo -u hdfs hadoop namenode -format 2>&1 | systemd-cat")

     

       
183
       
 
       
    nn1.succeed("systemctl start hdfs-namenode")

     

       
184
       
 
       
    nn1.wait_for_open_port(9870)

     

       
185
       
 
       
    nn1.wait_for_open_port(8022)

     

       
186
       
 
       
    nn1.wait_for_open_port(8020)

     

       
187
       
 
       


     

       
188
       
 
       
    # Bootstrap NN2 from NN1 and start it

     

       
189
       
-
       
    nn2.succeed("sudo -u hdfs hdfs namenode -bootstrapStandby 2>&1 | systemd-cat")

     

       
190
       
 
       
    nn2.succeed("systemctl start hdfs-namenode")

     

       
191
       
 
       
    nn2.wait_for_open_port(9870)

     

       
192
       
 
       
    nn2.wait_for_open_port(8022)

     

       
193
       
 
       
    nn2.wait_for_open_port(8020)

     

       
194
       
-
       
    nn1.succeed("netstat -tulpne | systemd-cat")

     

       
195
       
 
       


     

       
196
       
 
       
    # Start failover controllers

     

       
197
       
 
       
    nn1.succeed("systemctl start hdfs-zkfc")

     
···

       
200
       
 
       
    # DN should have started by now, but confirm anyway

     

       
201
       
 
       
    dn1.wait_for_unit("hdfs-datanode")

     

       
202
       
 
       
    # Print states of namenodes

     

       
203
       
-
       
    client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")

     

       
204
       
 
       
    # Wait for cluster to exit safemode

     

       
205
       
 
       
    client.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")

     

       
206
       
-
       
    client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")

     

       
207
       
 
       
    # test R/W

     

       
208
       
 
       
    client.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")

     

       
209
       
 
       
    assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")

     
···

       
211
       
 
       
    # Test NN failover

     

       
212
       
 
       
    nn1.succeed("systemctl stop hdfs-namenode")

     

       
213
       
 
       
    assert "active" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")

     

       
214
       
-
       
    client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")

     

       
215
       
 
       
    assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")

     

       
216
       
 
       


     

       
217
       
 
       
    nn1.succeed("systemctl start hdfs-namenode")

     
···

       
219
       
 
       
    nn1.wait_for_open_port(8022)

     

       
220
       
 
       
    nn1.wait_for_open_port(8020)

     

       
221
       
 
       
    assert "standby" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")

     

       
222
       
-
       
    client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")

     

       
223
       
 
       


     

       
224
       
 
       
    #### YARN tests ####

     

       
225
       
 
       


     
···

       
236
       
 
       
    nm1.wait_for_open_port(8042)

     

       
237
       
 
       
    nm1.wait_for_open_port(8040)

     

       
238
       
 
       
    client.wait_until_succeeds("yarn node -list | grep Nodes:1")

     

       
239
       
-
       
    client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")

     

       
240
       
-
       
    client.succeed("sudo -u yarn yarn node -list | systemd-cat")

     

       
241
       
 
       


     

       
242
       
 
       
    # Test RM failover

     

       
243
       
 
       
    rm1.succeed("systemctl stop yarn-resourcemanager")

     

       
244
       
 
       
    assert "standby" not in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")

     

       
245
       
-
       
    client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")

     

       
246
       
 
       
    rm1.succeed("systemctl start yarn-resourcemanager")

     

       
247
       
 
       
    rm1.wait_for_unit("yarn-resourcemanager")

     

       
248
       
 
       
    rm1.wait_for_open_port(8088)

     

       
249
       
 
       
    assert "standby" in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")

     

       
250
       
-
       
    client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")

     

       
251
       
 
       


     

       
252
       
 
       
    assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")

     

       
253
       
 
       
    assert "SUCCEEDED" in client.succeed("yarn application -list -appStates FINISHED")

     

···

       
176
       
 
       
    nn2.succeed("systemctl stop hdfs-zkfc")

     

       
177
       
 
       


     

       
178
       
 
       
    # Initialize zookeeper for failover controller

     

       
179
       
+
       
    nn1.succeed("sudo -u hdfs systemd-cat hdfs zkfc -formatZK")

     

       
180
       
 
       


     

       
181
       
 
       
    # Format NN1 and start it

     

       
182
       
+
       
    nn1.succeed("sudo -u hdfs systemd-cat hadoop namenode -format")

     

       
183
       
 
       
    nn1.succeed("systemctl start hdfs-namenode")

     

       
184
       
 
       
    nn1.wait_for_open_port(9870)

     

       
185
       
 
       
    nn1.wait_for_open_port(8022)

     

       
186
       
 
       
    nn1.wait_for_open_port(8020)

     

       
187
       
 
       


     

       
188
       
 
       
    # Bootstrap NN2 from NN1 and start it

     

       
189
       
+
       
    nn2.succeed("sudo -u hdfs systemd-cat hdfs namenode -bootstrapStandby")

     

       
190
       
 
       
    nn2.succeed("systemctl start hdfs-namenode")

     

       
191
       
 
       
    nn2.wait_for_open_port(9870)

     

       
192
       
 
       
    nn2.wait_for_open_port(8022)

     

       
193
       
 
       
    nn2.wait_for_open_port(8020)

     

       
194
       
+
       
    nn1.succeed("systemd-cat netstat -tulpne")

     

       
195
       
 
       


     

       
196
       
 
       
    # Start failover controllers

     

       
197
       
 
       
    nn1.succeed("systemctl start hdfs-zkfc")

     
···

       
200
       
 
       
    # DN should have started by now, but confirm anyway

     

       
201
       
 
       
    dn1.wait_for_unit("hdfs-datanode")

     

       
202
       
 
       
    # Print states of namenodes

     

       
203
       
+
       
    client.succeed("sudo -u hdfs systemd-cat hdfs haadmin -getAllServiceState")

     

       
204
       
 
       
    # Wait for cluster to exit safemode

     

       
205
       
 
       
    client.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")

     

       
206
       
+
       
    client.succeed("sudo -u hdfs systemd-cat hdfs haadmin -getAllServiceState")

     

       
207
       
 
       
    # test R/W

     

       
208
       
 
       
    client.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")

     

       
209
       
 
       
    assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")

     
···

       
211
       
 
       
    # Test NN failover

     

       
212
       
 
       
    nn1.succeed("systemctl stop hdfs-namenode")

     

       
213
       
 
       
    assert "active" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")

     

       
214
       
+
       
    client.succeed("sudo -u hdfs systemd-cat hdfs haadmin -getAllServiceState")

     

       
215
       
 
       
    assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")

     

       
216
       
 
       


     

       
217
       
 
       
    nn1.succeed("systemctl start hdfs-namenode")

     
···

       
219
       
 
       
    nn1.wait_for_open_port(8022)

     

       
220
       
 
       
    nn1.wait_for_open_port(8020)

     

       
221
       
 
       
    assert "standby" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")

     

       
222
       
+
       
    client.succeed("sudo -u hdfs systemd-cat hdfs haadmin -getAllServiceState")

     

       
223
       
 
       


     

       
224
       
 
       
    #### YARN tests ####

     

       
225
       
 
       


     
···

       
236
       
 
       
    nm1.wait_for_open_port(8042)

     

       
237
       
 
       
    nm1.wait_for_open_port(8040)

     

       
238
       
 
       
    client.wait_until_succeeds("yarn node -list | grep Nodes:1")

     

       
239
       
+
       
    client.succeed("sudo -u yarn systemd-cat yarn rmadmin -getAllServiceState")

     

       
240
       
+
       
    client.succeed("sudo -u yarn systemd-cat yarn node -list")

     

       
241
       
 
       


     

       
242
       
 
       
    # Test RM failover

     

       
243
       
 
       
    rm1.succeed("systemctl stop yarn-resourcemanager")

     

       
244
       
 
       
    assert "standby" not in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")

     

       
245
       
+
       
    client.succeed("sudo -u yarn systemd-cat yarn rmadmin -getAllServiceState")

     

       
246
       
 
       
    rm1.succeed("systemctl start yarn-resourcemanager")

     

       
247
       
 
       
    rm1.wait_for_unit("yarn-resourcemanager")

     

       
248
       
 
       
    rm1.wait_for_open_port(8088)

     

       
249
       
 
       
    assert "standby" in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")

     

       
250
       
+
       
    client.succeed("sudo -u yarn systemd-cat yarn rmadmin -getAllServiceState")

     

       
251
       
 
       


     

       
252
       
 
       
    assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")

     

       
253
       
 
       
    assert "SUCCEEDED" in client.succeed("yarn application -list -appStates FINISHED")

     

···

       
50
       
 
       
    namenode.wait_for_unit("hdfs-namenode")

     

       
51
       
 
       
    namenode.wait_for_unit("network.target")

     

       
52
       
 
       
    namenode.wait_for_open_port(8020)

     

       
53
       
-
       
    namenode.succeed("ss -tulpne | systemd-cat")

     

       
54
       
-
       
    namenode.succeed("cat /etc/hadoop*/hdfs-site.xml | systemd-cat")

     

       
55
       
 
       
    namenode.wait_for_open_port(9870)

     

       
56
       
 
       


     

       
57
       
 
       
    datanode.wait_for_unit("hdfs-datanode")

     

···

       
50
       
 
       
    namenode.wait_for_unit("hdfs-namenode")

     

       
51
       
 
       
    namenode.wait_for_unit("network.target")

     

       
52
       
 
       
    namenode.wait_for_open_port(8020)

     

       
53
       
+
       
    namenode.succeed("systemd-cat ss -tulpne")

     

       
54
       
+
       
    namenode.succeed("systemd-cat cat /etc/hadoop*/hdfs-site.xml")

     

       
55
       
 
       
    namenode.wait_for_open_port(9870)

     

       
56
       
 
       


     

       
57
       
 
       
    datanode.wait_for_unit("hdfs-datanode")

     

···

       
202
       
 
       
      initiatorAuto.succeed("umount /mnt")

     

       
203
       
 
       


     

       
204
       
 
       
      initiatorAuto.succeed("systemctl restart multipathd")

     

       
205
       
-
       
      initiatorAuto.succeed("multipath -ll | systemd-cat")

     

       
206
       
 
       


     

       
207
       
 
       
      # Install our RootDisk machine to 123456, the alias to the device that multipath is now managing

     

       
208
       
 
       
      initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")

     
···

       
223
       
 
       
      initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")

     

       
224
       
 
       
      initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")

     

       
225
       
 
       
      initiatorRootDisk.succeed("systemctl restart multipathd")

     

       
226
       
-
       
      initiatorRootDisk.succeed("multipath -ll | systemd-cat")

     

       
227
       
 
       


     

       
228
       
 
       
      # Verify we can write and sync the root disk

     

       
229
       
 
       
      initiatorRootDisk.succeed("mkdir /scratch")

     

···

       
202
       
 
       
      initiatorAuto.succeed("umount /mnt")

     

       
203
       
 
       


     

       
204
       
 
       
      initiatorAuto.succeed("systemctl restart multipathd")

     

       
205
       
+
       
      initiatorAuto.succeed("systemd-cat multipath -ll")

     

       
206
       
 
       


     

       
207
       
 
       
      # Install our RootDisk machine to 123456, the alias to the device that multipath is now managing

     

       
208
       
 
       
      initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")

     
···

       
223
       
 
       
      initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")

     

       
224
       
 
       
      initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")

     

       
225
       
 
       
      initiatorRootDisk.succeed("systemctl restart multipathd")

     

       
226
       
+
       
      initiatorRootDisk.succeed("systemd-cat multipath -ll")

     

       
227
       
 
       


     

       
228
       
 
       
      # Verify we can write and sync the root disk

     

       
229
       
 
       
      initiatorRootDisk.succeed("mkdir /scratch")

     

···

       
174
       
 
       
          )

     

       
175
       
 
       


     

       
176
       
 
       
      with subtest("use the web interface to sign up, log in, and save a password"):

     

       
177
       
-
       
          server.succeed("PYTHONUNBUFFERED=1 test-runner | systemd-cat -t test-runner")

     

       
178
       
 
       


     

       
179
       
 
       
      with subtest("log in with the cli"):

     

       
180
       
 
       
          key = client.succeed(

     

···

       
174
       
 
       
          )

     

       
175
       
 
       


     

       
176
       
 
       
      with subtest("use the web interface to sign up, log in, and save a password"):

     

       
177
       
+
       
          server.succeed("PYTHONUNBUFFERED=1 systemd-cat -t test-runner test-runner")

     

       
178
       
 
       


     

       
179
       
 
       
      with subtest("log in with the cli"):

     

       
180
       
 
       
          key = client.succeed(