pyrox.dev
1import ../make-test-python.nix ({...}: {
2 name = "spark";
3
4 nodes = {
5 worker = { nodes, pkgs, ... }: {
6 services.spark.worker = {
7 enable = true;
8 master = "master:7077";
9 };
10 virtualisation.memorySize = 2048;
11 };
12 master = { config, pkgs, ... }: {
13 services.spark.master = {
14 enable = true;
15 bind = "0.0.0.0";
16 };
17 networking.firewall.allowedTCPPorts = [ 22 7077 8080 ];
18 };
19 };
20
21 testScript = ''
22 master.wait_for_unit("spark-master.service")
23 worker.wait_for_unit("spark-worker.service")
24 worker.copy_from_host( "${./spark_sample.py}", "/spark_sample.py" )
25 assert "<title>Spark Master at spark://" in worker.succeed("curl -sSfkL http://master:8080/")
26 worker.succeed("spark-submit --master spark://master:7077 --executor-memory 512m --executor-cores 1 /spark_sample.py")
27 '';
28})