nix/vm: store service data in a shared folder on the host #466

···

       
15
       
 
       
.env

     

       
16
       
 
       
*.rdb

     

       
17
       
 
       
.envrc

     

       
0
       
 
       

     

···

       
15
       
 
       
.env

     

       
16
       
 
       
*.rdb

     

       
17
       
 
       
.envrc

     

       
18
       
+
       
/nix/vm-data

     

···

       
60
       
 
       
ideally in a `.envrc` with [direnv](https://direnv.net) so you

     

       
61
       
 
       
don't lose it.

     

       
62
       
 
       


     

       
63
       
-
       
You can now start a lightweight NixOS VM using

     

       
64
       
-
       
`nixos-shell` like so:

     

       
65
       
 
       


     

       
66
       
 
       
```bash

     

       
67
       
-
       
nix run .#vm

     

       
68
       
-
       
# or nixos-shell --flake .#vm

     

       
69
       
 
       


     

       
70
       
-
       
# hit Ctrl-a + c + q to exit the VM

     

       
71
       
 
       
```

     

       
72
       
 
       


     

       
73
       
 
       
This starts a knot on port 6000, a spindle on port 6555

     

···

       
60
       
 
       
ideally in a `.envrc` with [direnv](https://direnv.net) so you

     

       
61
       
 
       
don't lose it.

     

       
62
       
 
       


     

       
63
       
+
       
You can now start a lightweight NixOS VM like so:

     

       
0
       
 
       

     

       
64
       
 
       


     

       
65
       
 
       
```bash

     

       
66
       
+
       
nix run --impure .#vm

     

       
0
       
 
       

     

       
67
       
 
       


     

       
68
       
+
       
# type `poweroff` at the shell to exit the VM

     

       
69
       
 
       
```

     

       
70
       
 
       


     

       
71
       
 
       
This starts a knot on port 6000, a spindle on port 6555

     

···

       
175
       
 
       
        program = ''${tailwind-watcher}/bin/run'';

     

       
176
       
 
       
      };

     

       
177
       
 
       
      vm = let

     

       
178
       
-
       
        system =

     

       
179
       
 
       
          if pkgs.stdenv.hostPlatform.isAarch64

     

       
180
       
-
       
          then "aarch64"

     

       
181
       
-
       
          else "x86_64";

     

       
182
       
-
       


     

       
183
       
-
       
        nixos-shell = pkgs.nixos-shell.overrideAttrs (old: {

     

       
184
       
-
       
          patches =

     

       
185
       
-
       
            (old.patches or [])

     

       
186
       
-
       
            ++ [

     

       
187
       
-
       
              # https://github.com/Mic92/nixos-shell/pull/94

     

       
188
       
-
       
              (pkgs.fetchpatch {

     

       
189
       
-
       
                name = "fix-foreign-vm.patch";

     

       
190
       
-
       
                url = "https://github.com/Mic92/nixos-shell/commit/113e4cc55ae236b5b0b1fbd8b321e9b67c77580e.patch";

     

       
191
       
-
       
                hash = "sha256-eauetBK0wXAOcd9PYbExokNCiwz2QyFnZ4FnwGi9VCo=";

     

       
192
       
-
       
              })

     

       
193
       
-
       
            ];

     

       
194
       
-
       
        });

     

       
195
       
 
       
      in {

     

       
196
       
 
       
        type = "app";

     

       
197
       
-
       
        program = toString (pkgs.writeShellScript "vm" ''

     

       
198
       
-
       
          ${nixos-shell}/bin/nixos-shell --flake .#vm-${system} --guest-system ${system}-linux

     

       
199
       
-
       
        '');

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
200
       
 
       
      };

     

       
201
       
 
       
      gomod2nix = {

     

       
202
       
 
       
        type = "app";

     
···

       
258
       
 
       


     

       
259
       
 
       
      services.tangled-spindle.package = lib.mkDefault self.packages.${pkgs.system}.spindle;

     

       
260
       
 
       
    };

     

       
261
       
-
       
    nixosConfigurations.vm-x86_64 = import ./nix/vm.nix {

     

       
262
       
-
       
      inherit self nixpkgs;

     

       
263
       
-
       
      system = "x86_64-linux";

     

       
264
       
-
       
    };

     

       
265
       
-
       
    nixosConfigurations.vm-aarch64 = import ./nix/vm.nix {

     

       
266
       
-
       
      inherit self nixpkgs;

     

       
267
       
-
       
      system = "aarch64-linux";

     

       
268
       
-
       
    };

     

       
269
       
 
       
  };

     

       
270
       
 
       
}

     

···

       
175
       
 
       
        program = ''${tailwind-watcher}/bin/run'';

     

       
176
       
 
       
      };

     

       
177
       
 
       
      vm = let

     

       
178
       
+
       
        guestSystem =

     

       
179
       
 
       
          if pkgs.stdenv.hostPlatform.isAarch64

     

       
180
       
+
       
          then "aarch64-linux"

     

       
181
       
+
       
          else "x86_64-linux";

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
182
       
 
       
      in {

     

       
183
       
 
       
        type = "app";

     

       
184
       
+
       
        program =

     

       
185
       
+
       
          (pkgs.writeShellApplication {

     

       
186
       
+
       
            name = "launch-vm";

     

       
187
       
+
       
            text = ''

     

       
188
       
+
       
              rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1)

     

       
189
       
+
       
              cd "$rootDir"

     

       
190
       
+
       


     

       
191
       
+
       
              mkdir -p nix/vm-data/{knot,repos,spindle,spindle-logs}

     

       
192
       
+
       


     

       
193
       
+
       
              export TANGLED_VM_DATA_DIR="$rootDir/nix/vm-data"

     

       
194
       
+
       
              exec ${pkgs.lib.getExe

     

       
195
       
+
       
                (import ./nix/vm.nix {

     

       
196
       
+
       
                  inherit nixpkgs self;

     

       
197
       
+
       
                  system = guestSystem;

     

       
198
       
+
       
                  hostSystem = system;

     

       
199
       
+
       
                }).config.system.build.vm}

     

       
200
       
+
       
            '';

     

       
201
       
+
       
          })

     

       
202
       
+
       
          + /bin/launch-vm;

     

       
203
       
 
       
      };

     

       
204
       
 
       
      gomod2nix = {

     

       
205
       
 
       
        type = "app";

     
···

       
261
       
 
       


     

       
262
       
 
       
      services.tangled-spindle.package = lib.mkDefault self.packages.${pkgs.system}.spindle;

     

       
263
       
 
       
    };

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
264
       
 
       
  };

     

       
265
       
 
       
}

     

···

       
1
       
 
       
{

     

       
2
       
 
       
  nixpkgs,

     

       
3
       
 
       
  system,

     

       
0
       
 
       

     

       
4
       
 
       
  self,

     

       
5
       
 
       
}: let

     

       
6
       
 
       
  envVar = name: let

     
···

       
16
       
 
       
      self.nixosModules.knot

     

       
17
       
 
       
      self.nixosModules.spindle

     

       
18
       
 
       
      ({

     

       
0
       
 
       

     

       
19
       
 
       
        config,

     

       
20
       
 
       
        pkgs,

     

       
21
       
 
       
        ...

     

       
22
       
 
       
      }: {

     

       
23
       
-
       
        nixos-shell = {

     

       
24
       
-
       
          inheritPath = false;

     

       
25
       
-
       
          mounts = {

     

       
26
       
-
       
            mountHome = false;

     

       
27
       
-
       
            mountNixProfile = false;

     

       
28
       
-
       
          };

     

       
29
       
-
       
        };

     

       
30
       
-
       
        virtualisation = {

     

       
31
       
 
       
          memorySize = 2048;

     

       
32
       
 
       
          diskSize = 10 * 1024;

     

       
33
       
 
       
          cores = 2;

     
···

       
51
       
 
       
              guest.port = 6555;

     

       
52
       
 
       
            }

     

       
53
       
 
       
          ];

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
54
       
 
       
        };

     

       
55
       
 
       
        services.getty.autologinUser = "root";

     

       
56
       
 
       
        environment.systemPackages = with pkgs; [curl vim git sqlite litecli];

     

       
57
       
-
       
        systemd.tmpfiles.rules = let

     

       
58
       
-
       
          u = config.services.tangled-knot.gitUser;

     

       
59
       
-
       
          g = config.services.tangled-knot.gitUser;

     

       
60
       
-
       
        in [

     

       
61
       
-
       
          "d /var/lib/knot 0770 ${u} ${g} - -" # Create the directory first

     

       
62
       
-
       
          "f+ /var/lib/knot/secret 0660 ${u} ${g} - KNOT_SERVER_SECRET=${envVar "TANGLED_VM_KNOT_SECRET"}"

     

       
63
       
-
       
        ];

     

       
64
       
 
       
        services.tangled-knot = {

     

       
65
       
 
       
          enable = true;

     

       
66
       
 
       
          motd = "Welcome to the development knot!\n";

     

       
67
       
 
       
          server = {

     

       
68
       
-
       
            secretFile = "/var/lib/knot/secret";

     

       
69
       
 
       
            hostname = "localhost:6000";

     

       
70
       
 
       
            listenAddr = "0.0.0.0:6000";

     

       
71
       
 
       
          };

     
···

       
82
       
 
       
            };

     

       
83
       
 
       
          };

     

       
84
       
 
       
        };

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
85
       
 
       
      })

     

       
86
       
 
       
    ];

     

       
87
       
 
       
  }

     

···

       
1
       
 
       
{

     

       
2
       
 
       
  nixpkgs,

     

       
3
       
 
       
  system,

     

       
4
       
+
       
  hostSystem,

     

       
5
       
 
       
  self,

     

       
6
       
 
       
}: let

     

       
7
       
 
       
  envVar = name: let

     
···

       
17
       
 
       
      self.nixosModules.knot

     

       
18
       
 
       
      self.nixosModules.spindle

     

       
19
       
 
       
      ({

     

       
20
       
+
       
        lib,

     

       
21
       
 
       
        config,

     

       
22
       
 
       
        pkgs,

     

       
23
       
 
       
        ...

     

       
24
       
 
       
      }: {

     

       
25
       
+
       
        virtualisation.vmVariant.virtualisation = {

     

       
26
       
+
       
          host.pkgs = import nixpkgs {system = hostSystem;};

     

       
27
       
+
       


     

       
28
       
+
       
          graphics = false;

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
29
       
 
       
          memorySize = 2048;

     

       
30
       
 
       
          diskSize = 10 * 1024;

     

       
31
       
 
       
          cores = 2;

     
···

       
49
       
 
       
              guest.port = 6555;

     

       
50
       
 
       
            }

     

       
51
       
 
       
          ];

     

       
52
       
+
       
          sharedDirectories = {

     

       
53
       
+
       
            # We can't use the 9p mounts directly for most of these

     

       
54
       
+
       
            # as SQLite is incompatible with them. So instead we

     

       
55
       
+
       
            # mount the shared directories to a different location

     

       
56
       
+
       
            # and copy the contents around on service start/stop.

     

       
57
       
+
       
            knotData = {

     

       
58
       
+
       
              source = "$TANGLED_VM_DATA_DIR/knot";

     

       
59
       
+
       
              target = "/mnt/knot-data";

     

       
60
       
+
       
            };

     

       
61
       
+
       
            spindleData = {

     

       
62
       
+
       
              source = "$TANGLED_VM_DATA_DIR/spindle";

     

       
63
       
+
       
              target = "/mnt/spindle-data";

     

       
64
       
+
       
            };

     

       
65
       
+
       
            spindleLogs = {

     

       
66
       
+
       
              source = "$TANGLED_VM_DATA_DIR/spindle-logs";

     

       
67
       
+
       
              target = "/var/log/spindle";

     

       
68
       
+
       
            };

     

       
69
       
+
       
          };

     

       
70
       
 
       
        };

     

       
71
       
 
       
        services.getty.autologinUser = "root";

     

       
72
       
 
       
        environment.systemPackages = with pkgs; [curl vim git sqlite litecli];

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
73
       
 
       
        services.tangled-knot = {

     

       
74
       
 
       
          enable = true;

     

       
75
       
 
       
          motd = "Welcome to the development knot!\n";

     

       
76
       
 
       
          server = {

     

       
77
       
+
       
            secretFile = builtins.toFile "knot-secret" ("KNOT_SERVER_SECRET=" + (envVar "TANGLED_VM_KNOT_SECRET"));

     

       
78
       
 
       
            hostname = "localhost:6000";

     

       
79
       
 
       
            listenAddr = "0.0.0.0:6000";

     

       
80
       
 
       
          };

     
···

       
91
       
 
       
            };

     

       
92
       
 
       
          };

     

       
93
       
 
       
        };

     

       
94
       
+
       
        systemd.services = let

     

       
95
       
+
       
          mkDataSyncScripts = source: target: {

     

       
96
       
+
       
            enableStrictShellChecks = true;

     

       
97
       
+
       


     

       
98
       
+
       
            postStart = ''

     

       
99
       
+
       
              [[ -d ${target} ]] || exit 0

     

       
100
       
+
       
              ${lib.getExe pkgs.rsync} -a ${source}/ ${target}

     

       
101
       
+
       
            '';

     

       
102
       
+
       


     

       
103
       
+
       
            postStop = ''

     

       
104
       
+
       
              [[ -d ${target} ]] || exit 0

     

       
105
       
+
       
              ${lib.getExe pkgs.rsync} -a ${target}/ ${source}

     

       
106
       
+
       
            '';

     

       
107
       
+
       
          };

     

       
108
       
+
       
        in {

     

       
109
       
+
       
          knot = mkDataSyncScripts "/mnt/knot-data" config.services.tangled-knot.stateDir;

     

       
110
       
+
       
          spindle = mkDataSyncScripts "/mnt/spindle-data" (builtins.dirOf config.services.tangled-spindle.server.dbPath);

     

       
111
       
+
       
        };

     

       
112
       
 
       
      })

     

       
113
       
 
       
    ];

     

       
114
       
 
       
  }