commit d6b326d6590f23a2fc833e2649f3c7afc77130a4 · pyrox.dev/nixpkgs

+51

nixos/doc/manual/development/writing-nixos-tests.section.md

···

       340
       340
        
       list-id: test-options-list

     

       341
       341
        
       source: @NIXOS_TEST_OPTIONS_JSON@

     

       342
       342
        
       ```

     

       343
       343
       +
       

     

       344
       344
       +
       ## Accessing VMs in the sandbox with SSH {#sec-test-sandbox-breakpoint}

     

       345
       345
       +
       

     

       346
       346
       +
       As explained in [](#sec-nixos-test-ssh-access), it's possible to configure an

     

       347
       347
       +
       SSH backdoor based on AF_VSOCK. This can be used to SSH into a VM of a running

     

       348
       348
       +
       build in a sandbox.

     

       349
       349
       +
       

     

       350
       350
       +
       This can be done when something in the test fails, e.g.

     

       351
       351
       +
       

     

       352
       352
       +
       ```nix

     

       353
       353
       +
       {

     

       354
       354
       +
         nodes.machine = {};

     

       355
       355
       +
       

     

       356
       356
       +
         sshBackdoor.enable = true;

     

       357
       357
       +
         enableDebugHook = true;

     

       358
       358
       +
       

     

       359
       359
       +
         testScript = ''

     

       360
       360
       +
           start_all()

     

       361
       361
       +
           machine.succeed("false") # this will fail

     

       362
       362
       +
         '';

     

       363
       363
       +
       }

     

       364
       364
       +
       ```

     

       365
       365
       +
       

     

       366
       366
       +
       For the AF_VSOCK feature to work, `/dev/vhost-vsock` is needed in the sandbox

     

       367
       367
       +
       which can be done with e.g.

     

       368
       368
       +
       

     

       369
       369
       +
       ```

     

       370
       370
       +
       nix-build -A nixosTests.foo --option sandbox-paths /dev/vhost-vsock

     

       371
       371
       +
       ```

     

       372
       372
       +
       

     

       373
       373
       +
       This will halt the test execution on a test-failure and print instructions

     

       374
       374
       +
       on how to enter the sandbox shell of the VM test. Inside, one can log into

     

       375
       375
       +
       e.g. `machine` with

     

       376
       376
       +
       

     

       377
       377
       +
       ```

     

       378
       378
       +
       ssh -F ./ssh_config vsock/3

     

       379
       379
       +
       ```

     

       380
       380
       +
       

     

       381
       381
       +
       As described in [](#sec-nixos-test-ssh-access), the numbers for vsock start at

     

       382
       382
       +
       `3` instead of `1`. So the first VM in the network (sorted alphabetically) can

     

       383
       383
       +
       be accessed with `vsock/3`.

     

       384
       384
       +
       

     

       385
       385
       +
       Alternatively, it's possible to explicitly set a breakpoint with

     

       386
       386
       +
       `debug.breakpoint()`. This also has the benefit, that one can step through

     

       387
       387
       +
       `testScript` with `pdb` like this:

     

       388
       388
       +
       

     

       389
       389
       +
       ```

     

       390
       390
       +
       $ sudo /nix/store/eeeee-attach <id>

     

       391
       391
       +
       bash# telnet 127.0.0.1 4444

     

       392
       392
       +
       pdb$ …

     

       393
       393
       +
       ```

nixos/doc/manual/redirects.json

···

       1902
       1902
        
         "test-opt-sshBackdoor.vsockOffset": [

     

       1903
       1903
        
           "index.html#test-opt-sshBackdoor.vsockOffset"

     

       1904
       1904
        
         ],

     

       1905
       1905
       +
         "test-opt-enableDebugHook": [

     

       1906
       1906
       +
           "index.html#test-opt-enableDebugHook"

     

       1907
       1907
       +
         ],

     

       1905
       1908
        
         "test-opt-defaults": [

     

       1906
       1909
        
           "index.html#test-opt-defaults"

     

       1907
       1910
        
         ],

     
···

       2009
       2012
        
         ],

     

       2010
       2013
        
         "sec-nixos-test-testing-hardware-features": [

     

       2011
       2014
        
           "index.html#sec-nixos-test-testing-hardware-features"

     

       2015
       2015
       +
         ],

     

       2016
       2016
       +
         "sec-test-sandbox-breakpoint": [

     

       2017
       2017
       +
           "index.html#sec-test-sandbox-breakpoint"

     

       2012
       2018
        
         ],

     

       2013
       2019
        
         "chap-developing-the-test-driver": [

     

       2014
       2020
        
           "index.html#chap-developing-the-test-driver"

nixos/lib/test-driver/default.nix

···

       14
       14
        
         extraPythonPackages ? (_: [ ]),

     

       15
       15
        
         nixosTests,

     

       16
       16
        
       }:

     

       17
       17
       +
       

     

       17
       18
        
       python3Packages.buildPythonApplication {

     

       18
       19
        
         pname = "nixos-test-driver";

     

       19
       20
        
         version = "1.1";

     
···

       32
       33
        
             junit-xml

     

       33
       34
        
             ptpython

     

       34
       35
        
             ipython

     

       36
       36
       +
             remote-pdb

     

       35
       37
        
           ]

     

       36
       38
        
           ++ extraPythonPackages python3Packages;

     

       37
       39

+10

nixos/lib/test-driver/src/test_driver/__init__.py

···

       5
       5
        
       

     

       6
       6
        
       import ptpython.ipython

     

       7
       7
        
       

     

       8
       8
       +
       from test_driver.debug import Debug, DebugAbstract, DebugNop

     

       8
       9
        
       from test_driver.driver import Driver

     

       9
       10
        
       from test_driver.logger import (

     

       10
       11
        
           CompositeLogger,

     
···

       64
       65
        
               "--interactive",

     

       65
       66
        
               help="drop into a python repl and run the tests interactively",

     

       66
       67
        
               action=argparse.BooleanOptionalAction,

     

       68
       68
       +
           )

     

       69
       69
       +
           arg_parser.add_argument(

     

       70
       70
       +
               "--debug-hook-attach",

     

       71
       71
       +
               help="Enable interactive debugging breakpoints for sandboxed runs",

     

       67
       72
        
           )

     

       68
       73
        
           arg_parser.add_argument(

     

       69
       74
        
               "--start-scripts",

     
···

       129
       134
        
           if not args.keep_vm_state:

     

       130
       135
        
               logger.info("Machine state will be reset. To keep it, pass --keep-vm-state")

     

       131
       136
        
       

     

       137
       137
       +
           debugger: DebugAbstract = DebugNop()

     

       138
       138
       +
           if args.debug_hook_attach is not None:

     

       139
       139
       +
               debugger = Debug(logger, args.debug_hook_attach)

     

       140
       140
       +
       

     

       132
       141
        
           with Driver(

     

       133
       142
        
               args.start_scripts,

     

       134
       143
        
               args.vlans,

     
···

       137
       146
        
               logger,

     

       138
       147
        
               args.keep_vm_state,

     

       139
       148
        
               args.global_timeout,

     

       149
       149
       +
               debug=debugger,

     

       140
       150
        
           ) as driver:

     

       141
       151
        
               if args.interactive:

     

       142
       152
        
                   history_dir = os.getcwd()

+53

nixos/lib/test-driver/src/test_driver/debug.py

···

       1
       1
       +
       import logging

     

       2
       2
       +
       import os

     

       3
       3
       +
       import random

     

       4
       4
       +
       import shutil

     

       5
       5
       +
       import subprocess

     

       6
       6
       +
       import sys

     

       7
       7
       +
       from abc import ABC, abstractmethod

     

       8
       8
       +
       

     

       9
       9
       +
       from remote_pdb import RemotePdb  # type:ignore

     

       10
       10
       +
       

     

       11
       11
       +
       from test_driver.logger import AbstractLogger

     

       12
       12
       +
       

     

       13
       13
       +
       

     

       14
       14
       +
       class DebugAbstract(ABC):

     

       15
       15
       +
           @abstractmethod

     

       16
       16
       +
           def breakpoint(self, host: str = "127.0.0.1", port: int = 4444) -> None:

     

       17
       17
       +
               pass

     

       18
       18
       +
       

     

       19
       19
       +
       

     

       20
       20
       +
       class DebugNop(DebugAbstract):

     

       21
       21
       +
           def __init__(self) -> None:

     

       22
       22
       +
               pass

     

       23
       23
       +
       

     

       24
       24
       +
           def breakpoint(self, host: str = "127.0.0.1", port: int = 4444) -> None:

     

       25
       25
       +
               pass

     

       26
       26
       +
       

     

       27
       27
       +
       

     

       28
       28
       +
       class Debug(DebugAbstract):

     

       29
       29
       +
           def __init__(self, logger: AbstractLogger, attach_command: str) -> None:

     

       30
       30
       +
               self.breakpoint_on_failure = False

     

       31
       31
       +
               self.logger = logger

     

       32
       32
       +
               self.attach = attach_command

     

       33
       33
       +
       

     

       34
       34
       +
           def breakpoint(self, host: str = "127.0.0.1", port: int = 4444) -> None:

     

       35
       35
       +
               """

     

       36
       36
       +
               Call this function to stop execution and put the process on sleep while

     

       37
       37
       +
               at the same time have the test driver provide a debug shell on TCP port

     

       38
       38
       +
               `port`. This is meant to be used for sandboxed tests that have the test

     

       39
       39
       +
               driver feature `enableDebugHook` enabled.

     

       40
       40
       +
               """

     

       41
       41
       +
               pattern = str(random.randrange(999999, 9999999))

     

       42
       42
       +
               self.logger.log_test_error(

     

       43
       43
       +
                   f"Breakpoint reached, run 'sudo {self.attach} {pattern}'"

     

       44
       44
       +
               )

     

       45
       45
       +
               os.environ["bashInteractive"] = shutil.which("bash")  # type:ignore

     

       46
       46
       +
               if os.fork() == 0:

     

       47
       47
       +
                   subprocess.run(["sleep", pattern])

     

       48
       48
       +
               else:

     

       49
       49
       +
                   # RemotePdb writes log messages to both stderr AND the logger,

     

       50
       50
       +
                   # which is the same here. Hence, disabling the remote_pdb logger

     

       51
       51
       +
                   # to avoid duplicate messages in the build log.

     

       52
       52
       +
                   logging.root.manager.loggerDict["remote_pdb"].disabled = True  # type:ignore

     

       53
       53
       +
                   RemotePdb(host=host, port=port).set_trace(sys._getframe().f_back)

+11

nixos/lib/test-driver/src/test_driver/driver.py

···

       13
       13
        
       

     

       14
       14
        
       from colorama import Style

     

       15
       15
        
       

     

       16
       16
       +
       from test_driver.debug import DebugAbstract, DebugNop

     

       16
       17
        
       from test_driver.errors import MachineError, RequestedAssertionFailed

     

       17
       18
        
       from test_driver.logger import AbstractLogger

     

       18
       19
        
       from test_driver.machine import Machine, NixStartScript, retry

     
···

       67
       68
        
           global_timeout: int

     

       68
       69
        
           race_timer: threading.Timer

     

       69
       70
        
           logger: AbstractLogger

     

       71
       71
       +
           debug: DebugAbstract

     

       70
       72
        
       

     

       71
       73
        
           def __init__(

     

       72
       74
        
               self,

     
···

       77
       79
        
               logger: AbstractLogger,

     

       78
       80
        
               keep_vm_state: bool = False,

     

       79
       81
        
               global_timeout: int = 24 * 60 * 60 * 7,

     

       82
       82
       +
               debug: DebugAbstract = DebugNop(),

     

       80
       83
        
           ):

     

       81
       84
        
               self.tests = tests

     

       82
       85
        
               self.out_dir = out_dir

     

       83
       86
        
               self.global_timeout = global_timeout

     

       84
       87
        
               self.race_timer = threading.Timer(global_timeout, self.terminate_test)

     

       85
       88
        
               self.logger = logger

     

       89
       89
       +
               self.debug = debug

     

       86
       90
        
       

     

       87
       91
        
               tmp_dir = get_tmp_dir()

     

       88
       92
        
       

     
···

       159
       163
        
                   polling_condition=self.polling_condition,

     

       160
       164
        
                   Machine=Machine,  # for typing

     

       161
       165
        
                   t=AssertionTester(),

     

       166
       166
       +
                   debug=self.debug,

     

       162
       167
        
               )

     

       163
       168
        
               machine_symbols = {pythonize_name(m.name): m for m in self.machines}

     

       164
       169
        
               # If there's exactly one machine, make it available under the name

     
···

       224
       229
        
                       for line in f"{exc_prefix}: {exc}".splitlines():

     

       225
       230
        
                           self.logger.log_test_error(line)

     

       226
       231
        
       

     

       232
       232
       +
                       self.debug.breakpoint()

     

       233
       233
       +
       

     

       227
       234
        
                       sys.exit(1)

     

       235
       235
       +
       

     

       236
       236
       +
                   except Exception:

     

       237
       237
       +
                       self.debug.breakpoint()

     

       238
       238
       +
                       raise

     

       228
       239
        
       

     

       229
       240
        
           def run_tests(self) -> None:

     

       230
       241
        
               """Run the test script (for non-interactive test runs)"""

nixos/lib/test-script-prepend.py

···

       1
       1
        
       # This file contains type hints that can be prepended to Nix test scripts so they can be type

     

       2
       2
        
       # checked.

     

       3
       3
        
       

     

       4
       4
       +
       from test_driver.debug import DebugAbstract

     

       4
       5
        
       from test_driver.driver import Driver

     

       5
       6
        
       from test_driver.vlan import VLan

     

       6
       7
        
       from test_driver.machine import Machine

     
···

       52
       53
        
       serial_stdout_off: Callable[[], None]

     

       53
       54
        
       serial_stdout_on: Callable[[], None]

     

       54
       55
        
       polling_condition: PollingConditionProtocol

     

       56
       56
       +
       debug: DebugAbstract

     

       55
       57
        
       t: TestCase

+32 -7

nixos/lib/testing/run.nix

···

       7
       7
        
       }:

     

       8
       8
        
       let

     

       9
       9
        
         inherit (lib) types mkOption;

     

       10
       10
       +
         inherit (hostPkgs.stdenv.hostPlatform) isDarwin isLinux;

     

       10
       11
        
       

     

       11
       12
        
         # TODO (lib): Also use lib equivalent in nodes.nix

     

       12
       13
        
         /**

     
···

       26
       27
        
           */

     

       27
       28
        
           f:

     

       28
       29
        
           lib.mkOverride (opt.highestPrio - 1) (f opt.value);

     

       29
       29
       -
       

     

       30
       30
        
       in

     

       31
       31
        
       {

     

       32
       32
        
         options = {

     
···

       42
       42
        
             '';

     

       43
       43
        
           };

     

       44
       44
        
       

     

       45
       45
       +
           enableDebugHook = lib.mkEnableOption "" // {

     

       46
       46
       +
             description = ''

     

       47
       47
       +
               Halt test execution after any test fail and provide the possibility to

     

       48
       48
       +
               hook into the sandbox to connect with either the test driver via

     

       49
       49
       +
               `telnet localhost 4444` or with the VMs via SSH and vsocks (see also

     

       50
       50
       +
               `sshBackdoor.enable`).

     

       51
       51
       +
             '';

     

       52
       52
       +
           };

     

       53
       53
       +
       

     

       45
       54
        
           rawTestDerivation = mkOption {

     

       46
       55
        
             type = types.package;

     

       47
       56
        
             description = ''

     
···

       74
       83
        
           rawTestDerivation = hostPkgs.stdenv.mkDerivation config.rawTestDerivationArg;

     

       75
       84
        
           rawTestDerivationArg =

     

       76
       85
        
             finalAttrs:

     

       77
       77
       -
             assert lib.assertMsg (!config.sshBackdoor.enable)

     

       78
       78
       -
               "The SSH backdoor is currently not supported for non-interactive testing! Please make sure to only set `interactive.sshBackdoor.enable = true;`!";

     

       86
       86
       +
             assert lib.assertMsg (

     

       87
       87
       +
               config.sshBackdoor.enable -> isLinux

     

       88
       88
       +
             ) "The SSH backdoor is not supported for macOS host systems!";

     

       89
       89
       +
       

     

       90
       90
       +
             assert lib.assertMsg (

     

       91
       91
       +
               config.enableDebugHook -> isLinux

     

       92
       92
       +
             ) "The debugging hook is not supported for macOS host systems!";

     

       79
       93
        
             {

     

       80
       94
        
               name = "vm-test-run-${config.name}";

     

       81
       95
        
       

     

       82
       96
        
               requiredSystemFeatures =

     

       83
       83
       -
                 [ "nixos-test" ]

     

       84
       84
       -
                 ++ lib.optionals hostPkgs.stdenv.hostPlatform.isLinux [ "kvm" ]

     

       85
       85
       -
                 ++ lib.optionals hostPkgs.stdenv.hostPlatform.isDarwin [ "apple-virt" ];

     

       97
       97
       +
                 [ "nixos-test" ] ++ lib.optional isLinux "kvm" ++ lib.optional isDarwin "apple-virt";

     

       98
       98
       +
       

     

       99
       99
       +
               nativeBuildInputs = lib.optionals config.enableDebugHook [

     

       100
       100
       +
                 hostPkgs.openssh

     

       101
       101
       +
                 hostPkgs.inetutils

     

       102
       102
       +
               ];

     

       86
       103
        
       

     

       87
       104
        
               buildCommand = ''

     

       88
       105
        
                 mkdir -p $out

     
···

       90
       107
        
                 # effectively mute the XMLLogger

     

       91
       108
        
                 export LOGFILE=/dev/null

     

       92
       109
        
       

     

       93
       93
       -
                 ${config.driver}/bin/nixos-test-driver -o $out

     

       110
       110
       +
                 ${lib.optionalString config.enableDebugHook ''

     

       111
       111
       +
                   ln -sf \

     

       112
       112
       +
                     ${hostPkgs.systemd}/lib/systemd/ssh_config.d/20-systemd-ssh-proxy.conf \

     

       113
       113
       +
                     ssh_config

     

       114
       114
       +
                 ''}

     

       115
       115
       +
       

     

       116
       116
       +
                 ${config.driver}/bin/nixos-test-driver \

     

       117
       117
       +
                   -o $out \

     

       118
       118
       +
                   ${lib.optionalString config.enableDebugHook "--debug-hook=${hostPkgs.breakpointHook.attach}"}

     

       94
       119
        
               '';

     

       95
       120
        
       

     

       96
       121
        
               passthru = config.passthru;