commit 11ff96a6795889a3d59b2ec0e9d587b9cf15ed5c · pyrox.dev/nixpkgs

+1 -1

nixos/lib/test-driver/src/pyproject.toml

···

       21
       21
        
       line-length = 88

     

       22
       22
        
       

     

       23
       23
        
       lint.select = ["E", "F", "I", "U", "N"]

     

       24
       24
       -
       lint.ignore = ["E501"]

     

       24
       24
       +
       lint.ignore = ["E501", "N818"]

     

       25
       25
        
       

     

       26
       26
        
       # xxx: we can import https://pypi.org/project/types-colorama/ here

     

       27
       27
        
       [[tool.mypy.overrides]]

+19 -15

nixos/lib/test-driver/src/test_driver/driver.py

···

       11
       11
        
       from typing import Any

     

       12
       12
        
       from unittest import TestCase

     

       13
       13
        
       

     

       14
       14
       +
       from test_driver.errors import RequestedAssertionFailed, TestScriptError

     

       14
       15
        
       from test_driver.logger import AbstractLogger

     

       15
       16
        
       from test_driver.machine import Machine, NixStartScript, retry

     

       16
       17
        
       from test_driver.polling_condition import PollingCondition

     

       17
       18
        
       from test_driver.vlan import VLan

     

       18
       19
        
       

     

       19
       20
        
       SENTINEL = object()

     

       21
       21
       +
       

     

       22
       22
       +
       

     

       23
       23
       +
       class AssertionTester(TestCase):

     

       24
       24
       +
           """

     

       25
       25
       +
           Subclass of `unittest.TestCase` which is used in the

     

       26
       26
       +
           `testScript` to perform assertions.

     

       27
       27
       +
       

     

       28
       28
       +
           It throws a custom exception whose parent class

     

       29
       29
       +
           gets special treatment in the logs.

     

       30
       30
       +
           """

     

       31
       31
       +
       

     

       32
       32
       +
           failureException = RequestedAssertionFailed

     

       20
       33
        
       

     

       21
       34
        
       

     

       22
       35
        
       def get_tmp_dir() -> Path:

     
···

       41
       54
        
           return re.sub(r"^[^A-z_]|[^A-z0-9_]", "_", name)

     

       42
       55
        
       

     

       43
       56
        
       

     

       44
       44
       -
       class NixOSAssertionError(AssertionError):

     

       45
       45
       -
           pass

     

       46
       46
       -
       

     

       47
       47
       -
       

     

       48
       48
       -
       class Tester(TestCase):

     

       49
       49
       -
           failureException = NixOSAssertionError

     

       50
       50
       -
       

     

       51
       51
       -
       

     

       52
       57
        
       class Driver:

     

       53
       58
        
           """A handle to the driver that sets up the environment

     

       54
       59
        
           and runs the tests"""

     
···

       126
       131
        
                   try:

     

       127
       132
        
                       yield

     

       128
       133
        
                   except Exception as e:

     

       129
       129
       -
                       self.logger.error(f'Test "{name}" failed with error: "{e}"')

     

       134
       134
       +
                       self.logger.log_test_error(f'Test "{name}" failed with error: "{e}"')

     

       130
       135
        
                       raise e

     

       131
       136
        
       

     

       132
       137
        
           def test_symbols(self) -> dict[str, Any]:

     
···

       151
       156
        
                   serial_stdout_on=self.serial_stdout_on,

     

       152
       157
        
                   polling_condition=self.polling_condition,

     

       153
       158
        
                   Machine=Machine,  # for typing

     

       154
       154
       -
                   t=Tester(),

     

       159
       159
       +
                   t=AssertionTester(),

     

       155
       160
        
               )

     

       156
       161
        
               machine_symbols = {pythonize_name(m.name): m for m in self.machines}

     

       157
       162
        
               # If there's exactly one machine, make it available under the name

     
···

       177
       182
        
                   symbols = self.test_symbols()  # call eagerly

     

       178
       183
        
                   try:

     

       179
       184
        
                       exec(self.tests, symbols, None)

     

       180
       180
       -
                   except NixOSAssertionError:

     

       185
       185
       +
                   except TestScriptError:

     

       181
       186
        
                       exc_type, exc, tb = sys.exc_info()

     

       182
       187
        
                       filtered = [

     

       183
       188
        
                           frame

     
···

       186
       191
        
                       ]

     

       187
       192
        
       

     

       188
       193
        
                       self.logger.log_test_error("Traceback (most recent call last):")

     

       194
       194
       +
       

     

       189
       195
        
                       code = self.tests.splitlines()

     

       190
       196
        
                       for frame, line in zip(filtered, traceback.format_list(filtered)):

     

       191
       197
        
                           self.logger.log_test_error(line.rstrip())

     

       192
       198
        
                           if lineno := frame.lineno:

     

       193
       193
       -
                               self.logger.log_test_error(

     

       194
       194
       -
                                   f"    {code[lineno - 1].strip()}",

     

       195
       195
       -
                               )

     

       199
       199
       +
                               self.logger.log_test_error(f"    {code[lineno - 1].strip()}")

     

       196
       200
        
       

     

       197
       197
       -
                       self.logger.log_test_error("") # blank line for readability

     

       201
       201
       +
                       self.logger.log_test_error("")  # blank line for readability

     

       198
       202
        
                       exc_prefix = exc_type.__name__ if exc_type is not None else "Error"

     

       199
       203
        
                       for line in f"{exc_prefix}: {exc}".splitlines():

     

       200
       204
        
                           self.logger.log_test_error(line)

+23

nixos/lib/test-driver/src/test_driver/errors.py

···

       1
       1
       +
       class TestScriptError(Exception):

     

       2
       2
       +
           """

     

       3
       3
       +
           The base error class to indicate that the test script failed.

     

       4
       4
       +
           This (and its subclasses) get special treatment, i.e. only stack

     

       5
       5
       +
           frames from `testScript` are printed and the error gets prefixed

     

       6
       6
       +
           with `!!!` to make it easier to spot between other log-lines.

     

       7
       7
       +
       

     

       8
       8
       +
           This class is used for errors that aren't an actual test failure,

     

       9
       9
       +
           but also not a bug in the driver, e.g. failing OCR.

     

       10
       10
       +
           """

     

       11
       11
       +
       

     

       12
       12
       +
       

     

       13
       13
       +
       class RequestedAssertionFailed(TestScriptError):

     

       14
       14
       +
           """

     

       15
       15
       +
           Subclass of `TestScriptError` that gets special treatment.

     

       16
       16
       +
       

     

       17
       17
       +
           Exception raised when a requested assertion fails,

     

       18
       18
       +
           e.g. `machine.succeed(...)` or `t.assertEqual(...)`.

     

       19
       19
       +
       

     

       20
       20
       +
           This is separate from the base error class, to have a dedicated class name

     

       21
       21
       +
           that better represents this kind of failures.

     

       22
       22
       +
           (better readability in test output)

     

       23
       23
       +
           """

+19 -12

nixos/lib/test-driver/src/test_driver/machine.py

···

       18
       18
        
       from queue import Queue

     

       19
       19
        
       from typing import Any

     

       20
       20
        
       

     

       21
       21
       +
       from test_driver.errors import RequestedAssertionFailed, TestScriptError

     

       21
       22
        
       from test_driver.logger import AbstractLogger

     

       22
       23
        
       

     

       23
       24
        
       from .qmp import QMPSession

     
···

       128
       129
        
           )

     

       129
       130
        
       

     

       130
       131
        
           if ret.returncode != 0:

     

       131
       131
       -
               raise Exception(

     

       132
       132
       +
               raise TestScriptError(

     

       132
       133
        
                   f"Image processing failed with exit code {ret.returncode}, stdout: {ret.stdout.decode()}, stderr: {ret.stderr.decode()}"

     

       133
       134
        
               )

     

       134
       135
        
       

     
···

       139
       140
        
           screenshot_path: str, model_ids: Iterable[int]

     

       140
       141
        
       ) -> list[str]:

     

       141
       142
        
           if shutil.which("tesseract") is None:

     

       142
       142
       -
               raise Exception("OCR requested but enableOCR is false")

     

       143
       143
       +
               raise TestScriptError("OCR requested but enableOCR is false")

     

       143
       144
        
       

     

       144
       145
        
           processed_image = _preprocess_screenshot(screenshot_path, negate=False)

     

       145
       146
        
           processed_negative = _preprocess_screenshot(screenshot_path, negate=True)

     
···

       162
       163
        
                       capture_output=True,

     

       163
       164
        
                   )

     

       164
       165
        
                   if ret.returncode != 0:

     

       165
       165
       -
                       raise Exception(f"OCR failed with exit code {ret.returncode}")

     

       166
       166
       +
                       raise TestScriptError(f"OCR failed with exit code {ret.returncode}")

     

       166
       167
        
                   model_results.append(ret.stdout.decode("utf-8"))

     

       167
       168
        
       

     

       168
       169
        
           return model_results

     
···

       179
       180
        
               time.sleep(1)

     

       180
       181
        
       

     

       181
       182
        
           if not fn(True):

     

       182
       182
       -
               raise Exception(f"action timed out after {timeout} seconds")

     

       183
       183
       +
               raise RequestedAssertionFailed(

     

       184
       184
       +
                   f"action timed out after {timeout} tries with one-second pause in-between"

     

       185
       185
       +
               )

     

       183
       186
        
       

     

       184
       187
        
       

     

       185
       188
        
       class StartCommand:

     
···

       402
       405
        
               def check_active(_last_try: bool) -> bool:

     

       403
       406
        
                   state = self.get_unit_property(unit, "ActiveState", user)

     

       404
       407
        
                   if state == "failed":

     

       405
       405
       -
                       raise Exception(f'unit "{unit}" reached state "{state}"')

     

       408
       408
       +
                       raise RequestedAssertionFailed(f'unit "{unit}" reached state "{state}"')

     

       406
       409
        
       

     

       407
       410
        
                   if state == "inactive":

     

       408
       411
        
                       status, jobs = self.systemctl("list-jobs --full 2>&1", user)

     

       409
       412
        
                       if "No jobs" in jobs:

     

       410
       413
        
                           info = self.get_unit_info(unit, user)

     

       411
       414
        
                           if info["ActiveState"] == state:

     

       412
       412
       -
                               raise Exception(

     

       415
       415
       +
                               raise RequestedAssertionFailed(

     

       413
       416
        
                                   f'unit "{unit}" is inactive and there are no pending jobs'

     

       414
       417
        
                               )

     

       415
       418
        
       

     
···

       424
       427
        
           def get_unit_info(self, unit: str, user: str | None = None) -> dict[str, str]:

     

       425
       428
        
               status, lines = self.systemctl(f'--no-pager show "{unit}"', user)

     

       426
       429
        
               if status != 0:

     

       427
       427
       -
                   raise Exception(

     

       430
       430
       +
                   raise RequestedAssertionFailed(

     

       428
       431
        
                       f'retrieving systemctl info for unit "{unit}"'

     

       429
       432
        
                       + ("" if user is None else f' under user "{user}"')

     

       430
       433
        
                       + f" failed with exit code {status}"

     
···

       454
       457
        
                   user,

     

       455
       458
        
               )

     

       456
       459
        
               if status != 0:

     

       457
       457
       -
                   raise Exception(

     

       460
       460
       +
                   raise RequestedAssertionFailed(

     

       458
       461
        
                       f'retrieving systemctl property "{property}" for unit "{unit}"'

     

       459
       462
        
                       + ("" if user is None else f' under user "{user}"')

     

       460
       463
        
                       + f" failed with exit code {status}"

     
···

       502
       505
        
                   info = self.get_unit_info(unit)

     

       503
       506
        
                   state = info["ActiveState"]

     

       504
       507
        
                   if state != require_state:

     

       505
       505
       -
                       raise Exception(

     

       508
       508
       +
                       raise RequestedAssertionFailed(

     

       506
       509
        
                           f"Expected unit '{unit}' to to be in state "

     

       507
       510
        
                           f"'{require_state}' but it is in state '{state}'"

     

       508
       511
        
                       )

     
···

       656
       659
        
                       (status, out) = self.execute(command, timeout=timeout)

     

       657
       660
        
                       if status != 0:

     

       658
       661
        
                           self.log(f"output: {out}")

     

       659
       659
       -
                           raise Exception(f"command `{command}` failed (exit code {status})")

     

       662
       662
       +
                           raise RequestedAssertionFailed(

     

       663
       663
       +
                               f"command `{command}` failed (exit code {status})"

     

       664
       664
       +
                           )

     

       660
       665
        
                       output += out

     

       661
       666
        
               return output

     

       662
       667
        
       

     
···

       670
       675
        
                   with self.nested(f"must fail: {command}"):

     

       671
       676
        
                       (status, out) = self.execute(command, timeout=timeout)

     

       672
       677
        
                       if status == 0:

     

       673
       673
       -
                           raise Exception(f"command `{command}` unexpectedly succeeded")

     

       678
       678
       +
                           raise RequestedAssertionFailed(

     

       679
       679
       +
                               f"command `{command}` unexpectedly succeeded"

     

       680
       680
       +
                           )

     

       674
       681
        
                       output += out

     

       675
       682
        
               return output

     

       676
       683
        
       

     
···

       915
       922
        
                   ret = subprocess.run(f"pnmtopng '{tmp}' > '{filename}'", shell=True)

     

       916
       923
        
                   os.unlink(tmp)

     

       917
       924
        
                   if ret.returncode != 0:

     

       918
       918
       -
                       raise Exception("Cannot convert screenshot")

     

       925
       925
       +
                       raise TestScriptError("Cannot convert screenshot")

     

       919
       926
        
       

     

       920
       927
        
           def copy_from_host_via_shell(self, source: str, target: str) -> None:

     

       921
       928
        
               """Copy a file from the host into the guest by piping it over the