nixos/testing-python.nix: Add evalTest

This is a decomposition of the testing-python.nix and build-vms.nix
files into modules.

By refactoring the glue, we accomplish the following:

- NixOS tests can now use `imports` and other module system features.
- Network-wide test setup can now be reusable; example:
- A setup with all VMs configured to use a DNS server
- Split long, slow tests into multiple tests that import a
common module that has most of the setup.
- Type checking for the test arguments
- (TBD) "generated" options reference docs
- Aspects that had to be wired through all the glue are now in their
own files.
- Chief example: interactive.nix.
- Also: network.nix

In rewriting this, I've generally stuck as close as possible to the
existing code; copying pieces of logic and rewiring them, without
changing the logic itself.

I've made two exceptions to this rule

- Introduction of `extraDriverArgs` instead of hardcoded
interactivity logic.

- Incorporation of https://github.com/NixOS/nixpkgs/pull/144110
in testScript.nix.

I might revert the latter and split it into a new commit.

+20
nixos/lib/testing-python.nix
···
${lib.optionalString (interactive) "--add-flags --interactive"}
'');
+
evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; };
+
runTest = module: (evalTest module).config.run;
+
+
testModules = [
+
./testing/driver.nix
+
./testing/interactive.nix
+
./testing/legacy.nix
+
./testing/meta.nix
+
./testing/name.nix
+
./testing/network.nix
+
./testing/nodes.nix
+
./testing/run.nix
+
./testing/testScript.nix
+
{
+
config = {
+
hostPkgs = pkgs;
+
};
+
}
+
];
+
# Make a full-blown test
makeTest =
{ machine ? null
+16
nixos/lib/testing/call-test.nix
···
+
{ config, lib, ... }:
+
let
+
inherit (lib) mkOption types;
+
in
+
{
+
options = {
+
callTest = mkOption {
+
internal = true;
+
type = types.functionTo types.raw;
+
};
+
result = mkOption {
+
internal = true;
+
default = config;
+
};
+
};
+
}
+177
nixos/lib/testing/driver.nix
···
+
{ config, lib, hostPkgs, ... }:
+
let
+
inherit (lib) mkOption types;
+
+
# Reifies and correctly wraps the python test driver for
+
# the respective qemu version and with or without ocr support
+
testDriver = hostPkgs.callPackage ../test-driver {
+
inherit (config) enableOCR extraPythonPackages;
+
qemu_pkg = config.qemu.package;
+
imagemagick_light = hostPkgs.imagemagick_light.override { inherit (hostPkgs) libtiff; };
+
tesseract4 = hostPkgs.tesseract4.override { enableLanguages = [ "eng" ]; };
+
};
+
+
+
vlans = map (m: m.virtualisation.vlans) (lib.attrValues config.nodes);
+
vms = map (m: m.system.build.vm) (lib.attrValues config.nodes);
+
+
nodeHostNames =
+
let
+
nodesList = map (c: c.system.name) (lib.attrValues config.nodes);
+
in
+
nodesList ++ lib.optional (lib.length nodesList == 1 && !lib.elem "machine" nodesList) "machine";
+
+
# TODO: This is an implementation error and needs fixing
+
# the testing famework cannot legitimately restrict hostnames further
+
# beyond RFC1035
+
invalidNodeNames = lib.filter
+
(node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
+
nodeHostNames;
+
+
uniqueVlans = lib.unique (builtins.concatLists vlans);
+
vlanNames = map (i: "vlan${toString i}: VLan;") uniqueVlans;
+
machineNames = map (name: "${name}: Machine;") nodeHostNames;
+
+
withChecks =
+
if lib.length invalidNodeNames > 0 then
+
throw ''
+
Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
+
All machines are referenced as python variables in the testing framework which will break the
+
script when special characters are used.
+
+
This is an IMPLEMENTATION ERROR and needs to be fixed. Meanwhile,
+
please stick to alphanumeric chars and underscores as separation.
+
''
+
else
+
lib.warnIf config.skipLint "Linting is disabled";
+
+
driver =
+
hostPkgs.runCommand "nixos-test-driver-${config.name}"
+
{
+
# inherit testName; TODO (roberth): need this?
+
nativeBuildInputs = [
+
hostPkgs.makeWrapper
+
] ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ];
+
testScript = config.testScriptString;
+
preferLocalBuild = true;
+
passthru = config.passthru;
+
meta = config.meta // {
+
mainProgram = "nixos-test-driver";
+
};
+
}
+
''
+
mkdir -p $out/bin
+
+
vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
+
+
${lib.optionalString (!config.skipTypeCheck) ''
+
# prepend type hints so the test script can be type checked with mypy
+
cat "${../test-script-prepend.py}" >> testScriptWithTypes
+
echo "${builtins.toString machineNames}" >> testScriptWithTypes
+
echo "${builtins.toString vlanNames}" >> testScriptWithTypes
+
echo -n "$testScript" >> testScriptWithTypes
+
+
cat -n testScriptWithTypes
+
+
# set pythonpath so mypy knows where to find the imports. this requires the py.typed file.
+
export PYTHONPATH='${../test-driver}'
+
mypy --no-implicit-optional \
+
--pretty \
+
--no-color-output \
+
testScriptWithTypes
+
unset PYTHONPATH
+
''}
+
+
echo -n "$testScript" >> $out/test-script
+
+
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
+
+
${testDriver}/bin/generate-driver-symbols
+
${lib.optionalString (!config.skipLint) ''
+
PYFLAKES_BUILTINS="$(
+
echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
+
< ${lib.escapeShellArg "driver-symbols"}
+
)" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script
+
''}
+
+
# set defaults through environment
+
# see: ./test-driver/test-driver.py argparse implementation
+
wrapProgram $out/bin/nixos-test-driver \
+
--set startScripts "''${vmStartScripts[*]}" \
+
--set testScript "$out/test-script" \
+
--set vlans '${toString vlans}' \
+
${lib.escapeShellArgs (lib.concatMap (arg: ["--add-flags" arg]) config.extraDriverArgs)}
+
'';
+
+
in
+
{
+
options = {
+
+
driver = mkOption {
+
description = "Script that runs the test.";
+
type = types.package;
+
defaultText = lib.literalDocBook "set by the test framework";
+
};
+
+
hostPkgs = mkOption {
+
description = "Nixpkgs attrset used outside the nodes.";
+
type = types.raw;
+
example = lib.literalExpression ''
+
import nixpkgs { inherit system config overlays; }
+
'';
+
};
+
+
qemu.package = mkOption {
+
description = "Which qemu package to use.";
+
type = types.package;
+
default = hostPkgs.qemu_test;
+
defaultText = "hostPkgs.qemu_test";
+
};
+
+
enableOCR = mkOption {
+
description = ''
+
Whether to enable Optical Character Recognition functionality for
+
testing graphical programs.
+
'';
+
type = types.bool;
+
default = false;
+
};
+
+
extraPythonPackages = mkOption {
+
description = ''
+
Python packages to add to the test driver.
+
+
The argument is a Python package set, similar to `pkgs.pythonPackages`.
+
'';
+
type = types.functionTo (types.listOf types.package);
+
default = ps: [ ];
+
};
+
+
extraDriverArgs = mkOption {
+
description = ''
+
Extra arguments to pass to the test driver.
+
'';
+
type = types.listOf types.str;
+
default = [];
+
};
+
+
skipLint = mkOption {
+
type = types.bool;
+
default = false;
+
};
+
+
skipTypeCheck = mkOption {
+
type = types.bool;
+
default = false;
+
};
+
};
+
+
config = {
+
_module.args.hostPkgs = config.hostPkgs;
+
+
driver = withChecks driver;
+
+
# make available on the test runner
+
passthru.driver = config.driver;
+
};
+
}
+18
nixos/lib/testing/interactive.nix
···
+
{ config, lib, moduleType, hostPkgs, ... }:
+
let
+
inherit (lib) mkOption types;
+
in
+
{
+
options = {
+
interactive = mkOption {
+
description = "All the same options, but configured for interactive use.";
+
type = moduleType;
+
};
+
};
+
+
config = {
+
interactive.qemu.package = hostPkgs.qemu;
+
interactive.extraDriverArgs = [ "--interactive" ];
+
passthru.driverInteractive = config.interactive.driver;
+
};
+
}
+25
nixos/lib/testing/legacy.nix
···
+
{ config, options, lib, ... }:
+
let
+
inherit (lib) mkIf mkOption types;
+
in
+
{
+
# This needs options.warnings, which we don't have (yet?).
+
# imports = [
+
# (lib.mkRenamedOptionModule [ "machine" ] [ "nodes" "machine" ])
+
# ];
+
+
options = {
+
machine = mkOption {
+
internal = true;
+
type = types.raw;
+
};
+
};
+
+
config = {
+
nodes = mkIf options.machine.isDefined (
+
lib.warn
+
"In test `${config.name}': The `machine' attribute in NixOS tests (pkgs.nixosTest / make-test-python.nix / testing-python.nix / makeTest) is deprecated. Please set the equivalent `nodes.machine'."
+
{ inherit (config) machine; }
+
);
+
};
+
}
+12
nixos/lib/testing/meta.nix
···
+
{ lib, ... }:
+
let
+
inherit (lib) types mkOption;
+
in
+
{
+
options = {
+
meta.maintainers = lib.mkOption {
+
type = types.listOf types.raw;
+
default = [];
+
};
+
};
+
}
+7
nixos/lib/testing/name.nix
···
+
{ lib, ... }:
+
{
+
options.name = lib.mkOption {
+
description = "The name of the test.";
+
type = lib.types.str;
+
};
+
}
+75
nixos/lib/testing/network.nix
···
+
{ lib, nodes, ... }:
+
+
with lib;
+
+
+
let
+
machines = attrNames nodes;
+
+
machinesNumbered = zipLists machines (range 1 254);
+
+
nodes_ = forEach machinesNumbered (m: nameValuePair m.fst
+
[
+
({ config, nodes, pkgs, ... }:
+
let
+
interfacesNumbered = zipLists config.virtualisation.vlans (range 1 255);
+
interfaces = forEach interfacesNumbered ({ fst, snd }:
+
nameValuePair "eth${toString snd}" {
+
ipv4.addresses =
+
[{
+
address = "192.168.${toString fst}.${toString m.snd}";
+
prefixLength = 24;
+
}];
+
});
+
+
networkConfig =
+
{
+
networking.hostName = mkDefault m.fst;
+
+
networking.interfaces = listToAttrs interfaces;
+
+
networking.primaryIPAddress =
+
optionalString (interfaces != [ ]) (head (head interfaces).value.ipv4.addresses).address;
+
+
# Put the IP addresses of all VMs in this machine's
+
# /etc/hosts file. If a machine has multiple
+
# interfaces, use the IP address corresponding to
+
# the first interface (i.e. the first network in its
+
# virtualisation.vlans option).
+
networking.extraHosts = flip concatMapStrings machines
+
(m':
+
let config = (getAttr m' nodes).config; in
+
optionalString (config.networking.primaryIPAddress != "")
+
("${config.networking.primaryIPAddress} " +
+
optionalString (config.networking.domain != null)
+
"${config.networking.hostName}.${config.networking.domain} " +
+
"${config.networking.hostName}\n"));
+
+
virtualisation.qemu.options =
+
let qemu-common = import ../qemu-common.nix { inherit lib pkgs; };
+
in
+
flip concatMap interfacesNumbered
+
({ fst, snd }: qemu-common.qemuNICFlags snd fst m.snd);
+
};
+
+
in
+
{
+
key = "ip-address";
+
config = networkConfig // {
+
# Expose the networkConfig items for tests like nixops
+
# that need to recreate the network config.
+
system.build.networkConfig = networkConfig;
+
};
+
}
+
)
+
]);
+
+
extraNodeConfigs = lib.listToAttrs nodes_;
+
in
+
{
+
config = {
+
defaults = { config, name, ... }: {
+
imports = extraNodeConfigs.${name};
+
};
+
};
+
}
+91
nixos/lib/testing/nodes.nix
···
+
testModuleArgs@{ config, lib, hostPkgs, nodes, ... }:
+
+
let
+
inherit (lib) mkOption mkForce optional types mapAttrs mkDefault;
+
+
system = hostPkgs.stdenv.hostPlatform.system;
+
+
baseOS =
+
import ../eval-config.nix {
+
inherit system;
+
inherit (config.node) specialArgs;
+
modules = [ config.defaults ];
+
baseModules = (import ../../modules/module-list.nix) ++
+
[
+
../../modules/virtualisation/qemu-vm.nix
+
../../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs
+
{ key = "no-manual"; documentation.nixos.enable = false; }
+
{
+
key = "no-revision";
+
# Make the revision metadata constant, in order to avoid needless retesting.
+
# The human version (e.g. 21.05-pre) is left as is, because it is useful
+
# for external modules that test with e.g. testers.nixosTest and rely on that
+
# version number.
+
config.system.nixos.revision = mkForce "constant-nixos-revision";
+
}
+
{ key = "nodes"; _module.args.nodes = nodes; }
+
+
({ config, ... }:
+
{
+
virtualisation.qemu.package = testModuleArgs.config.qemu.package;
+
+
# Ensure we do not use aliases. Ideally this is only set
+
# when the test framework is used by Nixpkgs NixOS tests.
+
nixpkgs.config.allowAliases = false;
+
})
+
] ++ optional config.minimal ../../modules/testing/minimal-kernel.nix;
+
};
+
+
+
in
+
+
{
+
+
options = {
+
node.type = mkOption {
+
type = types.raw;
+
default = baseOS.type;
+
internal = true;
+
};
+
+
nodes = mkOption {
+
type = types.lazyAttrsOf config.node.type;
+
};
+
+
defaults = mkOption {
+
description = ''
+
NixOS configuration that is applied to all {option}`nodes`.
+
'';
+
type = types.deferredModule;
+
default = { };
+
};
+
+
node.specialArgs = mkOption {
+
type = types.lazyAttrsOf types.raw;
+
default = { };
+
};
+
+
minimal = mkOption {
+
type = types.bool;
+
default = false;
+
};
+
+
nodesCompat = mkOption {
+
internal = true;
+
};
+
};
+
+
config = {
+
_module.args.nodes = config.nodesCompat;
+
nodesCompat =
+
mapAttrs
+
(name: config: config // {
+
config = lib.warn
+
"Module argument `nodes.${name}.config` is deprecated. Use `nodes.${name}` instead."
+
config;
+
})
+
config.nodes;
+
+
passthru.nodes = config.nodesCompat;
+
};
+
}
+51
nixos/lib/testing/run.nix
···
+
{ config, hostPkgs, lib, ... }:
+
let
+
inherit (lib) types mkOption;
+
in
+
{
+
options = {
+
passthru = mkOption {
+
type = types.lazyAttrsOf types.raw;
+
description = ''
+
Attributes to add to the returned derivations,
+
which are not necessarily part of the build.
+
+
This is a bit like doing `drv // { myAttr = true; }` (which would be lost by `overrideAttrs`).
+
It does not change the actual derivation, but adds the attribute nonetheless, so that
+
consumers of what would be `drv` have more information.
+
'';
+
};
+
+
run = mkOption {
+
type = types.package;
+
description = ''
+
Derivation that runs the test.
+
'';
+
};
+
};
+
+
config = {
+
run = hostPkgs.stdenv.mkDerivation {
+
name = "vm-test-run-${config.name}";
+
+
requiredSystemFeatures = [ "kvm" "nixos-test" ];
+
+
buildCommand =
+
''
+
mkdir -p $out
+
+
# effectively mute the XMLLogger
+
export LOGFILE=/dev/null
+
+
${config.driver}/bin/nixos-test-driver -o $out
+
'';
+
+
passthru = config.passthru;
+
+
meta = config.meta;
+
};
+
+
# useful for inspection (debugging / exploration)
+
passthru.config = config;
+
};
+
}
+78
nixos/lib/testing/testScript.nix
···
+
testModuleArgs@{ config, lib, hostPkgs, nodes, moduleType, ... }:
+
let
+
inherit (lib) mkOption types;
+
inherit (types) either str functionTo;
+
in
+
{
+
options = {
+
testScript = mkOption {
+
type = either str (functionTo str);
+
};
+
testScriptString = mkOption {
+
type = str;
+
readOnly = true;
+
internal = true;
+
};
+
+
includeTestScriptReferences = mkOption {
+
type = types.bool;
+
default = true;
+
internal = true;
+
};
+
withoutTestScriptReferences = mkOption {
+
type = moduleType;
+
description = ''
+
A parallel universe where the testScript is invalid and has no references.
+
'';
+
};
+
};
+
config = {
+
withoutTestScriptReferences.includeTestScriptReferences = false;
+
withoutTestScriptReferences.testScript = lib.mkForce "testscript omitted";
+
+
testScriptString =
+
if lib.isFunction config.testScript
+
then
+
config.testScript
+
{
+
nodes =
+
lib.mapAttrs
+
(k: v:
+
if v.virtualisation.useNixStoreImage
+
then
+
# prevent infinite recursion when testScript would
+
# reference v's toplevel
+
config.withoutTestScriptReferences.nodesCompat.${k}
+
else
+
# reuse memoized config
+
v
+
)
+
config.nodesCompat;
+
}
+
else config.testScript;
+
+
defaults = { config, name, ... }: {
+
# Make sure all derivations referenced by the test
+
# script are available on the nodes. When the store is
+
# accessed through 9p, this isn't important, since
+
# everything in the store is available to the guest,
+
# but when building a root image it is, as all paths
+
# that should be available to the guest has to be
+
# copied to the image.
+
virtualisation.additionalPaths =
+
lib.optional
+
# A testScript may evaluate nodes, which has caused
+
# infinite recursions. The demand cycle involves:
+
# testScript -->
+
# nodes -->
+
# toplevel -->
+
# additionalPaths -->
+
# hasContext testScript' -->
+
# testScript (ad infinitum)
+
# If we don't need to build an image, we can break this
+
# cycle by short-circuiting when useNixStoreImage is false.
+
(config.virtualisation.useNixStoreImage && builtins.hasContext testModuleArgs.config.testScriptString && testModuleArgs.config.includeTestScriptReferences)
+
(hostPkgs.writeStringReferencesToFile testModuleArgs.config.testScriptString);
+
};
+
};
+
}