Merge pull request #324789 from NixOS/devShellTools-env

`devShellTools`: add environment functions

Changed files
+331 -99
doc
nixos
pkgs
build-support
dev-shell-tools
docker
+46
doc/build-helpers/dev-shell-tools.chapter.md
···
devShellTools.valueToString false
=> ""
```
+
+
:::
+
+
## `devShellTools.unstructuredDerivationInputEnv` {#sec-devShellTools-unstructuredDerivationInputEnv}
+
+
Convert a set of derivation attributes (as would be passed to [`derivation`]) to a set of environment variables that can be used in a shell script.
+
This function does not support `__structuredAttrs`, but does support `passAsFile`.
+
+
:::{.example}
+
## `unstructuredDerivationInputEnv` usage example
+
+
```nix
+
devShellTools.unstructuredDerivationInputEnv {
+
drvAttrs = {
+
name = "foo";
+
buildInputs = [ hello figlet ];
+
builder = bash;
+
args = [ "-c" "${./builder.sh}" ];
+
};
+
}
+
=> {
+
name = "foo";
+
buildInputs = "/nix/store/...-hello /nix/store/...-figlet";
+
builder = "/nix/store/...-bash";
+
}
+
```
+
+
Note that `args` is not included, because Nix does not added it to the builder process environment.
+
+
:::
+
+
## `devShellTools.derivationOutputEnv` {#sec-devShellTools-derivationOutputEnv}
+
+
Takes the relevant parts of a derivation and returns a set of environment variables, that would be present in the derivation.
+
+
:::{.example}
+
## `derivationOutputEnv` usage example
+
+
```nix
+
let
+
pkg = hello;
+
in
+
devShellTools.derivationOutputEnv { outputList = pkg.outputs; outputMap = pkg; }
+
```
+
+
:::
+1
nixos/tests/all-tests.nix
···
docker-rootless = handleTestOn ["aarch64-linux" "x86_64-linux"] ./docker-rootless.nix {};
docker-registry = handleTest ./docker-registry.nix {};
docker-tools = handleTestOn ["x86_64-linux"] ./docker-tools.nix {};
+
docker-tools-nix-shell = runTest ./docker-tools-nix-shell.nix;
docker-tools-cross = handleTestOn ["x86_64-linux" "aarch64-linux"] ./docker-tools-cross.nix {};
docker-tools-overlay = handleTestOn ["x86_64-linux"] ./docker-tools-overlay.nix {};
documize = handleTest ./documize.nix {};
+95
nixos/tests/docker-tools-nix-shell.nix
···
+
# nix-build -A nixosTests.docker-tools-nix-shell
+
{ config, lib, ... }:
+
let
+
inherit (config.node.pkgs.dockerTools) examples;
+
in
+
{
+
name = "docker-tools-nix-shell";
+
meta = with lib.maintainers; {
+
maintainers = [
+
infinisil
+
roberth
+
];
+
};
+
+
nodes = {
+
docker =
+
{ ... }:
+
{
+
virtualisation = {
+
diskSize = 3072;
+
docker.enable = true;
+
};
+
};
+
};
+
+
testScript = ''
+
docker.wait_for_unit("sockets.target")
+
+
with subtest("buildImageWithNixDB: Has a nix database"):
+
docker.succeed(
+
"docker load --input='${examples.nix}'",
+
"docker run --rm ${examples.nix.imageName} nix-store -q --references /bin/bash"
+
)
+
+
with subtest("buildNixShellImage: Can build a basic derivation"):
+
docker.succeed(
+
"${examples.nix-shell-basic} | docker load",
+
"docker run --rm nix-shell-basic bash -c 'buildDerivation && $out/bin/hello' | grep '^Hello, world!$'"
+
)
+
+
with subtest("buildNixShellImage: Runs the shell hook"):
+
docker.succeed(
+
"${examples.nix-shell-hook} | docker load",
+
"docker run --rm -it nix-shell-hook | grep 'This is the shell hook!'"
+
)
+
+
with subtest("buildNixShellImage: Sources stdenv, making build inputs available"):
+
docker.succeed(
+
"${examples.nix-shell-inputs} | docker load",
+
"docker run --rm -it nix-shell-inputs | grep 'Hello, world!'"
+
)
+
+
with subtest("buildNixShellImage: passAsFile works"):
+
docker.succeed(
+
"${examples.nix-shell-pass-as-file} | docker load",
+
"docker run --rm -it nix-shell-pass-as-file | grep 'this is a string'"
+
)
+
+
with subtest("buildNixShellImage: run argument works"):
+
docker.succeed(
+
"${examples.nix-shell-run} | docker load",
+
"docker run --rm -it nix-shell-run | grep 'This shell is not interactive'"
+
)
+
+
with subtest("buildNixShellImage: command argument works"):
+
docker.succeed(
+
"${examples.nix-shell-command} | docker load",
+
"docker run --rm -it nix-shell-command | grep 'This shell is interactive'"
+
)
+
+
with subtest("buildNixShellImage: home directory is writable by default"):
+
docker.succeed(
+
"${examples.nix-shell-writable-home} | docker load",
+
"docker run --rm -it nix-shell-writable-home"
+
)
+
+
with subtest("buildNixShellImage: home directory can be made non-existent"):
+
docker.succeed(
+
"${examples.nix-shell-nonexistent-home} | docker load",
+
"docker run --rm -it nix-shell-nonexistent-home"
+
)
+
+
with subtest("buildNixShellImage: can build derivations"):
+
docker.succeed(
+
"${examples.nix-shell-build-derivation} | docker load",
+
"docker run --rm -it nix-shell-build-derivation"
+
)
+
+
with subtest("streamLayeredImage: with nix db"):
+
docker.succeed(
+
"${examples.nix-layered} | docker load",
+
"docker run --rm ${examples.nix-layered.imageName} nix-store -q --references /bin/bash"
+
)
+
'';
+
}
+1 -67
nixos/tests/docker-tools.nix
···
};
nonRootTestImage =
-
pkgs.dockerTools.streamLayeredImage rec {
+
pkgs.dockerTools.streamLayeredImage {
name = "non-root-test";
tag = "latest";
uid = 1000;
···
docker.succeed("docker run --rm image-with-certs:latest test -r /etc/pki/tls/certs/ca-bundle.crt")
docker.succeed("docker image rm image-with-certs:latest")
-
with subtest("buildImageWithNixDB: Has a nix database"):
-
docker.succeed(
-
"docker load --input='${examples.nix}'",
-
"docker run --rm ${examples.nix.imageName} nix-store -q --references /bin/bash"
-
)
-
-
with subtest("buildNixShellImage: Can build a basic derivation"):
-
docker.succeed(
-
"${examples.nix-shell-basic} | docker load",
-
"docker run --rm nix-shell-basic bash -c 'buildDerivation && $out/bin/hello' | grep '^Hello, world!$'"
-
)
-
-
with subtest("buildNixShellImage: Runs the shell hook"):
-
docker.succeed(
-
"${examples.nix-shell-hook} | docker load",
-
"docker run --rm -it nix-shell-hook | grep 'This is the shell hook!'"
-
)
-
-
with subtest("buildNixShellImage: Sources stdenv, making build inputs available"):
-
docker.succeed(
-
"${examples.nix-shell-inputs} | docker load",
-
"docker run --rm -it nix-shell-inputs | grep 'Hello, world!'"
-
)
-
-
with subtest("buildNixShellImage: passAsFile works"):
-
docker.succeed(
-
"${examples.nix-shell-pass-as-file} | docker load",
-
"docker run --rm -it nix-shell-pass-as-file | grep 'this is a string'"
-
)
-
-
with subtest("buildNixShellImage: run argument works"):
-
docker.succeed(
-
"${examples.nix-shell-run} | docker load",
-
"docker run --rm -it nix-shell-run | grep 'This shell is not interactive'"
-
)
-
-
with subtest("buildNixShellImage: command argument works"):
-
docker.succeed(
-
"${examples.nix-shell-command} | docker load",
-
"docker run --rm -it nix-shell-command | grep 'This shell is interactive'"
-
)
-
-
with subtest("buildNixShellImage: home directory is writable by default"):
-
docker.succeed(
-
"${examples.nix-shell-writable-home} | docker load",
-
"docker run --rm -it nix-shell-writable-home"
-
)
-
-
with subtest("buildNixShellImage: home directory can be made non-existent"):
-
docker.succeed(
-
"${examples.nix-shell-nonexistent-home} | docker load",
-
"docker run --rm -it nix-shell-nonexistent-home"
-
)
-
-
with subtest("buildNixShellImage: can build derivations"):
-
docker.succeed(
-
"${examples.nix-shell-build-derivation} | docker load",
-
"docker run --rm -it nix-shell-build-derivation"
-
)
-
with subtest("streamLayeredImage: chown is persistent in fakeRootCommands"):
docker.succeed(
"${chownTestImage} | docker load",
···
docker.succeed(
"${nonRootTestImage} | docker load",
"docker run --rm ${chownTestImage.imageName} | diff /dev/stdin <(echo 12345:12345)"
-
)
-
-
with subtest("streamLayeredImage: with nix db"):
-
docker.succeed(
-
"${examples.nix-layered} | docker load",
-
"docker run --rm ${examples.nix-layered.imageName} nix-store -q --references /bin/bash"
)
'';
})
+50 -1
pkgs/build-support/dev-shell-tools/default.nix
···
-
{ lib }:
+
{
+
lib,
+
writeTextFile,
+
}:
let
inherit (builtins) typeOf;
in
rec {
+
# Docs: doc/build-helpers/dev-shell-tools.chapter.md
+
# Tests: ./tests/default.nix
# This function closely mirrors what this Nix code does:
# https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1102
# https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/eval.cc#L1981-L2036
···
if typeOf value == "path" then "${value}"
else if typeOf value == "list" then toString (map valueToString value)
else toString value;
+
+
+
# Docs: doc/build-helpers/dev-shell-tools.chapter.md
+
# Tests: ./tests/default.nix
+
# https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L992-L1004
+
unstructuredDerivationInputEnv = { drvAttrs }:
+
# FIXME: this should be `normalAttrs // passAsFileAttrs`
+
lib.mapAttrs'
+
(name: value:
+
let str = valueToString value;
+
in if lib.elem name (drvAttrs.passAsFile or [])
+
then
+
let
+
nameHash =
+
if builtins?convertHash
+
then builtins.convertHash {
+
hash = "sha256:" + builtins.hashString "sha256" name;
+
toHashFormat = "nix32";
+
}
+
else
+
builtins.hashString "sha256" name;
+
basename = ".attr-${nameHash}";
+
in
+
lib.nameValuePair "${name}Path" "${
+
writeTextFile {
+
name = "shell-passAsFile-${name}";
+
text = str;
+
destination = "/${basename}";
+
}
+
}/${basename}"
+
else lib.nameValuePair name str
+
)
+
(removeAttrs drvAttrs [
+
# TODO: there may be more of these
+
"args"
+
]);
+
+
# Docs: doc/build-helpers/dev-shell-tools.chapter.md
+
# Tests: ./tests/default.nix
+
derivationOutputEnv = { outputList, outputMap }:
+
# A mapping from output name to the nix store path where they should end up
+
# https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253
+
lib.genAttrs outputList (output: builtins.unsafeDiscardStringContext outputMap.${output}.outPath);
+
}
+122 -2
pkgs/build-support/dev-shell-tools/tests/default.nix
···
lib,
stdenv,
hello,
+
writeText,
+
zlib,
+
nixosTests,
}:
let
-
inherit (lib) escapeShellArg;
+
inherit (lib)
+
concatLines
+
escapeShellArg
+
isString
+
mapAttrsToList
+
;
in
-
{
+
lib.recurseIntoAttrs {
+
+
# nix-build -A tests.devShellTools.nixos
+
nixos = nixosTests.docker-tools-nix-shell;
+
# nix-build -A tests.devShellTools.valueToString
valueToString =
let inherit (devShellTools) valueToString; in
···
) >log 2>&1 || { cat log; exit 1; }
'';
};
+
+
# nix-build -A tests.devShellTools.valueToString
+
unstructuredDerivationInputEnv =
+
let
+
inherit (devShellTools) unstructuredDerivationInputEnv;
+
+
drvAttrs = {
+
one = 1;
+
boolTrue = true;
+
boolFalse = false;
+
foo = "foo";
+
list = [ 1 2 3 ];
+
pathDefaultNix = ./default.nix;
+
stringWithDep = "Exe: ${hello}/bin/hello";
+
aPackageAttrSet = hello;
+
anOutPath = hello.outPath;
+
anAnAlternateOutput = zlib.dev;
+
args = [ "args must not be added to the environment" "Nix doesn't do it." ];
+
+
passAsFile = [ "bar" ];
+
bar = ''
+
bar
+
${writeText "qux" "yea"}
+
'';
+
};
+
result = unstructuredDerivationInputEnv { inherit drvAttrs; };
+
in
+
assert result // { barPath = "<check later>"; } == {
+
one = "1";
+
boolTrue = "1";
+
boolFalse = "";
+
foo = "foo";
+
list = "1 2 3";
+
pathDefaultNix = "${./default.nix}";
+
stringWithDep = "Exe: ${hello}/bin/hello";
+
aPackageAttrSet = "${hello}";
+
anOutPath = "${hello.outPath}";
+
anAnAlternateOutput = "${zlib.dev}";
+
+
passAsFile = "bar";
+
barPath = "<check later>";
+
};
+
+
# Not runCommand, because it alters `passAsFile`
+
stdenv.mkDerivation ({
+
name = "devShellTools-unstructuredDerivationInputEnv-built-tests";
+
+
exampleBarPathString =
+
assert isString result.barPath;
+
result.barPath;
+
+
dontUnpack = true;
+
dontBuild = true;
+
dontFixup = true;
+
doCheck = true;
+
+
installPhase = "touch $out";
+
+
checkPhase = ''
+
fail() {
+
echo "$@" >&2
+
exit 1
+
}
+
checkAttr() {
+
echo checking attribute $1...
+
if [[ "$2" != "$3" ]]; then
+
echo "expected: $3"
+
echo "actual: $2"
+
exit 1
+
fi
+
}
+
${
+
concatLines
+
(mapAttrsToList
+
(name: value:
+
"checkAttr ${name} \"\$${name}\" ${escapeShellArg value}"
+
)
+
(removeAttrs
+
result
+
[
+
"args"
+
+
# Nix puts it in workdir, which is not a concept for
+
# unstructuredDerivationInputEnv, so we have to put it in the
+
# store instead. This means the full path won't match.
+
"barPath"
+
])
+
)
+
}
+
(
+
set -x
+
+
diff $exampleBarPathString $barPath
+
+
${lib.optionalString (builtins?convertHash) ''
+
[[ "$(basename $exampleBarPathString)" = "$(basename $barPath)" ]]
+
''}
+
)
+
+
''${args:+fail "args should not be set by Nix. We don't expect it to and unstructuredDerivationInputEnv removes it."}
+
if [[ "''${builder:-x}" == x ]]; then
+
fail "builder should be set by Nix. We don't remove it in unstructuredDerivationInputEnv."
+
fi
+
'';
+
} // removeAttrs drvAttrs [
+
# This would break the derivation. Instead, we have a check in the derivation to make sure Nix doesn't set it.
+
"args"
+
]);
}
+16 -29
pkgs/build-support/docker/default.nix
···
);
# This function streams a docker image that behaves like a nix-shell for a derivation
+
# Docs: doc/build-helpers/images/dockertools.section.md
+
# Tests: nixos/tests/docker-tools-nix-shell.nix
streamNixShellImage =
-
{ # The derivation whose environment this docker image should be based on
-
drv
-
, # Image Name
-
name ? drv.name + "-env"
-
, # Image tag, the Nix's output hash will be used if null
-
tag ? null
-
, # User id to run the container as. Defaults to 1000, because many
-
# binaries don't like to be run as root
-
uid ? 1000
-
, # Group id to run the container as, see also uid
-
gid ? 1000
-
, # The home directory of the user
-
homeDirectory ? "/build"
-
, # The path to the bash binary to use as the shell. See `NIX_BUILD_SHELL` in `man nix-shell`
-
shell ? bashInteractive + "/bin/bash"
-
, # Run this command in the environment of the derivation, in an interactive shell. See `--command` in `man nix-shell`
-
command ? null
-
, # Same as `command`, but runs the command in a non-interactive shell instead. See `--run` in `man nix-shell`
-
run ? null
+
{ drv
+
, name ? drv.name + "-env"
+
, tag ? null
+
, uid ? 1000
+
, gid ? 1000
+
, homeDirectory ? "/build"
+
, shell ? bashInteractive + "/bin/bash"
+
, command ? null
+
, run ? null
}:
assert lib.assertMsg (! (drv.drvAttrs.__structuredAttrs or false))
"streamNixShellImage: Does not work with the derivation ${drv.name} because it uses __structuredAttrs";
···
# https://github.com/NixOS/nix/blob/2.8.0/src/libstore/globals.hh#L464-L465
sandboxBuildDir = "/build";
-
# https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L992-L1004
-
drvEnv = lib.mapAttrs' (name: value:
-
let str = valueToString value;
-
in if lib.elem name (drv.drvAttrs.passAsFile or [])
-
then lib.nameValuePair "${name}Path" (writeText "pass-as-text-${name}" str)
-
else lib.nameValuePair name str
-
) drv.drvAttrs //
-
# A mapping from output name to the nix store path where they should end up
-
# https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253
-
lib.genAttrs drv.outputs (output: builtins.unsafeDiscardStringContext drv.${output}.outPath);
+
drvEnv =
+
devShellTools.unstructuredDerivationInputEnv { inherit (drv) drvAttrs; }
+
// devShellTools.derivationOutputEnv { outputList = drv.outputs; outputMap = drv; };
# Environment variables set in the image
envVars = {
···
};
# Wrapper around streamNixShellImage to build an image from the result
+
# Docs: doc/build-helpers/images/dockertools.section.md
+
# Tests: nixos/tests/docker-tools-nix-shell.nix
buildNixShellImage = { drv, compressor ? "gz", ... }@args:
let
stream = streamNixShellImage (builtins.removeAttrs args ["compressor"]);