top-level/release-outpaths: move to ci/eval & refactor (#439221)

+83
ci/eval/attrpaths.nix
···
+
# This expression will, as efficiently as possible, dump a
+
# *superset* of all attrpaths of derivations which might be
+
# part of a release on *any* platform.
+
#
+
# This expression runs single-threaded under all current Nix
+
# implementations, but much faster and with much less memory
+
# used than ./outpaths.nix itself.
+
#
+
# Once you have the list of attrnames you can split it up into
+
# $NUM_CORES batches and evaluate the outpaths separately for each
+
# batch, in parallel.
+
#
+
# To dump the attrnames:
+
#
+
# nix-instantiate --eval --strict --json ci/eval/attrpaths.nix -A names
+
#
+
{
+
lib ? import (path + "/lib"),
+
trace ? false,
+
path ? ./../..,
+
}:
+
let
+
+
# TODO: Use mapAttrsToListRecursiveCond when this PR lands:
+
# https://github.com/NixOS/nixpkgs/pull/395160
+
justAttrNames =
+
path: value:
+
let
+
result =
+
if path == [ "AAAAAASomeThingsFailToEvaluate" ] || !(lib.isAttrs value) then
+
[ ]
+
else if lib.isDerivation value then
+
[ path ]
+
else
+
lib.pipe value [
+
(lib.mapAttrsToList (
+
name: value:
+
lib.addErrorContext "while evaluating package set attribute path '${
+
lib.showAttrPath (path ++ [ name ])
+
}'" (justAttrNames (path ++ [ name ]) value)
+
))
+
lib.concatLists
+
];
+
in
+
lib.traceIf trace "** ${lib.showAttrPath path}" result;
+
+
outpaths = import ./outpaths.nix {
+
inherit path;
+
attrNamesOnly = true;
+
};
+
+
paths = [
+
# Some of the following are based on variants, which are disabled with `attrNamesOnly = true`.
+
# Until these have been removed from release.nix / hydra, we manually add them to the list.
+
[
+
"pkgsLLVM"
+
"stdenv"
+
]
+
[
+
"pkgsArocc"
+
"stdenv"
+
]
+
[
+
"pkgsZig"
+
"stdenv"
+
]
+
[
+
"pkgsStatic"
+
"stdenv"
+
]
+
[
+
"pkgsMusl"
+
"stdenv"
+
]
+
]
+
++ justAttrNames [ ] outpaths;
+
+
names = map lib.showAttrPath paths;
+
+
in
+
{
+
inherit paths names;
+
}
-1
ci/eval/compare/default.nix
···
) rebuildsByKernel
# Set the "11.by: package-maintainer" label to whether all packages directly
# changed are maintained by the PR's author.
-
# (https://github.com/NixOS/ofborg/blob/df400f44502d4a4a80fa283d33f2e55a4e43ee90/ofborg/src/tagger.rs#L83-L88)
// {
"11.by: package-maintainer" =
maintainers ? ${githubAuthorId}
-1
ci/eval/compare/maintainers.nix
···
{
lib,
}:
-
# Almost directly vendored from https://github.com/NixOS/ofborg/blob/5a4e743f192fb151915fcbe8789922fa401ecf48/ofborg/src/maintainers.nix
{
changedattrs,
changedpathsjson,
+7 -6
ci/eval/default.nix
···
root = ../..;
fileset = unions (
map (lib.path.append ../..) [
+
".version"
+
"ci/supportedSystems.json"
+
"ci/eval/attrpaths.nix"
+
"ci/eval/chunk.nix"
+
"ci/eval/outpaths.nix"
"default.nix"
"doc"
"lib"
···
"modules"
"nixos"
"pkgs"
-
".version"
-
"ci/supportedSystems.json"
]
);
};
···
export GC_INITIAL_HEAP_SIZE=4g
command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \
nix-instantiate --eval --strict --json --show-trace \
-
"$src/pkgs/top-level/release-attrpaths-superset.nix" \
+
"$src/ci/eval/attrpaths.nix" \
-A paths \
-I "$src" \
--option restrict-eval true \
···
attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json",
# The number of attributes per chunk, see ./README.md for more info.
chunkSize ? 5000,
-
checkMeta ? true,
# Don't try to eval packages marked as broken.
includeBroken ? false,
···
set +e
command time -o "$outputDir/timestats/$myChunk" \
-f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
-
nix-env -f "${nixpkgs}/pkgs/top-level/release-outpaths-parallel.nix" \
+
nix-env -f "${nixpkgs}/ci/eval/chunk.nix" \
--eval-system "$system" \
--option restrict-eval true \
--option allow-import-from-derivation false \
···
--arg myChunk "$myChunk" \
--arg attrpathFile "${attrpathFile}" \
--arg systems "[ \"$system\" ]" \
-
--arg checkMeta ${lib.boolToString checkMeta} \
--arg includeBroken ${lib.boolToString includeBroken} \
-I ${nixpkgs} \
-I ${attrpathFile} \
+1 -3
pkgs/test/default.nix
···
php = recurseIntoAttrs (callPackages ./php { });
-
pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { }) // {
-
__recurseIntoDerivationForReleaseJobs = true;
-
};
+
pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { });
buildRustCrate = recurseIntoAttrs (callPackage ../build-support/rust/build-rust-crate/test { });
importCargoLock = recurseIntoAttrs (callPackage ../build-support/rust/test/import-cargo-lock { });
+13 -20
pkgs/top-level/all-packages.nix
···
haskell.packages.ghc96
else
haskell.packages.ghc98
-
)
-
// {
-
__recurseIntoDerivationForReleaseJobs = true;
-
};
+
);
# haskellPackages.ghc is build->host (it exposes the compiler used to build the
# set, similarly to stdenv.cc), but pkgs.ghc should be host->target to be more
···
);
-
libsForQt5 =
-
(recurseIntoAttrs (
-
import ./qt5-packages.nix {
-
inherit
-
lib
-
config
-
__splicedPackages
-
makeScopeWithSplicing'
-
generateSplicesForMkScope
-
pkgsHostTarget
-
;
-
}
-
))
-
// {
-
__recurseIntoDerivationForReleaseJobs = true;
-
};
+
libsForQt5 = recurseIntoAttrs (
+
import ./qt5-packages.nix {
+
inherit
+
lib
+
config
+
__splicedPackages
+
makeScopeWithSplicing'
+
generateSplicesForMkScope
+
pkgsHostTarget
+
;
+
}
+
);
# plasma5Packages maps to the Qt5 packages set that is used to build the plasma5 desktop
plasma5Packages = libsForQt5;
-1
pkgs/top-level/linux-kernels.nix
···
linux_libre = recurseIntoAttrs (packagesFor kernels.linux_libre);
linux_latest_libre = recurseIntoAttrs (packagesFor kernels.linux_latest_libre);
-
__recurseIntoDerivationForReleaseJobs = true;
}
// lib.optionalAttrs config.allowAliases {
+1 -3
pkgs/top-level/pkg-config/tests.nix
···
};
in
lib.recurseIntoAttrs {
-
defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { } // {
-
__recurseIntoDerivationForReleaseJobs = true;
-
};
+
defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { };
}
-114
pkgs/top-level/release-attrpaths-superset.nix
···
-
# This expression will, as efficiently as possible, dump a
-
# *superset* of all attrpaths of derivations which might be
-
# part of a release on *any* platform.
-
#
-
# Both this expression and what ofborg uses (release-outpaths.nix)
-
# are essentially single-threaded (under the current cppnix
-
# implementation).
-
#
-
# This expression runs much, much, much faster and uses much, much
-
# less memory than the ofborg script by skipping the
-
# platform-relevance checks. The ofborg outpaths.nix script takes
-
# half an hour on a 3ghz core and peaks at 60gbytes of memory; this
-
# expression runs on the same machine in 44 seconds with peak memory
-
# usage of 5gbytes.
-
#
-
# Once you have the list of attrnames you can split it up into
-
# $NUM_CORES batches and run the platform checks separately for each
-
# batch, in parallel.
-
#
-
# To dump the attrnames:
-
#
-
# nix-instantiate --eval --strict --json pkgs/top-level/release-attrpaths-superset.nix -A names
-
#
-
{
-
lib ? import (path + "/lib"),
-
trace ? false,
-
checkMeta ? true,
-
path ? ./../..,
-
}:
-
let
-
-
# The intended semantics are that an attrpath rooted at pkgs is
-
# part of the (unfiltered) release jobset iff both of the following
-
# are true:
-
#
-
# 1. The attrpath leads to a value for which lib.isDerivation is true
-
#
-
# 2. Any proper prefix of the attrpath at which lib.isDerivation
-
# is true also has __recurseIntoDerivationForReleaseJobs=true.
-
#
-
# The second condition is unfortunately necessary because there are
-
# Hydra release jobnames which have proper prefixes which are
-
# attrnames of derivations (!). We should probably restructure
-
# the job tree so that this is not the case.
-
#
-
justAttrNames =
-
path: value:
-
let
-
result =
-
if path == [ "AAAAAASomeThingsFailToEvaluate" ] then
-
[ ]
-
else if
-
lib.isDerivation value
-
&&
-
# in some places we have *derivations* with jobsets as subattributes, ugh
-
!(value.__recurseIntoDerivationForReleaseJobs or false)
-
then
-
[ path ]
-
-
# Even wackier case: we have meta.broken==true jobs with
-
# !meta.broken jobs as subattributes with license=unfree, and
-
# check-meta.nix won't throw an "unfree" failure because the
-
# enclosing derivation is marked broken. Yeah. Bonkers.
-
# We should just forbid jobsets enclosed by derivations.
-
else if lib.isDerivation value && !value.meta.available then
-
[ ]
-
-
else if !(lib.isAttrs value) then
-
[ ]
-
else
-
lib.pipe value [
-
(builtins.mapAttrs (
-
name: value:
-
builtins.addErrorContext "while evaluating package set attribute path '${
-
lib.showAttrPath (path ++ [ name ])
-
}'" (justAttrNames (path ++ [ name ]) value)
-
))
-
builtins.attrValues
-
builtins.concatLists
-
];
-
in
-
if !trace then result else lib.trace "** ${lib.concatStringsSep "." path}" result;
-
-
releaseOutpaths = import ./release-outpaths.nix {
-
inherit checkMeta;
-
attrNamesOnly = true;
-
inherit path;
-
};
-
-
paths = [
-
# I am not entirely sure why these three packages end up in
-
# the Hydra jobset. But they do, and they don't meet the
-
# criteria above, so at the moment they are special-cased.
-
[
-
"pkgsLLVM"
-
"stdenv"
-
]
-
[
-
"pkgsStatic"
-
"stdenv"
-
]
-
[
-
"pkgsMusl"
-
"stdenv"
-
]
-
]
-
++ justAttrNames [ ] releaseOutpaths;
-
-
names = map (path: (lib.concatStringsSep "." path)) paths;
-
-
in
-
{
-
inherit paths names;
-
}
+3 -5
pkgs/top-level/release-outpaths-parallel.nix ci/eval/chunk.nix
···
-
# This file works in tandem with ../../ci/eval/default.nix
-
# It turns ./release-outpaths.nix into chunks of a fixed size
+
# This turns ./outpaths.nix into chunks of a fixed size.
{
lib ? import ../../lib,
path ? ../..,
···
attrpathFile,
chunkSize,
myChunk,
-
checkMeta,
includeBroken,
systems,
}:
···
attrpaths = lib.importJSON attrpathFile;
myAttrpaths = lib.sublist (chunkSize * myChunk) chunkSize attrpaths;
-
unfiltered = import ./release-outpaths.nix {
+
unfiltered = import ./outpaths.nix {
inherit path;
-
inherit checkMeta includeBroken systems;
+
inherit includeBroken systems;
};
# Turns the unfiltered recursive attribute set into one that is limited to myAttrpaths
+4 -7
pkgs/top-level/release-outpaths.nix ci/eval/outpaths.nix
···
#!/usr/bin/env nix-shell
# When using as a callable script, passing `--argstr path some/path` overrides $PWD.
-
#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path --arg checkMeta true -f pkgs/top-level/release-outpaths.nix"
+
#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path -f ci/eval/outpaths.nix"
-
# Vendored from:
-
# https://raw.githubusercontent.com/NixOS/ofborg/74f38efa7ef6f0e8e71ec3bfc675ae4fb57d7491/ofborg/src/outpaths.nix
{
-
checkMeta,
includeBroken ? true, # set this to false to exclude meta.broken packages from the output
path ? ./../..,
-
# used by pkgs/top-level/release-attrnames-superset.nix
+
# used by ./attrpaths.nix
attrNamesOnly ? false,
# Set this to `null` to build for builtins.currentSystem only
-
systems ? builtins.fromJSON (builtins.readFile ../../ci/supportedSystems.json),
+
systems ? builtins.fromJSON (builtins.readFile ../supportedSystems.json),
}:
let
lib = import (path + "/lib");
···
allowUnfree = true;
allowInsecurePredicate = x: true;
allowVariants = !attrNamesOnly;
-
checkMeta = checkMeta;
+
checkMeta = true;
handleEvalIssue =
reason: errormsg:
+1 -5
pkgs/top-level/release-python.nix
···
res = builtins.tryEval (
if isDerivation value then
value.meta.isBuildPythonPackage or [ ]
-
else if
-
value.recurseForDerivations or false
-
|| value.recurseForRelease or false
-
|| value.__recurseIntoDerivationForReleaseJobs or false
-
then
+
else if value.recurseForDerivations or false || value.recurseForRelease or false then
packagePython value
else
[ ]
+1 -5
pkgs/top-level/release-unfree-redistributable.nix
···
value.meta.hydraPlatforms
or (lib.subtractLists (value.meta.badPlatforms or [ ]) (value.meta.platforms or [ "x86_64-linux" ]))
)
-
else if
-
value.recurseForDerivations or false
-
|| value.recurseForRelease or false
-
|| value.__recurseIntoDerivationForReleaseJobs or false
-
then
+
else if value.recurseForDerivations or false || value.recurseForRelease or false then
# Recurse
packagesWith attrPath cond value
else
+4 -6
pkgs/top-level/release.nix
···
# resulting tree of attributes to *not* have a ".${system}"
# suffixed upon every job name like Hydra expects.
#
-
# This flag exists mainly for use by
-
# pkgs/top-level/release-attrnames-superset.nix; see that file for
-
# full details. The exact behavior of this flag may change; it
-
# should be considered an internal implementation detail of
-
# pkgs/top-level/.
-
#
+
# This flag exists mainly for use by ci/eval/attrpaths.nix; see
+
# that file for full details. The exact behavior of this flag
+
# may change; it should be considered an internal implementation
+
# detail of ci/eval.
attrNamesOnly ? false,
}: