pyrox.dev
1{
2 lib,
3 jq,
4 runCommand,
5 writeText,
6 python3,
7 ...
8}:
9{
10 beforeResultDir,
11 afterResultDir,
12 touchedFilesJson,
13 byName ? false,
14}:
15let
16 /*
17 Derivation that computes which packages are affected (added, changed or removed) between two revisions of nixpkgs.
18 Note: "platforms" are "x86_64-linux", "aarch64-darwin", ...
19
20 ---
21 Inputs:
22 - beforeResultDir, afterResultDir: The evaluation result from before and after the change.
23 They can be obtained by running `nix-build -A ci.eval.full` on both revisions.
24
25 ---
26 Outputs:
27 - changed-paths.json: Various information about the changes:
28 {
29 attrdiff: {
30 added: ["package1"],
31 changed: ["package2", "package3"],
32 removed: ["package4"],
33 },
34 labels: [
35 "10.rebuild-darwin: 1-10",
36 "10.rebuild-linux: 1-10"
37 ],
38 rebuildsByKernel: {
39 darwin: ["package1", "package2"],
40 linux: ["package1", "package2", "package3"]
41 },
42 rebuildCountByKernel: {
43 darwin: 2,
44 linux: 3,
45 },
46 rebuildsByPlatform: {
47 aarch64-darwin: ["package1", "package2"],
48 aarch64-linux: ["package1", "package2"],
49 x86_64-linux: ["package1", "package2", "package3"],
50 x86_64-darwin: ["package1"],
51 },
52 }
53 - step-summary.md: A markdown render of the changes
54
55 ---
56 Implementation details:
57
58 Helper functions can be found in ./utils.nix.
59 Two main "types" are important:
60
61 - `packagePlatformPath`: A string of the form "<PACKAGE_PATH>.<PLATFORM>"
62 Example: "python312Packages.numpy.x86_64-linux"
63
64 - `packagePlatformAttr`: An attrs representation of a packagePlatformPath:
65 Example: { name = "python312Packages.numpy"; platform = "x86_64-linux"; }
66 */
67 inherit (import ./utils.nix { inherit lib; })
68 diff
69 groupByKernel
70 convertToPackagePlatformAttrs
71 groupByPlatform
72 extractPackageNames
73 getLabels
74 ;
75
76 getAttrs =
77 dir:
78 let
79 raw = builtins.readFile "${dir}/outpaths.json";
80 # The file contains Nix paths; we need to ignore them for evaluation purposes,
81 # else there will be a "is not allowed to refer to a store path" error.
82 data = builtins.unsafeDiscardStringContext raw;
83 in
84 builtins.fromJSON data;
85 beforeAttrs = getAttrs beforeResultDir;
86 afterAttrs = getAttrs afterResultDir;
87
88 # Attrs
89 # - keys: "added", "changed" and "removed"
90 # - values: lists of `packagePlatformPath`s
91 diffAttrs = diff beforeAttrs afterAttrs;
92
93 rebuilds = diffAttrs.added ++ diffAttrs.changed;
94 rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds;
95
96 changed-paths =
97 let
98 rebuildsByPlatform = groupByPlatform rebuildsPackagePlatformAttrs;
99 rebuildsByKernel = groupByKernel rebuildsPackagePlatformAttrs;
100 rebuildCountByKernel = lib.mapAttrs (
101 kernel: kernelRebuilds: lib.length kernelRebuilds
102 ) rebuildsByKernel;
103 in
104 writeText "changed-paths.json" (
105 builtins.toJSON {
106 attrdiff = lib.mapAttrs (_: extractPackageNames) diffAttrs;
107 inherit
108 rebuildsByPlatform
109 rebuildsByKernel
110 rebuildCountByKernel
111 ;
112 labels =
113 (getLabels rebuildCountByKernel)
114 # Adds "10.rebuild-*-stdenv" label if the "stdenv" attribute was changed
115 ++ lib.mapAttrsToList (kernel: _: "10.rebuild-${kernel}-stdenv") (
116 lib.filterAttrs (_: kernelRebuilds: kernelRebuilds ? "stdenv") rebuildsByKernel
117 );
118 }
119 );
120
121 maintainers = import ./maintainers.nix {
122 changedattrs = lib.attrNames (lib.groupBy (a: a.name) rebuildsPackagePlatformAttrs);
123 changedpathsjson = touchedFilesJson;
124 inherit byName;
125 };
126in
127runCommand "compare"
128 {
129 nativeBuildInputs = [
130 jq
131 (python3.withPackages (
132 ps: with ps; [
133 numpy
134 pandas
135 scipy
136 ]
137 ))
138
139 ];
140 maintainers = builtins.toJSON maintainers;
141 passAsFile = [ "maintainers" ];
142 env = {
143 BEFORE_DIR = "${beforeResultDir}";
144 AFTER_DIR = "${afterResultDir}";
145 };
146 }
147 ''
148 mkdir $out
149
150 cp ${changed-paths} $out/changed-paths.json
151
152
153 if jq -e '(.attrdiff.added | length == 0) and (.attrdiff.removed | length == 0)' "${changed-paths}" > /dev/null; then
154 # Chunks have changed between revisions
155 # We cannot generate a performance comparison
156 {
157 echo
158 echo "# Performance comparison"
159 echo
160 echo "This compares the performance of this branch against its pull request base branch (e.g., 'master')"
161 echo
162 echo "For further help please refer to: [ci/README.md](https://github.com/NixOS/nixpkgs/blob/master/ci/README.md)"
163 echo
164 } >> $out/step-summary.md
165
166 python3 ${./cmp-stats.py} >> $out/step-summary.md
167
168 else
169 # Package chunks are the same in both revisions
170 # We can use the to generate a performance comparison
171 {
172 echo
173 echo "# Performance Comparison"
174 echo
175 echo "Performance stats were skipped because the package sets differ between the two revisions."
176 echo
177 echo "For further help please refer to: [ci/README.md](https://github.com/NixOS/nixpkgs/blob/master/ci/README.md)"
178 } >> $out/step-summary.md
179 fi
180
181 jq -r -f ${./generate-step-summary.jq} < ${changed-paths} >> $out/step-summary.md
182
183 cp "$maintainersPath" "$out/maintainers.json"
184 ''