pyrox.dev
1name: Eval
2
3on:
4 workflow_call:
5 inputs:
6 mergedSha:
7 required: true
8 type: string
9 targetSha:
10 type: string
11 systems:
12 required: true
13 type: string
14 testVersions:
15 required: false
16 default: false
17 type: boolean
18 secrets:
19 CACHIX_AUTH_TOKEN:
20 required: true
21
22permissions: {}
23
24defaults:
25 run:
26 shell: bash
27
28jobs:
29 versions:
30 if: inputs.testVersions
31 runs-on: ubuntu-24.04-arm
32 outputs:
33 versions: ${{ steps.versions.outputs.versions }}
34 steps:
35 - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
36 with:
37 path: trusted
38 sparse-checkout: |
39 ci/supportedVersions.nix
40
41 - name: Check out the PR at the test merge commit
42 uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
43 with:
44 ref: ${{ inputs.mergedSha }}
45 path: untrusted
46 sparse-checkout: |
47 ci/pinned.json
48
49 - name: Install Nix
50 uses: cachix/install-nix-action@9280e7aca88deada44c930f1e2c78e21c3ae3edd # v31
51
52 - name: Load supported versions
53 id: versions
54 run: |
55 echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
56
57 eval:
58 runs-on: ubuntu-24.04-arm
59 needs: versions
60 if: ${{ !cancelled() && !failure() }}
61 strategy:
62 fail-fast: false
63 matrix:
64 system: ${{ fromJSON(inputs.systems) }}
65 version:
66 - "" # Default Eval triggering rebuild labels and such.
67 - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
68 # Failures for versioned Evals will be collected in a separate job below
69 # to not interrupt main Eval's compare step.
70 continue-on-error: ${{ matrix.version != '' }}
71 name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
72 timeout-minutes: 15
73 steps:
74 # This is not supposed to be used and just acts as a fallback.
75 # Without swap, when Eval runs OOM, it will fail badly with a
76 # job that is sometimes not interruptible anymore.
77 # If Eval starts swapping, decrease chunkSize to keep it fast.
78 - name: Enable swap
79 run: |
80 sudo fallocate -l 10G /swap
81 sudo chmod 600 /swap
82 sudo mkswap /swap
83 sudo swapon /swap
84
85 - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
86 with:
87 sparse-checkout: .github/actions
88 - name: Check out the PR at merged and target commits
89 uses: ./.github/actions/checkout
90 with:
91 merged-as-untrusted-at: ${{ inputs.mergedSha }}
92 target-as-trusted-at: ${{ inputs.targetSha }}
93
94 - name: Install Nix
95 uses: cachix/install-nix-action@9280e7aca88deada44c930f1e2c78e21c3ae3edd # v31
96
97 - uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16
98 with:
99 # The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI.
100 name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }}
101 extraPullNames: nixpkgs-ci
102 authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
103 pushFilter: '(-source|-single-chunk)$'
104
105 - name: Evaluate the ${{ matrix.system }} output paths at the merge commit
106 env:
107 MATRIX_SYSTEM: ${{ matrix.system }}
108 MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
109 run: |
110 nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \
111 --argstr evalSystem "$MATRIX_SYSTEM" \
112 --arg chunkSize 8000 \
113 --argstr nixPath "$MATRIX_VERSION" \
114 --out-link merged
115 # If it uses too much memory, slightly decrease chunkSize.
116 # Note: Keep the same further down in sync!
117
118 # Running the attrpath generation step separately from the outpath step afterwards.
119 # The idea is that, *if* Eval on the target branch has not finished, yet, we will
120 # generate the attrpaths in the meantime - and the separate command command afterwards
121 # will check cachix again for whether Eval has finished. If no Eval result from the
122 # target branch can be found the second time, we proceed to run it in here. Attrpaths
123 # generation takes roughly 30 seconds, so for every normal use-case this should be more
124 # than enough of a head start for Eval on the target branch to finish.
125 # This edge-case, that Eval on the target branch is delayed is unlikely to happen anyway:
126 # For a commit to become the target commit of a PR, it must *already* be on the branch.
127 # Normally, CI should always start running on that push event *before* it starts running
128 # on the PR.
129 - name: Evaluate the ${{ matrix.system }} attribute paths at the target commit
130 if: inputs.targetSha
131 env:
132 MATRIX_SYSTEM: ${{ matrix.system }}
133 run: |
134 nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.attrpathsSuperset \
135 --argstr evalSystem "$MATRIX_SYSTEM" \
136 --argstr nixPath "nixVersions.latest"
137
138 - name: Evaluate the ${{ matrix.system }} output paths at the target commit
139 if: inputs.targetSha
140 env:
141 MATRIX_SYSTEM: ${{ matrix.system }}
142 # This should be very quick, because it pulls the eval results from Cachix.
143 run: |
144 nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
145 --argstr evalSystem "$MATRIX_SYSTEM" \
146 --arg chunkSize 8000 \
147 --argstr nixPath "nixVersions.latest" \
148 --out-link target
149
150 - name: Compare outpaths against the target branch
151 if: inputs.targetSha
152 env:
153 MATRIX_SYSTEM: ${{ matrix.system }}
154 run: |
155 nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
156 --arg beforeDir ./target \
157 --arg afterDir ./merged \
158 --argstr evalSystem "$MATRIX_SYSTEM" \
159 --out-link diff
160
161 - name: Upload outpaths diff and stats
162 if: inputs.targetSha
163 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
164 with:
165 name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
166 path: diff/*
167
168 compare:
169 runs-on: ubuntu-24.04-arm
170 needs: [eval]
171 if: inputs.targetSha && !cancelled() && !failure()
172 permissions:
173 statuses: write
174 timeout-minutes: 5
175 steps:
176 - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
177 with:
178 sparse-checkout: .github/actions
179 - name: Check out the PR at the target commit
180 uses: ./.github/actions/checkout
181 with:
182 merged-as-untrusted-at: ${{ inputs.mergedSha }}
183 target-as-trusted-at: ${{ inputs.targetSha }}
184
185 - name: Download output paths and eval stats for all systems
186 uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
187 with:
188 pattern: diff-*
189 path: diff
190 merge-multiple: true
191
192 - name: Install Nix
193 uses: cachix/install-nix-action@9280e7aca88deada44c930f1e2c78e21c3ae3edd # v31
194
195 - name: Combine all output paths and eval stats
196 run: |
197 nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
198 --arg diffDir ./diff \
199 --out-link combined
200
201 - name: Compare against the target branch
202 env:
203 AUTHOR_ID: ${{ github.event.pull_request.user.id }}
204 run: |
205 git -C nixpkgs/trusted diff --name-only ${{ inputs.mergedSha }} \
206 | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
207
208 # Use the target branch to get accurate maintainer info
209 nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
210 --arg combinedDir ./combined \
211 --arg touchedFilesJson ./touched-files.json \
212 --argstr githubAuthorId "$AUTHOR_ID" \
213 --out-link comparison
214
215 cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
216
217 - name: Upload the comparison results
218 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
219 with:
220 name: comparison
221 path: comparison/*
222
223 - name: Add eval summary to commit statuses
224 if: ${{ github.event_name == 'pull_request_target' }}
225 uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
226 with:
227 script: |
228 const { readFile } = require('node:fs/promises')
229 const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
230 const description =
231 'Package: ' + [
232 `added ${changed.attrdiff.added.length}`,
233 `removed ${changed.attrdiff.removed.length}`,
234 `changed ${changed.attrdiff.changed.length}`
235 ].join(', ') +
236 ' — Rebuild: ' + [
237 `linux ${changed.rebuildCountByKernel.linux}`,
238 `darwin ${changed.rebuildCountByKernel.darwin}`
239 ].join(', ')
240
241 const { serverUrl, repo, runId, payload } = context
242 const target_url =
243 `${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
244
245 await github.rest.repos.createCommitStatus({
246 ...repo,
247 sha: payload.pull_request.head.sha,
248 context: 'Eval Summary',
249 state: 'success',
250 description,
251 target_url
252 })
253
254 # Creates a matrix of Eval performance for various versions and systems.
255 report:
256 runs-on: ubuntu-24.04-arm
257 needs: [versions, eval]
258 steps:
259 - name: Download output paths and eval stats for all versions
260 uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
261 with:
262 pattern: "*-diff-*"
263 path: versions
264
265 - name: Add version comparison table to job summary
266 uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
267 env:
268 SYSTEMS: ${{ inputs.systems }}
269 VERSIONS: ${{ needs.versions.outputs.versions }}
270 with:
271 script: |
272 const { readFileSync } = require('node:fs')
273 const path = require('node:path')
274
275 const systems = JSON.parse(process.env.SYSTEMS)
276 const versions = JSON.parse(process.env.VERSIONS)
277
278 core.summary.addHeading('Lix/Nix version comparison')
279 core.summary.addTable(
280 [].concat(
281 [
282 [{ data: 'Version', header: true }].concat(
283 systems.map((system) => ({ data: system, header: true })),
284 ),
285 ],
286 versions.map((version) =>
287 [{ data: version }].concat(
288 systems.map((system) => {
289 try {
290 const artifact = path.join('versions', `${version}-diff-${system}`)
291 const time = Math.round(
292 parseFloat(
293 readFileSync(
294 path.join(artifact, 'after', system, 'total-time'),
295 'utf-8',
296 ),
297 ),
298 )
299 const diff = JSON.parse(
300 readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
301 )
302 const attrs = [].concat(
303 diff.added,
304 diff.removed,
305 diff.changed,
306 diff.rebuilds
307 ).filter(attr =>
308 // Exceptions related to dev shells, which changed at some time between 2.18 and 2.24.
309 !attr.startsWith('tests.devShellTools.nixos.') &&
310 !attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.')
311 )
312 if (attrs.length > 0) {
313 core.setFailed(
314 `${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
315 )
316 return { data: ':x:' }
317 }
318 return { data: time }
319 } catch {
320 core.warning(`${version} on ${system} did not produce artifact.`)
321 return { data: ':warning:' }
322 }
323 }),
324 ),
325 ),
326 ),
327 )
328 core.summary.addRaw(
329 '\n*Evaluation time in seconds without downloading dependencies.*',
330 true,
331 )
332 core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
333 core.summary.addRaw(
334 '\n*:x: Job produced different outpaths than the target branch.*',
335 true,
336 )
337 core.summary.write()
338
339 misc:
340 if: ${{ github.event_name != 'push' }}
341 runs-on: ubuntu-24.04-arm
342 timeout-minutes: 10
343 steps:
344 - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
345 with:
346 sparse-checkout: .github/actions
347 - name: Checkout the merge commit
348 uses: ./.github/actions/checkout
349 with:
350 merged-as-untrusted-at: ${{ inputs.mergedSha }}
351
352 - name: Install Nix
353 uses: cachix/install-nix-action@9280e7aca88deada44c930f1e2c78e21c3ae3edd # v31
354
355 - name: Run misc eval tasks in parallel
356 run: |
357 # Ensure flake outputs on all systems still evaluate
358 nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1' &
359 # Query nixpkgs with aliases enabled to check for basic syntax errors
360 nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null &
361 wait