pyrox.dev
1#!/usr/bin/env nix-shell
2#!nix-shell -I nixpkgs=./. -i bash -p curl jq nix gnused nixfmt
3# shellcheck shell=bash
4
5set -Eeuo pipefail
6shopt -s inherit_errexit
7
8trap 'exit 1' ERR
9
10rids=({linux-{,musl-}{arm,arm64,x64},osx-{arm64,x64},win-{arm64,x64,x86}})
11
12release () {
13 local content="$1"
14 local version="$2"
15
16 jq -er '.releases[] | select(."release-version" == "'"$version"'")' <<< "$content"
17}
18
19release_files () {
20 local release="$1"
21 local expr="$2"
22
23 jq -er '[('"$expr"').files[] | select(.name | test("^.*.tar.gz$"))]' <<< "$release"
24}
25
26release_platform_attr () {
27 local release_files="$1"
28 local platform="$2"
29 local attr="$3"
30
31 jq -r '.[] | select((.rid == "'"$platform"'") and (.name | contains("-composite-") or contains("-pack-") | not)) | ."'"$attr"'"' <<< "$release_files"
32}
33
34platform_sources () {
35 local release_files="$1"
36
37 echo "srcs = {"
38 for rid in "${rids[@]}"; do
39 local url hash
40
41 url=$(release_platform_attr "$release_files" "$rid" url)
42 hash=$(release_platform_attr "$release_files" "$rid" hash)
43
44 [[ -z "$url" || -z "$hash" ]] && continue
45
46 hash=$(nix --extra-experimental-features nix-command hash convert --to sri --hash-algo sha512 "$hash")
47
48 echo " $rid = {
49 url = \"$url\";
50 hash = \"$hash\";
51 };"
52 done
53 echo " };"
54}
55
56nuget_index="$(curl -fsSL "https://api.nuget.org/v3/index.json")"
57
58get_nuget_resource() {
59 jq -er '.resources[] | select(."@type" == "'"$1"'")."@id"' <<<"$nuget_index"
60}
61
62nuget_package_base_url="$(get_nuget_resource "PackageBaseAddress/3.0.0")"
63nuget_registration_base_url="$(get_nuget_resource "RegistrationsBaseUrl/3.6.0")"
64
65generate_package_list() {
66 local version="$1" indent="$2"
67 shift 2
68 local pkgs=( "$@" ) pkg url hash catalog_url catalog hash_algorithm
69
70 for pkg in "${pkgs[@]}"; do
71 url=${nuget_package_base_url}${pkg,,}/${version,,}/${pkg,,}.${version,,}.nupkg
72
73 if hash=$(curl -s --head "$url" -o /dev/null -w '%header{x-ms-meta-sha512}') && [[ -n "$hash" ]]; then
74 # Undocumented fast path for nuget.org
75 # https://github.com/NuGet/NuGetGallery/issues/9433#issuecomment-1472286080
76 hash=$(nix --extra-experimental-features nix-command hash convert --to sri --hash-algo sha512 "$hash")
77 elif {
78 catalog_url=$(curl -sL --compressed "${nuget_registration_base_url}${pkg,,}/${version,,}.json" | jq -r ".catalogEntry") && [[ -n "$catalog_url" ]] &&
79 catalog=$(curl -sL "$catalog_url") && [[ -n "$catalog" ]] &&
80 hash_algorithm="$(jq -er '.packageHashAlgorithm' <<<"$catalog")"&& [[ -n "$hash_algorithm" ]] &&
81 hash=$(jq -er '.packageHash' <<<"$catalog") && [[ -n "$hash" ]]
82 }; then
83 # Documented but slower path (requires 2 requests)
84 hash=$(nix --extra-experimental-features nix-command hash convert --to sri --hash-algo "${hash_algorithm,,}" "$hash")
85 elif hash=$(nix-prefetch-url "$url" --type sha512); then
86 # Fallback to downloading and hashing locally
87 echo "Failed to fetch hash from nuget for $url, falling back to downloading locally" >&2
88 hash=$(nix --extra-experimental-features nix-command hash convert --to sri --hash-algo sha512 "$hash")
89 else
90 echo "Failed to fetch hash for $url" >&2
91 exit 1
92 fi
93
94 echo "$indent(fetchNupkg { pname = \"${pkg}\"; version = \"${version}\"; hash = \"${hash}\"; })"
95 done
96}
97
98versionAtLeast () {
99 local cur_version=$1 min_version=$2
100 printf "%s\0%s" "$min_version" "$cur_version" | sort -zVC
101}
102
103# These packages are implicitly references by the build process,
104# based on the specific project configurations (RIDs, used features, etc.)
105# They are always referenced with the same version as the SDK used for building.
106# Since we lock nuget dependencies, when these packages are included in the generated
107# lock files (deps.nix), every update of SDK required those lock files to be
108# updated to reflect the new versions of these packages - otherwise, the build
109# would fail due to missing dependencies.
110#
111# Moving them to a separate list stored alongside the SDK package definitions,
112# and implicitly including them along in buildDotnetModule allows us
113# to make updating .NET SDK packages a lot easier - we now just update
114# the versions of these packages in one place, and all packages that
115# use buildDotnetModule continue building with the new .NET version without changes.
116#
117# Keep in mind that there is no canonical list of these implicitly
118# referenced packages - this list was created based on looking into
119# the deps.nix files of existing packages, and which dependencies required
120# updating after a SDK version bump.
121#
122# Due to this, make sure to check if new SDK versions introduce any new packages.
123# This should not happend in minor or bugfix updates, but probably happens
124# with every new major .NET release.
125aspnetcore_packages () {
126 local version=$1
127 local pkgs=(
128 Microsoft.AspNetCore.App.Ref
129 )
130
131 generate_package_list "$version" ' ' "${pkgs[@]}"
132}
133
134aspnetcore_target_packages () {
135 local version=$1
136 local rid=$2
137 local pkgs=(
138 "Microsoft.AspNetCore.App.Runtime.$rid"
139 )
140
141 generate_package_list "$version" ' ' "${pkgs[@]}"
142}
143
144netcore_packages () {
145 local version=$1
146 local pkgs=(
147 Microsoft.NETCore.DotNetAppHost
148 Microsoft.NETCore.App.Ref
149 )
150
151 if ! versionAtLeast "$version" 9; then
152 pkgs+=(
153 Microsoft.NETCore.DotNetHost
154 Microsoft.NETCore.DotNetHostPolicy
155 Microsoft.NETCore.DotNetHostResolver
156 )
157 fi
158
159 if versionAtLeast "$version" 7; then
160 pkgs+=(
161 Microsoft.DotNet.ILCompiler
162 )
163 fi
164
165 if versionAtLeast "$version" 8; then
166 pkgs+=(
167 Microsoft.NET.ILLink.Tasks
168 )
169 fi
170
171 generate_package_list "$version" ' ' "${pkgs[@]}"
172}
173
174netcore_host_packages () {
175 local version=$1
176 local rid=$2
177 local pkgs=(
178 "Microsoft.NETCore.App.Crossgen2.$rid"
179 )
180
181 local min_ilcompiler=
182 case "$rid" in
183 linux-musl-arm) ;;
184 linux-arm) ;;
185 win-x86) ;;
186 osx-arm64) min_ilcompiler=8 ;;
187 *) min_ilcompiler=7 ;;
188 esac
189
190 if [[ -n "$min_ilcompiler" ]] && versionAtLeast "$version" "$min_ilcompiler"; then
191 pkgs+=(
192 "runtime.$rid.Microsoft.DotNet.ILCompiler"
193 )
194 fi
195
196 generate_package_list "$version" ' ' "${pkgs[@]}"
197}
198
199netcore_target_packages () {
200 local version=$1
201 local rid=$2
202 local pkgs=(
203 "Microsoft.NETCore.App.Host.$rid"
204 "Microsoft.NETCore.App.Runtime.$rid"
205 "runtime.$rid.Microsoft.NETCore.DotNetAppHost"
206 )
207
208 if ! versionAtLeast "$version" 9; then
209 pkgs+=(
210 "runtime.$rid.Microsoft.NETCore.DotNetHost"
211 "runtime.$rid.Microsoft.NETCore.DotNetHostPolicy"
212 "runtime.$rid.Microsoft.NETCore.DotNetHostResolver"
213 )
214 case "$rid" in
215 linux-musl-arm*) ;;
216 win-arm64) ;;
217 *) pkgs+=(
218 "Microsoft.NETCore.App.Runtime.Mono.$rid"
219 ) ;;
220 esac
221 fi
222
223 if versionAtLeast "$version" 10; then
224 pkgs+=(
225 "Microsoft.NETCore.App.Runtime.NativeAOT.$rid"
226 )
227 fi
228
229 generate_package_list "$version" ' ' "${pkgs[@]}"
230}
231
232usage () {
233 echo "Usage: $pname [[--sdk] [-o output] sem-version] ...
234Get updated dotnet src (platform - url & sha512) expressions for specified versions
235
236Exit codes:
237 0 Success
238 1 Failure
239 2 Release not found
240
241Examples:
242 $pname 6.0.14 7.0.201 - specific x.y.z versions
243 $pname 6.0 7.0 - latest x.y versions
244" >&2
245}
246
247update() {
248 local -r sem_version=$1 sdk=$2
249 local output=$3
250
251 local patch_specified=false
252 # Check if a patch was specified as an argument.
253 # If so, generate file for the specific version.
254 # If only x.y version was provided, get the latest patch
255 # version of the given x.y version.
256 if [[ "$sem_version" =~ ^[0-9]{1,}\.[0-9]{1,}\.[0-9]{1,} ]]; then
257 patch_specified=true
258 elif [[ ! "$sem_version" =~ ^[0-9]{1,}\.[0-9]{1,}$ ]]; then
259 usage
260 return 1
261 fi
262
263 : ${output:="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"/versions/$sem_version.nix}
264 echo "Generating $output"
265
266 # Make sure the x.y version is properly passed to .NET release metadata url.
267 # Then get the json file and parse it to find the latest patch release.
268 local major_minor content major_minor_patch
269 major_minor=$(sed 's/^\([0-9]*\.[0-9]*\).*$/\1/' <<< "$sem_version")
270 content=$(curl -fsSL https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/"$major_minor"/releases.json)
271 if [[ -n $sdk ]]; then
272 trap '' ERR
273 major_minor_patch=$(
274 jq -er --arg version "$sem_version" '
275 .releases[] |
276 select(.sdks[].version == $version) |
277 ."release-version"' <<< "$content" || if [[ $? == 4 ]]; then exit 2; else exit 1; fi)
278 trap 'exit 1' ERR
279 else
280 major_minor_patch=$([ "$patch_specified" == true ] && echo "$sem_version" || jq -er '."latest-release"' <<< "$content")
281 fi
282 local major_minor_underscore=${major_minor/./_}
283
284 local release_content aspnetcore_version runtime_version
285 local -a sdk_versions
286
287 release_content=$(release "$content" "$major_minor_patch")
288 aspnetcore_version=$(jq -er '."aspnetcore-runtime".version' <<< "$release_content")
289 runtime_version=$(jq -er '.runtime.version' <<< "$release_content")
290
291 if [[ -n $sdk ]]; then
292 sdk_versions=("$sem_version")
293 else
294 mapfile -t sdk_versions < <(jq -er '.sdks[] | .version' <<< "$release_content" | sort -rn)
295 fi
296
297 # If patch was not specified, check if the package is already the latest version
298 # If it is, exit early
299 if [ "$patch_specified" == false ] && [ -f "$output" ]; then
300 local -a versions
301 IFS= readarray -d '' versions < <(
302 nix-instantiate --eval --json -E "{ output }: with (import output {
303 buildAspNetCore = { ... }: {};
304 buildNetSdk = { version, ... }: { inherit version; };
305 buildNetRuntime = { version, ... }: { inherit version; };
306 fetchNupkg = { ... }: {};
307 }); (x: builtins.deepSeq x x) [
308 runtime_${major_minor_underscore}.version
309 sdk_${major_minor_underscore}.version
310 ]" --argstr output "$output" | jq -e --raw-output0 .[])
311 if [[ "${versions[0]}" == "$major_minor_patch" && "${versions[1]}" == "${sdk_versions[0]}" ]]; then
312 echo "Nothing to update."
313 return
314 fi
315 fi
316
317 local aspnetcore_files runtime_files
318 aspnetcore_files="$(release_files "$release_content" .\"aspnetcore-runtime\")"
319 runtime_files="$(release_files "$release_content" .runtime)"
320
321 local channel_version support_phase
322 channel_version=$(jq -er '."channel-version"' <<< "$content")
323 support_phase=$(jq -er '."support-phase"' <<< "$content")
324
325 local aspnetcore_sources runtime_sources
326 aspnetcore_sources="$(platform_sources "$aspnetcore_files")"
327 runtime_sources="$(platform_sources "$runtime_files")"
328
329 result=$(mktemp -t dotnet-XXXXXX.nix)
330 trap "rm -f $result" TERM INT EXIT
331
332 (
333 echo "{ buildAspNetCore, buildNetRuntime, buildNetSdk, fetchNupkg }:
334
335# v$channel_version ($support_phase)
336
337let
338 commonPackages = ["
339 aspnetcore_packages "${aspnetcore_version}"
340 netcore_packages "${runtime_version}"
341 echo " ];
342
343 hostPackages = {"
344 for rid in "${rids[@]}"; do
345 echo " $rid = ["
346 netcore_host_packages "${runtime_version}" "$rid"
347 echo " ];"
348 done
349 echo " };
350
351 targetPackages = {"
352 for rid in "${rids[@]}"; do
353 echo " $rid = ["
354 aspnetcore_target_packages "${aspnetcore_version}" "$rid"
355 netcore_target_packages "${runtime_version}" "$rid"
356 echo " ];"
357 done
358 echo " };
359
360in rec {
361 release_$major_minor_underscore = \"$major_minor_patch\";
362
363 aspnetcore_$major_minor_underscore = buildAspNetCore {
364 version = \"${aspnetcore_version}\";
365 $aspnetcore_sources
366 };
367
368 runtime_$major_minor_underscore = buildNetRuntime {
369 version = \"${runtime_version}\";
370 $runtime_sources
371 };"
372
373 local -A feature_bands
374 unset latest_sdk
375
376 for sdk_version in "${sdk_versions[@]}"; do
377 local sdk_base_version=${sdk_version%-*}
378 local feature_band=${sdk_base_version:0:-2}xx
379 # sometimes one release has e.g. both 8.0.202 and 8.0.203
380 [[ ! ${feature_bands[$feature_band]+true} ]] || continue
381 feature_bands[$feature_band]=$sdk_version
382 local sdk_files sdk_sources
383 sdk_files="$(release_files "$release_content" ".sdks[] | select(.version == \"$sdk_version\")")"
384 sdk_sources="$(platform_sources "$sdk_files")"
385 local sdk_attrname=sdk_${feature_band//./_}
386 [[ -v latest_sdk ]] || local latest_sdk=$sdk_attrname
387
388 echo "
389 $sdk_attrname = buildNetSdk {
390 version = \"${sdk_version}\";
391 $sdk_sources
392 inherit commonPackages hostPackages targetPackages;
393 runtime = runtime_$major_minor_underscore;
394 aspnetcore = aspnetcore_$major_minor_underscore;
395 };"
396 done
397
398 if [[ -n $sdk ]]; then
399 echo "
400 sdk = sdk_$major_minor_underscore;
401"
402 fi
403
404 echo "
405 sdk_$major_minor_underscore = $latest_sdk;
406}"
407 )> "$result"
408
409 nixfmt "$result"
410 cp "$result" "$output"
411 echo "Generated $output"
412}
413
414main () {
415 local pname sdk output
416 pname=$(basename "$0")
417
418 sdk=
419 output=
420
421 while [ $# -gt 0 ]; do
422 case $1 in
423 --sdk)
424 shift
425 sdk=1
426 ;;
427 -o)
428 shift
429 output=$1
430 shift
431 ;;
432 *)
433 update "$1" "$sdk" "$output"
434 shift
435 ;;
436 esac
437 done
438}
439
440main "$@"