pyrox.dev
1#!/usr/bin/env nix-shell
2#! nix-shell --pure
3#! nix-shell -i bash
4#! nix-shell -p curl cacert
5#! nix-shell -p git
6#! nix-shell -p nix
7#! nix-shell -p jq
8
9set -o pipefail
10
11# How the refresher works:
12#
13# For a given list of <targets>:
14# 1. fetch latest successful '.build` job
15# 2. fetch oldest evaluation that contained that '.build', extract nixpkgs commit
16# 3. fetch all the `.build` artifacts from '$out/on-server/' directory
17# 4. calculate hashes and craft the commit message with the details on
18# how to upload the result to 'tarballs.nixos.org'
19
20scratch_dir=$(mktemp -d)
21trap 'rm -rf -- "${scratch_dir}"' EXIT
22
23usage() {
24 cat >&2 <<EOF
25Usage:
26 $0 [ --commit ] --targets=<target>[,<target>,...]
27
28 The tool must be ran from the root directory of 'nixpkgs' repository.
29
30Synopsis:
31 'refresh-tarballs.bash' script fetches latest bootstrapFiles built
32 by hydra, registers them in 'nixpkgs' and provides commands to
33 upload seed files to 'tarballs.nixos.org'.
34
35 This is usually done in the following cases:
36
37 1. Single target fix: current bootstrap files for a single target
38 are problematic for some reason (target-specific bug). In this
39 case we can refresh just that target as:
40
41 \$ $0 --commit --targets=i686-unknown-linux-gnu
42
43 2. Routine refresh: all bootstrap files should be refreshed to avoid
44 debugging problems that only occur on very old binaries.
45
46 \$ $0 --commit --all-targets
47
48To get help on uploading refreshed binaries to 'tarballs.nixos.org'
49please have a look at <maintainers/scripts/bootstrap-files/README.md>.
50EOF
51 exit 1
52}
53
54# log helpers
55
56die() {
57 echo "ERROR: $*" >&2
58 exit 1
59}
60
61info() {
62 echo "INFO: $*" >&2
63}
64
65[[ ${#@} -eq 0 ]] && usage
66
67# known targets
68
69NATIVE_TARGETS=(
70 aarch64-unknown-linux-gnu
71 aarch64-unknown-linux-musl
72 i686-unknown-linux-gnu
73 x86_64-unknown-linux-gnu
74 x86_64-unknown-linux-musl
75 aarch64-apple-darwin
76 x86_64-apple-darwin
77)
78
79is_native() {
80 local t target=$1
81 for t in "${NATIVE_TARGETS[@]}"; do
82 [[ $t == $target ]] && return 0
83 done
84 return 1
85}
86
87CROSS_TARGETS=(
88 armv5tel-unknown-linux-gnueabi
89 armv6l-unknown-linux-gnueabihf
90 armv6l-unknown-linux-musleabihf
91 armv7l-unknown-linux-gnueabihf
92 mips64el-unknown-linux-gnuabi64
93 mips64el-unknown-linux-gnuabin32
94 mipsel-unknown-linux-gnu
95 powerpc64-unknown-linux-gnuabielfv1
96 powerpc64-unknown-linux-gnuabielfv2
97 powerpc64le-unknown-linux-gnu
98 riscv64-unknown-linux-gnu
99 s390x-unknown-linux-gnu
100 x86_64-unknown-freebsd
101 loongarch64-unknown-linux-gnu
102)
103
104is_cross() {
105 local t target=$1
106 for t in "${CROSS_TARGETS[@]}"; do
107 [[ $t == $target ]] && return 0
108 done
109 return 1
110}
111
112nar_sri_get() {
113 local restore_path store_path
114 ((${#@} != 2)) && die "nar_sri_get /path/to/name.nar.xz name"
115 restore_path="${scratch_dir}/$2"
116 xz -d < "$1" | nix-store --restore "${restore_path}"
117 [[ $? -ne 0 ]] && die "Failed to unpack '$1'"
118
119 store_path=$(nix-store --add "${restore_path}")
120 [[ $? -ne 0 ]] && die "Failed to add '$restore_path' to store"
121 rm -rf -- "${restore_path}"
122
123 nix-hash --to-sri "$(nix-store --query --hash "${store_path}")"
124}
125
126# collect passed options
127
128targets=()
129commit=no
130
131for arg in "$@"; do
132 case "$arg" in
133 --all-targets)
134 targets+=(
135 ${CROSS_TARGETS[@]}
136 ${NATIVE_TARGETS[@]}
137 )
138 ;;
139 --targets=*)
140 # Convert "--targets=a,b,c" to targets=(a b c) bash array.
141 comma_targets=${arg#--targets=}
142 targets+=(${comma_targets//,/ })
143 ;;
144 --commit)
145 commit=yes
146 ;;
147 *)
148 usage
149 ;;
150 esac
151done
152
153for target in "${targets[@]}"; do
154 # Native and cross jobsets differ a bit. We'll have to pick the
155 # one based on target name:
156 if is_native $target; then
157 jobset=nixpkgs/trunk
158 job="stdenvBootstrapTools.${target}.build"
159 elif is_cross $target; then
160 jobset=nixpkgs/cross-trunk
161 job="bootstrapTools.${target}.build"
162 else
163 die "'$target' is not present in either of 'NATIVE_TARGETS' or 'CROSS_TARGETS'. Please add one."
164 fi
165
166 # 'nixpkgs' prefix where we will write new tarball hashes
167 case "$target" in
168 *linux*) nixpkgs_prefix="pkgs/stdenv/linux" ;;
169 *darwin*) nixpkgs_prefix="pkgs/stdenv/darwin" ;;
170 *freebsd*) nixpkgs_prefix="pkgs/stdenv/freebsd" ;;
171 *) die "don't know where to put '$target'" ;;
172 esac
173
174 # We enforce s3 prefix for all targets here. This slightly differs
175 # from manual uploads targets where names were chosen inconsistently.
176 s3_prefix="stdenv/$target"
177
178 # resolve 'latest' build to the build 'id', construct the link.
179 latest_build_uri="https://hydra.nixos.org/job/$jobset/$job/latest"
180 latest_build="$target.latest-build"
181 info "Fetching latest successful build from '${latest_build_uri}'"
182 curl -s -H "Content-Type: application/json" -L "$latest_build_uri" > "$latest_build"
183 [[ $? -ne 0 ]] && die "Failed to fetch latest successful build"
184 latest_build_id=$(jq '.id' < "$latest_build")
185 [[ $? -ne 0 ]] && die "Did not find 'id' in latest build"
186 build_uri="https://hydra.nixos.org/build/${latest_build_id}"
187
188 # We pick oldest jobset evaluation and extract the 'nicpkgs' commit.
189 #
190 # We use oldest instead of latest to make the result more stable
191 # across unrelated 'nixpkgs' updates. Ideally two subsequent runs of
192 # this refresher should produce the same output (provided there are
193 # no bootstrapTools updates committed between the two runs).
194 oldest_eval_id=$(jq '.jobsetevals|min' < "$latest_build")
195 [[ $? -ne 0 ]] && die "Did not find 'jobsetevals' in latest build"
196 eval_uri="https://hydra.nixos.org/eval/${oldest_eval_id}"
197 eval_meta="$target.eval-meta"
198 info "Fetching oldest eval details from '${eval_uri}' (can take a minute)"
199 curl -s -H "Content-Type: application/json" -L "${eval_uri}" > "$eval_meta"
200 [[ $? -ne 0 ]] && die "Failed to fetch eval metadata"
201 nixpkgs_revision=$(jq --raw-output ".jobsetevalinputs.nixpkgs.revision" < "$eval_meta")
202 [[ $? -ne 0 ]] && die "Failed to fetch revision"
203
204 # Extract the build paths out of the build metadata
205 drvpath=$(jq --raw-output '.drvpath' < "${latest_build}")
206 [[ $? -ne 0 ]] && die "Did not find 'drvpath' in latest build"
207 outpath=$(jq --raw-output '.buildoutputs.out.path' < "${latest_build}")
208 [[ $? -ne 0 ]] && die "Did not find 'buildoutputs' in latest build"
209 build_timestamp=$(jq --raw-output '.timestamp' < "${latest_build}")
210 [[ $? -ne 0 ]] && die "Did not find 'timestamp' in latest build"
211 build_time=$(TZ=UTC LANG=C date --date="@${build_timestamp}" --rfc-email)
212 [[ $? -ne 0 ]] && die "Failed to format timestamp"
213
214 info "Fetching bootstrap tools to calculate hashes from '${outpath}'"
215 nix-store --realize "$outpath"
216 [[ $? -ne 0 ]] && die "Failed to fetch '${outpath}' from hydra"
217
218 fnames=()
219
220 target_file="${nixpkgs_prefix}/bootstrap-files/${target}.nix"
221 info "Writing '${target_file}'"
222 {
223 # header
224 cat <<EOF
225# Autogenerated by maintainers/scripts/bootstrap-files/refresh-tarballs.bash as:
226# $ ./refresh-tarballs.bash --targets=${target}
227#
228# Metadata:
229# - nixpkgs revision: ${nixpkgs_revision}
230# - hydra build: ${latest_build_uri}
231# - resolved hydra build: ${build_uri}
232# - instantiated derivation: ${drvpath}
233# - output directory: ${outpath}
234# - build time: ${build_time}
235{
236EOF
237 for p in "${outpath}/on-server"/*; do
238 fname=$(basename "$p")
239 fnames+=("$fname")
240 case "$fname" in
241 bootstrap-tools.tar.xz) attr=bootstrapTools ;;
242 busybox) attr=$fname ;;
243 unpack.nar.xz) attr=unpack ;;
244 *) die "Don't know how to map '$fname' to attribute name. Please update me."
245 esac
246
247 executable_arg=
248 executable_nix=
249 if [[ -x "$p" ]]; then
250 executable_arg="--executable"
251 executable_nix="executable = true;"
252 fi
253 unpack_nix=
254 name_nix=
255 if [[ $fname = *.nar.xz ]]; then
256 unpack_nix="unpack = true;"
257 name_nix="name = \"${fname%.nar.xz}\";"
258 sri=$(nar_sri_get "$p" "${fname%.nar.xz}")
259 [[ $? -ne 0 ]] && die "Failed to get hash of '$p'"
260 else
261 sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p")
262 [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'"
263 sri=$(nix-hash --to-sri "sha256:$sha256")
264 [[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form"
265 fi
266
267 # individual file entries
268 cat <<EOF
269 $attr = import <nix/fetchurl.nix> {
270 url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname";
271 hash = "${sri}";$(
272 [[ -n ${executable_nix} ]] && printf "\n %s" "${executable_nix}"
273 [[ -n ${name_nix} ]] && printf "\n %s" "${name_nix}"
274 [[ -n ${unpack_nix} ]] && printf "\n %s" "${unpack_nix}"
275 )
276 };
277EOF
278 done
279 # footer
280 cat <<EOF
281}
282EOF
283 } > "${target_file}"
284
285 target_file_commit_msg=${target}.commit_message
286 cat > "$target_file_commit_msg" <<EOF
287${nixpkgs_prefix}: update ${target} bootstrap-files
288
289sha256sum of files to be uploaded:
290
291$(
292echo "$ sha256sum ${outpath}/on-server/*"
293sha256sum ${outpath}/on-server/*
294)
295
296Suggested commands to upload files to 'tarballs.nixos.org':
297
298 $ nix-store --realize ${outpath}
299 $ aws s3 cp --recursive --acl public-read ${outpath}/on-server/ s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision}
300 $ aws s3 cp --recursive s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision} ./
301 $ sha256sum ${fnames[*]}
302 $ sha256sum ${outpath}/on-server/*
303EOF
304
305 cat "$target_file_commit_msg"
306 if [[ $commit == yes ]]; then
307 git commit "${target_file}" -F "$target_file_commit_msg"
308 else
309 info "DRY RUN: git commit ${target_file} -F $target_file_commit_msg"
310 fi
311 rm -- "$target_file_commit_msg"
312
313 # delete temp files
314 rm -- "$latest_build" "$eval_meta"
315done