pyrox.dev
1#! /usr/bin/env nix-shell
2#! nix-shell -i python3 -p python3Packages.pyyaml
3
4import argparse
5import json
6import os
7import re
8import subprocess
9import sys
10import tempfile
11import urllib.request
12from pathlib import Path
13
14import yaml
15
16FAKE_HASH = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
17
18NIXPKGS_ROOT = (
19 subprocess.Popen(
20 ["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE, text=True
21 )
22 .communicate()[0]
23 .strip()
24)
25
26
27def load_code(name, **kwargs):
28 with Path(
29 f"{NIXPKGS_ROOT}/pkgs/development/compilers/flutter/update/{name}.in"
30 ).open("r", encoding="utf-8") as f:
31 code = f.read()
32
33 for key, value in kwargs.items():
34 code = code.replace(f"@{key}@", value)
35
36 return code
37
38
39# Return out paths
40def nix_build(code):
41 with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as temp:
42 temp.write(code)
43 temp.flush()
44 os.fsync(temp.fileno())
45 temp_name = temp.name
46
47 process = subprocess.Popen(
48 [
49 "nix-build",
50 "--impure",
51 "--no-out-link",
52 "--expr",
53 f"with import {NIXPKGS_ROOT} {{}}; callPackage {temp_name} {{}}",
54 ],
55 stdout=subprocess.PIPE,
56 text=True,
57 )
58
59 process.wait()
60 Path(temp_name).unlink() # Clean up the temporary file
61 return process.stdout.read().strip().splitlines()[0]
62
63
64# Return errors
65def nix_build_to_fail(code):
66 with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as temp:
67 temp.write(code)
68 temp.flush()
69 os.fsync(temp.fileno())
70 temp_name = temp.name
71
72 process = subprocess.Popen(
73 [
74 "nix-build",
75 "--impure",
76 "--keep-going",
77 "--no-link",
78 "--expr",
79 f"with import {NIXPKGS_ROOT} {{}}; callPackage {temp_name} {{}}",
80 ],
81 stderr=subprocess.PIPE,
82 text=True,
83 )
84
85 stderr = ""
86 while True:
87 line = process.stderr.readline()
88 if not line:
89 break
90 stderr += line
91 print(line.strip())
92
93 process.wait()
94 Path(temp_name).unlink() # Clean up the temporary file
95 return stderr
96
97
98def get_engine_hashes(engine_version, flutter_version):
99 code = load_code(
100 "get-engine-hashes.nix",
101 nixpkgs_root=NIXPKGS_ROOT,
102 flutter_version=flutter_version,
103 engine_version=engine_version,
104 )
105
106 stderr = nix_build_to_fail(code)
107
108 pattern = re.compile(
109 rf"/nix/store/.*-flutter-engine-source-{engine_version}-(.+?-.+?)-(.+?-.+?).drv':\n\s+specified: .*\n\s+got:\s+(.+?)\n"
110 )
111 matches = pattern.findall(stderr)
112 result_dict = {}
113
114 for match in matches:
115 flutter_platform, architecture, got = match
116 result_dict.setdefault(flutter_platform, {})[architecture] = got
117
118 def sort_dict_recursive(d):
119 return {
120 k: sort_dict_recursive(v) if isinstance(v, dict) else v
121 for k, v in sorted(d.items())
122 }
123
124 return sort_dict_recursive(result_dict)
125
126
127def get_artifact_hashes(flutter_compact_version):
128 code = load_code(
129 "get-artifact-hashes.nix",
130 nixpkgs_root=NIXPKGS_ROOT,
131 flutter_compact_version=flutter_compact_version,
132 )
133
134 stderr = nix_build_to_fail(code)
135
136 pattern = re.compile(
137 r"/nix/store/.*-flutter-artifacts-(.+?)-(.+?).drv':\n\s+specified: .*\n\s+got:\s+(.+?)\n"
138 )
139 matches = pattern.findall(stderr)
140 result_dict = {}
141
142 for match in matches:
143 flutter_platform, architecture, got = match
144 result_dict.setdefault(flutter_platform, {})[architecture] = got
145
146 def sort_dict_recursive(d):
147 return {
148 k: sort_dict_recursive(v) if isinstance(v, dict) else v
149 for k, v in sorted(d.items())
150 }
151
152 return sort_dict_recursive(result_dict)
153
154
155def get_dart_hashes(dart_version, channel):
156 platforms = ["x86_64-linux", "aarch64-linux", "x86_64-darwin", "aarch64-darwin"]
157 result_dict = {}
158 for platform in platforms:
159 code = load_code(
160 "get-dart-hashes.nix",
161 dart_version=dart_version,
162 channel=channel,
163 platform=platform,
164 )
165 stderr = nix_build_to_fail(code)
166
167 pattern = re.compile(r"got:\s+(.+?)\n")
168 result_dict[platform] = pattern.findall(stderr)[0]
169
170 return result_dict
171
172
173def get_flutter_hash_and_src(flutter_version):
174 code = load_code("get-flutter.nix", flutter_version=flutter_version, hash="")
175
176 stderr = nix_build_to_fail(code)
177 pattern = re.compile(r"got:\s+(.+?)\n")
178 flutter_hash_value = pattern.findall(stderr)[0]
179
180 code = load_code(
181 "get-flutter.nix", flutter_version=flutter_version, hash=flutter_hash_value
182 )
183
184 return (flutter_hash_value, nix_build(code))
185
186
187def get_pubspec_lock(flutter_compact_version, flutter_src):
188 code = load_code(
189 "get-pubspec-lock.nix",
190 flutter_compact_version=flutter_compact_version,
191 flutter_src=flutter_src,
192 hash="",
193 )
194
195 stderr = nix_build_to_fail(code)
196 pattern = re.compile(r"got:\s+(.+?)\n")
197 pubspec_lock_hash = pattern.findall(stderr)[0]
198
199 code = load_code(
200 "get-pubspec-lock.nix",
201 flutter_compact_version=flutter_compact_version,
202 flutter_src=flutter_src,
203 hash=pubspec_lock_hash,
204 )
205
206 pubspec_lock_file = nix_build(code)
207
208 with Path(pubspec_lock_file).open("r", encoding="utf-8") as f:
209 pubspec_lock_yaml = f.read()
210
211 return yaml.safe_load(pubspec_lock_yaml)
212
213
214def get_engine_swiftshader_rev(engine_version):
215 with urllib.request.urlopen(
216 f"https://github.com/flutter/flutter/raw/{engine_version}/DEPS"
217 ) as f:
218 deps = f.read().decode("utf-8")
219 pattern = re.compile(
220 r"Var\('swiftshader_git'\) \+ '\/SwiftShader\.git' \+ '@' \+ \'([0-9a-fA-F]{40})\'\,"
221 )
222 return pattern.findall(deps)[0]
223
224
225def get_engine_swiftshader_hash(engine_swiftshader_rev):
226 code = load_code(
227 "get-engine-swiftshader.nix",
228 engine_swiftshader_rev=engine_swiftshader_rev,
229 hash="",
230 )
231
232 stderr = nix_build_to_fail(code)
233 pattern = re.compile(r"got:\s+(.+?)\n")
234 return pattern.findall(stderr)[0]
235
236
237def write_data(
238 nixpkgs_flutter_version_directory,
239 flutter_version,
240 channel,
241 engine_hash,
242 engine_hashes,
243 engine_swiftshader_hash,
244 engine_swiftshader_rev,
245 dart_version,
246 dart_hash,
247 flutter_hash,
248 artifact_hashes,
249 pubspec_lock,
250):
251 with Path(f"{nixpkgs_flutter_version_directory}/data.json").open(
252 "w", encoding="utf-8"
253 ) as f:
254 f.write(
255 json.dumps(
256 {
257 "version": flutter_version,
258 "engineVersion": engine_hash,
259 "engineSwiftShaderHash": engine_swiftshader_hash,
260 "engineSwiftShaderRev": engine_swiftshader_rev,
261 "channel": channel,
262 "engineHashes": engine_hashes,
263 "dartVersion": dart_version,
264 "dartHash": dart_hash,
265 "flutterHash": flutter_hash,
266 "artifactHashes": artifact_hashes,
267 "pubspecLock": pubspec_lock,
268 },
269 indent=2,
270 ).strip()
271 + "\n"
272 )
273
274
275def update_all_packages():
276 versions_directory = f"{NIXPKGS_ROOT}/pkgs/development/compilers/flutter/versions"
277 versions = [d.name for d in Path(versions_directory).iterdir()]
278 versions = sorted(
279 versions,
280 key=lambda x: (int(x.split("_")[0]), int(x.split("_")[1])),
281 reverse=True,
282 )
283
284 new_content = [
285 "flutterPackages-bin = recurseIntoAttrs (callPackage ../development/compilers/flutter { });",
286 "flutterPackages-source = recurseIntoAttrs (",
287 " callPackage ../development/compilers/flutter { useNixpkgsEngine = true; }",
288 ");",
289 "flutterPackages = flutterPackages-bin;",
290 "flutter = flutterPackages.stable;",
291 ] + [
292 f"flutter{version.replace('_', '')} = flutterPackages.v{version};"
293 for version in versions
294 ]
295
296 with Path(f"{NIXPKGS_ROOT}/pkgs/top-level/all-packages.nix").open(
297 "r", encoding="utf-8"
298 ) as file:
299 lines = file.read().splitlines(keepends=True)
300
301 start = -1
302 end = -1
303 for i, line in enumerate(lines):
304 if (
305 "flutterPackages-bin = recurseIntoAttrs (callPackage ../development/compilers/flutter { });"
306 in line
307 ):
308 start = i
309 if start != -1 and len(line.strip()) == 0:
310 end = i
311 break
312
313 if start != -1 and end != -1:
314 del lines[start:end]
315 lines[start:start] = [f" {line}\n" for line in new_content]
316
317 with Path(f"{NIXPKGS_ROOT}/pkgs/top-level/all-packages.nix").open(
318 "w", encoding="utf-8"
319 ) as file:
320 file.write("".join(lines))
321
322
323# Finds Flutter version, Dart version, and Engine hash.
324# If the Flutter version is given, it uses that. Otherwise finds the
325# latest stable Flutter version.
326def find_versions(flutter_version=None, channel=None):
327 engine_hash = None
328 dart_version = None
329
330 releases = json.load(
331 urllib.request.urlopen(
332 "https://storage.googleapis.com/flutter_infra_release/releases/releases_linux.json"
333 )
334 )
335
336 if not channel:
337 channel = "stable"
338
339 if not flutter_version:
340 release_hash = releases["current_release"][channel]
341 release = next(
342 filter(
343 lambda release: release["hash"] == release_hash, releases["releases"]
344 )
345 )
346 flutter_version = release["version"]
347
348 tags = (
349 subprocess.Popen(
350 ["git", "ls-remote", "--tags", "https://github.com/flutter/flutter.git"],
351 stdout=subprocess.PIPE,
352 text=True,
353 )
354 .communicate()[0]
355 .strip()
356 )
357
358 try:
359 flutter_hash = (
360 next(
361 filter(
362 lambda line: line.endswith(f"refs/tags/{flutter_version}"),
363 tags.splitlines(),
364 )
365 )
366 .split("refs")[0]
367 .strip()
368 )
369
370 engine_hash = (
371 urllib.request.urlopen(
372 f"https://github.com/flutter/flutter/raw/{flutter_hash}/bin/internal/engine.version"
373 )
374 .read()
375 .decode("utf-8")
376 .strip()
377 )
378 except StopIteration:
379 sys.exit(f"Couldn't find Engine hash for Flutter version: {flutter_version}")
380
381 try:
382 dart_version = next(
383 filter(
384 lambda release: release["version"] == flutter_version,
385 releases["releases"],
386 )
387 )["dart_sdk_version"]
388
389 if " " in dart_version:
390 dart_version = dart_version.split(" ")[2][:-1]
391 except StopIteration:
392 sys.exit(f"Couldn't find Dart version for Flutter version: {flutter_version}")
393
394 return (flutter_version, engine_hash, dart_version, channel)
395
396
397def main():
398 parser = argparse.ArgumentParser(description="Update Flutter in Nixpkgs")
399 parser.add_argument("--version", type=str, help="Specify Flutter version")
400 parser.add_argument("--channel", type=str, help="Specify Flutter release channel")
401 parser.add_argument(
402 "--artifact-hashes", action="store_true", help="Whether to get artifact hashes"
403 )
404 args = parser.parse_args()
405
406 (flutter_version, engine_hash, dart_version, channel) = find_versions(
407 args.version, args.channel
408 )
409
410 flutter_compact_version = "_".join(flutter_version.split(".")[:2])
411
412 if args.artifact_hashes:
413 print(
414 json.dumps(get_artifact_hashes(flutter_compact_version), indent=2).strip()
415 + "\n"
416 )
417 return
418
419 print(
420 f"Flutter version: {flutter_version} ({flutter_compact_version}) on ({channel})"
421 )
422 print(f"Engine hash: {engine_hash}")
423 print(f"Dart version: {dart_version}")
424
425 dart_hash = get_dart_hashes(dart_version, channel)
426 (flutter_hash, flutter_src) = get_flutter_hash_and_src(flutter_version)
427
428 nixpkgs_flutter_version_directory = f"{NIXPKGS_ROOT}/pkgs/development/compilers/flutter/versions/{flutter_compact_version}"
429
430 if Path(f"{nixpkgs_flutter_version_directory}/data.json").exists():
431 Path(f"{nixpkgs_flutter_version_directory}/data.json").unlink()
432 Path(nixpkgs_flutter_version_directory).mkdir(parents=True, exist_ok=True)
433
434 update_all_packages()
435
436 common_data_args = {
437 "nixpkgs_flutter_version_directory": nixpkgs_flutter_version_directory,
438 "flutter_version": flutter_version,
439 "channel": channel,
440 "dart_version": dart_version,
441 "engine_hash": engine_hash,
442 "flutter_hash": flutter_hash,
443 "dart_hash": dart_hash,
444 }
445
446 write_data(
447 pubspec_lock={},
448 artifact_hashes={},
449 engine_hashes={},
450 engine_swiftshader_hash=FAKE_HASH,
451 engine_swiftshader_rev="0",
452 **common_data_args,
453 )
454
455 pubspec_lock = get_pubspec_lock(flutter_compact_version, flutter_src)
456
457 write_data(
458 pubspec_lock=pubspec_lock,
459 artifact_hashes={},
460 engine_hashes={},
461 engine_swiftshader_hash=FAKE_HASH,
462 engine_swiftshader_rev="0",
463 **common_data_args,
464 )
465
466 artifact_hashes = get_artifact_hashes(flutter_compact_version)
467
468 write_data(
469 pubspec_lock=pubspec_lock,
470 artifact_hashes=artifact_hashes,
471 engine_hashes={},
472 engine_swiftshader_hash=FAKE_HASH,
473 engine_swiftshader_rev="0",
474 **common_data_args,
475 )
476
477 engine_hashes = get_engine_hashes(engine_hash, flutter_version)
478
479 write_data(
480 pubspec_lock=pubspec_lock,
481 artifact_hashes=artifact_hashes,
482 engine_hashes=engine_hashes,
483 engine_swiftshader_hash=FAKE_HASH,
484 engine_swiftshader_rev="0",
485 **common_data_args,
486 )
487
488 engine_swiftshader_rev = get_engine_swiftshader_rev(engine_hash)
489 engine_swiftshader_hash = get_engine_swiftshader_hash(engine_swiftshader_rev)
490
491 write_data(
492 pubspec_lock=pubspec_lock,
493 artifact_hashes=artifact_hashes,
494 engine_hashes=engine_hashes,
495 engine_swiftshader_hash=engine_swiftshader_hash,
496 engine_swiftshader_rev=engine_swiftshader_rev,
497 **common_data_args,
498 )
499
500
501if __name__ == "__main__":
502 main()