pyrox.dev
1
2import json
3from pathlib import Path
4import multiprocessing
5import subprocess
6import sys
7import toml
8from urllib.parse import urlparse
9import yaml
10
11import dag
12
13# This should match the behavior of the default unpackPhase.
14# See https://github.com/NixOS/nixpkgs/blob/59fa082abdbf462515facc8800d517f5728c909d/pkgs/stdenv/generic/setup.sh#L1044
15archive_extensions = [
16 # xz extensions
17 ".tar.xz",
18 ".tar.lzma",
19 ".txz",
20
21 # *.tar or *.tar.*
22 ".tar",
23 ".tar.Z",
24 ".tar.bz2",
25 ".tar.gz",
26
27 # Other tar extensions
28 ".tgz",
29 ".tbz2",
30 ".tbz",
31
32 ".zip"
33 ]
34
35def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin):
36 depends_on = set()
37 if closure_dependencies_dag.has_node(uuid):
38 depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids)
39
40 other_libs = extra_libs.get(uuid, [])
41
42 if is_darwin:
43 fixup = f"""fixupPhase = let
44 libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
45 [{" ".join(["uuid-" + x for x in depends_on])}];
46 in ''
47
48 ''"""
49 else:
50 # We provide gcc.cc.lib by default in order to get some common libraries
51 # like libquadmath.so. A number of packages expect this to be available and
52 # will give linker errors if it isn't.
53 fixup = f"""fixupPhase = let
54 libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
55 [{" ".join(["uuid-" + x for x in depends_on])}];
56 in ''
57 find $out -type f -executable -exec \
58 patchelf --set-rpath \\$ORIGIN:\\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc gcc.cc.lib] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \\;
59 find $out -type f -executable -exec \
60 patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \\;
61 ''"""
62
63 return f"""stdenv.mkDerivation {{
64 name = "{artifact_name}";
65 src = fetchurl {{
66 url = "{url}";
67 sha256 = "{sha256}";
68 }};
69 preUnpack = ''
70 mkdir unpacked
71 cd unpacked
72 '';
73 sourceRoot = ".";
74 dontConfigure = true;
75 dontBuild = true;
76 installPhase = "cp -r . $out";
77 {fixup};
78 }}"""
79
80def get_plain_derivation(url, sha256):
81 return f"""fetchurl {{
82 url = "{url}";
83 sha256 = "{sha256}";
84 }}"""
85
86def process_item(args):
87 item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin = args
88 uuid, src = item
89 lines = []
90
91 artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())
92 if not artifacts:
93 return f' uuid-{uuid} = {{}};\n'
94
95 lines.append(f' uuid-{uuid} = {{')
96
97 for artifact_name, details in artifacts.items():
98 if len(details["download"]) == 0:
99 continue
100 download = details["download"][0]
101 url = download["url"]
102 sha256 = download["sha256"]
103
104 git_tree_sha1 = details["git-tree-sha1"]
105
106 parsed_url = urlparse(url)
107 if any(parsed_url.path.endswith(x) for x in archive_extensions):
108 derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
109 else:
110 derivation = get_plain_derivation(url, sha256)
111
112 lines.append(f""" "{artifact_name}" = {{
113 sha1 = "{git_tree_sha1}";
114 path = {derivation};
115 }};\n""")
116
117 lines.append(' };\n')
118
119 return "\n".join(lines)
120
121def main():
122 dependencies_path = Path(sys.argv[1])
123 closure_yaml_path = Path(sys.argv[2])
124 julia_path = Path(sys.argv[3])
125 extract_artifacts_script = Path(sys.argv[4])
126 extra_libs = json.loads(sys.argv[5])
127 is_darwin = json.loads(sys.argv[6])
128 out_path = Path(sys.argv[7])
129
130 with open(dependencies_path, "r") as f:
131 dependencies = yaml.safe_load(f)
132 dependency_uuids = list(dependencies.keys()) # Convert dict_keys to list
133
134 with open(closure_yaml_path, "r") as f:
135 # Build up a map of UUID -> closure information
136 closure_yaml_list = yaml.safe_load(f) or []
137 closure_yaml = {}
138 for item in closure_yaml_list:
139 closure_yaml[item["uuid"]] = item
140
141 # Build up a dependency graph of UUIDs
142 closure_dependencies_dag = dag.DAG()
143 for uuid, contents in closure_yaml.items():
144 if contents.get("depends_on"):
145 closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
146
147 with open(out_path, "w") as f:
148 if is_darwin:
149 f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n")
150 else:
151 f.write("{ lib, fetchurl, gcc, glibc, pkgs, stdenv }:\n\n")
152
153 f.write("rec {\n")
154
155 with multiprocessing.Pool(10) as pool:
156 # Create args tuples for each item
157 process_args = [
158 (item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
159 for item in dependencies.items()
160 ]
161 for s in pool.map(process_item, process_args):
162 f.write(s)
163
164 f.write(f"""
165}}\n""")
166
167if __name__ == "__main__":
168 main()