pyrox.dev
1#!/usr/bin/env nix-shell
2#!nix-shell -p "python3.withPackages (p: with p; [ tomli tomli-w packaging license-expression])" -i python3
3
4# This file is formatted with `ruff format`.
5
6import os
7import re
8import tomli
9import tomli_w
10import subprocess
11import concurrent.futures
12import argparse
13import tempfile
14import tarfile
15from string import punctuation
16from packaging.version import Version
17from urllib import request
18from collections import OrderedDict
19
20
21class TypstPackage:
22 def __init__(self, **kwargs):
23 self.pname = kwargs["pname"]
24 self.version = kwargs["version"]
25 self.meta = kwargs["meta"]
26 self.path = kwargs["path"]
27 self.repo = (
28 None
29 if "repository" not in self.meta["package"]
30 else self.meta["package"]["repository"]
31 )
32 self.description = self.meta["package"]["description"].rstrip(punctuation)
33 self.license = self.meta["package"]["license"]
34 self.params = "" if "params" not in kwargs else kwargs["params"]
35 self.deps = [] if "deps" not in kwargs else kwargs["deps"]
36
37 @classmethod
38 def package_name_full(cls, package_name, version):
39 version_number = map(lambda x: int(x), version.split("."))
40 version_nix = "_".join(map(lambda x: str(x), version_number))
41 return "_".join((package_name, version_nix))
42
43 def license_tokens(self):
44 import license_expression as le
45
46 try:
47 # FIXME: ad hoc conversion
48 exception_list = [("EUPL-1.2+", "EUPL-1.2")]
49
50 def sanitize_license_string(license_string, lookups):
51 if not lookups:
52 return license_string
53 return sanitize_license_string(
54 license_string.replace(lookups[0][0], lookups[0][1]), lookups[1:]
55 )
56
57 sanitized = sanitize_license_string(self.license, exception_list)
58 licensing = le.get_spdx_licensing()
59 parsed = licensing.parse(sanitized, validate=True)
60 return [s.key for s in licensing.license_symbols(parsed)]
61 except le.ExpressionError as e:
62 print(
63 f'Failed to parse license string "{self.license}" because of {str(e)}'
64 )
65 exit(1)
66
67 def source(self):
68 url = f"https://packages.typst.org/preview/{self.pname}-{self.version}.tar.gz"
69 cmd = [
70 "nix",
71 "store",
72 "prefetch-file",
73 "--unpack",
74 "--hash-type",
75 "sha256",
76 "--refresh",
77 "--extra-experimental-features",
78 "nix-command",
79 ]
80 result = subprocess.run(cmd + [url], capture_output=True, text=True)
81 hash = re.search(r"hash\s+\'(sha256-.{44})\'", result.stderr).groups()[0]
82 return url, hash
83
84 def to_name_full(self):
85 return self.package_name_full(self.pname, self.version)
86
87 def to_attrs(self):
88 deps = set()
89 excludes = list(map(
90 lambda e: os.path.join(self.path, e),
91 self.meta["package"]["exclude"] if "exclude" in self.meta["package"] else [],
92 ))
93 for root, _, files in os.walk(self.path):
94 for file in filter(lambda f: f.split(".")[-1] == "typ", files):
95 file_path = os.path.join(root, file)
96 if file_path in excludes:
97 continue
98 with open(file_path, "r") as f:
99 deps.update(
100 set(
101 re.findall(
102 r"^\s*#import\s+\"@preview/([\w|-]+):(\d+.\d+.\d+)\"",
103 f.read(),
104 re.MULTILINE,
105 )
106 )
107 )
108 self.deps = list(
109 filter(lambda p: p[0] != self.pname or p[1] != self.version, deps)
110 )
111 source_url, source_hash = self.source()
112
113 return dict(
114 url=source_url,
115 hash=source_hash,
116 typstDeps=[
117 self.package_name_full(p, v)
118 for p, v in sorted(self.deps, key=lambda x: (x[0], Version(x[1])))
119 ],
120 description=self.description,
121 license=self.license_tokens(),
122 ) | (dict(homepage=self.repo) if self.repo else dict())
123
124
125def generate_typst_packages(preview_dir, output_file):
126 package_tree = dict()
127
128 print("Parsing metadata... from", preview_dir)
129 for p in os.listdir(preview_dir):
130 package_dir = os.path.join(preview_dir, p)
131 for v in os.listdir(package_dir):
132 package_version_dir = os.path.join(package_dir, v)
133 with open(
134 os.path.join(package_version_dir, "typst.toml"), "rb"
135 ) as meta_file:
136 try:
137 package = TypstPackage(
138 pname=p,
139 version=v,
140 meta=tomli.load(meta_file),
141 path=package_version_dir,
142 )
143 if package.pname in package_tree:
144 package_tree[package.pname][v] = package
145 else:
146 package_tree[package.pname] = dict({v: package})
147 except tomli.TOMLDecodeError:
148 print("Invalid typst.toml:", package_version_dir)
149
150 with open(output_file, "wb") as typst_packages:
151
152 def generate_package(pname, package_subtree):
153 sorted_keys = sorted(package_subtree.keys(), key=Version, reverse=True)
154 print(f"Generating metadata for {pname}")
155 return {
156 pname: OrderedDict(
157 (k, package_subtree[k].to_attrs()) for k in sorted_keys
158 )
159 }
160
161 with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
162 sorted_packages = sorted(package_tree.items(), key=lambda x: x[0])
163 futures = list()
164 for pname, psubtree in sorted_packages:
165 futures.append(executor.submit(generate_package, pname, psubtree))
166 packages = OrderedDict(
167 (package, subtree)
168 for future in futures
169 for package, subtree in future.result().items()
170 )
171 print(f"Writing metadata... to {output_file}")
172 tomli_w.dump(packages, typst_packages)
173
174
175def main(args):
176 PREVIEW_DIR = "packages/preview"
177 TYPST_PACKAGE_TARBALL_URL = (
178 "https://github.com/typst/packages/archive/refs/heads/main.tar.gz"
179 )
180
181 directory = args.directory
182 if not directory:
183 tempdir = tempfile.mkdtemp()
184 print(tempdir)
185 typst_tarball = os.path.join(tempdir, "main.tar.gz")
186
187 print(
188 "Downloading Typst packages source from {} to {}".format(
189 TYPST_PACKAGE_TARBALL_URL, typst_tarball
190 )
191 )
192 with request.urlopen(
193 request.Request(TYPST_PACKAGE_TARBALL_URL), timeout=15.0
194 ) as response:
195 if response.status == 200:
196 with open(typst_tarball, "wb+") as f:
197 f.write(response.read())
198 else:
199 print("Download failed")
200 exit(1)
201 with tarfile.open(typst_tarball) as tar:
202 tar.extractall(path=tempdir, filter="data")
203 directory = os.path.join(tempdir, "packages-main")
204 directory = os.path.abspath(directory)
205
206 generate_typst_packages(
207 os.path.join(directory, PREVIEW_DIR),
208 args.output,
209 )
210
211 exit(0)
212
213
214if __name__ == "__main__":
215 parser = argparse.ArgumentParser()
216 parser.add_argument(
217 "-d", "--directory", help="Local Typst Universe repository", default=None
218 )
219 parser.add_argument(
220 "-o",
221 "--output",
222 help="Output file",
223 default=os.path.join(os.path.abspath("."), "typst-packages-from-universe.toml"),
224 )
225 args = parser.parse_args()
226 main(args)