pyrox.dev
1# Used by pkgs/misc/vim-plugins/update.py and pkgs/applications/editors/kakoune/plugins/update.py
2
3# format:
4# $ nix run nixpkgs.python3Packages.black -c black update.py
5# type-check:
6# $ nix run nixpkgs.python3Packages.mypy -c mypy update.py
7# linted:
8# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
9
10import argparse
11import functools
12import http
13import json
14import os
15import subprocess
16import logging
17import sys
18import time
19import traceback
20import urllib.error
21import urllib.parse
22import urllib.request
23import xml.etree.ElementTree as ET
24from datetime import datetime
25from functools import wraps
26from multiprocessing.dummy import Pool
27from pathlib import Path
28from typing import Dict, List, Optional, Tuple, Union, Any, Callable
29from urllib.parse import urljoin, urlparse
30from tempfile import NamedTemporaryFile
31from dataclasses import dataclass
32
33import git
34
35ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" # " vim gets confused here
36ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # "
37ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # "
38
39LOG_LEVELS = {
40 logging.getLevelName(level): level for level in [
41 logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR ]
42}
43
44log = logging.getLogger()
45
46def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
47 """Retry calling the decorated function using an exponential backoff.
48 http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
49 original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
50 (BSD licensed)
51 :param ExceptionToCheck: the exception on which to retry
52 :param tries: number of times to try (not retry) before giving up
53 :param delay: initial delay between retries in seconds
54 :param backoff: backoff multiplier e.g. value of 2 will double the delay
55 each retry
56 """
57
58 def deco_retry(f: Callable) -> Callable:
59 @wraps(f)
60 def f_retry(*args: Any, **kwargs: Any) -> Any:
61 mtries, mdelay = tries, delay
62 while mtries > 1:
63 try:
64 return f(*args, **kwargs)
65 except ExceptionToCheck as e:
66 print(f"{str(e)}, Retrying in {mdelay} seconds...")
67 time.sleep(mdelay)
68 mtries -= 1
69 mdelay *= backoff
70 return f(*args, **kwargs)
71
72 return f_retry # true decorator
73
74 return deco_retry
75
76
77def make_request(url: str) -> urllib.request.Request:
78 token = os.getenv("GITHUB_API_TOKEN")
79 headers = {}
80 if token is not None:
81 headers["Authorization"] = f"token {token}"
82 return urllib.request.Request(url, headers=headers)
83
84@dataclass
85class PluginDesc:
86 owner: str
87 repo: str
88 branch: str
89 alias: Optional[str]
90
91
92class Repo:
93 def __init__(
94 self, owner: str, name: str, branch: str, alias: Optional[str]
95 ) -> None:
96 self.owner = owner
97 self.name = name
98 self.branch = branch
99 self.alias = alias
100 self.redirect: Dict[str, str] = {}
101
102 def url(self, path: str) -> str:
103 return urljoin(f"https://github.com/{self.owner}/{self.name}/", path)
104
105 def __repr__(self) -> str:
106 return f"Repo({self.owner}, {self.name})"
107
108 @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
109 def has_submodules(self) -> bool:
110 try:
111 req = make_request(self.url(f"blob/{self.branch}/.gitmodules"))
112 urllib.request.urlopen(req, timeout=10).close()
113 except urllib.error.HTTPError as e:
114 if e.code == 404:
115 return False
116 else:
117 raise
118 return True
119
120 @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
121 def latest_commit(self) -> Tuple[str, datetime]:
122 commit_url = self.url(f"commits/{self.branch}.atom")
123 commit_req = make_request(commit_url)
124 with urllib.request.urlopen(commit_req, timeout=10) as req:
125 self.check_for_redirect(commit_url, req)
126 xml = req.read()
127 root = ET.fromstring(xml)
128 latest_entry = root.find(ATOM_ENTRY)
129 assert latest_entry is not None, f"No commits found in repository {self}"
130 commit_link = latest_entry.find(ATOM_LINK)
131 assert commit_link is not None, f"No link tag found feed entry {xml}"
132 url = urlparse(commit_link.get("href"))
133 updated_tag = latest_entry.find(ATOM_UPDATED)
134 assert (
135 updated_tag is not None and updated_tag.text is not None
136 ), f"No updated tag found feed entry {xml}"
137 updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ")
138 return Path(str(url.path)).name, updated
139
140 def check_for_redirect(self, url: str, req: http.client.HTTPResponse):
141 response_url = req.geturl()
142 if url != response_url:
143 new_owner, new_name = (
144 urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
145 )
146 end_line = "\n" if self.alias is None else f" as {self.alias}\n"
147 plugin_line = "{owner}/{name}" + end_line
148
149 old_plugin = plugin_line.format(owner=self.owner, name=self.name)
150 new_plugin = plugin_line.format(owner=new_owner, name=new_name)
151 self.redirect[old_plugin] = new_plugin
152
153 def prefetch_git(self, ref: str) -> str:
154 data = subprocess.check_output(
155 ["nix-prefetch-git", "--fetch-submodules", self.url(""), ref]
156 )
157 return json.loads(data)["sha256"]
158
159 def prefetch_github(self, ref: str) -> str:
160 data = subprocess.check_output(
161 ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
162 )
163 return data.strip().decode("utf-8")
164
165
166class Plugin:
167 def __init__(
168 self,
169 name: str,
170 commit: str,
171 has_submodules: bool,
172 sha256: str,
173 date: Optional[datetime] = None,
174 ) -> None:
175 self.name = name
176 self.commit = commit
177 self.has_submodules = has_submodules
178 self.sha256 = sha256
179 self.date = date
180
181 @property
182 def normalized_name(self) -> str:
183 return self.name.replace(".", "-")
184
185 @property
186 def version(self) -> str:
187 assert self.date is not None
188 return self.date.strftime("%Y-%m-%d")
189
190 def as_json(self) -> Dict[str, str]:
191 copy = self.__dict__.copy()
192 del copy["date"]
193 return copy
194
195
196class Editor:
197 """The configuration of the update script."""
198
199 def __init__(
200 self,
201 name: str,
202 root: Path,
203 get_plugins: str,
204 default_in: Optional[Path] = None,
205 default_out: Optional[Path] = None,
206 deprecated: Optional[Path] = None,
207 cache_file: Optional[str] = None,
208 ):
209 log.debug("get_plugins:", get_plugins)
210 self.name = name
211 self.root = root
212 self.get_plugins = get_plugins
213 self.default_in = default_in or root.joinpath(f"{name}-plugin-names")
214 self.default_out = default_out or root.joinpath("generated.nix")
215 self.deprecated = deprecated or root.joinpath("deprecated.json")
216 self.cache_file = cache_file or f"{name}-plugin-cache.json"
217
218 def get_current_plugins(self):
219 """To fill the cache"""
220 return get_current_plugins(self)
221
222 def load_plugin_spec(self, plugin_file) -> List[PluginDesc]:
223 return load_plugin_spec(plugin_file)
224
225 def generate_nix(self, plugins, outfile: str):
226 '''Returns nothing for now, writes directly to outfile'''
227 raise NotImplementedError()
228
229 def get_update(self, input_file: str, outfile: str, proc: int):
230 return get_update(input_file, outfile, proc, editor=self)
231
232 @property
233 def attr_path(self):
234 return self.name + "Plugins"
235
236 def get_drv_name(self, name: str):
237 return self.attr_path + "." + name
238
239 def rewrite_input(self, *args, **kwargs):
240 return rewrite_input(*args, **kwargs)
241
242 def create_parser(self):
243 parser = argparse.ArgumentParser(
244 description=(
245 f"Updates nix derivations for {self.name} plugins"
246 f"By default from {self.default_in} to {self.default_out}"
247 )
248 )
249 parser.add_argument(
250 "--add",
251 dest="add_plugins",
252 default=[],
253 action="append",
254 help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo",
255 )
256 parser.add_argument(
257 "--input-names",
258 "-i",
259 dest="input_file",
260 default=self.default_in,
261 help="A list of plugins in the form owner/repo",
262 )
263 parser.add_argument(
264 "--out",
265 "-o",
266 dest="outfile",
267 default=self.default_out,
268 help="Filename to save generated nix code",
269 )
270 parser.add_argument(
271 "--proc",
272 "-p",
273 dest="proc",
274 type=int,
275 default=30,
276 help="Number of concurrent processes to spawn.",
277 )
278 parser.add_argument(
279 "--no-commit", "-n", action="store_true", default=False,
280 help="Whether to autocommit changes"
281 )
282 parser.add_argument(
283 "--debug", "-d", choices=LOG_LEVELS.keys(),
284 default=logging.getLevelName(logging.WARN),
285 help="Adjust log level"
286 )
287 return parser
288
289
290
291class CleanEnvironment(object):
292 def __enter__(self) -> None:
293 self.old_environ = os.environ.copy()
294 local_pkgs = str(Path(__file__).parent.parent.parent)
295 os.environ["NIX_PATH"] = f"localpkgs={local_pkgs}"
296 self.empty_config = NamedTemporaryFile()
297 self.empty_config.write(b"{}")
298 self.empty_config.flush()
299 os.environ["NIXPKGS_CONFIG"] = self.empty_config.name
300
301 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
302 os.environ.update(self.old_environ)
303 self.empty_config.close()
304
305
306def get_current_plugins(editor: Editor) -> List[Plugin]:
307 with CleanEnvironment():
308 cmd = ["nix", "eval", "--impure", "--json", "--expr", editor.get_plugins]
309 log.debug("Running command %s", cmd)
310 out = subprocess.check_output(cmd)
311 data = json.loads(out)
312 plugins = []
313 for name, attr in data.items():
314 p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"])
315 plugins.append(p)
316 return plugins
317
318
319def prefetch_plugin(
320 p: PluginDesc,
321 cache: "Optional[Cache]" = None,
322) -> Tuple[Plugin, Dict[str, str]]:
323 user, repo_name, branch, alias = p.owner, p.repo, p.branch, p.alias
324 log.info(f"Fetching last commit for plugin {user}/{repo_name}@{branch}")
325 repo = Repo(user, repo_name, branch, alias)
326 commit, date = repo.latest_commit()
327 has_submodules = repo.has_submodules()
328 cached_plugin = cache[commit] if cache else None
329 if cached_plugin is not None:
330 log.debug("Cache hit !")
331 cached_plugin.name = alias or repo_name
332 cached_plugin.date = date
333 return cached_plugin, repo.redirect
334
335 print(f"prefetch {user}/{repo_name}")
336 if has_submodules:
337 sha256 = repo.prefetch_git(commit)
338 else:
339 sha256 = repo.prefetch_github(commit)
340
341 return (
342 Plugin(alias or repo_name, commit, has_submodules, sha256, date=date),
343 repo.redirect,
344 )
345
346
347def fetch_plugin_from_pluginline(plugin_line: str) -> Plugin:
348 plugin, _ = prefetch_plugin(parse_plugin_line(plugin_line))
349 return plugin
350
351
352def print_download_error(plugin: str, ex: Exception):
353 print(f"{plugin}: {ex}", file=sys.stderr)
354 ex_traceback = ex.__traceback__
355 tb_lines = [
356 line.rstrip("\n")
357 for line in traceback.format_exception(ex.__class__, ex, ex_traceback)
358 ]
359 print("\n".join(tb_lines))
360
361
362def check_results(
363 results: List[Tuple[str, str, Union[Exception, Plugin], Dict[str, str]]]
364) -> Tuple[List[Tuple[str, str, Plugin]], Dict[str, str]]:
365 failures: List[Tuple[str, Exception]] = []
366 plugins = []
367 redirects: Dict[str, str] = {}
368 for (owner, name, result, redirect) in results:
369 if isinstance(result, Exception):
370 failures.append((name, result))
371 else:
372 plugins.append((owner, name, result))
373 redirects.update(redirect)
374
375 print(f"{len(results) - len(failures)} plugins were checked", end="")
376 if len(failures) == 0:
377 print()
378 return plugins, redirects
379 else:
380 print(f", {len(failures)} plugin(s) could not be downloaded:\n")
381
382 for (plugin, exception) in failures:
383 print_download_error(plugin, exception)
384
385 sys.exit(1)
386
387def parse_plugin_line(line: str) -> PluginDesc:
388 branch = "master"
389 alias = None
390 name, repo = line.split("/")
391 if " as " in repo:
392 repo, alias = repo.split(" as ")
393 alias = alias.strip()
394 if "@" in repo:
395 repo, branch = repo.split("@")
396
397 return PluginDesc(name.strip(), repo.strip(), branch.strip(), alias)
398
399
400def load_plugin_spec(plugin_file: str) -> List[PluginDesc]:
401 plugins = []
402 with open(plugin_file) as f:
403 for line in f:
404 plugin = parse_plugin_line(line)
405 if not plugin.owner:
406 msg = f"Invalid repository {line}, must be in the format owner/repo[ as alias]"
407 print(msg, file=sys.stderr)
408 sys.exit(1)
409 plugins.append(plugin)
410 return plugins
411
412
413def get_cache_path(cache_file_name: str) -> Optional[Path]:
414 xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
415 if xdg_cache is None:
416 home = os.environ.get("HOME", None)
417 if home is None:
418 return None
419 xdg_cache = str(Path(home, ".cache"))
420
421 return Path(xdg_cache, cache_file_name)
422
423
424class Cache:
425 def __init__(self, initial_plugins: List[Plugin], cache_file_name: str) -> None:
426 self.cache_file = get_cache_path(cache_file_name)
427
428 downloads = {}
429 for plugin in initial_plugins:
430 downloads[plugin.commit] = plugin
431 downloads.update(self.load())
432 self.downloads = downloads
433
434 def load(self) -> Dict[str, Plugin]:
435 if self.cache_file is None or not self.cache_file.exists():
436 return {}
437
438 downloads: Dict[str, Plugin] = {}
439 with open(self.cache_file) as f:
440 data = json.load(f)
441 for attr in data.values():
442 p = Plugin(
443 attr["name"], attr["commit"], attr["has_submodules"], attr["sha256"]
444 )
445 downloads[attr["commit"]] = p
446 return downloads
447
448 def store(self) -> None:
449 if self.cache_file is None:
450 return
451
452 os.makedirs(self.cache_file.parent, exist_ok=True)
453 with open(self.cache_file, "w+") as f:
454 data = {}
455 for name, attr in self.downloads.items():
456 data[name] = attr.as_json()
457 json.dump(data, f, indent=4, sort_keys=True)
458
459 def __getitem__(self, key: str) -> Optional[Plugin]:
460 return self.downloads.get(key, None)
461
462 def __setitem__(self, key: str, value: Plugin) -> None:
463 self.downloads[key] = value
464
465
466def prefetch(
467 pluginDesc: PluginDesc, cache: Cache
468) -> Tuple[str, str, Union[Exception, Plugin], dict]:
469 owner, repo = pluginDesc.owner, pluginDesc.repo
470 try:
471 plugin, redirect = prefetch_plugin(pluginDesc, cache)
472 cache[plugin.commit] = plugin
473 return (owner, repo, plugin, redirect)
474 except Exception as e:
475 return (owner, repo, e, {})
476
477
478def rewrite_input(
479 input_file: Path,
480 deprecated: Path,
481 redirects: Dict[str, str] = None,
482 append: Tuple = (),
483):
484 with open(input_file, "r") as f:
485 lines = f.readlines()
486
487 lines.extend(append)
488
489 if redirects:
490 lines = [redirects.get(line, line) for line in lines]
491
492 cur_date_iso = datetime.now().strftime("%Y-%m-%d")
493 with open(deprecated, "r") as f:
494 deprecations = json.load(f)
495 for old, new in redirects.items():
496 old_plugin = fetch_plugin_from_pluginline(old)
497 new_plugin = fetch_plugin_from_pluginline(new)
498 if old_plugin.normalized_name != new_plugin.normalized_name:
499 deprecations[old_plugin.normalized_name] = {
500 "new": new_plugin.normalized_name,
501 "date": cur_date_iso,
502 }
503 with open(deprecated, "w") as f:
504 json.dump(deprecations, f, indent=4, sort_keys=True)
505 f.write("\n")
506
507 lines = sorted(lines, key=str.casefold)
508
509 with open(input_file, "w") as f:
510 f.writelines(lines)
511
512
513def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
514 repo.index.add([str(f.resolve()) for f in files])
515
516 if repo.index.diff("HEAD"):
517 print(f'committing to nixpkgs "{message}"')
518 repo.index.commit(message)
519 else:
520 print("no changes in working tree to commit")
521
522
523def get_update(input_file: str, outfile: str, proc: int, editor: Editor):
524 cache: Cache = Cache(editor.get_current_plugins(), editor.cache_file)
525 _prefetch = functools.partial(prefetch, cache=cache)
526
527 def update() -> dict:
528 plugin_names = editor.load_plugin_spec(input_file)
529
530 try:
531 pool = Pool(processes=proc)
532 results = pool.map(_prefetch, plugin_names)
533 finally:
534 cache.store()
535
536 plugins, redirects = check_results(results)
537
538 editor.generate_nix(plugins, outfile)
539
540 return redirects
541
542 return update
543
544
545def update_plugins(editor: Editor, args):
546 """The main entry function of this module. All input arguments are grouped in the `Editor`."""
547
548 log.setLevel(LOG_LEVELS[args.debug])
549 log.info("Start updating plugins")
550 update = editor.get_update(args.input_file, args.outfile, args.proc)
551
552 redirects = update()
553 editor.rewrite_input(args.input_file, editor.deprecated, redirects)
554
555 autocommit = not args.no_commit
556
557 if autocommit:
558 nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True)
559 commit(nixpkgs_repo, f"{editor.attr_path}: update", [args.outfile])
560
561 if redirects:
562 update()
563 if autocommit:
564 commit(
565 nixpkgs_repo,
566 f"{editor.attr_path}: resolve github repository redirects",
567 [args.outfile, args.input_file, editor.deprecated],
568 )
569
570 for plugin_line in args.add_plugins:
571 editor.rewrite_input(args.input_file, editor.deprecated, append=(plugin_line + "\n",))
572 update()
573 plugin = fetch_plugin_from_pluginline(plugin_line)
574 if autocommit:
575 commit(
576 nixpkgs_repo,
577 "{drv_name}: init at {version}".format(
578 drv_name=editor.get_drv_name(plugin.normalized_name),
579 version=plugin.version
580 ),
581 [args.outfile, args.input_file],
582 )