···
1
+
#!/usr/bin/env nix-shell
2
+
#!nix-shell update-octave-shell.nix -i python3
5
+
Update a Octave package expression by passing in the `.nix` file, or the directory containing it.
6
+
You can pass in multiple files or paths.
8
+
You'll likely want to use
10
+
$ ./update-octave-libraries ../../pkgs/development/octave-modules/**/default.nix
12
+
to update all non-pinned libraries in that folder.
21
+
from concurrent.futures import ThreadPoolExecutor as Pool
22
+
from packaging.version import Version as _Version
23
+
from packaging.version import InvalidVersion
24
+
from packaging.specifiers import SpecifierSet
29
+
INDEX = "https://raw.githubusercontent.com/gnu-octave/packages/main/packages"
30
+
"""url of Octave packages' source on GitHub"""
32
+
EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip']
33
+
"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned."""
39
+
NIXPGKS_ROOT = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode('utf-8').strip()
42
+
logging.basicConfig(level=logging.INFO)
45
+
class Version(_Version, collections.abc.Sequence):
47
+
def __init__(self, version):
48
+
super().__init__(version)
49
+
# We cannot use `str(Version(0.04.21))` because that becomes `0.4.21`
50
+
# https://github.com/avian2/unidecode/issues/13#issuecomment-354538882
51
+
self.raw_version = version
53
+
def __getitem__(self, i):
54
+
return self._version.release[i]
57
+
return len(self._version.release)
60
+
yield from self._version.release
63
+
def _get_values(attribute, text):
64
+
"""Match attribute in text and return all matches.
66
+
:returns: List of matches.
68
+
regex = '{}\s+=\s+"(.*)";'.format(attribute)
69
+
regex = re.compile(regex)
70
+
values = regex.findall(text)
73
+
def _get_unique_value(attribute, text):
74
+
"""Match attribute in text and return unique match.
76
+
:returns: Single match.
78
+
values = _get_values(attribute, text)
81
+
raise ValueError("found too many values for {}".format(attribute))
85
+
raise ValueError("no value found for {}".format(attribute))
87
+
def _get_line_and_value(attribute, text):
88
+
"""Match attribute in text. Return the line and the value of the attribute."""
89
+
regex = '({}\s+=\s+"(.*)";)'.format(attribute)
90
+
regex = re.compile(regex)
91
+
value = regex.findall(text)
94
+
raise ValueError("found too many values for {}".format(attribute))
98
+
raise ValueError("no value found for {}".format(attribute))
101
+
def _replace_value(attribute, value, text):
102
+
"""Search and replace value of attribute in text."""
103
+
old_line, old_value = _get_line_and_value(attribute, text)
104
+
new_line = old_line.replace(old_value, value)
105
+
new_text = text.replace(old_line, new_line)
109
+
def _fetch_page(url):
110
+
r = requests.get(url)
111
+
if r.status_code == requests.codes.ok:
112
+
return list(yaml.safe_load_all(r.content))[0]
114
+
raise ValueError("request for {} failed".format(url))
117
+
def _fetch_github(url):
119
+
token = os.environ.get('GITHUB_API_TOKEN')
121
+
headers["Authorization"] = f"token {token}"
122
+
r = requests.get(url, headers=headers)
124
+
if r.status_code == requests.codes.ok:
127
+
raise ValueError("request for {} failed".format(url))
137
+
def _determine_latest_version(current_version, target, versions):
138
+
"""Determine latest version, given `target`, returning the more recent version.
140
+
current_version = Version(current_version)
142
+
def _parse_versions(versions):
146
+
except InvalidVersion:
149
+
versions = _parse_versions(versions)
151
+
index = SEMVER[target]
153
+
ceiling = list(current_version[0:index])
154
+
if len(ceiling) == 0:
158
+
ceiling = Version(".".join(map(str, ceiling)))
160
+
# We do not want prereleases
161
+
versions = SpecifierSet(prereleases=PRERELEASES).filter(versions)
163
+
if ceiling is not None:
164
+
versions = SpecifierSet(f"<{ceiling}").filter(versions)
166
+
return (max(sorted(versions))).raw_version
169
+
def _get_latest_version_octave_packages(package, extension, current_version, target):
170
+
"""Get latest version and hash from Octave Packages."""
171
+
url = "{}/{}.yaml".format(INDEX, package)
172
+
yaml = _fetch_page(url)
174
+
versions = list(map(lambda pv: pv['id'], yaml['versions']))
175
+
version = _determine_latest_version(current_version, target, versions)
178
+
releases = [v if v['id'] == version else None for v in yaml['versions']]
179
+
except KeyError as e:
180
+
raise KeyError('Could not find version {} for {}'.format(version, package)) from e
181
+
for release in releases:
182
+
if release['url'].endswith(extension):
183
+
sha256 = release['sha256']
187
+
return version, sha256, None
190
+
def _get_latest_version_github(package, extension, current_version, target):
191
+
def strip_prefix(tag):
192
+
return re.sub("^[^0-9]*", "", tag)
194
+
def get_prefix(string):
195
+
matches = re.findall(r"^([^0-9]*)", string)
196
+
return next(iter(matches), "")
198
+
# when invoked as an updateScript, UPDATE_NIX_ATTR_PATH will be set
199
+
# this allows us to work with packages which live outside of octave-modules
200
+
attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"octavePackages.{package}")
202
+
homepage = subprocess.check_output(
203
+
["nix", "eval", "-f", f"{NIXPGKS_ROOT}/default.nix", "--raw", f"{attr_path}.src.meta.homepage"])\
205
+
except Exception as e:
206
+
raise ValueError(f"Unable to determine homepage: {e}")
207
+
owner_repo = homepage[len("https://github.com/"):] # remove prefix
208
+
owner, repo = owner_repo.split("/")
210
+
url = f"https://api.github.com/repos/{owner}/{repo}/releases"
211
+
all_releases = _fetch_github(url)
212
+
releases = list(filter(lambda x: not x['prerelease'], all_releases))
214
+
if len(releases) == 0:
215
+
raise ValueError(f"{homepage} does not contain any stable releases")
217
+
versions = map(lambda x: strip_prefix(x['tag_name']), releases)
218
+
version = _determine_latest_version(current_version, target, versions)
220
+
release = next(filter(lambda x: strip_prefix(x['tag_name']) == version, releases))
221
+
prefix = get_prefix(release['tag_name'])
223
+
sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", f"{release['tarball_url']}"], stderr=subprocess.DEVNULL)\
224
+
.decode('utf-8').strip()
226
+
# this may fail if they have both a branch and a tag of the same name, attempt tag name
227
+
tag_url = str(release['tarball_url']).replace("tarball","tarball/refs/tags")
228
+
sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", tag_url], stderr=subprocess.DEVNULL)\
229
+
.decode('utf-8').strip()
232
+
return version, sha256, prefix
234
+
def _get_latest_version_git(package, extension, current_version, target):
235
+
"""NOTE: Unimplemented!"""
236
+
# attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"octavePackages.{package}")
238
+
# download_url = subprocess.check_output(
239
+
# ["nix", "--extra-experimental-features", "nix-command", "eval", "-f", f"{NIXPGKS_ROOT}/default.nix", "--raw", f"{attr_path}.src.url"])\
241
+
# except Exception as e:
242
+
# raise ValueError(f"Unable to determine download link: {e}")
244
+
# with tempfile.TemporaryDirectory(prefix=attr_path) as new_clone_location:
245
+
# subprocess.run(["git", "clone", download_url, new_clone_location])
246
+
# newest_commit = subprocess.check_output(
247
+
# ["git" "rev-parse" "$(git branch -r)" "|" "tail" "-n" "1"]).decode('utf-8')
252
+
'fetchFromGitHub' : _get_latest_version_github,
253
+
'fetchurl' : _get_latest_version_octave_packages,
254
+
'fetchgit' : _get_latest_version_git,
258
+
DEFAULT_SETUPTOOLS_EXTENSION = 'tar.gz'
262
+
'setuptools' : DEFAULT_SETUPTOOLS_EXTENSION,
265
+
def _determine_fetcher(text):
266
+
# Count occurrences of fetchers.
267
+
nfetchers = sum(text.count('src = {}'.format(fetcher)) for fetcher in FETCHERS.keys())
269
+
raise ValueError("no fetcher.")
270
+
elif nfetchers > 1:
271
+
raise ValueError("multiple fetchers.")
273
+
# Then we check which fetcher to use.
274
+
for fetcher in FETCHERS.keys():
275
+
if 'src = {}'.format(fetcher) in text:
279
+
def _determine_extension(text, fetcher):
280
+
"""Determine what extension is used in the expression.
283
+
- fetchPypi, we check if format is specified.
284
+
- fetchurl, we determine the extension from the url.
285
+
- fetchFromGitHub we simply use `.tar.gz`.
287
+
if fetcher == 'fetchurl':
288
+
url = _get_unique_value('url', text)
289
+
extension = os.path.splitext(url)[1]
291
+
elif fetcher == 'fetchFromGitHub' or fetcher == 'fetchgit':
292
+
if "fetchSubmodules" in text:
293
+
raise ValueError("fetchFromGitHub fetcher doesn't support submodules")
294
+
extension = "tar.gz"
299
+
def _update_package(path, target):
301
+
# Read the expression
302
+
with open(path, 'r') as f:
305
+
# Determine pname. Many files have more than one pname
306
+
pnames = _get_values('pname', text)
308
+
# Determine version.
309
+
version = _get_unique_value('version', text)
311
+
# First we check how many fetchers are mentioned.
312
+
fetcher = _determine_fetcher(text)
314
+
extension = _determine_extension(text, fetcher)
316
+
# Attempt a fetch using each pname, e.g. backports-zoneinfo vs backports.zoneinfo
317
+
successful_fetch = False
318
+
for pname in pnames:
319
+
if fetcher == "fetchgit":
320
+
logging.warning(f"You must update {pname} MANUALLY!")
321
+
return { 'path': path, 'target': target, 'pname': pname,
322
+
'old_version': version, 'new_version': version }
324
+
new_version, new_sha256, prefix = FETCHERS[fetcher](pname, extension, version, target)
325
+
successful_fetch = True
330
+
if not successful_fetch:
331
+
raise ValueError(f"Unable to find correct package using these pnames: {pnames}")
333
+
if new_version == version:
334
+
logging.info("Path {}: no update available for {}.".format(path, pname))
336
+
elif Version(new_version) <= Version(version):
337
+
raise ValueError("downgrade for {}.".format(pname))
339
+
raise ValueError("no file available for {}.".format(pname))
341
+
text = _replace_value('version', new_version, text)
342
+
# hashes from pypi are 16-bit encoded sha256's, normalize it to sri to avoid merge conflicts
343
+
# sri hashes have been the default format since nix 2.4+
344
+
sri_hash = subprocess.check_output(["nix", "--extra-experimental-features", "nix-command", "hash", "to-sri", "--type", "sha256", new_sha256]).decode('utf-8').strip()
347
+
# fetchers can specify a sha256, or a sri hash
349
+
text = _replace_value('sha256', sri_hash, text)
351
+
text = _replace_value('hash', sri_hash, text)
353
+
if fetcher == 'fetchFromGitHub':
354
+
# in the case of fetchFromGitHub, it's common to see `rev = version;` or `rev = "v${version}";`
355
+
# in which no string value is meant to be substituted. However, we can just overwrite the previous value.
356
+
regex = '(rev\s+=\s+[^;]*;)'
357
+
regex = re.compile(regex)
358
+
matches = regex.findall(text)
362
+
raise ValueError("Unable to find rev value for {}.".format(pname))
364
+
# forcefully rewrite rev, incase tagging conventions changed for a release
366
+
text = text.replace(match, f'rev = "refs/tags/{prefix}${{version}}";')
367
+
# incase there's no prefix, just rewrite without interpolation
368
+
text = text.replace('"${version}";', 'version;')
370
+
with open(path, 'w') as f:
373
+
logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version))
379
+
'old_version' : version,
380
+
'new_version' : new_version,
381
+
#'fetcher' : fetcher,
387
+
def _update(path, target):
389
+
# We need to read and modify a Nix expression.
390
+
if os.path.isdir(path):
391
+
path = os.path.join(path, 'default.nix')
393
+
# If a default.nix does not exist, we quit.
394
+
if not os.path.isfile(path):
395
+
logging.info("Path {}: does not exist.".format(path))
398
+
# If file is not a Nix expression, we quit.
399
+
if not path.endswith(".nix"):
400
+
logging.info("Path {}: does not end with `.nix`.".format(path))
404
+
return _update_package(path, target)
405
+
except ValueError as e:
406
+
logging.warning("Path {}: {}".format(path, e))
410
+
def _commit(path, pname, old_version, new_version, pkgs_prefix="octave: ", **kwargs):
414
+
msg = f'{pkgs_prefix}{pname}: {old_version} -> {new_version}'
417
+
subprocess.check_call([GIT, 'add', path])
418
+
subprocess.check_call([GIT, 'commit', '-m', msg])
419
+
except subprocess.CalledProcessError as e:
420
+
subprocess.check_call([GIT, 'checkout', path])
421
+
raise subprocess.CalledProcessError(f'Could not commit {path}') from e
429
+
environment variables:
430
+
GITHUB_API_TOKEN\tGitHub API token used when updating github packages
432
+
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog)
433
+
parser.add_argument('package', type=str, nargs='+')
434
+
parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major')
435
+
parser.add_argument('--commit', action='store_true', help='Create a commit for each package update')
436
+
parser.add_argument('--use-pkgs-prefix', action='store_true', help='Use octavePackages.${pname}: instead of octave: ${pname}: when making commits')
438
+
args = parser.parse_args()
439
+
target = args.target
441
+
packages = list(map(os.path.abspath, args.package))
443
+
logging.info("Updating packages...")
445
+
# Use threads to update packages concurrently
447
+
results = list(filter(bool, p.map(lambda pkg: _update(pkg, target), packages)))
449
+
logging.info("Finished updating packages.")
451
+
commit_options = {}
452
+
if args.use_pkgs_prefix:
453
+
logging.info("Using octavePackages. prefix for commits")
454
+
commit_options["pkgs_prefix"] = "octavePackages."
456
+
# Commits are created sequentially.
458
+
logging.info("Committing updates...")
459
+
# list forces evaluation
460
+
list(map(lambda x: _commit(**x, **commit_options), results))
461
+
logging.info("Finished committing updates")
463
+
count = len(results)
464
+
logging.info("{} package(s) updated".format(count))
467
+
if __name__ == '__main__':
pyrox.dev