···
1
+
#! /usr/bin/env nix-shell
2
+
#! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ requests toolz ])'
5
+
Update a Python package expression by passing in the `.nix` file, or the directory containing it.
6
+
You can pass in multiple files or paths.
8
+
You'll likely want to use
10
+
$ ./update-python-libraries ../../pkgs/development/python-modules/*
12
+
to update all libraries in that folder.
22
+
INDEX = "https://pypi.io/pypi"
25
+
EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl']
26
+
"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned."""
28
+
def _get_value(attribute, text):
29
+
"""Match attribute in text and return it."""
30
+
regex = '{}\s+=\s+"(.*)";'.format(attribute)
31
+
regex = re.compile(regex)
32
+
value = regex.findall(text)
35
+
raise ValueError("Found too many values for {}".format(attribute))
39
+
raise ValueError("No value found for {}".format(attribute))
41
+
def _get_line_and_value(attribute, text):
42
+
"""Match attribute in text. Return the line and the value of the attribute."""
43
+
regex = '({}\s+=\s+"(.*)";)'.format(attribute)
44
+
regex = re.compile(regex)
45
+
value = regex.findall(text)
48
+
raise ValueError("Found too many values for {}".format(attribute))
52
+
raise ValueError("No value found for {}".format(attribute))
55
+
def _replace_value(attribute, value, text):
56
+
"""Search and replace value of attribute in text."""
57
+
old_line, old_value = _get_line_and_value(attribute, text)
58
+
new_line = old_line.replace(old_value, value)
59
+
new_text = text.replace(old_line, new_line)
62
+
def _fetch_page(url):
63
+
r = requests.get(url)
64
+
if r.status_code == requests.codes.ok:
67
+
logging.warning("Request for {} failed".format(url))
69
+
def _get_latest_version(package, extension):
72
+
url = "{}/{}/json".format(INDEX, package)
73
+
json = _fetch_page(url)
75
+
data = extract_relevant_nix_data(json)[1]
77
+
version = data['latest_version']
78
+
if version in data['versions']:
79
+
sha256 = data['versions'][version]['sha256']
81
+
sha256 = None # Its possible that no file was uploaded to PyPI
83
+
return version, sha256
86
+
def extract_relevant_nix_data(json):
87
+
"""Extract relevant Nix data from the JSON of a package obtained from PyPI.
89
+
:param json: JSON obtained from PyPI
91
+
def _extract_license(json):
92
+
"""Extract license from JSON."""
93
+
return json['info']['license']
95
+
def _available_versions(json):
96
+
return json['releases'].keys()
98
+
def _extract_latest_version(json):
99
+
return json['info']['version']
101
+
def _get_src_and_hash(json, version, extensions):
102
+
"""Obtain url and hash for a given version and list of allowable extensions."""
103
+
if not json['releases']:
104
+
msg = "Package {}: No releases available.".format(json['info']['name'])
105
+
raise ValueError(msg)
107
+
# We use ['releases'] and not ['urls'] because we want to have the possibility for different version.
108
+
for possible_file in json['releases'][version]:
109
+
for extension in extensions:
110
+
if possible_file['filename'].endswith(extension):
111
+
src = {'url': str(possible_file['url']),
112
+
'sha256': str(possible_file['digests']['sha256']),
116
+
msg = "Package {}: No release with valid file extension available.".format(json['info']['name'])
119
+
#raise ValueError(msg)
121
+
def _get_sources(json, extensions):
122
+
versions = _available_versions(json)
123
+
releases = {version: _get_src_and_hash(json, version, extensions) for version in versions}
124
+
releases = toolz.itemfilter(lambda x: x[1] is not None, releases)
128
+
name = str(json['info']['name'])
129
+
latest_version = str(_extract_latest_version(json))
130
+
#src = _get_src_and_hash(json, latest_version, EXTENSIONS)
131
+
sources = _get_sources(json, EXTENSIONS)
133
+
# Collect meta data
134
+
license = str(_extract_license(json))
135
+
license = license if license != "UNKNOWN" else None
136
+
summary = str(json['info'].get('summary')).strip('.')
137
+
summary = summary if summary != "UNKNOWN" else None
138
+
#description = str(json['info'].get('description'))
139
+
#description = description if description != "UNKNOWN" else None
140
+
homepage = json['info'].get('home_page')
143
+
'latest_version' : latest_version,
144
+
'versions' : sources,
147
+
'description' : summary if summary else None,
148
+
#'longDescription' : description,
149
+
'license' : license,
150
+
'homepage' : homepage,
156
+
def _update_package(path):
158
+
# We need to read and modify a Nix expression.
159
+
if os.path.isdir(path):
160
+
path = os.path.join(path, 'default.nix')
162
+
if not os.path.isfile(path):
163
+
logging.warning("Path does not exist: {}".format(path))
166
+
if not path.endswith(".nix"):
167
+
logging.warning("Path does not end with `.nix`, skipping: {}".format(path))
170
+
with open(path, 'r') as f:
174
+
pname = _get_value('pname', text)
175
+
except ValueError as e:
176
+
logging.warning("Path {}: {}".format(path, str(e)))
180
+
version = _get_value('version', text)
181
+
except ValueError as e:
182
+
logging.warning("Path {}: {}".format(path, str(e)))
185
+
# If we use a wheel, then we need to request a wheel as well
187
+
format = _get_value('format', text)
188
+
except ValueError as e:
189
+
# No format mentioned, then we assume we have setuptools
190
+
# and use a .tar.gz
191
+
logging.warning("Path {}: {}".format(path, str(e)))
192
+
extension = ".tar.gz"
194
+
if format == 'wheel':
198
+
url = _get_value('url', text)
199
+
extension = os.path.splitext(url)[1]
200
+
except ValueError as e:
201
+
logging.warning("Path {}: {}".format(path, str(e)))
202
+
extension = ".tar.gz"
204
+
new_version, new_sha256 = _get_latest_version(pname, extension)
206
+
logging.warning("Path has no valid file available: {}".format(path))
209
+
if new_version != version:
212
+
text = _replace_value('version', new_version, text)
213
+
except ValueError as e:
214
+
logging.warning("Path {}: {}".format(path, str(e)))
216
+
text = _replace_value('sha256', new_sha256, text)
217
+
except ValueError as e:
218
+
logging.warning("Path {}: {}".format(path, str(e)))
220
+
with open(path, 'w') as f:
223
+
logging.info("Updated {} from {} to {}".format(pname, version, new_version))
226
+
logging.info("No update available for {} at {}".format(pname, version))
233
+
parser = argparse.ArgumentParser()
234
+
parser.add_argument('package', type=str, nargs='+')
236
+
args = parser.parse_args()
238
+
packages = args.package
240
+
count = list(map(_update_package, packages))
242
+
#logging.info("{} package(s) updated".format(sum(count)))
244
+
if __name__ == '__main__':
pyrox.dev