···
1
+
#!/usr/bin/env nix-shell
2
+
#!nix-shell -i ruby -p "ruby.withPackages (ps: with ps; [ slop curb nokogiri ])"
14
+
# Returns a repo URL for a given package name.
16
+
if value && value.start_with?('http')
19
+
"https://dl.google.com/android/repository/#{value}"
25
+
# Returns a system image URL for a given system image name.
26
+
def image_url value, dir
30
+
if value && value.start_with?('http')
33
+
"https://dl.google.com/android/repository/sys-img/#{dir}/#{value}"
39
+
# Runs a GET with curl.
41
+
curl = Curl::Easy.new(url) do |http|
42
+
http.headers['User-Agent'] = 'nixpkgs androidenv update bot'
43
+
yield http if block_given?
45
+
STDERR.print "GET #{url}"
47
+
STDERR.puts "... #{curl.response_code}"
49
+
StringIO.new(curl.body_str)
52
+
# Retrieves a repo from the filesystem or a URL.
54
+
uri = URI.parse(location)
57
+
_curl_get repo_url("#{uri.host}#{uri.fragment}.xml")
59
+
_curl_get image_url("sys-img#{uri.fragment}.xml", uri.host)
61
+
if File.exist?(uri.path)
62
+
File.open(uri.path, 'rt')
64
+
raise "Repository #{uri} was neither a file nor a repo URL"
69
+
# Returns a JSON with the data and structure of the input XML
70
+
def to_json_collector doc
73
+
doc.element_children.each { |node|
74
+
if node.children.length == 1 and node.children.first.text?
75
+
json["#{node.name}:#{index}"] ||= node.content
79
+
json["#{node.name}:#{index}"] ||= to_json_collector node
82
+
element_attributes = {}
83
+
doc.attribute_nodes.each do |attr|
84
+
if attr.name == "type"
85
+
type = attr.value.split(':', 2).last
87
+
when 'generic:genericDetailsType'
88
+
element_attributes["xsi:type"] ||= "ns5:#{type}"
89
+
when 'addon:extraDetailsType'
90
+
element_attributes["xsi:type"] ||= "ns8:#{type}"
91
+
when 'addon:mavenType'
92
+
element_attributes["xsi:type"] ||= "ns8:#{type}"
93
+
when 'sdk:platformDetailsType'
94
+
element_attributes["xsi:type"] ||= "ns11:#{type}"
95
+
when 'sdk:sourceDetailsType'
96
+
element_attributes["xsi:type"] ||= "ns11:#{type}"
97
+
when 'sys-img:sysImgDetailsType'
98
+
element_attributes["xsi:type"] ||= "ns12:#{type}"
99
+
when 'addon:addonDetailsType' then
100
+
element_attributes["xsi:type"] ||= "ns8:#{type}"
103
+
element_attributes[attr.name] ||= attr.value
106
+
if !element_attributes.empty?
107
+
json['element-attributes'] ||= element_attributes
112
+
# Returns a tuple of [type, revision, revision components] for a package node.
113
+
def package_revision package
114
+
type_details = package.at_css('> type-details')
115
+
type = type_details.attributes['type']
116
+
type &&= type.value
122
+
when 'generic:genericDetailsType', 'addon:extraDetailsType', 'addon:mavenType'
123
+
major = text package.at_css('> revision > major')
124
+
minor = text package.at_css('> revision > minor')
125
+
micro = text package.at_css('> revision > micro')
126
+
preview = text package.at_css('> revision > preview')
130
+
unless empty?(major)
132
+
components << major
135
+
unless empty?(minor)
136
+
revision << ".#{minor}"
137
+
components << minor
140
+
unless empty?(micro)
141
+
revision << ".#{micro}"
142
+
components << micro
145
+
unless empty?(preview)
146
+
revision << "-rc#{preview}"
147
+
components << preview
149
+
when 'sdk:platformDetailsType'
150
+
codename = text type_details.at_css('> codename')
151
+
api_level = text type_details.at_css('> api-level')
152
+
revision = empty?(codename) ? api_level : codename
153
+
components = [revision]
154
+
when 'sdk:sourceDetailsType'
155
+
api_level = text type_details.at_css('> api-level')
156
+
revision, components = api_level, [api_level]
157
+
when 'sys-img:sysImgDetailsType'
158
+
codename = text type_details.at_css('> codename')
159
+
api_level = text type_details.at_css('> api-level')
160
+
id = text type_details.at_css('> tag > id')
161
+
abi = text type_details.at_css('> abi')
165
+
if empty?(codename)
166
+
revision << api_level
167
+
components << api_level
169
+
revision << codename
170
+
components << codename
174
+
revision << "-#{id}"
179
+
revision << "-#{abi}"
182
+
when 'addon:addonDetailsType' then
183
+
api_level = text type_details.at_css('> api-level')
184
+
id = text type_details.at_css('> tag > id')
185
+
revision = api_level
186
+
components = [api_level, id]
189
+
[type, revision, components]
192
+
# Returns a hash of archives for the specified package node.
193
+
def package_archives package
195
+
package.css('> archives > archive').each do |archive|
196
+
host_os = text archive.at_css('> host-os')
197
+
host_arch = text archive.at_css('> host-arch')
198
+
host_os = 'all' if empty?(host_os)
199
+
host_arch = 'all' if empty?(host_arch)
200
+
archives[host_os + host_arch] = {
202
+
'arch' => host_arch,
203
+
'size' => Integer(text(archive.at_css('> complete > size'))),
204
+
'sha1' => text(archive.at_css('> complete > checksum')),
205
+
'url' => yield(text(archive.at_css('> complete > url')))
211
+
# Returns the text from a node, or nil.
213
+
node ? node.text : nil
216
+
# Nil or empty helper.
218
+
!value || value.empty?
221
+
# Fixes up returned hashes by converting archives like
222
+
# (e.g. {'linux' => {'sha1' => ...}, 'macosx' => ...} to
223
+
# [{'os' => 'linux', 'sha1' => ...}, {'os' => 'macosx', ...}, ...].
225
+
Hash[value.map do |k, v|
226
+
if k == 'archives' && v.is_a?(Hash)
227
+
[k, v.map do |os, archive|
230
+
elsif v.is_a?(Hash)
238
+
# Today since Unix Epoch, January 1, 1970.
240
+
Time.now.utc.to_i / 24 / 60 / 60
243
+
# The expiration strategy. Expire if the last available day was before the `oldest_valid_day`.
244
+
def expire_records record, oldest_valid_day
245
+
if record.is_a?(Hash)
246
+
if record.has_key?('last-available-day') &&
247
+
record['last-available-day'] < oldest_valid_day
251
+
# This should only happen in the first run of this scrip after adding the `expire_record` function.
252
+
if record.has_key?('displayName') &&
253
+
!record.has_key?('last-available-day')
254
+
update['last-available-day'] = today
256
+
record.each {|key, value|
257
+
v = expire_records value, oldest_valid_day
258
+
update[key] = v if v
266
+
# Normalize the specified license text.
267
+
# See: https://brash-snapper.glitch.me/ for how the munging works.
268
+
def normalize_license license
269
+
license = license.dup
270
+
license.gsub!(/([^\n])\n([^\n])/m, '\1 \2')
271
+
license.gsub!(/ +/, ' ')
276
+
# Gets all license texts, deduplicating them.
277
+
def get_licenses doc
279
+
doc.css('license[type="text"]').each do |license_node|
280
+
license_id = license_node['id']
282
+
licenses[license_id] ||= []
283
+
licenses[license_id] |= [normalize_license(text(license_node))]
289
+
def parse_package_xml doc
290
+
licenses = get_licenses doc
292
+
# check https://github.com/NixOS/nixpkgs/issues/373785
295
+
doc.css('remotePackage').each do |package|
296
+
name, _, version = package['path'].partition(';')
297
+
next if version == 'latest'
299
+
is_extras = name == 'extras'
301
+
name = package['path'].tr(';', '-')
304
+
type, revision, _ = package_revision(package)
305
+
next unless revision
307
+
path = package['path'].tr(';', '/')
308
+
display_name = text package.at_css('> display-name')
309
+
uses_license = package.at_css('> uses-license')
310
+
uses_license &&= uses_license['ref']
311
+
obsolete ||= package['obsolete']
312
+
type_details = to_json_collector package.at_css('> type-details')
313
+
revision_details = to_json_collector package.at_css('> revision')
314
+
archives = package_archives(package) {|url| repo_url url}
315
+
dependencies_xml = package.at_css('> dependencies')
316
+
dependencies = to_json_collector dependencies_xml if dependencies_xml
320
+
component = package['path']
321
+
target = (target[component] ||= {})
323
+
target = (packages[name] ||= {})
324
+
target = (target[revision] ||= {})
327
+
target['name'] ||= name
328
+
target['path'] ||= path
329
+
target['revision'] ||= revision
330
+
target['displayName'] ||= display_name
331
+
target['license'] ||= uses_license if uses_license
332
+
target['obsolete'] ||= obsolete if obsolete == 'true'
333
+
target['type-details'] ||= type_details
334
+
target['revision-details'] ||= revision_details
335
+
target['dependencies'] ||= dependencies if dependencies
336
+
target['archives'] ||= {}
337
+
merge target['archives'], archives
338
+
target['last-available-day'] = today
341
+
[licenses, packages, extras]
344
+
def parse_image_xml doc
345
+
licenses = get_licenses doc
348
+
doc.css('remotePackage[path^="system-images;"]').each do |package|
349
+
type, revision, components = package_revision(package)
350
+
next unless revision
352
+
path = package['path'].tr(';', '/')
353
+
display_name = text package.at_css('> display-name')
354
+
uses_license = package.at_css('> uses-license')
355
+
uses_license &&= uses_license['ref']
356
+
obsolete &&= package['obsolete']
357
+
type_details = to_json_collector package.at_css('> type-details')
358
+
revision_details = to_json_collector package.at_css('> revision')
359
+
archives = package_archives(package) {|url| image_url url, components[-2]}
360
+
dependencies_xml = package.at_css('> dependencies')
361
+
dependencies = to_json_collector dependencies_xml if dependencies_xml
364
+
components.each do |component|
365
+
target[component] ||= {}
366
+
target = target[component]
369
+
target['name'] ||= "system-image-#{revision}"
370
+
target['path'] ||= path
371
+
target['revision'] ||= revision
372
+
target['displayName'] ||= display_name
373
+
target['license'] ||= uses_license if uses_license
374
+
target['obsolete'] ||= obsolete if obsolete
375
+
target['type-details'] ||= type_details
376
+
target['revision-details'] ||= revision_details
377
+
target['dependencies'] ||= dependencies if dependencies
378
+
target['archives'] ||= {}
379
+
merge target['archives'], archives
380
+
target['last-available-day'] = today
386
+
def parse_addon_xml doc
387
+
licenses = get_licenses doc
388
+
addons, extras = {}, {}
390
+
doc.css('remotePackage').each do |package|
391
+
type, revision, components = package_revision(package)
392
+
next unless revision
394
+
path = package['path'].tr(';', '/')
395
+
display_name = text package.at_css('> display-name')
396
+
uses_license = package.at_css('> uses-license')
397
+
uses_license &&= uses_license['ref']
398
+
obsolete &&= package['obsolete']
399
+
type_details = to_json_collector package.at_css('> type-details')
400
+
revision_details = to_json_collector package.at_css('> revision')
401
+
archives = package_archives(package) {|url| repo_url url}
402
+
dependencies_xml = package.at_css('> dependencies')
403
+
dependencies = to_json_collector dependencies_xml if dependencies_xml
406
+
when 'addon:addonDetailsType'
407
+
name = components.last
410
+
# Hack for Google APIs 25 r1, which displays as 23 for some reason
411
+
archive_name = text package.at_css('> archives > archive > complete > url')
412
+
if archive_name == 'google_apis-25_r1.zip'
413
+
path = 'add-ons/addon-google_apis-google-25'
415
+
components = [revision, components.last]
417
+
when 'addon:extraDetailsType', 'addon:mavenType'
418
+
name = package['path'].tr(';', '-')
419
+
components = [package['path']]
423
+
components.each do |component|
424
+
target = (target[component] ||= {})
427
+
target['name'] ||= name
428
+
target['path'] ||= path
429
+
target['revision'] ||= revision
430
+
target['displayName'] ||= display_name
431
+
target['license'] ||= uses_license if uses_license
432
+
target['obsolete'] ||= obsolete if obsolete
433
+
target['type-details'] ||= type_details
434
+
target['revision-details'] ||= revision_details
435
+
target['dependencies'] ||= dependencies if dependencies
436
+
target['archives'] ||= {}
437
+
merge target['archives'], archives
438
+
target['last-available-day'] = today
441
+
[licenses, addons, extras]
444
+
# Make the clean diff by always sorting the result before puting it in the stdout.
445
+
def sort_recursively value
446
+
if value.is_a?(Hash)
448
+
value.map do |k, v|
449
+
[k, sort_recursively(v)]
450
+
end.sort_by {|(k, v)| k }
452
+
elsif value.is_a?(Array)
453
+
value.map do |v| sort_recursively(v) end
459
+
def merge_recursively a, b
460
+
a.merge!(b) {|key, a_item, b_item|
461
+
if a_item.is_a?(Hash) && b_item.is_a?(Hash)
462
+
merge_recursively(a_item, b_item)
463
+
elsif b_item != nil
470
+
def merge dest, src
471
+
merge_recursively dest, src
474
+
opts = Slop.parse do |o|
475
+
o.array '-p', '--packages', 'packages repo XMLs to parse', default: %w[repo://repository#2-3]
476
+
o.array '-i', '--images', 'system image repo XMLs to parse', default: %w[
477
+
image://android#2-3
478
+
image://android-tv#2-3
479
+
image://android-wear#2-3
480
+
image://android-wear-cn#2-3
481
+
image://android-automotive#2-3
482
+
image://google_apis#2-3
483
+
image://google_apis_playstore#2-3
485
+
o.array '-a', '--addons', 'addon repo XMLs to parse', default: %w[repo://addon#2-3]
486
+
o.string '-I', '--input', 'input JSON file for repo', default: File.join(__dir__, 'repo.json')
487
+
o.string '-O', '--output', 'output JSON file for repo', default: File.join(__dir__, 'repo.json')
491
+
result['licenses'] = {}
492
+
result['packages'] = {}
493
+
result['images'] = {}
494
+
result['addons'] = {}
495
+
result['extras'] = {}
497
+
opts[:packages].each do |filename|
498
+
licenses, packages, extras = parse_package_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
499
+
merge result['licenses'], licenses
500
+
merge result['packages'], packages
501
+
merge result['extras'], extras
504
+
opts[:images].each do |filename|
505
+
licenses, images = parse_image_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
506
+
merge result['licenses'], licenses
507
+
merge result['images'], images
510
+
opts[:addons].each do |filename|
511
+
licenses, addons, extras = parse_addon_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
512
+
merge result['licenses'], licenses
513
+
merge result['addons'], addons
514
+
merge result['extras'], extras
517
+
result['latest'] = {}
518
+
result['packages'].each do |name, versions|
519
+
max_version = Gem::Version.new('0')
520
+
versions.each do |version, package|
521
+
if package['license'] == 'android-sdk-license' && Gem::Version.correct?(package['revision'])
522
+
package_version = Gem::Version.new(package['revision'])
523
+
max_version = package_version if package_version > max_version
526
+
result['latest'][name] = max_version.to_s
529
+
# As we keep the old packages in the repo JSON file, we should have
530
+
# a strategy to remove them at some point!
531
+
# So with this variable we claim it's okay to remove them from the
532
+
# JSON after two years that they are not available.
533
+
two_years_ago = today - 2 * 365
538
+
input_json = if File.exist?(opts[:input])
539
+
STDERR.puts "Reading #{opts[:input]}"
540
+
File.read(opts[:input])
542
+
STDERR.puts "Creating new repo"
546
+
if input_json != nil && !input_json.empty?
547
+
input = expire_records(JSON.parse(input_json), two_years_ago)
549
+
# Just create a new set of latest packages.
550
+
prev_latest = input['latest'] || {}
551
+
input['latest'] = {}
553
+
rescue JSON::ParserError => e
554
+
STDERR.write(e.message)
558
+
fixup_result = fixup(result)
560
+
# Regular installation of Android SDK would keep the previously installed packages even if they are not
561
+
# in the uptodate XML files, so here we try to support this logic by keeping un-available packages,
562
+
# therefore the old packages will work as long as the links are working on the Google servers.
563
+
output = merge input, fixup_result
565
+
# Write the repository. Append a \n to keep nixpkgs Github Actions happy.
566
+
STDERR.puts "Writing #{opts[:output]}"
567
+
File.write opts[:output], (JSON.pretty_generate(sort_recursively(output)) + "\n")
569
+
# Output metadata for the nixpkgs update script.
570
+
if ENV['UPDATE_NIX_ATTR_PATH']
571
+
# See if there are any changes in the latest versions.
572
+
cur_latest = output['latest'] || {}
579
+
cur_latest.each do |k, v|
580
+
prev = prev_latest[k]
581
+
if prev && prev != v
582
+
old_versions << "#{k}:#{prev}"
583
+
new_versions << "#{k}:#{v}"
584
+
changes << "#{k}: #{prev} -> #{v}"
592
+
test_result = `NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ACCEPT_ANDROID_SDK_LICENSE=1 nix-build #{Shellwords.escape(File.realpath(File.join(__dir__, '..', '..', '..', '..', 'default.nix')))} -A #{Shellwords.join [ENV['UPDATE_NIX_ATTR_PATH']]} 2>&1`
593
+
test_status = $?.exitstatus
595
+
template = ERB.new(<<-EOF, trim_mode: '<>-')
596
+
androidenv: <%= changes.join('; ') %>
598
+
Performed the following automatic androidenv updates:
600
+
<% changes.each do |change| %>
604
+
Tests exited with status: <%= test_status -%>
606
+
<% if !test_result.empty? %>
607
+
Last 100 lines of output:
609
+
<%= test_result.lines.last(100).join -%>
615
+
attrPath: 'androidenv.androidPkgs.androidsdk',
616
+
oldVersion: old_versions.join('; '),
617
+
newVersion: new_versions.join('; '),
621
+
commitMessage: template.result(binding)
625
+
# nix-update info is on stdout
626
+
STDOUT.puts JSON.pretty_generate(changed_paths)
pyrox.dev