pyrox.dev
1#!/usr/bin/env nix-shell
2#!nix-shell -i ruby -p "ruby.withPackages (ps: with ps; [ slop curb nokogiri ])"
3
4require 'json'
5require 'rubygems'
6require 'shellwords'
7require 'erb'
8require 'uri'
9require 'stringio'
10require 'slop'
11require 'curb'
12require 'nokogiri'
13
14# Returns a repo URL for a given package name.
15def repo_url value
16 if value && value.start_with?('http')
17 value
18 elsif value
19 "https://dl.google.com/android/repository/#{value}"
20 else
21 nil
22 end
23end
24
25# Returns a system image URL for a given system image name.
26def image_url value, dir
27 if dir == "default"
28 dir = "android"
29 end
30 if value && value.start_with?('http')
31 value
32 elsif value
33 "https://dl.google.com/android/repository/sys-img/#{dir}/#{value}"
34 else
35 nil
36 end
37end
38
39# Runs a GET with curl.
40def _curl_get url
41 curl = Curl::Easy.new(url) do |http|
42 http.headers['User-Agent'] = 'nixpkgs androidenv update bot'
43 yield http if block_given?
44 end
45 STDERR.print "GET #{url}"
46 curl.perform
47 STDERR.puts "... #{curl.response_code}"
48
49 StringIO.new(curl.body_str)
50end
51
52# Retrieves a repo from the filesystem or a URL.
53def get location
54 uri = URI.parse(location)
55 case uri.scheme
56 when 'repo'
57 _curl_get repo_url("#{uri.host}#{uri.fragment}.xml")
58 when 'image'
59 _curl_get image_url("sys-img#{uri.fragment}.xml", uri.host)
60 else
61 if File.exist?(uri.path)
62 File.open(uri.path, 'rt')
63 else
64 raise "Repository #{uri} was neither a file nor a repo URL"
65 end
66 end
67end
68
69# Returns a JSON with the data and structure of the input XML
70def to_json_collector doc
71 json = {}
72 index = 0
73 doc.element_children.each { |node|
74 if node.children.length == 1 and node.children.first.text?
75 json["#{node.name}:#{index}"] ||= node.content
76 index += 1
77 next
78 end
79 json["#{node.name}:#{index}"] ||= to_json_collector node
80 index += 1
81 }
82 element_attributes = {}
83 doc.attribute_nodes.each do |attr|
84 if attr.name == "type"
85 type = attr.value.split(':', 2).last
86 case attr.value
87 when 'generic:genericDetailsType'
88 element_attributes["xsi:type"] ||= "ns5:#{type}"
89 when 'addon:extraDetailsType'
90 element_attributes["xsi:type"] ||= "ns8:#{type}"
91 when 'addon:mavenType'
92 element_attributes["xsi:type"] ||= "ns8:#{type}"
93 when 'sdk:platformDetailsType'
94 element_attributes["xsi:type"] ||= "ns11:#{type}"
95 when 'sdk:sourceDetailsType'
96 element_attributes["xsi:type"] ||= "ns11:#{type}"
97 when 'sys-img:sysImgDetailsType'
98 element_attributes["xsi:type"] ||= "ns12:#{type}"
99 when 'addon:addonDetailsType' then
100 element_attributes["xsi:type"] ||= "ns8:#{type}"
101 end
102 else
103 element_attributes[attr.name] ||= attr.value
104 end
105 end
106 if !element_attributes.empty?
107 json['element-attributes'] ||= element_attributes
108 end
109 json
110end
111
112# Returns a tuple of [type, revision, revision components] for a package node.
113def package_revision package
114 type_details = package.at_css('> type-details')
115 type = type_details.attributes['type']
116 type &&= type.value
117
118 revision = nil
119 components = nil
120
121 case type
122 when 'generic:genericDetailsType', 'addon:extraDetailsType', 'addon:mavenType'
123 major = text package.at_css('> revision > major')
124 minor = text package.at_css('> revision > minor')
125 micro = text package.at_css('> revision > micro')
126 preview = text package.at_css('> revision > preview')
127
128 revision = ''
129 components = []
130 unless empty?(major)
131 revision << major
132 components << major
133 end
134
135 unless empty?(minor)
136 revision << ".#{minor}"
137 components << minor
138 end
139
140 unless empty?(micro)
141 revision << ".#{micro}"
142 components << micro
143 end
144
145 unless empty?(preview)
146 revision << "-rc#{preview}"
147 components << preview
148 end
149 when 'sdk:platformDetailsType'
150 codename = text type_details.at_css('> codename')
151 api_level = text type_details.at_css('> api-level')
152 revision = empty?(codename) ? api_level : codename
153 components = [revision]
154 when 'sdk:sourceDetailsType'
155 api_level = text type_details.at_css('> api-level')
156 revision, components = api_level, [api_level]
157 when 'sys-img:sysImgDetailsType'
158 codename = text type_details.at_css('> codename')
159 api_level = text type_details.at_css('> api-level')
160 id = text type_details.at_css('> tag > id')
161 abi = text type_details.at_css('> abi')
162
163 revision = ''
164 components = []
165 if empty?(codename)
166 revision << api_level
167 components << api_level
168 else
169 revision << codename
170 components << codename
171 end
172
173 unless empty?(id)
174 revision << "-#{id}"
175 components << id
176 end
177
178 unless empty?(abi)
179 revision << "-#{abi}"
180 components << abi
181 end
182 when 'addon:addonDetailsType' then
183 api_level = text type_details.at_css('> api-level')
184 id = text type_details.at_css('> tag > id')
185 revision = api_level
186 components = [api_level, id]
187 end
188
189 [type, revision, components]
190end
191
192# Returns a hash of archives for the specified package node.
193def package_archives package
194 archives = {}
195 package.css('> archives > archive').each do |archive|
196 host_os = text archive.at_css('> host-os')
197 host_arch = text archive.at_css('> host-arch')
198 host_os = 'all' if empty?(host_os)
199 host_arch = 'all' if empty?(host_arch)
200 archives[host_os + host_arch] = {
201 'os' => host_os,
202 'arch' => host_arch,
203 'size' => Integer(text(archive.at_css('> complete > size'))),
204 'sha1' => text(archive.at_css('> complete > checksum')),
205 'url' => yield(text(archive.at_css('> complete > url')))
206 }
207 end
208 archives
209end
210
211# Returns the text from a node, or nil.
212def text node
213 node ? node.text : nil
214end
215
216# Nil or empty helper.
217def empty? value
218 !value || value.empty?
219end
220
221# Fixes up returned hashes by converting archives like
222# (e.g. {'linux' => {'sha1' => ...}, 'macosx' => ...} to
223# [{'os' => 'linux', 'sha1' => ...}, {'os' => 'macosx', ...}, ...].
224def fixup value
225 Hash[value.map do |k, v|
226 if k == 'archives' && v.is_a?(Hash)
227 [k, v.map do |os, archive|
228 fixup(archive)
229 end]
230 elsif v.is_a?(Hash)
231 [k, fixup(v)]
232 else
233 [k, v]
234 end
235 end]
236end
237
238# Today since Unix Epoch, January 1, 1970.
239def today
240 Time.now.utc.to_i / 24 / 60 / 60
241end
242
243# The expiration strategy. Expire if the last available day was before the `oldest_valid_day`.
244def expire_records record, oldest_valid_day
245 if record.is_a?(Hash)
246 if record.has_key?('last-available-day') &&
247 record['last-available-day'] < oldest_valid_day
248 return nil
249 end
250 update = {}
251 # This should only happen in the first run of this scrip after adding the `expire_record` function.
252 if record.has_key?('displayName') &&
253 !record.has_key?('last-available-day')
254 update['last-available-day'] = today
255 end
256 record.each {|key, value|
257 v = expire_records value, oldest_valid_day
258 update[key] = v if v
259 }
260 update
261 else
262 record
263 end
264end
265
266# Normalize the specified license text.
267# See: https://brash-snapper.glitch.me/ for how the munging works.
268def normalize_license license
269 license = license.dup
270 license.gsub!(/([^\n])\n([^\n])/m, '\1 \2')
271 license.gsub!(/ +/, ' ')
272 license.strip!
273 license
274end
275
276# Gets all license texts, deduplicating them.
277def get_licenses doc
278 licenses = {}
279 doc.css('license[type="text"]').each do |license_node|
280 license_id = license_node['id']
281 if license_id
282 licenses[license_id] ||= []
283 licenses[license_id] |= [normalize_license(text(license_node))]
284 end
285 end
286 licenses
287end
288
289def parse_package_xml doc
290 licenses = get_licenses doc
291 packages = {}
292 # check https://github.com/NixOS/nixpkgs/issues/373785
293 extras = {}
294
295 doc.css('remotePackage').each do |package|
296 name, _, version = package['path'].partition(';')
297 next if version == 'latest'
298
299 is_extras = name == 'extras'
300 if is_extras
301 name = package['path'].tr(';', '-')
302 end
303
304 type, revision, _ = package_revision(package)
305 next unless revision
306
307 path = package['path'].tr(';', '/')
308 display_name = text package.at_css('> display-name')
309 uses_license = package.at_css('> uses-license')
310 uses_license &&= uses_license['ref']
311 obsolete ||= package['obsolete']
312 type_details = to_json_collector package.at_css('> type-details')
313 revision_details = to_json_collector package.at_css('> revision')
314 archives = package_archives(package) {|url| repo_url url}
315 dependencies_xml = package.at_css('> dependencies')
316 dependencies = to_json_collector dependencies_xml if dependencies_xml
317
318 if is_extras
319 target = extras
320 component = package['path']
321 target = (target[component] ||= {})
322 else
323 target = (packages[name] ||= {})
324 target = (target[revision] ||= {})
325 end
326
327 target['name'] ||= name
328 target['path'] ||= path
329 target['revision'] ||= revision
330 target['displayName'] ||= display_name
331 target['license'] ||= uses_license if uses_license
332 target['obsolete'] ||= obsolete if obsolete == 'true'
333 target['type-details'] ||= type_details
334 target['revision-details'] ||= revision_details
335 target['dependencies'] ||= dependencies if dependencies
336 target['archives'] ||= {}
337 merge target['archives'], archives
338 target['last-available-day'] = today
339 end
340
341 [licenses, packages, extras]
342end
343
344def parse_image_xml doc
345 licenses = get_licenses doc
346 images = {}
347
348 doc.css('remotePackage[path^="system-images;"]').each do |package|
349 type, revision, components = package_revision(package)
350 next unless revision
351
352 path = package['path'].tr(';', '/')
353 display_name = text package.at_css('> display-name')
354 uses_license = package.at_css('> uses-license')
355 uses_license &&= uses_license['ref']
356 obsolete &&= package['obsolete']
357 type_details = to_json_collector package.at_css('> type-details')
358 revision_details = to_json_collector package.at_css('> revision')
359 archives = package_archives(package) {|url| image_url url, components[-2]}
360 dependencies_xml = package.at_css('> dependencies')
361 dependencies = to_json_collector dependencies_xml if dependencies_xml
362
363 target = images
364 components.each do |component|
365 target[component] ||= {}
366 target = target[component]
367 end
368
369 target['name'] ||= "system-image-#{revision}"
370 target['path'] ||= path
371 target['revision'] ||= revision
372 target['displayName'] ||= display_name
373 target['license'] ||= uses_license if uses_license
374 target['obsolete'] ||= obsolete if obsolete
375 target['type-details'] ||= type_details
376 target['revision-details'] ||= revision_details
377 target['dependencies'] ||= dependencies if dependencies
378 target['archives'] ||= {}
379 merge target['archives'], archives
380 target['last-available-day'] = today
381 end
382
383 [licenses, images]
384end
385
386def parse_addon_xml doc
387 licenses = get_licenses doc
388 addons, extras = {}, {}
389
390 doc.css('remotePackage').each do |package|
391 type, revision, components = package_revision(package)
392 next unless revision
393
394 path = package['path'].tr(';', '/')
395 display_name = text package.at_css('> display-name')
396 uses_license = package.at_css('> uses-license')
397 uses_license &&= uses_license['ref']
398 obsolete &&= package['obsolete']
399 type_details = to_json_collector package.at_css('> type-details')
400 revision_details = to_json_collector package.at_css('> revision')
401 archives = package_archives(package) {|url| repo_url url}
402 dependencies_xml = package.at_css('> dependencies')
403 dependencies = to_json_collector dependencies_xml if dependencies_xml
404
405 case type
406 when 'addon:addonDetailsType'
407 name = components.last
408 target = addons
409
410 # Hack for Google APIs 25 r1, which displays as 23 for some reason
411 archive_name = text package.at_css('> archives > archive > complete > url')
412 if archive_name == 'google_apis-25_r1.zip'
413 path = 'add-ons/addon-google_apis-google-25'
414 revision = '25'
415 components = [revision, components.last]
416 end
417 when 'addon:extraDetailsType', 'addon:mavenType'
418 name = package['path'].tr(';', '-')
419 components = [package['path']]
420 target = extras
421 end
422
423 components.each do |component|
424 target = (target[component] ||= {})
425 end
426
427 target['name'] ||= name
428 target['path'] ||= path
429 target['revision'] ||= revision
430 target['displayName'] ||= display_name
431 target['license'] ||= uses_license if uses_license
432 target['obsolete'] ||= obsolete if obsolete
433 target['type-details'] ||= type_details
434 target['revision-details'] ||= revision_details
435 target['dependencies'] ||= dependencies if dependencies
436 target['archives'] ||= {}
437 merge target['archives'], archives
438 target['last-available-day'] = today
439 end
440
441 [licenses, addons, extras]
442end
443
444# Make the clean diff by always sorting the result before puting it in the stdout.
445def sort_recursively value
446 if value.is_a?(Hash)
447 Hash[
448 value.map do |k, v|
449 [k, sort_recursively(v)]
450 end.sort_by {|(k, v)| k }
451 ]
452 elsif value.is_a?(Array)
453 value.map do |v| sort_recursively(v) end
454 else
455 value
456 end
457end
458
459def merge_recursively a, b
460 a.merge!(b) {|key, a_item, b_item|
461 if a_item.is_a?(Hash) && b_item.is_a?(Hash)
462 merge_recursively(a_item, b_item)
463 elsif b_item != nil
464 b_item
465 end
466 }
467 a
468end
469
470def merge dest, src
471 merge_recursively dest, src
472end
473
474opts = Slop.parse do |o|
475 o.array '-p', '--packages', 'packages repo XMLs to parse', default: %w[repo://repository#2-3]
476 o.array '-i', '--images', 'system image repo XMLs to parse', default: %w[
477 image://android#2-3
478 image://android-tv#2-3
479 image://android-wear#2-3
480 image://android-wear-cn#2-3
481 image://android-automotive#2-3
482 image://google_apis#2-3
483 image://google_apis_playstore#2-3
484 ]
485 o.array '-a', '--addons', 'addon repo XMLs to parse', default: %w[repo://addon#2-3]
486 o.string '-I', '--input', 'input JSON file for repo', default: File.join(__dir__, 'repo.json')
487 o.string '-O', '--output', 'output JSON file for repo', default: File.join(__dir__, 'repo.json')
488end
489
490result = {}
491result['licenses'] = {}
492result['packages'] = {}
493result['images'] = {}
494result['addons'] = {}
495result['extras'] = {}
496
497opts[:packages].each do |filename|
498 licenses, packages, extras = parse_package_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
499 merge result['licenses'], licenses
500 merge result['packages'], packages
501 merge result['extras'], extras
502end
503
504opts[:images].each do |filename|
505 licenses, images = parse_image_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
506 merge result['licenses'], licenses
507 merge result['images'], images
508end
509
510opts[:addons].each do |filename|
511 licenses, addons, extras = parse_addon_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
512 merge result['licenses'], licenses
513 merge result['addons'], addons
514 merge result['extras'], extras
515end
516
517result['latest'] = {}
518result['packages'].each do |name, versions|
519 max_version = Gem::Version.new('0')
520 versions.each do |version, package|
521 if package['license'] == 'android-sdk-license' && Gem::Version.correct?(package['revision'])
522 package_version = Gem::Version.new(package['revision'])
523 max_version = package_version if package_version > max_version
524 end
525 end
526 result['latest'][name] = max_version.to_s
527end
528
529# As we keep the old packages in the repo JSON file, we should have
530# a strategy to remove them at some point!
531# So with this variable we claim it's okay to remove them from the
532# JSON after two years that they are not available.
533two_years_ago = today - 2 * 365
534
535input = {}
536prev_latest = {}
537begin
538 input_json = if File.exist?(opts[:input])
539 STDERR.puts "Reading #{opts[:input]}"
540 File.read(opts[:input])
541 else
542 STDERR.puts "Creating new repo"
543 "{}"
544 end
545
546 if input_json != nil && !input_json.empty?
547 input = expire_records(JSON.parse(input_json), two_years_ago)
548
549 # Just create a new set of latest packages.
550 prev_latest = input['latest'] || {}
551 input['latest'] = {}
552 end
553rescue JSON::ParserError => e
554 STDERR.write(e.message)
555 return
556end
557
558fixup_result = fixup(result)
559
560# Regular installation of Android SDK would keep the previously installed packages even if they are not
561# in the uptodate XML files, so here we try to support this logic by keeping un-available packages,
562# therefore the old packages will work as long as the links are working on the Google servers.
563output = merge input, fixup_result
564
565# Write the repository. Append a \n to keep nixpkgs Github Actions happy.
566STDERR.puts "Writing #{opts[:output]}"
567File.write opts[:output], (JSON.pretty_generate(sort_recursively(output)) + "\n")
568
569# Output metadata for the nixpkgs update script.
570if ENV['UPDATE_NIX_ATTR_PATH']
571 # See if there are any changes in the latest versions.
572 cur_latest = output['latest'] || {}
573
574 old_versions = []
575 new_versions = []
576 changes = []
577 changed = false
578
579 cur_latest.each do |k, v|
580 prev = prev_latest[k]
581 if prev && prev != v
582 old_versions << "#{k}:#{prev}"
583 new_versions << "#{k}:#{v}"
584 changes << "#{k}: #{prev} -> #{v}"
585 changed = true
586 end
587 end
588
589 changed_paths = []
590 if changed
591 # Instantiate it.
592 test_result = `NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ACCEPT_ANDROID_SDK_LICENSE=1 nix-build #{Shellwords.escape(File.realpath(File.join(__dir__, '..', '..', '..', '..', 'default.nix')))} -A #{Shellwords.join [ENV['UPDATE_NIX_ATTR_PATH']]} 2>&1`
593 test_status = $?.exitstatus
594
595 template = ERB.new(<<-EOF, trim_mode: '<>-')
596androidenv: <%= changes.join('; ') %>
597
598Performed the following automatic androidenv updates:
599
600<% changes.each do |change| %>
601- <%= change -%>
602<% end %>
603
604Tests exited with status: <%= test_status -%>
605
606<% if !test_result.empty? %>
607Last 100 lines of output:
608```
609<%= test_result.lines.last(100).join -%>
610```
611<% end %>
612EOF
613
614 changed_paths << {
615 attrPath: 'androidenv.androidPkgs.androidsdk',
616 oldVersion: old_versions.join('; '),
617 newVersion: new_versions.join('; '),
618 files: [
619 opts[:output]
620 ],
621 commitMessage: template.result(binding)
622 }
623 end
624
625 # nix-update info is on stdout
626 STDOUT.puts JSON.pretty_generate(changed_paths)
627end