pyrox.dev
1#! /usr/bin/env nix-shell
2#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nix
3
4# This command uploads tarballs to tarballs.nixos.org, the
5# content-addressed cache used by fetchurl as a fallback for when
6# upstream tarballs disappear or change. Usage:
7#
8# 1) To upload one or more files:
9#
10# $ copy-tarballs.pl --file /path/to/tarball.tar.gz
11#
12# 2) To upload all files obtained via calls to fetchurl in a Nix derivation:
13#
14# $ copy-tarballs.pl --expr '(import <nixpkgs> {}).hello'
15
16use strict;
17use warnings;
18use File::Basename;
19use File::Path;
20use File::Slurp;
21use JSON;
22use Net::Amazon::S3;
23
24sub usage {
25 die "Syntax: $0 [--dry-run] [--exclude REGEXP] [--expr EXPR | --file FILES...]\n";
26}
27
28sub computeFixedOutputPath {
29 my ($name, $algo, $hash) = @_;
30 my $expr = <<'EXPR';
31{ name, outputHashAlgo, outputHash }:
32builtins.toString (derivation {
33 inherit name outputHashAlgo outputHash;
34 builder = "false";
35 system = "dontcare";
36 outputHashMode = "flat";
37})
38EXPR
39 open(my $fh, "-|",
40 "nix-instantiate",
41 "--eval",
42 "--strict",
43 "-E", $expr,
44 "--argstr", "name", $name,
45 "--argstr", "outputHashAlgo", $algo,
46 "--argstr", "outputHash", $hash) or die "Failed to run nix-instantiate: $!";
47
48 my $storePathJson = <$fh>;
49 chomp $storePathJson;
50 my $storePath = decode_json($storePathJson);
51 close $fh;
52 return $storePath;
53}
54
55sub nixHash {
56 my ($algo, $base16, $path) = @_;
57 open(my $fh, "-|",
58 "nix-hash",
59 "--type", $algo,
60 "--flat",
61 ($base16 ? "--base16" : ()),
62 $path) or die "Failed to run nix-hash: $!";
63 my $hash = <$fh>;
64 chomp $hash;
65 return $hash;
66}
67
68my $dryRun = 0;
69my $expr;
70my @fileNames;
71my $exclude;
72
73while (@ARGV) {
74 my $flag = shift @ARGV;
75
76 if ($flag eq "--expr") {
77 $expr = shift @ARGV or die "--expr requires an argument";
78 } elsif ($flag eq "--file") {
79 @fileNames = @ARGV;
80 last;
81 } elsif ($flag eq "--dry-run") {
82 $dryRun = 1;
83 } elsif ($flag eq "--exclude") {
84 $exclude = shift @ARGV or die "--exclude requires an argument";
85 } else {
86 usage();
87 }
88}
89
90my $bucket;
91
92if (not defined $ENV{DEBUG}) {
93 # S3 setup.
94 my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "AWS_ACCESS_KEY_ID not set\n";
95 my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "AWS_SECRET_ACCESS_KEY not set\n";
96
97 my $s3 = Net::Amazon::S3->new(
98 { aws_access_key_id => $aws_access_key_id,
99 aws_secret_access_key => $aws_secret_access_key,
100 retry => 1,
101 host => "s3-eu-west-1.amazonaws.com",
102 });
103
104 $bucket = $s3->bucket("nixpkgs-tarballs") or die;
105}
106
107my $doWrite = 0;
108my $cacheFile = ($ENV{"HOME"} or die "\$HOME is not set") . "/.cache/nix/copy-tarballs";
109my %cache;
110$cache{$_} = 1 foreach read_file($cacheFile, err_mode => 'quiet', chomp => 1);
111$doWrite = 1;
112
113END() {
114 File::Path::mkpath(dirname($cacheFile), 0, 0755);
115 write_file($cacheFile, map { "$_\n" } keys %cache) if $doWrite;
116}
117
118sub alreadyMirrored {
119 my ($algo, $hash) = @_;
120 my $key = "$algo/$hash";
121 return 1 if defined $cache{$key};
122 my $res = defined $bucket->get_key($key);
123 $cache{$key} = 1 if $res;
124 return $res;
125}
126
127sub uploadFile {
128 my ($fn, $name) = @_;
129
130 my $md5_16 = nixHash("md5", 0, $fn) or die;
131 my $sha1_16 = nixHash("sha1", 0, $fn) or die;
132 my $sha256_32 = nixHash("sha256", 1, $fn) or die;
133 my $sha256_16 = nixHash("sha256", 0, $fn) or die;
134 my $sha512_32 = nixHash("sha512", 1, $fn) or die;
135 my $sha512_16 = nixHash("sha512", 0, $fn) or die;
136
137 my $mainKey = "sha512/$sha512_16";
138
139 # Create redirects from the other hash types.
140 sub redirect {
141 my ($name, $dest) = @_;
142 #print STDERR "linking $name to $dest...\n";
143 $bucket->add_key($name, "", {
144 'x-amz-website-redirect-location' => "/" . $dest,
145 'x-amz-acl' => "public-read"
146 })
147 or die "failed to create redirect from $name to $dest\n";
148 $cache{$name} = 1;
149 }
150 redirect "md5/$md5_16", $mainKey;
151 redirect "sha1/$sha1_16", $mainKey;
152 redirect "sha256/$sha256_32", $mainKey;
153 redirect "sha256/$sha256_16", $mainKey;
154 redirect "sha512/$sha512_32", $mainKey;
155
156 # Upload the file as sha512/<hash-in-base-16>.
157 print STDERR "uploading $fn to $mainKey...\n";
158 $bucket->add_key_filename($mainKey, $fn, {
159 'x-amz-meta-original-name' => $name,
160 'x-amz-acl' => "public-read"
161 })
162 or die "failed to upload $fn to $mainKey\n";
163 $cache{$mainKey} = 1;
164}
165
166if (scalar @fileNames) {
167 my $res = 0;
168 foreach my $fn (@fileNames) {
169 eval {
170 if (alreadyMirrored("sha512", nixHash("sha512", 0, $fn))) {
171 print STDERR "$fn is already mirrored\n";
172 } else {
173 uploadFile($fn, basename $fn);
174 }
175 };
176 if ($@) {
177 warn "$@";
178 $res = 1;
179 }
180 }
181 exit $res;
182}
183
184elsif (defined $expr) {
185
186 # Evaluate find-tarballs.nix.
187 my $pid = open(JSON, "-|", "nix-instantiate", "--eval", "--json", "--strict",
188 "<nixpkgs/maintainers/scripts/find-tarballs.nix>",
189 "--arg", "expr", $expr);
190 my $stdout = <JSON>;
191 waitpid($pid, 0);
192 die "$0: evaluation failed\n" if $?;
193 close JSON;
194
195 my $fetches = decode_json($stdout);
196
197 print STDERR "evaluation returned ", scalar(@{$fetches}), " tarballs\n";
198
199 # Check every fetchurl call discovered by find-tarballs.nix.
200 my $mirrored = 0;
201 my $have = 0;
202 foreach my $fetch (sort { $a->{urls}->[0] cmp $b->{urls}->[0] } @{$fetches}) {
203 my $urls = $fetch->{urls};
204 my $algo = $fetch->{type};
205 my $hash = $fetch->{hash};
206 my $name = $fetch->{name};
207 my $isPatch = $fetch->{isPatch};
208
209 if ($isPatch) {
210 print STDERR "skipping $urls->[0] (support for patches is missing)\n";
211 next;
212 }
213
214 if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) {
215 $algo = $1;
216 open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", $hash) or die;
217 $hash = <$fh>;
218 close $fh;
219 chomp $hash;
220 }
221
222 next unless $algo =~ /^[a-z0-9]+$/;
223
224 # Convert non-SRI base-64 to base-16.
225 if ($hash =~ /^[A-Za-z0-9+\/=]+$/) {
226 open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", "--hash-algo", $algo, $hash) or die;
227 $hash = <$fh>;
228 close $fh;
229 chomp $hash;
230 }
231
232 my $storePath = computeFixedOutputPath($name, $algo, $hash);
233
234 for my $url (@$urls) {
235 if (defined $ENV{DEBUG}) {
236 print "$url $algo $hash\n";
237 next;
238 }
239
240 if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
241 print STDERR "skipping $url (unsupported scheme)\n";
242 next;
243 }
244
245 next if defined $exclude && $url =~ /$exclude/;
246
247 if (alreadyMirrored($algo, $hash)) {
248 $have++;
249 last;
250 }
251
252 print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
253
254
255 if ($dryRun) {
256 $mirrored++;
257 last;
258 }
259 my $isValidPath = system("nix-store", "-r", $storePath) == 0;
260
261 # Otherwise download the file using nix-prefetch-url.
262 if (!$isValidPath) {
263 $ENV{QUIET} = 1;
264 $ENV{PRINT_PATH} = 1;
265 my $fh;
266 my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
267 waitpid($pid, 0) or die;
268 if ($? != 0) {
269 print STDERR "failed to fetch $url: $?\n";
270 next;
271 }
272 <$fh>; my $storePath2 = <$fh>; chomp $storePath2;
273 if ($storePath ne $storePath2) {
274 warn "strange: $storePath != $storePath2\n";
275 next;
276 }
277 }
278
279 uploadFile($storePath, $url);
280 $mirrored++;
281 last;
282 }
283 }
284
285 print STDERR "mirrored $mirrored files, already have $have files\n";
286}
287
288else {
289 usage();
290}