pyrox.dev
1#! /usr/bin/env nix-shell
2#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nix nix.perl-bindings
3
4# This command uploads tarballs to tarballs.nixos.org, the
5# content-addressed cache used by fetchurl as a fallback for when
6# upstream tarballs disappear or change. Usage:
7#
8# 1) To upload one or more files:
9#
10# $ copy-tarballs.pl --file /path/to/tarball.tar.gz
11#
12# 2) To upload all files obtained via calls to fetchurl in a Nix derivation:
13#
14# $ copy-tarballs.pl --expr '(import <nixpkgs> {}).hello'
15
16use strict;
17use warnings;
18use File::Basename;
19use File::Path;
20use File::Slurp;
21use JSON;
22use Net::Amazon::S3;
23use Nix::Store;
24
25isValidPath("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"); # FIXME: forces Nix::Store initialisation
26
27sub usage {
28 die "Syntax: $0 [--dry-run] [--exclude REGEXP] [--expr EXPR | --file FILES...]\n";
29}
30
31my $dryRun = 0;
32my $expr;
33my @fileNames;
34my $exclude;
35
36while (@ARGV) {
37 my $flag = shift @ARGV;
38
39 if ($flag eq "--expr") {
40 $expr = shift @ARGV or die "--expr requires an argument";
41 } elsif ($flag eq "--file") {
42 @fileNames = @ARGV;
43 last;
44 } elsif ($flag eq "--dry-run") {
45 $dryRun = 1;
46 } elsif ($flag eq "--exclude") {
47 $exclude = shift @ARGV or die "--exclude requires an argument";
48 } else {
49 usage();
50 }
51}
52
53my $bucket;
54
55if (not defined $ENV{DEBUG}) {
56 # S3 setup.
57 my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "AWS_ACCESS_KEY_ID not set\n";
58 my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "AWS_SECRET_ACCESS_KEY not set\n";
59
60 my $s3 = Net::Amazon::S3->new(
61 { aws_access_key_id => $aws_access_key_id,
62 aws_secret_access_key => $aws_secret_access_key,
63 retry => 1,
64 host => "s3-eu-west-1.amazonaws.com",
65 });
66
67 $bucket = $s3->bucket("nixpkgs-tarballs") or die;
68}
69
70my $doWrite = 0;
71my $cacheFile = ($ENV{"HOME"} or die "\$HOME is not set") . "/.cache/nix/copy-tarballs";
72my %cache;
73$cache{$_} = 1 foreach read_file($cacheFile, err_mode => 'quiet', chomp => 1);
74$doWrite = 1;
75
76END() {
77 File::Path::mkpath(dirname($cacheFile), 0, 0755);
78 write_file($cacheFile, map { "$_\n" } keys %cache) if $doWrite;
79}
80
81sub alreadyMirrored {
82 my ($algo, $hash) = @_;
83 my $key = "$algo/$hash";
84 return 1 if defined $cache{$key};
85 my $res = defined $bucket->get_key($key);
86 $cache{$key} = 1 if $res;
87 return $res;
88}
89
90sub uploadFile {
91 my ($fn, $name) = @_;
92
93 my $md5_16 = hashFile("md5", 0, $fn) or die;
94 my $sha1_16 = hashFile("sha1", 0, $fn) or die;
95 my $sha256_32 = hashFile("sha256", 1, $fn) or die;
96 my $sha256_16 = hashFile("sha256", 0, $fn) or die;
97 my $sha512_32 = hashFile("sha512", 1, $fn) or die;
98 my $sha512_16 = hashFile("sha512", 0, $fn) or die;
99
100 my $mainKey = "sha512/$sha512_16";
101
102 # Create redirects from the other hash types.
103 sub redirect {
104 my ($name, $dest) = @_;
105 #print STDERR "linking $name to $dest...\n";
106 $bucket->add_key($name, "", {
107 'x-amz-website-redirect-location' => "/" . $dest,
108 'x-amz-acl' => "public-read"
109 })
110 or die "failed to create redirect from $name to $dest\n";
111 $cache{$name} = 1;
112 }
113 redirect "md5/$md5_16", $mainKey;
114 redirect "sha1/$sha1_16", $mainKey;
115 redirect "sha256/$sha256_32", $mainKey;
116 redirect "sha256/$sha256_16", $mainKey;
117 redirect "sha512/$sha512_32", $mainKey;
118
119 # Upload the file as sha512/<hash-in-base-16>.
120 print STDERR "uploading $fn to $mainKey...\n";
121 $bucket->add_key_filename($mainKey, $fn, {
122 'x-amz-meta-original-name' => $name,
123 'x-amz-acl' => "public-read"
124 })
125 or die "failed to upload $fn to $mainKey\n";
126 $cache{$mainKey} = 1;
127}
128
129if (scalar @fileNames) {
130 my $res = 0;
131 foreach my $fn (@fileNames) {
132 eval {
133 if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) {
134 print STDERR "$fn is already mirrored\n";
135 } else {
136 uploadFile($fn, basename $fn);
137 }
138 };
139 if ($@) {
140 warn "$@";
141 $res = 1;
142 }
143 }
144 exit $res;
145}
146
147elsif (defined $expr) {
148
149 # Evaluate find-tarballs.nix.
150 my $pid = open(JSON, "-|", "nix-instantiate", "--eval", "--json", "--strict",
151 "<nixpkgs/maintainers/scripts/find-tarballs.nix>",
152 "--arg", "expr", $expr);
153 my $stdout = <JSON>;
154 waitpid($pid, 0);
155 die "$0: evaluation failed\n" if $?;
156 close JSON;
157
158 my $fetches = decode_json($stdout);
159
160 print STDERR "evaluation returned ", scalar(@{$fetches}), " tarballs\n";
161
162 # Check every fetchurl call discovered by find-tarballs.nix.
163 my $mirrored = 0;
164 my $have = 0;
165 foreach my $fetch (sort { $a->{urls}->[0] cmp $b->{urls}->[0] } @{$fetches}) {
166 my $urls = $fetch->{urls};
167 my $algo = $fetch->{type};
168 my $hash = $fetch->{hash};
169 my $name = $fetch->{name};
170 my $isPatch = $fetch->{isPatch};
171
172 if ($isPatch) {
173 print STDERR "skipping $urls->[0] (support for patches is missing)\n";
174 next;
175 }
176
177 if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) {
178 $algo = $1;
179 $hash = `nix hash to-base16 $hash` or die;
180 chomp $hash;
181 }
182
183 next unless $algo =~ /^[a-z0-9]+$/;
184
185 # Convert non-SRI base-64 to base-16.
186 if ($hash =~ /^[A-Za-z0-9+\/=]+$/) {
187 $hash = `nix hash to-base16 --type '$algo' $hash` or die;
188 chomp $hash;
189 }
190
191 my $storePath = makeFixedOutputPath(0, $algo, $hash, $name);
192
193 for my $url (@$urls) {
194 if (defined $ENV{DEBUG}) {
195 print "$url $algo $hash\n";
196 next;
197 }
198
199 if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
200 print STDERR "skipping $url (unsupported scheme)\n";
201 next;
202 }
203
204 next if defined $exclude && $url =~ /$exclude/;
205
206 if (alreadyMirrored($algo, $hash)) {
207 $have++;
208 last;
209 }
210
211 print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
212
213 if ($dryRun) {
214 $mirrored++;
215 last;
216 }
217
218 # Substitute the output.
219 if (!isValidPath($storePath)) {
220 system("nix-store", "-r", $storePath);
221 }
222
223 # Otherwise download the file using nix-prefetch-url.
224 if (!isValidPath($storePath)) {
225 $ENV{QUIET} = 1;
226 $ENV{PRINT_PATH} = 1;
227 my $fh;
228 my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
229 waitpid($pid, 0) or die;
230 if ($? != 0) {
231 print STDERR "failed to fetch $url: $?\n";
232 next;
233 }
234 <$fh>; my $storePath2 = <$fh>; chomp $storePath2;
235 if ($storePath ne $storePath2) {
236 warn "strange: $storePath != $storePath2\n";
237 next;
238 }
239 }
240
241 uploadFile($storePath, $url);
242 $mirrored++;
243 last;
244 }
245 }
246
247 print STDERR "mirrored $mirrored files, already have $have files\n";
248}
249
250else {
251 usage();
252}