commit e6146272ea972699225ea6d9c5f416d335f43ced · pyrox.dev/nixpkgs

+58 -20
maintainers/scripts/copy-tarballs.pl
···

       1
       1
        
       #! /usr/bin/env nix-shell

     

       2
       2
       -
       #! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nix nix.perl-bindings

     

       2
       2
       +
       #! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nix

     

       3
       3
        
       

     

       4
       4
        
       # This command uploads tarballs to tarballs.nixos.org, the

     

       5
       5
        
       # content-addressed cache used by fetchurl as a fallback for when

     
···

       20
       20
        
       use File::Slurp;

     

       21
       21
        
       use JSON;

     

       22
       22
        
       use Net::Amazon::S3;

     

       23
       23
       -
       use Nix::Store;

     

       24
       24
       -
       

     

       25
       25
       -
       isValidPath("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"); # FIXME: forces Nix::Store initialisation

     

       26
       23
        
       

     

       27
       24
        
       sub usage {

     

       28
       25
        
           die "Syntax: $0 [--dry-run] [--exclude REGEXP] [--expr EXPR | --file FILES...]\n";

     

       29
       26
        
       }

     

       30
       27
        
       

     

       28
       28
       +
       sub computeFixedOutputPath {

     

       29
       29
       +
           my ($name, $algo, $hash) = @_;

     

       30
       30
       +
           my $expr = <<'EXPR';

     

       31
       31
       +
       { name, outputHashAlgo, outputHash }:

     

       32
       32
       +
       builtins.toString (derivation {

     

       33
       33
       +
         inherit name outputHashAlgo outputHash;

     

       34
       34
       +
         builder = "false";

     

       35
       35
       +
         system = "dontcare";

     

       36
       36
       +
         outputHashMode = "flat";

     

       37
       37
       +
       })

     

       38
       38
       +
       EXPR

     

       39
       39
       +
           open(my $fh, "-|",

     

       40
       40
       +
               "nix-instantiate",

     

       41
       41
       +
               "--eval",

     

       42
       42
       +
               "--strict",

     

       43
       43
       +
               "-E", $expr,

     

       44
       44
       +
               "--argstr", "name", $name,

     

       45
       45
       +
               "--argstr", "outputHashAlgo", $algo,

     

       46
       46
       +
               "--argstr", "outputHash", $hash) or die "Failed to run nix-instantiate: $!";

     

       47
       47
       +
       

     

       48
       48
       +
           my $storePathJson = <$fh>;

     

       49
       49
       +
           chomp $storePathJson;

     

       50
       50
       +
           my $storePath = decode_json($storePathJson);

     

       51
       51
       +
           close $fh;

     

       52
       52
       +
           return $storePath;

     

       53
       53
       +
       }

     

       54
       54
       +
       

     

       55
       55
       +
       sub nixHash {

     

       56
       56
       +
           my ($algo, $base16, $path) = @_;

     

       57
       57
       +
           open(my $fh, "-|",

     

       58
       58
       +
               "nix-hash",

     

       59
       59
       +
               "--type", $algo,

     

       60
       60
       +
               "--flat",

     

       61
       61
       +
               ($base16 ? "--base16" : ()),

     

       62
       62
       +
               $path) or die "Failed to run nix-hash: $!";

     

       63
       63
       +
           my $hash = <$fh>;

     

       64
       64
       +
           chomp $hash;

     

       65
       65
       +
           return $hash;

     

       66
       66
       +
       }

     

       67
       67
       +
       

     

       31
       68
        
       my $dryRun = 0;

     

       32
       69
        
       my $expr;

     

       33
       70
        
       my @fileNames;

     
···

       90
       127
        
       sub uploadFile {

     

       91
       128
        
           my ($fn, $name) = @_;

     

       92
       129
        
       

     

       93
       93
       -
           my $md5_16 = hashFile("md5", 0, $fn) or die;

     

       94
       94
       -
           my $sha1_16 = hashFile("sha1", 0, $fn) or die;

     

       95
       95
       -
           my $sha256_32 = hashFile("sha256", 1, $fn) or die;

     

       96
       96
       -
           my $sha256_16 = hashFile("sha256", 0, $fn) or die;

     

       97
       97
       -
           my $sha512_32 = hashFile("sha512", 1, $fn) or die;

     

       98
       98
       -
           my $sha512_16 = hashFile("sha512", 0, $fn) or die;

     

       130
       130
       +
           my $md5_16 = nixHash("md5", 0, $fn) or die;

     

       131
       131
       +
           my $sha1_16 = nixHash("sha1", 0, $fn) or die;

     

       132
       132
       +
           my $sha256_32 = nixHash("sha256", 1, $fn) or die;

     

       133
       133
       +
           my $sha256_16 = nixHash("sha256", 0, $fn) or die;

     

       134
       134
       +
           my $sha512_32 = nixHash("sha512", 1, $fn) or die;

     

       135
       135
       +
           my $sha512_16 = nixHash("sha512", 0, $fn) or die;

     

       99
       136
        
       

     

       100
       137
        
           my $mainKey = "sha512/$sha512_16";

     

       101
       138
        
       

     
···

       130
       167
        
           my $res = 0;

     

       131
       168
        
           foreach my $fn (@fileNames) {

     

       132
       169
        
               eval {

     

       133
       133
       -
                   if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) {

     

       170
       170
       +
                   if (alreadyMirrored("sha512", nixHash("sha512", 0, $fn))) {

     

       134
       171
        
                       print STDERR "$fn is already mirrored\n";

     

       135
       172
        
                   } else {

     

       136
       173
        
                       uploadFile($fn, basename $fn);

     
···

       176
       213
        
       

     

       177
       214
        
               if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) {

     

       178
       215
        
                   $algo = $1;

     

       179
       179
       -
                   $hash = `nix --extra-experimental-features nix-command hash to-base16 $hash` or die;

     

       216
       216
       +
                   open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", $hash) or die;

     

       217
       217
       +
                   $hash = <$fh>;

     

       218
       218
       +
                   close $fh;

     

       180
       219
        
                   chomp $hash;

     

       181
       220
        
               }

     

       182
       221
        
       

     
···

       184
       223
        
       

     

       185
       224
        
               # Convert non-SRI base-64 to base-16.

     

       186
       225
        
               if ($hash =~ /^[A-Za-z0-9+\/=]+$/) {

     

       187
       187
       -
                   $hash = `nix --extra-experimental-features nix-command hash to-base16 --type '$algo' $hash` or die;

     

       226
       226
       +
                   open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", "--hash-algo", $algo, $hash) or die;

     

       227
       227
       +
                   $hash = <$fh>;

     

       228
       228
       +
                   close $fh;

     

       188
       229
        
                   chomp $hash;

     

       189
       230
        
               }

     

       190
       231
        
       

     

       191
       191
       -
               my $storePath = makeFixedOutputPath(0, $algo, $hash, $name);

     

       232
       232
       +
               my $storePath = computeFixedOutputPath($name, $algo, $hash);

     

       192
       233
        
       

     

       193
       234
        
               for my $url (@$urls) {

     

       194
       235
        
                   if (defined $ENV{DEBUG}) {

     
···

       210
       251
        
       

     

       211
       252
        
                   print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";

     

       212
       253
        
       

     

       254
       254
       +
       

     

       213
       255
        
                   if ($dryRun) {

     

       214
       256
        
                       $mirrored++;

     

       215
       257
        
                       last;

     

       216
       258
        
                   }

     

       217
       217
       -
       

     

       218
       218
       -
                   # Substitute the output.

     

       219
       219
       -
                   if (!isValidPath($storePath)) {

     

       220
       220
       -
                       system("nix-store", "-r", $storePath);

     

       221
       221
       -
                   }

     

       259
       259
       +
                   my $isValidPath = system("nix-store", "-r", $storePath) == 0;

     

       222
       260
        
       

     

       223
       261
        
                   # Otherwise download the file using nix-prefetch-url.

     

       224
       224
       -
                   if (!isValidPath($storePath)) {

     

       262
       262
       +
                   if (!$isValidPath) {

     

       225
       263
        
                       $ENV{QUIET} = 1;

     

       226
       264
        
                       $ENV{PRINT_PATH} = 1;

     

       227
       265
        
                       my $fh;