commit 6ea4ff601eecb63a781a8f29b28270820126310f · pyrox.dev/nixpkgs

+8

doc/build-helpers/images/binarycache.section.md

···

       11
       11
        
       `rootPaths` must be a list of derivations.

     

       12
       12
        
       The transitive closure of these derivations' outputs will be copied into the cache.

     

       13
       13
        
       

     

       14
       14
       +
       ## Optional arguments {#sec-pkgs-binary-cache-arguments}

     

       15
       15
       +
       

     

       16
       16
       +
       `compression` (`"none"` or `"xz"` or `"zstd"`; _optional_)

     

       17
       17
       +
       

     

       18
       18
       +
       : The compression algorithm to use.

     

       19
       19
       +
       

     

       20
       20
       +
         _Default value:_ `zstd`.

     

       21
       21
       +
       

     

       14
       22
        
       ::: {.note}

     

       15
       23
        
       This function is meant for advanced use cases.

     

       16
       24
        
       The more idiomatic way to work with flat-file binary caches is via the [nix-copy-closure](https://nixos.org/manual/nix/stable/command-ref/nix-copy-closure.html) command.

+3

doc/redirects.json

···

       1971
       1971
        
         "sec-pkgs-binary-cache": [

     

       1972
       1972
        
           "index.html#sec-pkgs-binary-cache"

     

       1973
       1973
        
         ],

     

       1974
       1974
       +
         "sec-pkgs-binary-cache-arguments": [

     

       1975
       1975
       +
           "index.html#sec-pkgs-binary-cache-arguments"

     

       1976
       1976
       +
         ],

     

       1974
       1977
        
         "sec-pkgs-binary-cache-example": [

     

       1975
       1978
        
           "index.html#sec-pkgs-binary-cache-example"

     

       1976
       1979
        
         ],

+3 -1

nixos/tests/all-tests.nix

···

       196
       196
        
         beanstalkd = handleTest ./beanstalkd.nix {};

     

       197
       197
        
         bees = handleTest ./bees.nix {};

     

       198
       198
        
         benchexec = handleTest ./benchexec.nix {};

     

       199
       199
       -
         binary-cache = handleTest ./binary-cache.nix {};

     

       199
       199
       +
         binary-cache = handleTest ./binary-cache.nix { compression = "zstd"; };

     

       200
       200
       +
         binary-cache-no-compression = handleTest ./binary-cache.nix { compression = "none"; };

     

       201
       201
       +
         binary-cache-xz = handleTest ./binary-cache.nix { compression = "xz"; };

     

       200
       202
        
         bind = handleTest ./bind.nix {};

     

       201
       203
        
         bird = handleTest ./bird.nix {};

     

       202
       204
        
         birdwatcher = handleTest ./birdwatcher.nix {};

+10 -3

nixos/tests/binary-cache.nix

···

       1
       1
       +
       { compression, ... }@args:

     

       2
       2
       +
       

     

       1
       3
        
       import ./make-test-python.nix (

     

       2
       4
        
         { lib, pkgs, ... }:

     

       3
       5
        
       

     

       4
       6
        
         {

     

       5
       5
       -
           name = "binary-cache";

     

       7
       7
       +
           name = "binary-cache-" + compression;

     

       6
       8
        
           meta.maintainers = with lib.maintainers; [ thomasjm ];

     

       7
       9
        
       

     

       8
       10
        
           nodes.machine =

     
···

       24
       26
        
                     nativeBuildInputs = [ openssl ];

     

       25
       27
        
                   }

     

       26
       28
        
                   ''

     

       27
       27
       -
                     tar -czf tmp.tar.gz -C "${mkBinaryCache { rootPaths = [ hello ]; }}" .

     

       29
       29
       +
                     tar -czf tmp.tar.gz -C "${

     

       30
       30
       +
                       mkBinaryCache {

     

       31
       31
       +
                         rootPaths = [ hello ];

     

       32
       32
       +
                         inherit compression;

     

       33
       33
       +
                       }

     

       34
       34
       +
                     }" .

     

       28
       35
        
                     openssl enc -aes-256-cbc -salt -in tmp.tar.gz -out $out -k mysecretpassword

     

       29
       36
        
                   '';

     

       30
       37
        
       

     
···

       78
       85
        
             machine.succeed("[ -d %s ] || exit 1" % storePath)

     

       79
       86
        
           '';

     

       80
       87
        
         }

     

       81
       81
       -
       )

     

       88
       88
       +
       ) args

+18 -8

pkgs/build-support/binary-cache/default.nix

···

       6
       6
        
         python3,

     

       7
       7
        
         nix,

     

       8
       8
        
         xz,

     

       9
       9
       +
         zstd,

     

       9
       10
        
       }:

     

       10
       11
        
       

     

       11
       12
        
       # This function is for creating a flat-file binary cache, i.e. the kind created by

     
···

       16
       17
        
       

     

       17
       18
        
       {

     

       18
       19
        
         name ? "binary-cache",

     

       20
       20
       +
         compression ? "zstd", # one of ["none" "xz" "zstd"]

     

       19
       21
        
         rootPaths,

     

       20
       22
        
       }:

     

       21
       23
        
       

     

       24
       24
       +
       assert lib.elem compression [

     

       25
       25
       +
         "none"

     

       26
       26
       +
         "xz"

     

       27
       27
       +
         "zstd"

     

       28
       28
       +
       ];

     

       29
       29
       +
       

     

       22
       30
        
       stdenv.mkDerivation {

     

       23
       31
        
         inherit name;

     

       24
       32
        
       

     
···

       28
       36
        
       

     

       29
       37
        
         preferLocalBuild = true;

     

       30
       38
        
       

     

       31
       31
       -
         nativeBuildInputs = [

     

       32
       32
       -
           coreutils

     

       33
       33
       -
           jq

     

       34
       34
       -
           python3

     

       35
       35
       -
           nix

     

       36
       36
       -
           xz

     

       37
       37
       -
         ];

     

       39
       39
       +
         nativeBuildInputs =

     

       40
       40
       +
           [

     

       41
       41
       +
             coreutils

     

       42
       42
       +
             jq

     

       43
       43
       +
             python3

     

       44
       44
       +
             nix

     

       45
       45
       +
           ]

     

       46
       46
       +
           ++ lib.optional (compression == "xz") xz

     

       47
       47
       +
           ++ lib.optional (compression == "zstd") zstd;

     

       38
       48
        
       

     

       39
       49
        
         buildCommand = ''

     

       40
       50
        
           mkdir -p $out/nar

     

       41
       51
        
       

     

       42
       42
       -
           python ${./make-binary-cache.py}

     

       52
       52
       +
           python ${./make-binary-cache.py} --compression "${compression}"

     

       43
       53
        
       

     

       44
       54
        
           # These directories must exist, or Nix might try to create them in LocalBinaryCacheStore::init(),

     

       45
       55
        
           # which fails if mounted read-only

+40 -12

pkgs/build-support/binary-cache/make-binary-cache.py

···

       1
       1
       +
       import argparse

     

       1
       2
        
       from functools import partial

     

       2
       3
        
       import json

     

       3
       4
        
       from multiprocessing import Pool

     
···

       10
       11
        
           return path[len(nixPrefix + "/") :]

     

       11
       12
        
       

     

       12
       13
        
       

     

       13
       13
       -
       def processItem(item, nixPrefix, outDir):

     

       14
       14
       +
       def processItem(

     

       15
       15
       +
           item, nixPrefix, outDir, compression, compressionCommand, compressionExtension

     

       16
       16
       +
       ):

     

       14
       17
        
           narInfoHash = dropPrefix(item["path"], nixPrefix).split("-")[0]

     

       15
       18
        
       

     

       16
       16
       -
           xzFile = outDir / "nar" / f"{narInfoHash}.nar.xz"

     

       17
       17
       -
           with open(xzFile, "wb") as f:

     

       19
       19
       +
           narFile = outDir / "nar" / f"{narInfoHash}{compressionExtension}"

     

       20
       20
       +
           with open(narFile, "wb") as f:

     

       18
       21
        
               subprocess.run(

     

       19
       19
       -
                   f"nix-store --dump {item['path']} | xz -c",

     

       22
       22
       +
                   f"nix-store --dump {item['path']} {compressionCommand}",

     

       20
       23
        
                   stdout=f,

     

       21
       24
        
                   shell=True,

     

       22
       25
        
                   check=True,

     
···

       24
       27
        
       

     

       25
       28
        
           fileHash = (

     

       26
       29
        
               subprocess.run(

     

       27
       27
       -
                   ["nix-hash", "--base32", "--type", "sha256", "--flat", xzFile],

     

       30
       30
       +
                   ["nix-hash", "--base32", "--type", "sha256", "--flat", narFile],

     

       28
       31
        
                   capture_output=True,

     

       29
       32
        
                   check=True,

     

       30
       33
        
               )

     

       31
       34
        
               .stdout.decode()

     

       32
       35
        
               .strip()

     

       33
       36
        
           )

     

       34
       34
       -
           fileSize = os.path.getsize(xzFile)

     

       37
       37
       +
           fileSize = os.path.getsize(narFile)

     

       35
       38
        
       

     

       36
       36
       -
           finalXzFileName = Path("nar") / f"{fileHash}.nar.xz"

     

       37
       37
       -
           os.rename(xzFile, outDir / finalXzFileName)

     

       39
       39
       +
           finalNarFileName = Path("nar") / f"{fileHash}{compressionExtension}"

     

       40
       40
       +
           os.rename(narFile, outDir / finalNarFileName)

     

       38
       41
        
       

     

       39
       42
        
           with open(outDir / f"{narInfoHash}.narinfo", "wt") as f:

     

       40
       43
        
               f.write(f"StorePath: {item['path']}\n")

     

       41
       41
       -
               f.write(f"URL: {finalXzFileName}\n")

     

       42
       42
       -
               f.write("Compression: xz\n")

     

       44
       44
       +
               f.write(f"URL: {finalNarFileName}\n")

     

       45
       45
       +
               f.write(f"Compression: {compression}\n")

     

       43
       46
        
               f.write(f"FileHash: sha256:{fileHash}\n")

     

       44
       47
        
               f.write(f"FileSize: {fileSize}\n")

     

       45
       48
        
               f.write(f"NarHash: {item['narHash']}\n")

     

       46
       49
        
               f.write(f"NarSize: {item['narSize']}\n")

     

       47
       47
       -
               f.write(f"References: {' '.join(dropPrefix(ref, nixPrefix) for ref in item['references'])}\n")

     

       50
       50
       +
               f.write(

     

       51
       51
       +
                   f"References: {' '.join(dropPrefix(ref, nixPrefix) for ref in item['references'])}\n"

     

       52
       52
       +
               )

     

       48
       53
        
       

     

       49
       54
        
       

     

       50
       55
        
       def main():

     

       56
       56
       +
           parser = argparse.ArgumentParser()

     

       57
       57
       +
           parser.add_argument("--compression", choices=["none", "xz", "zstd"])

     

       58
       58
       +
           args = parser.parse_args()

     

       59
       59
       +
       

     

       60
       60
       +
           compressionCommand = {

     

       61
       61
       +
               "none": "",

     

       62
       62
       +
               "xz": "| xz -c",

     

       63
       63
       +
               "zstd": "| zstd",

     

       64
       64
       +
           }[args.compression]

     

       65
       65
       +
       

     

       66
       66
       +
           compressionExtension = {

     

       67
       67
       +
               "none": "",

     

       68
       68
       +
               "xz": ".xz",

     

       69
       69
       +
               "zstd": ".zst",

     

       70
       70
       +
           }[args.compression]

     

       71
       71
       +
       

     

       51
       72
        
           outDir = Path(os.environ["out"])

     

       52
       73
        
           nixPrefix = os.environ["NIX_STORE"]

     

       53
       74
        
           numWorkers = int(os.environ.get("NIX_BUILD_CORES", "4"))

     
···

       61
       82
        
               f.write(f"StoreDir: {nixPrefix}\n")

     

       62
       83
        
       

     

       63
       84
        
           with Pool(processes=numWorkers) as pool:

     

       64
       64
       -
               worker = partial(processItem, nixPrefix=nixPrefix, outDir=outDir)

     

       85
       85
       +
               worker = partial(

     

       86
       86
       +
                   processItem,

     

       87
       87
       +
                   nixPrefix=nixPrefix,

     

       88
       88
       +
                   outDir=outDir,

     

       89
       89
       +
                   compression=args.compression,

     

       90
       90
       +
                   compressionCommand=compressionCommand,

     

       91
       91
       +
                   compressionExtension=compressionExtension,

     

       92
       92
       +
               )

     

       65
       93
        
               pool.map(worker, closures)

     

       66
       94
        
       

     

       67
       95