commit d1a2a16a3a0220e8f571c07001a2ada4a664c492 · pyrox.dev/nixpkgs

doc/builders/images.xml

···

       11
       11
        
        <xi:include href="images/snaptools.section.xml" />

     

       12
       12
        
        <xi:include href="images/portableservice.section.xml" />

     

       13
       13
        
        <xi:include href="images/makediskimage.section.xml" />

     

       14
       14
       +
        <xi:include href="images/binarycache.section.xml" />

     

       14
       15
        
       </chapter>

+49

doc/builders/images/binarycache.section.md

···

       1
       1
       +
       # pkgs.mkBinaryCache {#sec-pkgs-binary-cache}

     

       2
       2
       +
       

     

       3
       3
       +
       `pkgs.mkBinaryCache` is a function for creating Nix flat-file binary caches. Such a cache exists as a directory on disk, and can be used as a Nix substituter by passing `--substituter file:///path/to/cache` to Nix commands.

     

       4
       4
       +
       

     

       5
       5
       +
       Nix packages are most commonly shared between machines using [HTTP, SSH, or S3](https://nixos.org/manual/nix/stable/package-management/sharing-packages.html), but a flat-file binary cache can still be useful in some situations. For example, you can copy it directly to another machine, or make it available on a network file system. It can also be a convenient way to make some Nix packages available inside a container via bind-mounting.

     

       6
       6
       +
       

     

       7
       7
       +
       Note that this function is meant for advanced use-cases. The more idiomatic way to work with flat-file binary caches is via the [nix-copy-closure](https://nixos.org/manual/nix/stable/command-ref/nix-copy-closure.html) command. You may also want to consider [dockerTools](#sec-pkgs-dockerTools) for your containerization needs.

     

       8
       8
       +
       

     

       9
       9
       +
       ## Example

     

       10
       10
       +
       

     

       11
       11
       +
       The following derivation will construct a flat-file binary cache containing the closure of `hello`.

     

       12
       12
       +
       

     

       13
       13
       +
       ```nix

     

       14
       14
       +
       mkBinaryCache {

     

       15
       15
       +
         rootPaths = [hello];

     

       16
       16
       +
       }

     

       17
       17
       +
       ```

     

       18
       18
       +
       

     

       19
       19
       +
       - `rootPaths` specifies a list of root derivations. The transitive closure of these derivations' outputs will be copied into the cache.

     

       20
       20
       +
       

     

       21
       21
       +
       Here's an example of building and using the cache.

     

       22
       22
       +
       

     

       23
       23
       +
       Build the cache on one machine, `host1`:

     

       24
       24
       +
       

     

       25
       25
       +
       ```shellSession

     

       26
       26
       +
       nix-build -E 'with import <nixpkgs> {}; mkBinaryCache { rootPaths = [hello]; }'

     

       27
       27
       +
       ```

     

       28
       28
       +
       

     

       29
       29
       +
       ```shellSession

     

       30
       30
       +
       /nix/store/cc0562q828rnjqjyfj23d5q162gb424g-binary-cache

     

       31
       31
       +
       ```

     

       32
       32
       +
       

     

       33
       33
       +
       Copy the resulting directory to the other machine, `host2`:

     

       34
       34
       +
       

     

       35
       35
       +
       ```shellSession

     

       36
       36
       +
       scp result host2:/tmp/hello-cache

     

       37
       37
       +
       ```

     

       38
       38
       +
       

     

       39
       39
       +
       Build the derivation using the flat-file binary cache on the other machine, `host2`:

     

       40
       40
       +
       ```shellSession

     

       41
       41
       +
       nix-build -A hello '<nixpkgs>' \

     

       42
       42
       +
         --option require-sigs false \

     

       43
       43
       +
         --option trusted-substituters file:///tmp/hello-cache \

     

       44
       44
       +
         --option substituters file:///tmp/hello-cache

     

       45
       45
       +
       ```

     

       46
       46
       +
       

     

       47
       47
       +
       ```shellSession

     

       48
       48
       +
       /nix/store/gl5a41azbpsadfkfmbilh9yk40dh5dl0-hello-2.12.1

     

       49
       49
       +
       ```

nixos/tests/all-tests.nix

···

       92
       92
        
         bcachefs = handleTestOn ["x86_64-linux" "aarch64-linux"] ./bcachefs.nix {};

     

       93
       93
        
         beanstalkd = handleTest ./beanstalkd.nix {};

     

       94
       94
        
         bees = handleTest ./bees.nix {};

     

       95
       95
       +
         binary-cache = handleTest ./binary-cache.nix {};

     

       95
       96
        
         bind = handleTest ./bind.nix {};

     

       96
       97
        
         bird = handleTest ./bird.nix {};

     

       97
       98
        
         bitcoind = handleTest ./bitcoind.nix {};

+62

nixos/tests/binary-cache.nix

···

       1
       1
       +
       import ./make-test-python.nix ({ lib, ... }:

     

       2
       2
       +
       

     

       3
       3
       +
       with lib;

     

       4
       4
       +
       

     

       5
       5
       +
       {

     

       6
       6
       +
         name = "binary-cache";

     

       7
       7
       +
         meta.maintainers = with maintainers; [ thomasjm ];

     

       8
       8
       +
       

     

       9
       9
       +
         nodes.machine =

     

       10
       10
       +
           { pkgs, ... }: {

     

       11
       11
       +
             imports = [ ../modules/installer/cd-dvd/channel.nix ];

     

       12
       12
       +
             environment.systemPackages = with pkgs; [python3];

     

       13
       13
       +
             system.extraDependencies = with pkgs; [hello.inputDerivation];

     

       14
       14
       +
             nix.extraOptions = ''

     

       15
       15
       +
               experimental-features = nix-command

     

       16
       16
       +
             '';

     

       17
       17
       +
           };

     

       18
       18
       +
       

     

       19
       19
       +
         testScript = ''

     

       20
       20
       +
           # Build the cache, then remove it from the store

     

       21
       21
       +
           cachePath = machine.succeed("nix-build --no-out-link -E 'with import <nixpkgs> {}; mkBinaryCache { rootPaths = [hello]; }'").strip()

     

       22
       22
       +
           machine.succeed("cp -r %s/. /tmp/cache" % cachePath)

     

       23
       23
       +
           machine.succeed("nix-store --delete " + cachePath)

     

       24
       24
       +
       

     

       25
       25
       +
           # Sanity test of cache structure

     

       26
       26
       +
           status, stdout = machine.execute("ls /tmp/cache")

     

       27
       27
       +
           cache_files = stdout.split()

     

       28
       28
       +
           assert ("nix-cache-info" in cache_files)

     

       29
       29
       +
           assert ("nar" in cache_files)

     

       30
       30
       +
       

     

       31
       31
       +
           # Nix store ping should work

     

       32
       32
       +
           machine.succeed("nix store ping --store file:///tmp/cache")

     

       33
       33
       +
       

     

       34
       34
       +
           # Cache should contain a .narinfo referring to "hello"

     

       35
       35
       +
           grepLogs = machine.succeed("grep -l 'StorePath: /nix/store/[[:alnum:]]*-hello-.*' /tmp/cache/*.narinfo")

     

       36
       36
       +
       

     

       37
       37
       +
           # Get the store path referenced by the .narinfo

     

       38
       38
       +
           narInfoFile = grepLogs.strip()

     

       39
       39
       +
           narInfoContents = machine.succeed("cat " + narInfoFile)

     

       40
       40
       +
           import re

     

       41
       41
       +
           match = re.match(r"^StorePath: (/nix/store/[a-z0-9]*-hello-.*)$", narInfoContents, re.MULTILINE)

     

       42
       42
       +
           if not match: raise Exception("Couldn't find hello store path in cache")

     

       43
       43
       +
           storePath = match[1]

     

       44
       44
       +
       

     

       45
       45
       +
           # Delete the store path

     

       46
       46
       +
           machine.succeed("nix-store --delete " + storePath)

     

       47
       47
       +
           machine.succeed("[ ! -d %s ] || exit 1" % storePath)

     

       48
       48
       +
       

     

       49
       49
       +
           # Should be able to build hello using the cache

     

       50
       50
       +
           logs = machine.succeed("nix-build -A hello '<nixpkgs>' --option require-sigs false --option trusted-substituters file:///tmp/cache --option substituters file:///tmp/cache 2>&1")

     

       51
       51
       +
           logLines = logs.split("\n")

     

       52
       52
       +
           if not "this path will be fetched" in logLines[0]: raise Exception("Unexpected first log line")

     

       53
       53
       +
           def shouldBe(got, desired):

     

       54
       54
       +
             if got != desired: raise Exception("Expected '%s' but got '%s'" % (desired, got))

     

       55
       55
       +
           shouldBe(logLines[1], "  " + storePath)

     

       56
       56
       +
           shouldBe(logLines[2], "copying path '%s' from 'file:///tmp/cache'..." % storePath)

     

       57
       57
       +
           shouldBe(logLines[3], storePath)

     

       58
       58
       +
       

     

       59
       59
       +
           # Store path should exist in the store now

     

       60
       60
       +
           machine.succeed("[ -d %s ] || exit 1" % storePath)

     

       61
       61
       +
         '';

     

       62
       62
       +
       })

+40

pkgs/build-support/binary-cache/default.nix

···

       1
       1
       +
       { stdenv, buildPackages }:

     

       2
       2
       +
       

     

       3
       3
       +
       # This function is for creating a flat-file binary cache, i.e. the kind created by

     

       4
       4
       +
       # nix copy --to file:///some/path and usable as a substituter (with the file:// prefix).

     

       5
       5
       +
       

     

       6
       6
       +
       # For example, in the Nixpkgs repo:

     

       7
       7
       +
       # nix-build -E 'with import ./. {}; mkBinaryCache { rootPaths = [hello]; }'

     

       8
       8
       +
       

     

       9
       9
       +
       { name ? "binary-cache"

     

       10
       10
       +
       , rootPaths

     

       11
       11
       +
       }:

     

       12
       12
       +
       

     

       13
       13
       +
       stdenv.mkDerivation {

     

       14
       14
       +
         inherit name;

     

       15
       15
       +
       

     

       16
       16
       +
         __structuredAttrs = true;

     

       17
       17
       +
       

     

       18
       18
       +
         exportReferencesGraph.closure = rootPaths;

     

       19
       19
       +
       

     

       20
       20
       +
         preferLocalBuild = true;

     

       21
       21
       +
       

     

       22
       22
       +
         PATH = "${buildPackages.coreutils}/bin:${buildPackages.jq}/bin:${buildPackages.python3}/bin:${buildPackages.nix}/bin:${buildPackages.xz}/bin";

     

       23
       23
       +
       

     

       24
       24
       +
         builder = builtins.toFile "builder" ''

     

       25
       25
       +
           . .attrs.sh

     

       26
       26
       +
       

     

       27
       27
       +
           export out=''${outputs[out]}

     

       28
       28
       +
       

     

       29
       29
       +
           mkdir $out

     

       30
       30
       +
           mkdir $out/nar

     

       31
       31
       +
       

     

       32
       32
       +
           python ${./make-binary-cache.py}

     

       33
       33
       +
       

     

       34
       34
       +
           # These directories must exist, or Nix might try to create them in LocalBinaryCacheStore::init(),

     

       35
       35
       +
           # which fails if mounted read-only

     

       36
       36
       +
           mkdir $out/realisations

     

       37
       37
       +
           mkdir $out/debuginfo

     

       38
       38
       +
           mkdir $out/log

     

       39
       39
       +
         '';

     

       40
       40
       +
       }

+43

pkgs/build-support/binary-cache/make-binary-cache.py

···

       1
       1
       +
       

     

       2
       2
       +
       import json

     

       3
       3
       +
       import os

     

       4
       4
       +
       import subprocess

     

       5
       5
       +
       

     

       6
       6
       +
       with open(".attrs.json", "r") as f:

     

       7
       7
       +
         closures = json.load(f)["closure"]

     

       8
       8
       +
       

     

       9
       9
       +
       os.chdir(os.environ["out"])

     

       10
       10
       +
       

     

       11
       11
       +
       nixPrefix = os.environ["NIX_STORE"] # Usually /nix/store

     

       12
       12
       +
       

     

       13
       13
       +
       with open("nix-cache-info", "w") as f:

     

       14
       14
       +
         f.write("StoreDir: " + nixPrefix + "\n")

     

       15
       15
       +
       

     

       16
       16
       +
       def dropPrefix(path):

     

       17
       17
       +
         return path[len(nixPrefix + "/"):]

     

       18
       18
       +
       

     

       19
       19
       +
       for item in closures:

     

       20
       20
       +
         narInfoHash = dropPrefix(item["path"]).split("-")[0]

     

       21
       21
       +
       

     

       22
       22
       +
         xzFile = "nar/" + narInfoHash + ".nar.xz"

     

       23
       23
       +
         with open(xzFile, "w") as f:

     

       24
       24
       +
           subprocess.run("nix-store --dump %s | xz -c" % item["path"], stdout=f, shell=True)

     

       25
       25
       +
       

     

       26
       26
       +
         fileHash = subprocess.run(["nix-hash", "--base32", "--type", "sha256", item["path"]], capture_output=True).stdout.decode().strip()

     

       27
       27
       +
         fileSize = os.path.getsize(xzFile)

     

       28
       28
       +
       

     

       29
       29
       +
         # Rename the .nar.xz file to its own hash to match "nix copy" behavior

     

       30
       30
       +
         finalXzFile = "nar/" + fileHash + ".nar.xz"

     

       31
       31
       +
         os.rename(xzFile, finalXzFile)

     

       32
       32
       +
       

     

       33
       33
       +
         with open(narInfoHash + ".narinfo", "w") as f:

     

       34
       34
       +
           f.writelines((x + "\n" for x in [

     

       35
       35
       +
             "StorePath: " + item["path"],

     

       36
       36
       +
             "URL: " + finalXzFile,

     

       37
       37
       +
             "Compression: xz",

     

       38
       38
       +
             "FileHash: sha256:" + fileHash,

     

       39
       39
       +
             "FileSize: " + str(fileSize),

     

       40
       40
       +
             "NarHash: " + item["narHash"],

     

       41
       41
       +
             "NarSize: " + str(item["narSize"]),

     

       42
       42
       +
             "References: " + " ".join(dropPrefix(ref) for ref in item["references"]),

     

       43
       43
       +
           ]))

pkgs/top-level/all-packages.nix

···

       1031
       1031
        
             inherit kernel firmware rootModules allowMissing;

     

       1032
       1032
        
           };

     

       1033
       1033
        
       

     

       1034
       1034
       +
         mkBinaryCache = callPackage ../build-support/binary-cache { };

     

       1035
       1035
       +
       

     

       1034
       1036
        
         mkShell = callPackage ../build-support/mkshell { };

     

       1035
       1037
        
         mkShellNoCC = mkShell.override { stdenv = stdenvNoCC; };

     

       1036
       1038