commit 0f78afdf253e3f53e76ec3379518d208a18c568c · pyrox.dev/nixpkgs

nixos/release.nix

···

       274
       274
        
         tests.hibernate = callTest tests/hibernate.nix {};

     

       275
       275
        
         tests.home-assistant = callTest tests/home-assistant.nix { };

     

       276
       276
        
         tests.hound = callTest tests/hound.nix {};

     

       277
       277
       +
         tests.hocker-fetchdocker = callTest tests/hocker-fetchdocker {};

     

       277
       278
        
         tests.i3wm = callTest tests/i3wm.nix {};

     

       278
       279
        
         tests.initrd-network-ssh = callTest tests/initrd-network-ssh {};

     

       279
       280
        
         tests.installer = callSubTests tests/installer.nix {};

+15

nixos/tests/hocker-fetchdocker/default.nix

···

       1
       1
       +
       import ../make-test.nix ({ pkgs, ...} : {

     

       2
       2
       +
         name = "test-hocker-fetchdocker";

     

       3
       3
       +
         meta = with pkgs.stdenv.lib.maintainers; {

     

       4
       4
       +
           maintainers = [ ixmatus ];

     

       5
       5
       +
         };

     

       6
       6
       +
       

     

       7
       7
       +
         machine = import ./machine.nix;

     

       8
       8
       +
       

     

       9
       9
       +
         testScript = ''

     

       10
       10
       +
           startAll;

     

       11
       11
       +
       

     

       12
       12
       +
           $machine->waitForUnit("sockets.target");

     

       13
       13
       +
           $machine->waitUntilSucceeds("docker run registry-1.docker.io/v2/library/hello-world:latest");

     

       14
       14
       +
         '';

     

       15
       15
       +
       })

+19

nixos/tests/hocker-fetchdocker/hello-world-container.nix

···

       1
       1
       +
       { fetchDockerConfig, fetchDockerLayer, fetchdocker }:

     

       2
       2
       +
       fetchdocker rec {

     

       3
       3
       +
           name = "hello-world";

     

       4
       4
       +
           registry = "https://registry-1.docker.io/v2/";

     

       5
       5
       +
           repository = "library";

     

       6
       6
       +
           imageName = "hello-world";

     

       7
       7
       +
           tag = "latest";

     

       8
       8
       +
           imageConfig = fetchDockerConfig {

     

       9
       9
       +
             inherit tag registry repository imageName;

     

       10
       10
       +
             sha256 = "1ivbd23hyindkahzfw4kahgzi6ibzz2ablmgsz6340vc6qr1gagj";

     

       11
       11
       +
           };

     

       12
       12
       +
           imageLayers = let

     

       13
       13
       +
             layer0 = fetchDockerLayer {

     

       14
       14
       +
               inherit registry repository imageName;

     

       15
       15
       +
               layerDigest = "ca4f61b1923c10e9eb81228bd46bee1dfba02b9c7dac1844527a734752688ede";

     

       16
       16
       +
               sha256 = "1plfd194fwvsa921ib3xkhms1yqxxrmx92r2h7myj41wjaqn2kya";

     

       17
       17
       +
             };

     

       18
       18
       +
             in [ layer0 ];

     

       19
       19
       +
         }

+26

nixos/tests/hocker-fetchdocker/machine.nix

···

       1
       1
       +
       { config, pkgs, ... }:

     

       2
       2
       +
       { nixpkgs.config.packageOverrides = pkgs': {

     

       3
       3
       +
           hello-world-container = pkgs'.callPackage ./hello-world-container.nix { };

     

       4
       4
       +
         };

     

       5
       5
       +
       

     

       6
       6
       +
         virtualisation.docker = {

     

       7
       7
       +
           enable  = true;

     

       8
       8
       +
           package = pkgs.docker;

     

       9
       9
       +
         };

     

       10
       10
       +
       

     

       11
       11
       +
         systemd.services.docker-load-fetchdocker-image = {

     

       12
       12
       +
           description = "Docker load hello-world-container";

     

       13
       13
       +
           wantedBy    = [ "multi-user.target" ];

     

       14
       14
       +
           wants       = [ "docker.service" "local-fs.target" ];

     

       15
       15
       +
           after       = [ "docker.service" "local-fs.target" ];

     

       16
       16
       +
       

     

       17
       17
       +
           script = ''

     

       18
       18
       +
             ${pkgs.hello-world-container}/compositeImage.sh | ${pkgs.docker}/bin/docker load

     

       19
       19
       +
           '';

     

       20
       20
       +
       

     

       21
       21
       +
           serviceConfig = {

     

       22
       22
       +
             Type = "oneshot";

     

       23
       23
       +
           };

     

       24
       24
       +
         };

     

       25
       25
       +
       }

     

       26
       26
       +

+38

pkgs/build-support/fetchdocker/credentials.nix

···

       1
       1
       +
       # We provide three paths to get the credentials into the builder's

     

       2
       2
       +
       # environment:

     

       3
       3
       +
       #

     

       4
       4
       +
       # 1. Via impureEnvVars. This method is difficult for multi-user Nix

     

       5
       5
       +
       #    installations (but works very well for single-user Nix

     

       6
       6
       +
       #    installations!) because it requires setting the environment

     

       7
       7
       +
       #    variables on the nix-daemon which is either complicated or unsafe

     

       8
       8
       +
       #    (i.e: configuring via Nix means the secrets will be persisted

     

       9
       9
       +
       #    into the store)

     

       10
       10
       +
       #

     

       11
       11
       +
       # 2. If the DOCKER_CREDENTIALS key with a path to a credentials file

     

       12
       12
       +
       #    is added to the NIX_PATH (usually via the '-I ' argument to most

     

       13
       13
       +
       #    Nix tools) then an attempt will be made to read credentials from

     

       14
       14
       +
       #    it. The semantics are simple, the file should contain two lines

     

       15
       15
       +
       #    for the username and password based authentication:

     

       16
       16
       +
       #

     

       17
       17
       +
       # $ cat ./credentials-file.txt

     

       18
       18
       +
       # DOCKER_USER=myusername

     

       19
       19
       +
       # DOCKER_PASS=mypassword

     

       20
       20
       +
       #

     

       21
       21
       +
       #    ... and a single line for the token based authentication:

     

       22
       22
       +
       #

     

       23
       23
       +
       # $ cat ./credentials-file.txt

     

       24
       24
       +
       # DOCKER_TOKEN=mytoken

     

       25
       25
       +
       #

     

       26
       26
       +
       # 3. A credential file at /etc/nix-docker-credentials.txt with the

     

       27
       27
       +
       #    same format as the file described in #2 can also be used to

     

       28
       28
       +
       #    communicate credentials to the builder. This is necessary for

     

       29
       29
       +
       #    situations (like Hydra) where you cannot customize the NIX_PATH

     

       30
       30
       +
       #    given to the nix-build invocation to provide it with the

     

       31
       31
       +
       #    DOCKER_CREDENTIALS path

     

       32
       32
       +
       let

     

       33
       33
       +
         pathParts =

     

       34
       34
       +
          (builtins.filter

     

       35
       35
       +
           ({path, prefix}: "DOCKER_CREDENTIALS" == prefix)

     

       36
       36
       +
           builtins.nixPath);

     

       37
       37
       +
       in

     

       38
       38
       +
         if (pathParts != []) then (builtins.head pathParts).path else ""

+61

pkgs/build-support/fetchdocker/default.nix

···

       1
       1
       +
       { stdenv, lib, coreutils, bash, gnutar, jq, writeText }:

     

       2
       2
       +
       let

     

       3
       3
       +
         stripScheme =

     

       4
       4
       +
           builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];

     

       5
       5
       +
         stripNixStore =

     

       6
       6
       +
           s: lib.removePrefix "/nix/store/" s;

     

       7
       7
       +
       in

     

       8
       8
       +
       { name

     

       9
       9
       +
       , registry         ? "https://registry-1.docker.io/v2/"

     

       10
       10
       +
       , repository       ? "library"

     

       11
       11
       +
       , imageName

     

       12
       12
       +
       , tag

     

       13
       13
       +
       , imageLayers

     

       14
       14
       +
       , imageConfig

     

       15
       15
       +
       , image            ? "${stripScheme registry}/${repository}/${imageName}:${tag}"

     

       16
       16
       +
       }:

     

       17
       17
       +
       

     

       18
       18
       +
       # Make sure there are *no* slashes in the repository or container

     

       19
       19
       +
       # names since we use these to make the output derivation name for the

     

       20
       20
       +
       # nix-store path.

     

       21
       21
       +
       assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);

     

       22
       22
       +
       assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);

     

       23
       23
       +
       

     

       24
       24
       +
       let

     

       25
       25
       +
         # Abuse `builtins.toPath` to collapse possible double slashes

     

       26
       26
       +
         repoTag0 = builtins.toString (builtins.toPath "/${stripScheme registry}/${repository}/${imageName}");

     

       27
       27
       +
         repoTag1 = lib.removePrefix "/" repoTag0;

     

       28
       28
       +
       

     

       29
       29
       +
         layers = builtins.map stripNixStore imageLayers;

     

       30
       30
       +
       

     

       31
       31
       +
         manifest =

     

       32
       32
       +
           writeText "manifest.json" (builtins.toJSON [

     

       33
       33
       +
             { Config   = stripNixStore imageConfig;

     

       34
       34
       +
               Layers   = layers;

     

       35
       35
       +
               RepoTags = [ "${repoTag1}:${tag}" ];

     

       36
       36
       +
             }]);

     

       37
       37
       +
       

     

       38
       38
       +
         repositories =

     

       39
       39
       +
           writeText "repositories" (builtins.toJSON {

     

       40
       40
       +
             "${repoTag1}" = {

     

       41
       41
       +
               "${tag}" = lib.last layers;

     

       42
       42
       +
             };

     

       43
       43
       +
           });

     

       44
       44
       +
       

     

       45
       45
       +
         imageFileStorePaths =

     

       46
       46
       +
           writeText "imageFileStorePaths.txt"

     

       47
       47
       +
             (lib.concatStringsSep "\n" ((lib.unique imageLayers) ++ [imageConfig]));

     

       48
       48
       +
       in

     

       49
       49
       +
       stdenv.mkDerivation {

     

       50
       50
       +
         builder     = ./fetchdocker-builder.sh;

     

       51
       51
       +
         buildInputs = [ coreutils ];

     

       52
       52
       +
         preferLocalBuild = true;

     

       53
       53
       +
       

     

       54
       54
       +
         inherit name imageName repository tag;

     

       55
       55
       +
         inherit bash gnutar manifest repositories;

     

       56
       56
       +
         inherit imageFileStorePaths;

     

       57
       57
       +
       

     

       58
       58
       +
         passthru = {

     

       59
       59
       +
           inherit image;

     

       60
       60
       +
         };

     

       61
       61
       +
       }

+13

pkgs/build-support/fetchdocker/fetchDockerConfig.nix

···

       1
       1
       +
       pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:

     

       2
       2
       +
       let

     

       3
       3
       +
         generic-fetcher =

     

       4
       4
       +
           import ./generic-fetcher.nix pkgargs;

     

       5
       5
       +
       in

     

       6
       6
       +
       

     

       7
       7
       +
       args@{ repository ? "library", imageName, tag, ... }:

     

       8
       8
       +
       

     

       9
       9
       +
       generic-fetcher ({

     

       10
       10
       +
         fetcher = "hocker-config";

     

       11
       11
       +
         name    = "${repository}_${imageName}_${tag}-config.json";

     

       12
       12
       +
         tag     = "unused";

     

       13
       13
       +
       } // args)

+13

pkgs/build-support/fetchdocker/fetchDockerLayer.nix

···

       1
       1
       +
       pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:

     

       2
       2
       +
       let

     

       3
       3
       +
         generic-fetcher =

     

       4
       4
       +
           import ./generic-fetcher.nix pkgargs;

     

       5
       5
       +
       in

     

       6
       6
       +
       

     

       7
       7
       +
       args@{ layerDigest, ... }:

     

       8
       8
       +
       

     

       9
       9
       +
       generic-fetcher ({

     

       10
       10
       +
         fetcher = "hocker-layer";

     

       11
       11
       +
         name    = "docker-layer-${layerDigest}.tar.gz";

     

       12
       12
       +
         tag     = "unused";

     

       13
       13
       +
       } // args)

+28

pkgs/build-support/fetchdocker/fetchdocker-builder.sh

···

       1
       1
       +
       source "${stdenv}/setup"

     

       2
       2
       +
       header "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}"

     

       3
       3
       +
       mkdir -p "${out}"

     

       4
       4
       +
       

     

       5
       5
       +
       cat <<EOF > "${out}/compositeImage.sh"

     

       6
       6
       +
       #! ${bash}/bin/bash

     

       7
       7
       +
       #

     

       8
       8
       +
       # Create a tar archive of a docker image's layers, docker image config

     

       9
       9
       +
       # json, manifest.json, and repositories json; this streams directly to

     

       10
       10
       +
       # stdout and is intended to be used in concert with docker load, i.e:

     

       11
       11
       +
       # 

     

       12
       12
       +
       # ${out}/compositeImage.sh | docker load

     

       13
       13
       +
       

     

       14
       14
       +
       # The first character follow the 's' command for sed becomes the

     

       15
       15
       +
       # delimiter sed will use; this makes the transformation regex easy to

     

       16
       16
       +
       # read. We feed tar a file listing the files we want in the archive,

     

       17
       17
       +
       # because the paths are absolute and docker load wants them flattened in

     

       18
       18
       +
       # the archive, we need to transform all of the paths going in by

     

       19
       19
       +
       # stripping everything *including* the last solidus so that we end up

     

       20
       20
       +
       # with the basename of the path.

     

       21
       21
       +
       ${gnutar}/bin/tar \

     

       22
       22
       +
         --transform='s=.*/==' \

     

       23
       23
       +
         --transform="s=.*-manifest.json=manifest.json=" \

     

       24
       24
       +
         --transform="s=.*-repositories=repositories=" \

     

       25
       25
       +
         -c "${manifest}" "${repositories}" -T "${imageFileStorePaths}"

     

       26
       26
       +
       EOF

     

       27
       27
       +
       chmod +x "${out}/compositeImage.sh"

     

       28
       28
       +
       stopNest

+97

pkgs/build-support/fetchdocker/generic-fetcher.nix

···

       1
       1
       +
       { stdenv, lib, haskellPackages, writeText, gawk }:

     

       2
       2
       +
       let

     

       3
       3
       +
         awk                   = "${gawk}/bin/awk";

     

       4
       4
       +
         dockerCredentialsFile = import ./credentials.nix;

     

       5
       5
       +
         stripScheme           =

     

       6
       6
       +
           builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];

     

       7
       7
       +
       in

     

       8
       8
       +
       { fetcher

     

       9
       9
       +
       , name

     

       10
       10
       +
        , registry    ? "https://registry-1.docker.io/v2/"

     

       11
       11
       +
        , repository  ? "library"

     

       12
       12
       +
        , imageName

     

       13
       13
       +
        , sha256

     

       14
       14
       +
        , tag         ? ""

     

       15
       15
       +
        , layerDigest ? ""

     

       16
       16
       +
       }:

     

       17
       17
       +
       

     

       18
       18
       +
       # There must be no slashes in the repository or container names since

     

       19
       19
       +
       # we use these to make the output derivation name for the nix store

     

       20
       20
       +
       # path

     

       21
       21
       +
       assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);

     

       22
       22
       +
       assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);

     

       23
       23
       +
       

     

       24
       24
       +
       # Only allow hocker-config and hocker-layer as fetchers for now

     

       25
       25
       +
       assert (builtins.elem fetcher ["hocker-config" "hocker-layer"]);

     

       26
       26
       +
       

     

       27
       27
       +
       # If layerDigest is non-empty then it must not have a 'sha256:' prefix!

     

       28
       28
       +
       assert

     

       29
       29
       +
         (if layerDigest != ""

     

       30
       30
       +
          then !lib.hasPrefix "sha256:" layerDigest

     

       31
       31
       +
          else true);

     

       32
       32
       +
       

     

       33
       33
       +
       let

     

       34
       34
       +
         layerDigestFlag =

     

       35
       35
       +
           lib.optionalString (layerDigest != "") "--layer ${layerDigest}";

     

       36
       36
       +
       in

     

       37
       37
       +
       stdenv.mkDerivation {

     

       38
       38
       +
         inherit name;

     

       39
       39
       +
         builder = writeText "${fetcher}-builder.sh" ''

     

       40
       40
       +
           source "$stdenv/setup"

     

       41
       41
       +
           header "${fetcher} exporting to $out"

     

       42
       42
       +
       

     

       43
       43
       +
           declare -A creds

     

       44
       44
       +
       

     

       45
       45
       +
           # This is a hack for Hydra since we have no way of adding values

     

       46
       46
       +
           # to the NIX_PATH for Hydra jobsets!!

     

       47
       47
       +
           staticCredentialsFile="/etc/nix-docker-credentials.txt"

     

       48
       48
       +
           if [ ! -f "$dockerCredentialsFile" -a -f "$staticCredentialsFile" ]; then

     

       49
       49
       +
             echo "credentials file not set, falling back on static credentials file at: $staticCredentialsFile"

     

       50
       50
       +
             dockerCredentialsFile=$staticCredentialsFile

     

       51
       51
       +
           fi

     

       52
       52
       +
       

     

       53
       53
       +
           if [ -f "$dockerCredentialsFile" ]; then

     

       54
       54
       +
             header "using credentials from $dockerCredentialsFile"

     

       55
       55
       +
       

     

       56
       56
       +
             CREDSFILE=$(cat "$dockerCredentialsFile")

     

       57
       57
       +
             creds[token]=$(${awk} -F'=' '/DOCKER_TOKEN/ {print $2}' <<< "$CREDSFILE" | head -n1)

     

       58
       58
       +
       

     

       59
       59
       +
             # Prefer DOCKER_TOKEN over the username and password

     

       60
       60
       +
             # authentication method

     

       61
       61
       +
             if [ -z "''${creds[token]}" ]; then

     

       62
       62
       +
               creds[user]=$(${awk} -F'=' '/DOCKER_USER/  {print $2}' <<< "$CREDSFILE" | head -n1)

     

       63
       63
       +
               creds[pass]=$(${awk} -F'=' '/DOCKER_PASS/  {print $2}' <<< "$CREDSFILE" | head -n1)

     

       64
       64
       +
             fi

     

       65
       65
       +
           fi

     

       66
       66
       +
       

     

       67
       67
       +
           # These variables will be filled in first by the impureEnvVars, if

     

       68
       68
       +
           # those variables are empty then they will default to the

     

       69
       69
       +
           # credentials that may have been read in from the 'DOCKER_CREDENTIALS'

     

       70
       70
       +
           DOCKER_USER="''${DOCKER_USER:-''${creds[user]}}"

     

       71
       71
       +
           DOCKER_PASS="''${DOCKER_PASS:-''${creds[pass]}}"

     

       72
       72
       +
           DOCKER_TOKEN="''${DOCKER_TOKEN:-''${creds[token]}}"

     

       73
       73
       +
       

     

       74
       74
       +
           ${fetcher} --out="$out" \

     

       75
       75
       +
             ''${registry:+--registry "$registry"} \

     

       76
       76
       +
             ''${DOCKER_USER:+--username "$DOCKER_USER"} \

     

       77
       77
       +
             ''${DOCKER_PASS:+--password "$DOCKER_PASS"} \

     

       78
       78
       +
             ''${DOCKER_TOKEN:+--token "$DOCKER_TOKEN"} \

     

       79
       79
       +
             ${layerDigestFlag} \

     

       80
       80
       +
             "${repository}/${imageName}" \

     

       81
       81
       +
             "${tag}"

     

       82
       82
       +
       

     

       83
       83
       +
           stopNest

     

       84
       84
       +
         '';

     

       85
       85
       +
       

     

       86
       86
       +
         buildInputs = [ haskellPackages.hocker ];

     

       87
       87
       +
       

     

       88
       88
       +
         outputHashAlgo = "sha256";

     

       89
       89
       +
         outputHashMode = "flat";

     

       90
       90
       +
         outputHash = sha256;

     

       91
       91
       +
       

     

       92
       92
       +
         preferLocalBuild = true;

     

       93
       93
       +
       

     

       94
       94
       +
         impureEnvVars = [ "DOCKER_USER" "DOCKER_PASS" "DOCKER_TOKEN" ];

     

       95
       95
       +
       

     

       96
       96
       +
         inherit registry dockerCredentialsFile;

     

       97
       97
       +
       }

pkgs/top-level/all-packages.nix

···

       160
       160
        
       

     

       161
       161
        
         fetchdarcs = callPackage ../build-support/fetchdarcs { };

     

       162
       162
        
       

     

       163
       163
       +
         fetchdocker = callPackage ../build-support/fetchdocker { };

     

       164
       164
       +
       

     

       165
       165
       +
         fetchDockerConfig = callPackage ../build-support/fetchdocker/fetchDockerConfig.nix { };

     

       166
       166
       +
       

     

       167
       167
       +
         fetchDockerLayer = callPackage ../build-support/fetchdocker/fetchDockerLayer.nix { };

     

       168
       168
       +
       

     

       163
       169
        
         fetchfossil = callPackage ../build-support/fetchfossil { };

     

       164
       170
        
       

     

       165
       171
        
         fetchgit = callPackage ../build-support/fetchgit {