commit 5eed5416ffa70d1a6c1ff00548c746014df7e2c6 · pyrox.dev/nixpkgs

doc/build-helpers/special.md

···

       7
        
       special/makesetuphook.section.md

     

       8
        
       special/mkshell.section.md

     

       9
        
       special/vm-tools.section.md

     

       0
        
       
     

       10
        
       ```

···

       7
        
       special/makesetuphook.section.md

     

       8
        
       special/mkshell.section.md

     

       9
        
       special/vm-tools.section.md

     

       10
       +
       special/checkpoint-build.section.md

     

       11
        
       ```

+36

doc/build-helpers/special/checkpoint-build.section.md

···

       1
       +
       # pkgs.checkpointBuildTools  {#sec-checkpoint-build}

     

       2
       +
       

     

       3
       +
       `pkgs.checkpointBuildTools` provides a way to build derivations incrementally. It consists of two functions to make checkpoint builds using Nix possible.

     

       4
       +
       

     

       5
       +
       For hermeticity, Nix derivations do not allow any state to carry over between builds, making a transparent incremental build within a derivation impossible.

     

       6
       +
       

     

       7
       +
       However, we can tell Nix explicitly what the previous build state was, by representing that previous state as a derivation output. This allows the passed build state to be used for an incremental build.

     

       8
       +
       

     

       9
       +
       To change a normal derivation to a checkpoint based build, these steps must be taken:

     

       10
       +
         - apply `prepareCheckpointBuild` on the desired derivation

     

       11
       +
           e.g.:

     

       12
       +
       ```nix

     

       13
       +
       checkpointArtifacts = (pkgs.checkpointBuildTools.prepareCheckpointBuild pkgs.virtualbox);

     

       14
       +
       ```

     

       15
       +
         - change something you want in the sources of the package. (e.g. using a source override)

     

       16
       +
       ```nix

     

       17
       +
       changedVBox = pkgs.virtualbox.overrideAttrs (old: {

     

       18
       +
         src = path/to/vbox/sources;

     

       19
       +
       }

     

       20
       +
       ```

     

       21
       +
         - use `mkCheckpointedBuild changedVBox buildOutput`

     

       22
       +
         - enjoy shorter build times

     

       23
       +
       

     

       24
       +
       ## Example {#sec-checkpoint-build-example}

     

       25
       +
       ```nix

     

       26
       +
       { pkgs ? import <nixpkgs> {} }: with (pkgs) checkpointBuildTools;

     

       27
       +
       let

     

       28
       +
         helloCheckpoint = checkpointBuildTools.prepareCheckpointBuild pkgs.hello;

     

       29
       +
         changedHello = pkgs.hello.overrideAttrs (_: {

     

       30
       +
           doCheck = false;

     

       31
       +
           patchPhase = ''

     

       32
       +
             sed -i 's/Hello, world!/Hello, Nix!/g' src/hello.c

     

       33
       +
           '';

     

       34
       +
         });

     

       35
       +
       in checkpointBuildTools.mkCheckpointBuild changedHello helloCheckpoint

     

       36
       +
       ```

+69

pkgs/build-support/checkpoint-build.nix

···

       1
       +
       { pkgs }:

     

       2
       +
       rec {

     

       3
       +
         /* Prepare a derivation for local builds.

     

       4
       +
           *

     

       5
       +
           * This function prepares checkpoint builds by provinding,

     

       6
       +
           * containing the build output and the sources for cross checking.

     

       7
       +
           * The build output can be used later to allow checkpoint builds

     

       8
       +
           * by passing the derivation output to the `mkCheckpointBuild` function.

     

       9
       +
           *

     

       10
       +
           * To build a project with checkpoints follow these steps:

     

       11
       +
           * - run prepareIncrementalBuild on the desired derivation

     

       12
       +
           *   e.G `incrementalBuildArtifacts = (pkgs.checkpointBuildTools.prepareCheckpointBuild pkgs.virtualbox);`

     

       13
       +
           * - change something you want in the sources of the package( e.G using source override)

     

       14
       +
           *   changedVBox = pkgs.virtuabox.overrideAttrs (old: {

     

       15
       +
           *      src = path/to/vbox/sources;

     

       16
       +
           *   }

     

       17
       +
           * - use `mkCheckpointedBuild changedVBox buildOutput`

     

       18
       +
           * - enjoy shorter build times

     

       19
       +
         */

     

       20
       +
         prepareCheckpointBuild = drv: drv.overrideAttrs (old: {

     

       21
       +
           outputs = [ "out" ];

     

       22
       +
           name = drv.name + "-checkpointArtifacts";

     

       23
       +
           # To determine differences between the state of the build directory

     

       24
       +
           # from an earlier build and  a later one we store the state of the build

     

       25
       +
           # directory before build, but after patch phases.

     

       26
       +
           # This way, the same derivation can be used multiple times and only changes are detected.

     

       27
       +
           # Additionally Removed files are handled correctly in later builds.

     

       28
       +
           preBuild = (old.preBuild or "") + ''

     

       29
       +
             mkdir -p $out/sources

     

       30
       +
             cp -r ./* $out/sources/

     

       31
       +
           '';

     

       32
       +
       

     

       33
       +
           # After the build the build directory is copied again

     

       34
       +
           # to get the output files.

     

       35
       +
           # We copy the complete build folder, to take care for

     

       36
       +
           # Build tools, building in the source directory, instead of

     

       37
       +
           # having a build root directory, e.G the Linux kernel.

     

       38
       +
           installPhase = ''

     

       39
       +
             runHook preCheckpointInstall

     

       40
       +
             mkdir -p $out/outputs

     

       41
       +
             cp -r ./* $out/outputs/

     

       42
       +
             runHook postCheckpointInstall

     

       43
       +
           '';

     

       44
       +
         });

     

       45
       +
       

     

       46
       +
         /* Build a derivation based on the checkpoint output generated by

     

       47
       +
           * the `prepareCheckpointBuild function.

     

       48
       +
           *

     

       49
       +
           * Usage:

     

       50
       +
           * let

     

       51
       +
           *   checkpointArtifacts = prepareCheckpointBuild drv

     

       52
       +
           * in mkCheckpointedBuild drv checkpointArtifacts

     

       53
       +
         */

     

       54
       +
         mkCheckpointedBuild = drv: previousBuildArtifacts: drv.overrideAttrs (old: {

     

       55
       +
           # The actual checkpoint build phase.

     

       56
       +
           # We compare the changed sources from a previous build with the current and create a patch

     

       57
       +
           # Afterwards we clean the build directory to copy the previous output files (Including the sources)

     

       58
       +
           # The source difference patch is applied to get the latest changes again to allow short build times.

     

       59
       +
           preBuild = (old.preBuild or "") + ''

     

       60
       +
             set +e

     

       61
       +
             diff -ur ${previousBuildArtifacts}/sources ./ > sourceDifference.patch

     

       62
       +
             set -e

     

       63
       +
             shopt -s extglob dotglob

     

       64
       +
             rm -r !("sourceDifference.patch")

     

       65
       +
             ${pkgs.rsync}/bin/rsync -cutU --chown=$USER:$USER --chmod=+w -r ${previousBuildArtifacts}/outputs/* .

     

       66
       +
             patch -p 1 -i sourceDifference.patch

     

       67
       +
           '';

     

       68
       +
         });

     

       69
       +
       }

+57

pkgs/test/checkpointBuild/default.nix

···

       1
       +
       { hello, checkpointBuildTools, runCommandNoCC, texinfo, stdenv, rsync }:

     

       2
       +
       let

     

       3
       +
         baseHelloArtifacts = checkpointBuildTools.prepareCheckpointBuild hello;

     

       4
       +
         patchedHello = hello.overrideAttrs (old: {

     

       5
       +
           buildInputs = [ texinfo ];

     

       6
       +
           src = runCommandNoCC "patch-hello-src" { } ''

     

       7
       +
             mkdir -p $out

     

       8
       +
             cd $out

     

       9
       +
             tar xf ${hello.src} --strip-components=1

     

       10
       +
             patch -p1 < ${./hello.patch}

     

       11
       +
           '';

     

       12
       +
         });

     

       13
       +
         checkpointBuiltHello = checkpointBuildTools.mkCheckpointedBuild patchedHello baseHelloArtifacts;

     

       14
       +
       

     

       15
       +
         checkpointBuiltHelloWithCheck = checkpointBuiltHello.overrideAttrs (old: {

     

       16
       +
           doCheck = true;

     

       17
       +
           checkPhase = ''

     

       18
       +
             echo "checking if unchanged source file is not recompiled"

     

       19
       +
               [ "$(stat --format="%Y" lib/exitfail.o)" = "$(stat --format="%Y" ${baseHelloArtifacts}/outputs/lib/exitfail.o)" ]

     

       20
       +
           '';

     

       21
       +
         });

     

       22
       +
       

     

       23
       +
         baseHelloRemoveFileArtifacts = checkpointBuildTools.prepareCheckpointBuild (hello.overrideAttrs (old: {

     

       24
       +
           patches = [ ./hello-additionalFile.patch ];

     

       25
       +
         }));

     

       26
       +
       

     

       27
       +
         preparedHelloRemoveFileSrc = runCommandNoCC "patch-hello-src" { } ''

     

       28
       +
           mkdir -p $out

     

       29
       +
           cd $out

     

       30
       +
           tar xf ${hello.src} --strip-components=1

     

       31
       +
           patch -p1 < ${./hello-additionalFile.patch}

     

       32
       +
         '';

     

       33
       +
       

     

       34
       +
         patchedHelloRemoveFile = hello.overrideAttrs (old: {

     

       35
       +
           buildInputs = [ texinfo ];

     

       36
       +
           src = runCommandNoCC "patch-hello-src" { } ''

     

       37
       +
             mkdir -p $out

     

       38
       +
             cd $out

     

       39
       +
             ${rsync}/bin/rsync -cutU --chown=$USER:$USER --chmod=+w -r ${preparedHelloRemoveFileSrc}/* .

     

       40
       +
             patch -p1 < ${./hello-removeFile.patch}

     

       41
       +
           '';

     

       42
       +
         });

     

       43
       +
       

     

       44
       +
         checkpointBuiltHelloWithRemovedFile = checkpointBuildTools.mkCheckpointedBuild patchedHelloRemoveFile baseHelloRemoveFileArtifacts;

     

       45
       +
       in

     

       46
       +
       stdenv.mkDerivation {

     

       47
       +
         name = "patched-hello-returns-correct-output";

     

       48
       +
         buildCommand = ''

     

       49
       +
           touch $out

     

       50
       +
       

     

       51
       +
           echo "testing output of hello binary"

     

       52
       +
           [ "$(${checkpointBuiltHelloWithCheck}/bin/hello)" = "Hello, incremental world!" ]

     

       53
       +
           echo "testing output of hello with removed file"

     

       54
       +
           [ "$(${checkpointBuiltHelloWithRemovedFile}/bin/hello)" = "Hello, incremental world!" ]

     

       55
       +
         '';

     

       56
       +
       }

     

       57
       +

+67

pkgs/test/checkpointBuild/hello-additionalFile.patch

···

       1
       +
       :100644 100644 0000000 0000000 M	Makefile.in

     

       2
       +
       :000000 100644 0000000 0000000 A	src/additionalFile.c

     

       3
       +
       :100644 100644 0000000 0000000 M	src/hello.c

     

       4
       +
       :100644 100644 0000000 0000000 M	src/system.h

     

       5
       +
       

     

       6
       +
       diff --git a/Makefile.in b/Makefile.in

     

       7
       +
       index 1597d39..f63f830 100644

     

       8
       +
       --- a/Makefile.in

     

       9
       +
       +++ b/Makefile.in

     

       10
       +
       @@ -312,7 +312,7 @@ am_lib_libhello_a_OBJECTS = lib/basename-lgpl.$(OBJEXT) \

     

       11
       +
        	lib/version-etc.$(OBJEXT) lib/version-etc-fsf.$(OBJEXT) \

     

       12
       +
        	lib/wctype-h.$(OBJEXT) lib/xmalloc.$(OBJEXT) \

     

       13
       +
        	lib/xalloc-die.$(OBJEXT) lib/xstriconv.$(OBJEXT) \

     

       14
       +
       -	lib/xstrndup.$(OBJEXT)

     

       15
       +
       +	lib/xstrndup.$(OBJEXT) src/additionalFile.$(OBJEXT)

     

       16
       +
        lib_libhello_a_OBJECTS = $(am_lib_libhello_a_OBJECTS)

     

       17
       +
        am_hello_OBJECTS = src/hello.$(OBJEXT)

     

       18
       +
        hello_OBJECTS = $(am_hello_OBJECTS)

     

       19
       +
       @@ -1842,7 +1842,7 @@ lib_libhello_a_SOURCES = lib/basename-lgpl.c lib/c-ctype.h \

     

       20
       +
        	$(am__append_4) $(am__append_5) lib/version-etc.h \

     

       21
       +
        	lib/version-etc.c lib/version-etc-fsf.c lib/wctype-h.c \

     

       22
       +
        	lib/xmalloc.c lib/xalloc-die.c lib/xstriconv.h lib/xstriconv.c \

     

       23
       +
       -	lib/xstrndup.h lib/xstrndup.c

     

       24
       +
       +	lib/xstrndup.h lib/xstrndup.c src/additionalFile.c

     

       25
       +
        lib_libhello_a_LIBADD = $(gl_LIBOBJS)

     

       26
       +
        lib_libhello_a_DEPENDENCIES = $(gl_LIBOBJS)

     

       27
       +
        EXTRA_lib_libhello_a_SOURCES = lib/close.c lib/stripslash.c lib/dup2.c \

     

       28
       +
       diff --git a/src/additionalFile.c b/src/additionalFile.c

     

       29
       +
       new file mode 100644

     

       30
       +
       index 0000000..34d683d

     

       31
       +
       --- /dev/null

     

       32
       +
       +++ b/src/additionalFile.c

     

       33
       +
       @@ -0,0 +1,6 @@

     

       34
       +
       +#include "config.h"

     

       35
       +
       +#include "system.h"

     

       36
       +
       +

     

       37
       +
       +int somefunc() {

     

       38
       +
       +    return 0;

     

       39
       +
       +}

     

       40
       +
       diff --git a/src/hello.c b/src/hello.c

     

       41
       +
       index 2e7d38e..a8e36dc 100644

     

       42
       +
       --- a/src/hello.c

     

       43
       +
       +++ b/src/hello.c

     

       44
       +
       @@ -146,7 +146,11 @@ main (int argc, char *argv[])

     

       45
       +
        #endif

     

       46
       +
        

     

       47
       +
          /* Having initialized gettext, get the default message. */

     

       48
       +
       -  greeting_msg = _("Hello, world!");

     

       49
       +
       +  if (somefunc() == 0) {

     

       50
       +
       +    greeting_msg = _("Hello, world!");

     

       51
       +
       +  } else {

     

       52
       +
       +    greeting_msg = _("Hello, incremental world!");

     

       53
       +
       +  }

     

       54
       +
        

     

       55
       +
          /* Even exiting has subtleties.  On exit, if any writes failed, change

     

       56
       +
             the exit status.  The /dev/full device on GNU/Linux can be used for

     

       57
       +
       diff --git a/src/system.h b/src/system.h

     

       58
       +
       index d39cdb9..dc425d2 100644

     

       59
       +
       --- a/src/system.h

     

       60
       +
       +++ b/src/system.h

     

       61
       +
       @@ -59,4 +59,6 @@

     

       62
       +
            } \

     

       63
       +
          while (0)

     

       64
       +
        

     

       65
       +
       +int somefunc();

     

       66
       +
       +

     

       67
       +
        #endif /* HELLO_SYSTEM_H */

+67

pkgs/test/checkpointBuild/hello-removeFile.patch

···

       1
       +
       :100644 100644 0000000 0000000 M	Makefile.in

     

       2
       +
       :100644 000000 0000000 0000000 D	src/additionalFile.c

     

       3
       +
       :100644 100644 0000000 0000000 M	src/hello.c

     

       4
       +
       :100755 100755 0000000 0000000 M	tests/hello-1

     

       5
       +
       

     

       6
       +
       diff --git a/Makefile.in b/Makefile.in

     

       7
       +
       index f63f830..1597d39 100644

     

       8
       +
       --- a/Makefile.in

     

       9
       +
       +++ b/Makefile.in

     

       10
       +
       @@ -312,7 +312,7 @@ am_lib_libhello_a_OBJECTS = lib/basename-lgpl.$(OBJEXT) \

     

       11
       +
        	lib/version-etc.$(OBJEXT) lib/version-etc-fsf.$(OBJEXT) \

     

       12
       +
        	lib/wctype-h.$(OBJEXT) lib/xmalloc.$(OBJEXT) \

     

       13
       +
        	lib/xalloc-die.$(OBJEXT) lib/xstriconv.$(OBJEXT) \

     

       14
       +
       -	lib/xstrndup.$(OBJEXT) src/additionalFile.$(OBJEXT)

     

       15
       +
       +	lib/xstrndup.$(OBJEXT)

     

       16
       +
        lib_libhello_a_OBJECTS = $(am_lib_libhello_a_OBJECTS)

     

       17
       +
        am_hello_OBJECTS = src/hello.$(OBJEXT)

     

       18
       +
        hello_OBJECTS = $(am_hello_OBJECTS)

     

       19
       +
       @@ -1842,7 +1842,7 @@ lib_libhello_a_SOURCES = lib/basename-lgpl.c lib/c-ctype.h \

     

       20
       +
        	$(am__append_4) $(am__append_5) lib/version-etc.h \

     

       21
       +
        	lib/version-etc.c lib/version-etc-fsf.c lib/wctype-h.c \

     

       22
       +
        	lib/xmalloc.c lib/xalloc-die.c lib/xstriconv.h lib/xstriconv.c \

     

       23
       +
       -	lib/xstrndup.h lib/xstrndup.c src/additionalFile.c

     

       24
       +
       +	lib/xstrndup.h lib/xstrndup.c

     

       25
       +
        lib_libhello_a_LIBADD = $(gl_LIBOBJS)

     

       26
       +
        lib_libhello_a_DEPENDENCIES = $(gl_LIBOBJS)

     

       27
       +
        EXTRA_lib_libhello_a_SOURCES = lib/close.c lib/stripslash.c lib/dup2.c \

     

       28
       +
       diff --git a/src/additionalFile.c b/src/additionalFile.c

     

       29
       +
       deleted file mode 100644

     

       30
       +
       index 34d683d..0000000

     

       31
       +
       --- a/src/additionalFile.c

     

       32
       +
       +++ /dev/null

     

       33
       +
       @@ -1,6 +0,0 @@

     

       34
       +
       -#include "config.h"

     

       35
       +
       -#include "system.h"

     

       36
       +
       -

     

       37
       +
       -int somefunc() {

     

       38
       +
       -    return 0;

     

       39
       +
       -}

     

       40
       +
       diff --git a/src/hello.c b/src/hello.c

     

       41
       +
       index a8e36dc..53722d9 100644

     

       42
       +
       --- a/src/hello.c

     

       43
       +
       +++ b/src/hello.c

     

       44
       +
       @@ -126,6 +126,10 @@ parse_options (int argc, char *argv[], const char **greeting_msg)

     

       45
       +
            }

     

       46
       +
        }

     

       47
       +
        

     

       48
       +
       +int somefunc() {

     

       49
       +
       +    return 1;

     

       50
       +
       +}

     

       51
       +
       +

     

       52
       +
        int

     

       53
       +
        main (int argc, char *argv[])

     

       54
       +
        {

     

       55
       +
       diff --git a/tests/hello-1 b/tests/hello-1

     

       56
       +
       index 96ffef8..f0b9f8d 100755

     

       57
       +
       --- a/tests/hello-1

     

       58
       +
       +++ b/tests/hello-1

     

       59
       +
       @@ -21,7 +21,7 @@ export LANGUAGE LC_ALL LC_MESSAGES LANG

     

       60
       +
        

     

       61
       +
        tmpfiles="hello-test1.ok"

     

       62
       +
        cat <<EOF > hello-test1.ok

     

       63
       +
       -Hello, world!

     

       64
       +
       +Hello, incremental world!

     

       65
       +
        EOF

     

       66
       +
        

     

       67
       +
        tmpfiles="$tmpfiles hello-test1.out"

+26

pkgs/test/checkpointBuild/hello.patch

···

       1
       +
       diff --git a/src/hello.c b/src/hello.c

     

       2
       +
       index 182303c..453962f 100644

     

       3
       +
       --- a/src/hello.c

     

       4
       +
       +++ b/src/hello.c

     

       5
       +
       @@ -57,7 +57,7 @@ main (int argc, char *argv[])

     

       6
       +
        #endif

     

       7
       +
        

     

       8
       +
          /* Having initialized gettext, get the default message. */

     

       9
       +
       -  greeting_msg = _("Hello, world!");

     

       10
       +
       +  greeting_msg = _("Hello, incremental world!");

     

       11
       +
        

     

       12
       +
          /* Even exiting has subtleties.  On exit, if any writes failed, change

     

       13
       +
             the exit status.  The /dev/full device on GNU/Linux can be used for

     

       14
       +
       diff --git a/tests/hello-1 b/tests/hello-1

     

       15
       +
       index 3b7a815..e15fa95 100755

     

       16
       +
       --- a/tests/hello-1

     

       17
       +
       +++ b/tests/hello-1

     

       18
       +
       @@ -21,7 +21,7 @@ export LANGUAGE LC_ALL LC_MESSAGES LANG

     

       19
       +
        

     

       20
       +
        tmpfiles="hello-test1.ok"

     

       21
       +
        cat <<EOF > hello-test1.ok

     

       22
       +
       -Hello, world!

     

       23
       +
       +Hello, incremental world!

     

       24
       +
        EOF

     

       25
       +
        

     

       26
       +
        tmpfiles="$tmpfiles hello-test1.out"

pkgs/test/default.nix

···

       113
        
       

     

       114
        
         install-shell-files = callPackage ./install-shell-files {};

     

       115
        
       

     

       0
        
       
     

       0
        
       
     

       116
        
         kernel-config = callPackage ./kernel.nix {};

     

       117
        
       

     

       118
        
         ld-library-path = callPackage ./ld-library-path {};

···

       113
        
       

     

       114
        
         install-shell-files = callPackage ./install-shell-files {};

     

       115
        
       

     

       116
       +
         checkpoint-build = callPackage ./checkpointBuild {};

     

       117
       +
       

     

       118
        
         kernel-config = callPackage ./kernel.nix {};

     

       119
        
       

     

       120
        
         ld-library-path = callPackage ./ld-library-path {};

pkgs/top-level/all-packages.nix

···

       430
        
       

     

       431
        
         camunda-modeler = callPackage ../applications/misc/camunda-modeler { };

     

       432
        
       

     

       0
        
       
     

       0
        
       
     

       433
        
         caroline = callPackage ../development/libraries/caroline { };

     

       434
        
       

     

       435
        
         cartridges = callPackage ../applications/misc/cartridges { };

···

       430
        
       

     

       431
        
         camunda-modeler = callPackage ../applications/misc/camunda-modeler { };

     

       432
        
       

     

       433
       +
         checkpointBuildTools = callPackage ../build-support/checkpoint-build.nix {};

     

       434
       +
       

     

       435
        
         caroline = callPackage ../development/libraries/caroline { };

     

       436
        
       

     

       437
        
         cartridges = callPackage ../applications/misc/cartridges { };