commit 36bf6afd42e755bf51ce9678ce2710e1b1d68f99 · pyrox.dev/nixpkgs

+20

pkgs/development/compilers/julia/default.nix

···

       1
       1
       +
       { callPackage }:

     

       2
       2
       +
       

     

       3
       3
       +
       let

     

       4
       4
       +
         juliaWithPackages = callPackage ../../julia-modules {};

     

       5
       5
       +
       

     

       6
       6
       +
         wrapJulia = julia: julia.overrideAttrs (oldAttrs: {

     

       7
       7
       +
           passthru = (oldAttrs.passthru or {}) // {

     

       8
       8
       +
             withPackages = juliaWithPackages.override { inherit julia; };

     

       9
       9
       +
           };

     

       10
       10
       +
         });

     

       11
       11
       +
       

     

       12
       12
       +
       in

     

       13
       13
       +
       

     

       14
       14
       +
       {

     

       15
       15
       +
         julia_16-bin = wrapJulia (callPackage ./1.6-bin.nix {});

     

       16
       16
       +
         julia_18-bin = wrapJulia (callPackage ./1.8-bin.nix {});

     

       17
       17
       +
         julia_19-bin = wrapJulia (callPackage ./1.9-bin.nix {});

     

       18
       18
       +
         julia_18 = wrapJulia (callPackage ./1.8.nix {});

     

       19
       19
       +
         julia_19 = wrapJulia (callPackage ./1.9.nix {});

     

       20
       20
       +
       }

+193

pkgs/development/julia-modules/default.nix

···

       1
       1
       +
       { lib

     

       2
       2
       +
       , callPackage

     

       3
       3
       +
       , runCommand

     

       4
       4
       +
       , fetchFromGitHub

     

       5
       5
       +
       , fetchgit

     

       6
       6
       +
       , fontconfig

     

       7
       7
       +
       , git

     

       8
       8
       +
       , makeWrapper

     

       9
       9
       +
       , writeText

     

       10
       10
       +
       , writeTextFile

     

       11
       11
       +
       , python3

     

       12
       12
       +
       

     

       13
       13
       +
       # Artifacts dependencies

     

       14
       14
       +
       , fetchurl

     

       15
       15
       +
       , glibc

     

       16
       16
       +
       , pkgs

     

       17
       17
       +
       , stdenv

     

       18
       18
       +
       

     

       19
       19
       +
       , julia

     

       20
       20
       +
       

     

       21
       21
       +
       # Special registry which is equal to JuliaRegistries/General, but every Versions.toml

     

       22
       22
       +
       # entry is augmented with a Nix sha256 hash

     

       23
       23
       +
       , augmentedRegistry ? callPackage ./registry.nix {}

     

       24
       24
       +
       

     

       25
       25
       +
       # Other overridable arguments

     

       26
       26
       +
       , extraLibs ? []

     

       27
       27
       +
       , precompile ? true

     

       28
       28
       +
       , setDefaultDepot ? true

     

       29
       29
       +
       , makeWrapperArgs ? ""

     

       30
       30
       +
       , packageOverrides ? {}

     

       31
       31
       +
       , makeTransitiveDependenciesImportable ? false # Used to support symbol indexing

     

       32
       32
       +
       }:

     

       33
       33
       +
       

     

       34
       34
       +
       packageNames:

     

       35
       35
       +
       

     

       36
       36
       +
       let

     

       37
       37
       +
         util = callPackage ./util.nix {};

     

       38
       38
       +
       

     

       39
       39
       +
       in

     

       40
       40
       +
       

     

       41
       41
       +
       let

     

       42
       42
       +
         # Some Julia packages require access to Python. Provide a Nixpkgs version so it

     

       43
       43
       +
         # doesn't try to install its own.

     

       44
       44
       +
         pythonToUse = let

     

       45
       45
       +
           extraPythonPackages = ((callPackage ./extra-python-packages.nix { inherit python3; }).getExtraPythonPackages packageNames);

     

       46
       46
       +
         in (if extraPythonPackages == [] then python3

     

       47
       47
       +
             else util.addPackagesToPython python3 (map (pkg: lib.getAttr pkg python3.pkgs) extraPythonPackages));

     

       48
       48
       +
       

     

       49
       49
       +
         # Start by wrapping Julia so it has access to Python and any other extra libs.

     

       50
       50
       +
         # Also, prevent various packages (CondaPkg.jl, PythonCall.jl) from trying to do network calls.

     

       51
       51
       +
         juliaWrapped = runCommand "julia-${julia.version}-wrapped" { buildInputs = [makeWrapper]; inherit makeWrapperArgs; } ''

     

       52
       52
       +
           mkdir -p $out/bin

     

       53
       53
       +
           makeWrapper ${julia}/bin/julia $out/bin/julia \

     

       54
       54
       +
             --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath extraLibs}" \

     

       55
       55
       +
             --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf \

     

       56
       56
       +
             --set PYTHONHOME "${pythonToUse}" \

     

       57
       57
       +
             --prefix PYTHONPATH : "${pythonToUse}/${pythonToUse.sitePackages}" \

     

       58
       58
       +
             --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \

     

       59
       59
       +
             --set JULIA_CONDAPKG_OFFLINE yes \

     

       60
       60
       +
             --set JULIA_CONDAPKG_BACKEND Null \

     

       61
       61
       +
             --set JULIA_PYTHONCALL_EXE "@PyCall"

     

       62
       62
       +
         '';

     

       63
       63
       +
       

     

       64
       64
       +
         # If our closure ends up with certain packages, add others.

     

       65
       65
       +
         packageImplications = {

     

       66
       66
       +
           # Because we want to put PythonCall in PyCall mode so it doesn't try to download

     

       67
       67
       +
           # Python packages

     

       68
       68
       +
           PythonCall = ["PyCall"];

     

       69
       69
       +
         };

     

       70
       70
       +
       

     

       71
       71
       +
         # Invoke Julia resolution logic to determine the full dependency closure

     

       72
       72
       +
         packageOverridesRepoified = lib.mapAttrs util.repoifySimple packageOverrides;

     

       73
       73
       +
         closureYaml = callPackage ./package-closure.nix {

     

       74
       74
       +
           inherit augmentedRegistry julia packageNames packageImplications;

     

       75
       75
       +
           packageOverrides = packageOverridesRepoified;

     

       76
       76
       +
         };

     

       77
       77
       +
       

     

       78
       78
       +
         # Generate a Nix file consisting of a map from dependency UUID --> package info with fetchgit call:

     

       79
       79
       +
         # {

     

       80
       80
       +
         #   "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" = {

     

       81
       81
       +
         #     src = fetchgit {...};

     

       82
       82
       +
         #     name = "...";

     

       83
       83
       +
         #     version = "...";

     

       84
       84
       +
         #     treehash = "...";

     

       85
       85
       +
         #   };

     

       86
       86
       +
         #   ...

     

       87
       87
       +
         # }

     

       88
       88
       +
         dependencies = runCommand "julia-sources.nix" { buildInputs = [(python3.withPackages (ps: with ps; [toml pyyaml])) git]; } ''

     

       89
       89
       +
           python ${./python}/sources_nix.py \

     

       90
       90
       +
             "${augmentedRegistry}" \

     

       91
       91
       +
             '${lib.generators.toJSON {} packageOverridesRepoified}' \

     

       92
       92
       +
             "${closureYaml}" \

     

       93
       93
       +
             "$out"

     

       94
       94
       +
         '';

     

       95
       95
       +
       

     

       96
       96
       +
         # Import the Nix file from the previous step (IFD) and turn each dependency repo into

     

       97
       97
       +
         # a dummy Git repository, as Julia expects. Format the results as a YAML map from

     

       98
       98
       +
         # dependency UUID -> Nix store location:

     

       99
       99
       +
         # {

     

       100
       100
       +
         #   "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3":"/nix/store/...-NaNMath.jl-0877504",

     

       101
       101
       +
         #   ...

     

       102
       102
       +
         # }

     

       103
       103
       +
         # This is also the point where we apply the packageOverrides.

     

       104
       104
       +
         dependencyUuidToInfo = import dependencies { inherit fetchgit; };

     

       105
       105
       +
         fillInOverrideSrc = uuid: info:

     

       106
       106
       +
           if lib.hasAttr info.name packageOverrides then (info // { src = lib.getAttr info.name packageOverrides; }) else info;

     

       107
       107
       +
         dependencyUuidToRepo = lib.mapAttrs util.repoifyInfo (lib.mapAttrs fillInOverrideSrc dependencyUuidToInfo);

     

       108
       108
       +
         dependencyUuidToRepoYaml = writeTextFile {

     

       109
       109
       +
           name = "dependency-uuid-to-repo.yml";

     

       110
       110
       +
           text = lib.generators.toYAML {} dependencyUuidToRepo;

     

       111
       111
       +
         };

     

       112
       112
       +
       

     

       113
       113
       +
         # Given the augmented registry, closure info yaml, and dependency path yaml, construct a complete

     

       114
       114
       +
         # Julia registry containing all the necessary packages

     

       115
       115
       +
         dependencyUuidToInfoYaml = writeTextFile {

     

       116
       116
       +
           name = "dependency-uuid-to-info.yml";

     

       117
       117
       +
           text = lib.generators.toYAML {} dependencyUuidToInfo;

     

       118
       118
       +
         };

     

       119
       119
       +
         fillInOverrideSrc' = uuid: info:

     

       120
       120
       +
           if lib.hasAttr info.name packageOverridesRepoified then (info // { src = lib.getAttr info.name packageOverridesRepoified; }) else info;

     

       121
       121
       +
         overridesOnly = lib.mapAttrs fillInOverrideSrc' (lib.filterAttrs (uuid: info: info.src == null) dependencyUuidToInfo);

     

       122
       122
       +
         minimalRegistry = runCommand "minimal-julia-registry" { buildInputs = [(python3.withPackages (ps: with ps; [toml pyyaml])) git]; } ''

     

       123
       123
       +
           python ${./python}/minimal_registry.py \

     

       124
       124
       +
             "${augmentedRegistry}" \

     

       125
       125
       +
             "${closureYaml}" \

     

       126
       126
       +
             '${lib.generators.toJSON {} overridesOnly}' \

     

       127
       127
       +
             "${dependencyUuidToRepoYaml}" \

     

       128
       128
       +
             "$out"

     

       129
       129
       +
         '';

     

       130
       130
       +
       

     

       131
       131
       +
         # Next, deal with artifacts. Scan each artifacts file individually and generate a Nix file that

     

       132
       132
       +
         # produces the desired Overrides.toml.

     

       133
       133
       +
         artifactsNix = runCommand "julia-artifacts.nix" { buildInputs = [(python3.withPackages (ps: with ps; [toml pyyaml]))]; } ''

     

       134
       134
       +
           python ${./python}/extract_artifacts.py \

     

       135
       135
       +
             "${dependencyUuidToRepoYaml}" \

     

       136
       136
       +
             "${closureYaml}" \

     

       137
       137
       +
             "${juliaWrapped}/bin/julia" \

     

       138
       138
       +
             "${if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl}" \

     

       139
       139
       +
             '${lib.generators.toJSON {} (import ./extra-libs.nix)}' \

     

       140
       140
       +
             "$out"

     

       141
       141
       +
         '';

     

       142
       142
       +
       

     

       143
       143
       +
         # Import the artifacts Nix to build Overrides.toml (IFD)

     

       144
       144
       +
         artifacts = import artifactsNix { inherit lib fetchurl pkgs glibc stdenv; };

     

       145
       145
       +
         overridesJson = writeTextFile {

     

       146
       146
       +
           name = "Overrides.json";

     

       147
       147
       +
           text = lib.generators.toJSON {} artifacts;

     

       148
       148
       +
         };

     

       149
       149
       +
         overridesToml = runCommand "Overrides.toml" { buildInputs = [(python3.withPackages (ps: with ps; [toml]))]; } ''

     

       150
       150
       +
           python ${./python}/format_overrides.py \

     

       151
       151
       +
             "${overridesJson}" \

     

       152
       152
       +
             "$out"

     

       153
       153
       +
         '';

     

       154
       154
       +
       

     

       155
       155
       +
         # Build a Julia project and depot. The project contains Project.toml/Manifest.toml, while the

     

       156
       156
       +
         # depot contains package build products (including the precompiled libraries, if precompile=true)

     

       157
       157
       +
         projectAndDepot = callPackage ./depot.nix {

     

       158
       158
       +
           inherit closureYaml extraLibs overridesToml packageImplications precompile;

     

       159
       159
       +
           julia = juliaWrapped;

     

       160
       160
       +
           registry = minimalRegistry;

     

       161
       161
       +
           packageNames = if makeTransitiveDependenciesImportable

     

       162
       162
       +
             then lib.mapAttrsToList (uuid: info: info.name) dependencyUuidToInfo

     

       163
       163
       +
             else packageNames;

     

       164
       164
       +
         };

     

       165
       165
       +
       

     

       166
       166
       +
       in

     

       167
       167
       +
       

     

       168
       168
       +
       runCommand "julia-${julia.version}-env" {

     

       169
       169
       +
         buildInputs = [makeWrapper];

     

       170
       170
       +
       

     

       171
       171
       +
         inherit julia;

     

       172
       172
       +
         inherit juliaWrapped;

     

       173
       173
       +
       

     

       174
       174
       +
         # Expose the steps we used along the way in case the user wants to use them, for example to build

     

       175
       175
       +
         # expressions and build them separately to avoid IFD.

     

       176
       176
       +
         inherit dependencies;

     

       177
       177
       +
         inherit closureYaml;

     

       178
       178
       +
         inherit dependencyUuidToInfoYaml;

     

       179
       179
       +
         inherit dependencyUuidToRepoYaml;

     

       180
       180
       +
         inherit minimalRegistry;

     

       181
       181
       +
         inherit artifactsNix;

     

       182
       182
       +
         inherit overridesJson;

     

       183
       183
       +
         inherit overridesToml;

     

       184
       184
       +
         inherit projectAndDepot;

     

       185
       185
       +
       } (''

     

       186
       186
       +
         mkdir -p $out/bin

     

       187
       187
       +
         makeWrapper ${juliaWrapped}/bin/julia $out/bin/julia \

     

       188
       188
       +
           --suffix JULIA_DEPOT_PATH : "${projectAndDepot}/depot" \

     

       189
       189
       +
           --set-default JULIA_PROJECT "${projectAndDepot}/project" \

     

       190
       190
       +
           --set-default JULIA_LOAD_PATH '@:${projectAndDepot}/project/Project.toml:@v#.#:@stdlib'

     

       191
       191
       +
       '' + lib.optionalString setDefaultDepot ''

     

       192
       192
       +
         sed -i '2 i\JULIA_DEPOT_PATH=''${JULIA_DEPOT_PATH-"$HOME/.julia"}' $out/bin/julia

     

       193
       193
       +
       '')

+85

pkgs/development/julia-modules/depot.nix

···

       1
       1
       +
       { lib

     

       2
       2
       +
       , runCommand

     

       3
       3
       +
       

     

       4
       4
       +
       , cacert

     

       5
       5
       +
       , curl

     

       6
       6
       +
       , git

     

       7
       7
       +
       , julia

     

       8
       8
       +
       , python3

     

       9
       9
       +
       

     

       10
       10
       +
       , closureYaml

     

       11
       11
       +
       , extraLibs

     

       12
       12
       +
       , overridesToml

     

       13
       13
       +
       , packageNames

     

       14
       14
       +
       , packageImplications

     

       15
       15
       +
       , precompile

     

       16
       16
       +
       , registry

     

       17
       17
       +
       }:

     

       18
       18
       +
       

     

       19
       19
       +
       runCommand "julia-depot" {

     

       20
       20
       +
           nativeBuildInputs = [curl git julia (python3.withPackages (ps: with ps; [pyyaml]))] ++ extraLibs;

     

       21
       21
       +
           inherit precompile registry;

     

       22
       22
       +
         } ''

     

       23
       23
       +
         export HOME=$(pwd)

     

       24
       24
       +
       

     

       25
       25
       +
         echo "Building Julia depot and project with the following inputs"

     

       26
       26
       +
         echo "Julia: ${julia}"

     

       27
       27
       +
         echo "Registry: $registry"

     

       28
       28
       +
         echo "Overrides ${overridesToml}"

     

       29
       29
       +
       

     

       30
       30
       +
         mkdir -p $out/project

     

       31
       31
       +
         export JULIA_PROJECT="$out/project"

     

       32
       32
       +
       

     

       33
       33
       +
         mkdir -p $out/depot/artifacts

     

       34
       34
       +
         export JULIA_DEPOT_PATH="$out/depot"

     

       35
       35
       +
         cp ${overridesToml} $out/depot/artifacts/Overrides.toml

     

       36
       36
       +
       

     

       37
       37
       +
         # These can be useful to debug problems

     

       38
       38
       +
         # export JULIA_DEBUG=Pkg

     

       39
       39
       +
         # export JULIA_DEBUG=loading

     

       40
       40
       +
       

     

       41
       41
       +
         export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt"

     

       42
       42
       +
       

     

       43
       43
       +
         # Only precompile if configured to below

     

       44
       44
       +
         export JULIA_PKG_PRECOMPILE_AUTO=0

     

       45
       45
       +
       

     

       46
       46
       +
         # Prevent a warning where Julia tries to download package server info

     

       47
       47
       +
         export JULIA_PKG_SERVER=""

     

       48
       48
       +
       

     

       49
       49
       +
         # See if we need to add any extra package names based on the closure

     

       50
       50
       +
         # and the packageImplications. We're using the full closure YAML here since

     

       51
       51
       +
         # it's available, which is slightly weird, but it should work just as well

     

       52
       52
       +
         # for finding the extra packages we need to add

     

       53
       53
       +
         python ${./python}/find_package_implications.py "${closureYaml}" '${lib.generators.toJSON {} packageImplications}' extra_package_names.txt

     

       54
       54
       +
       

     

       55
       55
       +
         # git config --global --add safe.directory '/nix'

     

       56
       56
       +
         export JULIA_PKG_USE_CLI_GIT="true"

     

       57
       57
       +
       

     

       58
       58
       +
         julia -e ' \

     

       59
       59
       +
           import Pkg

     

       60
       60
       +
           import Pkg.Types: PRESERVE_NONE

     

       61
       61
       +
       

     

       62
       62
       +
           Pkg.Registry.add(Pkg.RegistrySpec(path="${registry}"))

     

       63
       63
       +
       

     

       64
       64
       +
           input = ${lib.generators.toJSON {} packageNames} ::Vector{String}

     

       65
       65
       +
       

     

       66
       66
       +
           if isfile("extra_package_names.txt")

     

       67
       67
       +
             append!(input, readlines("extra_package_names.txt"))

     

       68
       68
       +
           end

     

       69
       69
       +
       

     

       70
       70
       +
           input = unique(input)

     

       71
       71
       +
       

     

       72
       72
       +
           if !isempty(input)

     

       73
       73
       +
             println("Adding packages: " * join(input, " "))

     

       74
       74
       +
             Pkg.add(input; preserve=PRESERVE_NONE)

     

       75
       75
       +
             Pkg.instantiate()

     

       76
       76
       +
       

     

       77
       77
       +
             if "precompile" in keys(ENV) && ENV["precompile"] != "0" && ENV["precompile"] != ""

     

       78
       78
       +
               Pkg.precompile()

     

       79
       79
       +
             end

     

       80
       80
       +
           end

     

       81
       81
       +
       

     

       82
       82
       +
           # Remove the registry to save space

     

       83
       83
       +
           Pkg.Registry.rm("General")

     

       84
       84
       +
         '

     

       85
       85
       +
       ''

+15

pkgs/development/julia-modules/extra-libs.nix

···

       1
       1
       +
       # A map from a Julia package (typically a JLL package) to extra libraries

     

       2
       2
       +
       # that they require from Nix.

     

       3
       3
       +
       # The libraries should be strings evaluated in a "with pkgs" context.

     

       4
       4
       +
       

     

       5
       5
       +
       {

     

       6
       6
       +
         # Qt5Base_jll

     

       7
       7
       +
         # Needs access to dbus or you get "Cannot find libdbus-1 in your system"

     

       8
       8
       +
         # Repro: build environment with ["Plots"]

     

       9
       9
       +
         # > using Plots; plot(cos, 0, 2pi)

     

       10
       10
       +
         "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1" = ["dbus.lib"];

     

       11
       11
       +
       

     

       12
       12
       +
         # Qt6Base_jll

     

       13
       13
       +
         # Same reason as Qt5Base_jll

     

       14
       14
       +
         "c0090381-4147-56d7-9ebc-da0b1113ec56" = ["dbus.lib"];

     

       15
       15
       +
       }

+22

pkgs/development/julia-modules/extra-python-packages.nix

···

       1
       1
       +
       { lib

     

       2
       2
       +
       , python3

     

       3
       3
       +
       }:

     

       4
       4
       +
       

     

       5
       5
       +
       # This file contains an extra mapping from Julia packages to the Python packages they depend on.

     

       6
       6
       +
       

     

       7
       7
       +
       with lib;

     

       8
       8
       +
       

     

       9
       9
       +
       rec {

     

       10
       10
       +
         packageMapping = {

     

       11
       11
       +
           ExcelFiles = ["xlrd"];

     

       12
       12
       +
           PyPlot = ["matplotlib"];

     

       13
       13
       +
           PythonPlot = ["matplotlib"];

     

       14
       14
       +
           SymPy = ["sympy"];

     

       15
       15
       +
         };

     

       16
       16
       +
       

     

       17
       17
       +
         getExtraPythonPackages = names: concatMap (name: let

     

       18
       18
       +
           allCandidates = if hasAttr name packageMapping then getAttr name packageMapping else [];

     

       19
       19
       +
         in

     

       20
       20
       +
           filter (x: hasAttr x python3.pkgs) allCandidates

     

       21
       21
       +
         ) names;

     

       22
       22
       +
       }

+63

pkgs/development/julia-modules/extract_artifacts.jl

···

       1
       1
       +
       

     

       2
       2
       +
       import Base: UUID

     

       3
       3
       +
       import Pkg.Artifacts: artifact_meta, artifact_names, find_artifacts_toml, load_artifacts_toml, select_downloadable_artifacts

     

       4
       4
       +
       import Pkg.BinaryPlatforms: AbstractPlatform, platform_key_abi, triplet

     

       5
       5
       +
       import Pkg.Operations: gen_build_code

     

       6
       6
       +
       import TOML

     

       7
       7
       +
       

     

       8
       8
       +
       pkg_uuid = UUID(ARGS[1])

     

       9
       9
       +
       dir = ARGS[2]

     

       10
       10
       +
       

     

       11
       11
       +
       artifacts_toml = find_artifacts_toml(dir)

     

       12
       12
       +
       

     

       13
       13
       +
       if artifacts_toml == nothing

     

       14
       14
       +
           print("")

     

       15
       15
       +
           exit()

     

       16
       16
       +
       end

     

       17
       17
       +
       

     

       18
       18
       +
       platform = platform_key_abi()

     

       19
       19
       +
       

     

       20
       20
       +
       # Using collect_artifacts (from Pkg.jl) is more reliable than calling select_downloadable_artifacts directly.

     

       21
       21
       +
       # collect_artifacts includes support for .pkg/select_artifacts.jl, which may produce different results.

     

       22
       22
       +
       # If we use select_downloadable_artifacts here, then at depot build time it may try to download a different artifact

     

       23
       23
       +
       # and fail.

     

       24
       24
       +
       

     

       25
       25
       +
       # However! The collect_artifacts from Pkg.jl doesn't allow us to pass lazy to select_downloadable_artifacts.

     

       26
       26
       +
       # So we have to paste our own version in here :(

     

       27
       27
       +
       

     

       28
       28
       +
       function collect_artifacts(pkg_root::String; platform::AbstractPlatform)

     

       29
       29
       +
           # Check to see if this package has an (Julia)Artifacts.toml

     

       30
       30
       +
           artifacts_tomls = Tuple{String,Base.TOML.TOMLDict}[]

     

       31
       31
       +
           for f in artifact_names

     

       32
       32
       +
               artifacts_toml = joinpath(pkg_root, f)

     

       33
       33
       +
               if isfile(artifacts_toml)

     

       34
       34
       +
                   selector_path = joinpath(pkg_root, ".pkg", "select_artifacts.jl")

     

       35
       35
       +
       

     

       36
       36
       +
                   # If there is a dynamic artifact selector, run that in an appropriate sandbox to select artifacts

     

       37
       37
       +
                   if isfile(selector_path)

     

       38
       38
       +
                       # Despite the fact that we inherit the project, since the in-memory manifest

     

       39
       39
       +
                       # has not been updated yet, if we try to load any dependencies, it may fail.

     

       40
       40
       +
                       # Therefore, this project inheritance is really only for Preferences, not dependencies.

     

       41
       41
       +
                       select_cmd = Cmd(`$(gen_build_code(selector_path; inherit_project=true)) --startup-file=no $(triplet(platform))`)

     

       42
       42
       +
                       meta_toml = String(read(select_cmd))

     

       43
       43
       +
                       res = TOML.tryparse(meta_toml)

     

       44
       44
       +
                       if res isa TOML.ParserError

     

       45
       45
       +
                           errstr = sprint(showerror, res; context=stderr)

     

       46
       46
       +
                           pkgerror("failed to parse TOML output from running $(repr(selector_path)), got: \n$errstr")

     

       47
       47
       +
                       else

     

       48
       48
       +
                           push!(artifacts_tomls, (artifacts_toml, TOML.parse(meta_toml)))

     

       49
       49
       +
                       end

     

       50
       50
       +
                   else

     

       51
       51
       +
                       # Otherwise, use the standard selector from `Artifacts`

     

       52
       52
       +
                       artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy=true)

     

       53
       53
       +
                       push!(artifacts_tomls, (artifacts_toml, artifacts))

     

       54
       54
       +
                   end

     

       55
       55
       +
                   break

     

       56
       56
       +
               end

     

       57
       57
       +
           end

     

       58
       58
       +
           return artifacts_tomls

     

       59
       59
       +
       end

     

       60
       60
       +
       

     

       61
       61
       +
       for (artifacts_toml, artifacts) in collect_artifacts(dir; platform)

     

       62
       62
       +
           TOML.print(artifacts)

     

       63
       63
       +
       end

+33

pkgs/development/julia-modules/extract_artifacts_16.jl

···

       1
       1
       +
       

     

       2
       2
       +
       import Base: UUID

     

       3
       3
       +
       import Pkg.Artifacts: artifact_meta, find_artifacts_toml, load_artifacts_toml

     

       4
       4
       +
       import Pkg.BinaryPlatforms: platform_key_abi

     

       5
       5
       +
       import TOML

     

       6
       6
       +
       

     

       7
       7
       +
       pkg_uuid = UUID(ARGS[1])

     

       8
       8
       +
       dir = ARGS[2]

     

       9
       9
       +
       

     

       10
       10
       +
       artifacts_toml = find_artifacts_toml(dir)

     

       11
       11
       +
       

     

       12
       12
       +
       if artifacts_toml == nothing

     

       13
       13
       +
           print("")

     

       14
       14
       +
           exit()

     

       15
       15
       +
       end

     

       16
       16
       +
       

     

       17
       17
       +
       platform = platform_key_abi()

     

       18
       18
       +
       

     

       19
       19
       +
       # Older Julia doesn't provide select_downloadable_artifacts or .pkg/select_artifacts.jl,

     

       20
       20
       +
       # so gather the artifacts the old-fashioned way

     

       21
       21
       +
       artifact_dict = load_artifacts_toml(artifacts_toml; pkg_uuid=pkg_uuid)

     

       22
       22
       +
       

     

       23
       23
       +
       results = Dict()

     

       24
       24
       +
       for name in keys(artifact_dict)

     

       25
       25
       +
           # Get the metadata about this name for the requested platform

     

       26
       26
       +
           meta = artifact_meta(name, artifact_dict, artifacts_toml; platform=platform)

     

       27
       27
       +
       

     

       28
       28
       +
           # If there are no instances of this name for the desired platform, skip it

     

       29
       29
       +
           meta === nothing && continue

     

       30
       30
       +
       

     

       31
       31
       +
           results[name] = meta

     

       32
       32
       +
       end

     

       33
       33
       +
       TOML.print(results)

+180

pkgs/development/julia-modules/package-closure.nix

···

       1
       1
       +
       { lib

     

       2
       2
       +
       , julia

     

       3
       3
       +
       , python3

     

       4
       4
       +
       , runCommand

     

       5
       5
       +
       

     

       6
       6
       +
       , augmentedRegistry

     

       7
       7
       +
       , packageNames

     

       8
       8
       +
       , packageOverrides

     

       9
       9
       +
       , packageImplications

     

       10
       10
       +
       }:

     

       11
       11
       +
       

     

       12
       12
       +
       let

     

       13
       13
       +
         # The specific package resolution code depends on the Julia version

     

       14
       14
       +
         # These are pretty similar and could be combined to reduce duplication

     

       15
       15
       +
         resolveCode = if lib.versionOlder julia.version "1.7" then resolveCode1_6 else resolveCode1_8;

     

       16
       16
       +
       

     

       17
       17
       +
         resolveCode1_6 = ''

     

       18
       18
       +
           import Pkg.API: check_package_name

     

       19
       19
       +
           import Pkg.Types: Context!, PRESERVE_NONE, manifest_info, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved

     

       20
       20
       +
           import Pkg.Operations: _resolve, assert_can_add, is_dep, update_package_add

     

       21
       21
       +
       

     

       22
       22
       +
           foreach(pkg -> check_package_name(pkg.name, :add), pkgs)

     

       23
       23
       +
           pkgs = deepcopy(pkgs)  # deepcopy for avoid mutating PackageSpec members

     

       24
       24
       +
           Context!(ctx)

     

       25
       25
       +
       

     

       26
       26
       +
           project_deps_resolve!(ctx, pkgs)

     

       27
       27
       +
           registry_resolve!(ctx, pkgs)

     

       28
       28
       +
           stdlib_resolve!(pkgs)

     

       29
       29
       +
           ensure_resolved(ctx, pkgs, registry=true)

     

       30
       30
       +
       

     

       31
       31
       +
           assert_can_add(ctx, pkgs)

     

       32
       32
       +
       

     

       33
       33
       +
           for (i, pkg) in pairs(pkgs)

     

       34
       34
       +
               entry = manifest_info(ctx, pkg.uuid)

     

       35
       35
       +
               pkgs[i] = update_package_add(ctx, pkg, entry, is_dep(ctx, pkg))

     

       36
       36
       +
           end

     

       37
       37
       +
       

     

       38
       38
       +
           foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs)

     

       39
       39
       +
       

     

       40
       40
       +
           pkgs, deps_map = _resolve(ctx, pkgs, PRESERVE_NONE)

     

       41
       41
       +
       '';

     

       42
       42
       +
       

     

       43
       43
       +
         resolveCode1_8 = ''

     

       44
       44
       +
           import Pkg.API: handle_package_input!

     

       45
       45
       +
           import Pkg.Types: PRESERVE_NONE, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved

     

       46
       46
       +
           import Pkg.Operations: _resolve, assert_can_add, update_package_add

     

       47
       47
       +
       

     

       48
       48
       +
           foreach(handle_package_input!, pkgs)

     

       49
       49
       +
       

     

       50
       50
       +
           # The handle_package_input! call above clears pkg.path, so we have to apply package overrides after

     

       51
       51
       +
           overrides = Dict{String, String}(${builtins.concatStringsSep ", " (lib.mapAttrsToList (name: path: ''"${name}" => "${path}"'') packageOverrides)})

     

       52
       52
       +
           println("Package overrides: ")

     

       53
       53
       +
           println(overrides)

     

       54
       54
       +
           for pkg in pkgs

     

       55
       55
       +
             if pkg.name in keys(overrides)

     

       56
       56
       +
               pkg.path = overrides[pkg.name]

     

       57
       57
       +
             end

     

       58
       58
       +
           end

     

       59
       59
       +
       

     

       60
       60
       +
           project_deps_resolve!(ctx.env, pkgs)

     

       61
       61
       +
           registry_resolve!(ctx.registries, pkgs)

     

       62
       62
       +
           stdlib_resolve!(pkgs)

     

       63
       63
       +
           ensure_resolved(ctx, ctx.env.manifest, pkgs, registry=true)

     

       64
       64
       +
       

     

       65
       65
       +
           assert_can_add(ctx, pkgs)

     

       66
       66
       +
       

     

       67
       67
       +
           for (i, pkg) in pairs(pkgs)

     

       68
       68
       +
               entry = Pkg.Types.manifest_info(ctx.env.manifest, pkg.uuid)

     

       69
       69
       +
               is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps])

     

       70
       70
       +
               pkgs[i] = update_package_add(ctx, pkg, entry, is_dep)

     

       71
       71
       +
           end

     

       72
       72
       +
       

     

       73
       73
       +
           foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs)

     

       74
       74
       +
       

     

       75
       75
       +
           # Save the original pkgs for later. We might need to augment it with the weak dependencies

     

       76
       76
       +
           orig_pkgs = pkgs

     

       77
       77
       +
       

     

       78
       78
       +
           pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, PRESERVE_NONE, ctx.julia_version)

     

       79
       79
       +
       

     

       80
       80
       +
           if VERSION >= VersionNumber("1.9")

     

       81
       81
       +
               # Check for weak dependencies, which appear on the RHS of the deps_map but not in pkgs.

     

       82
       82
       +
               # Build up weak_name_to_uuid

     

       83
       83
       +
               uuid_to_name = Dict()

     

       84
       84
       +
               for pkg in pkgs

     

       85
       85
       +
                   uuid_to_name[pkg.uuid] = pkg.name

     

       86
       86
       +
               end

     

       87
       87
       +
               weak_name_to_uuid = Dict()

     

       88
       88
       +
               for (uuid, deps) in pairs(deps_map)

     

       89
       89
       +
                   for (dep_name, dep_uuid) in pairs(deps)

     

       90
       90
       +
                       if !haskey(uuid_to_name, dep_uuid)

     

       91
       91
       +
                           weak_name_to_uuid[dep_name] = dep_uuid

     

       92
       92
       +
                       end

     

       93
       93
       +
                   end

     

       94
       94
       +
               end

     

       95
       95
       +
       

     

       96
       96
       +
               # If we have nontrivial weak dependencies, add each one to the initial pkgs and then re-run _resolve

     

       97
       97
       +
               if !isempty(weak_name_to_uuid)

     

       98
       98
       +
                   println("Found weak dependencies: $(keys(weak_name_to_uuid))")

     

       99
       99
       +
       

     

       100
       100
       +
                   orig_uuids = Set([pkg.uuid for pkg in orig_pkgs])

     

       101
       101
       +
       

     

       102
       102
       +
                   for (name, uuid) in pairs(weak_name_to_uuid)

     

       103
       103
       +
                       if uuid in orig_uuids

     

       104
       104
       +
                           continue

     

       105
       105
       +
                       end

     

       106
       106
       +
       

     

       107
       107
       +
                       pkg = PackageSpec(name, uuid)

     

       108
       108
       +
       

     

       109
       109
       +
                       push!(orig_uuids, uuid)

     

       110
       110
       +
                       push!(orig_pkgs, pkg)

     

       111
       111
       +
                       ctx.env.project.deps[name] = uuid

     

       112
       112
       +
                       entry = Pkg.Types.manifest_info(ctx.env.manifest, uuid)

     

       113
       113
       +
                       orig_pkgs[length(orig_pkgs)] = update_package_add(ctx, pkg, entry, false)

     

       114
       114
       +
                   end

     

       115
       115
       +
       

     

       116
       116
       +
                   pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, orig_pkgs, PRESERVE_NONE, ctx.julia_version)

     

       117
       117
       +
               end

     

       118
       118
       +
           end

     

       119
       119
       +
         '';

     

       120
       120
       +
       

     

       121
       121
       +
         juliaExpression = packageNames: ''

     

       122
       122
       +
           import Pkg

     

       123
       123
       +
           Pkg.Registry.add(Pkg.RegistrySpec(path="${augmentedRegistry}"))

     

       124
       124
       +
       

     

       125
       125
       +
           import Pkg.Types: Context, PackageSpec

     

       126
       126
       +
       

     

       127
       127
       +
           input = ${lib.generators.toJSON {} packageNames}

     

       128
       128
       +
       

     

       129
       129
       +
           if isfile("extra_package_names.txt")

     

       130
       130
       +
             append!(input, readlines("extra_package_names.txt"))

     

       131
       131
       +
           end

     

       132
       132
       +
       

     

       133
       133
       +
           input = unique(input)

     

       134
       134
       +
       

     

       135
       135
       +
           println("Resolving packages: " * join(input, " "))

     

       136
       136
       +
       

     

       137
       137
       +
           pkgs = [PackageSpec(pkg) for pkg in input]

     

       138
       138
       +
       

     

       139
       139
       +
           ctx = Context()

     

       140
       140
       +
       

     

       141
       141
       +
           ${resolveCode}

     

       142
       142
       +
       

     

       143
       143
       +
           open(ENV["out"], "w") do io

     

       144
       144
       +
             for spec in pkgs

     

       145
       145
       +
               println(io, "- name: " * spec.name)

     

       146
       146
       +
               println(io, "  uuid: " * string(spec.uuid))

     

       147
       147
       +
               println(io, "  version: " * string(spec.version))

     

       148
       148
       +
               if endswith(spec.name, "_jll") && haskey(deps_map, spec.uuid)

     

       149
       149
       +
                 println(io, "  depends_on: ")

     

       150
       150
       +
                 for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid])

     

       151
       151
       +
                   println(io, "    \"$(dep_name)\": \"$(dep_uuid)\"")

     

       152
       152
       +
                 end

     

       153
       153
       +
               end

     

       154
       154
       +
             end

     

       155
       155
       +
           end

     

       156
       156
       +
         '';

     

       157
       157
       +
       in

     

       158
       158
       +
       

     

       159
       159
       +
       runCommand "julia-package-closure.yml" { buildInputs = [julia (python3.withPackages (ps: with ps; [pyyaml]))]; } ''

     

       160
       160
       +
         mkdir home

     

       161
       161
       +
         export HOME=$(pwd)/home

     

       162
       162
       +
       

     

       163
       163
       +
         echo "Resolving Julia packages with the following inputs"

     

       164
       164
       +
         echo "Julia: ${julia}"

     

       165
       165
       +
         echo "Registry: ${augmentedRegistry}"

     

       166
       166
       +
       

     

       167
       167
       +
         # Prevent a warning where Julia tries to download package server info

     

       168
       168
       +
         export JULIA_PKG_SERVER=""

     

       169
       169
       +
       

     

       170
       170
       +
         julia -e '${juliaExpression packageNames}';

     

       171
       171
       +
       

     

       172
       172
       +
         # See if we need to add any extra package names based on the closure

     

       173
       173
       +
         # and the packageImplications

     

       174
       174
       +
         python ${./python}/find_package_implications.py "$out" '${lib.generators.toJSON {} packageImplications}' extra_package_names.txt

     

       175
       175
       +
       

     

       176
       176
       +
         if [ -f extra_package_names.txt ]; then

     

       177
       177
       +
           echo "Re-resolving with additional package names"

     

       178
       178
       +
           julia -e '${juliaExpression packageNames}';

     

       179
       179
       +
         fi

     

       180
       180
       +
       ''

+59

pkgs/development/julia-modules/python/dag.py

···

       1
       1
       +
       

     

       2
       2
       +
       # This file based on a ChatGPT reponse for the following prompt:

     

       3
       3
       +
       # "can you write code in python to build up a DAG representing

     

       4
       4
       +
       # a dependency tree, and then a function that can return all the

     

       5
       5
       +
       # dependencies of a given node?"

     

       6
       6
       +
       

     

       7
       7
       +
       class Node:

     

       8
       8
       +
           def __init__(self, name):

     

       9
       9
       +
               self.name = name

     

       10
       10
       +
               self.dependencies = set()

     

       11
       11
       +
       

     

       12
       12
       +
       

     

       13
       13
       +
       class DAG:

     

       14
       14
       +
           def __init__(self):

     

       15
       15
       +
               self.nodes = {}

     

       16
       16
       +
       

     

       17
       17
       +
           def add_node(self, node_name, dependencies=None):

     

       18
       18
       +
               if node_name in self.nodes:

     

       19
       19
       +
                   raise ValueError(f"Node '{node_name}' already exists in the graph.")

     

       20
       20
       +
       

     

       21
       21
       +
               node = Node(node_name)

     

       22
       22
       +
               if dependencies:

     

       23
       23
       +
                   node.dependencies.update(dependencies)

     

       24
       24
       +
       

     

       25
       25
       +
               self.nodes[node_name] = node

     

       26
       26
       +
       

     

       27
       27
       +
           def add_dependency(self, node_name, dependency_name):

     

       28
       28
       +
               if node_name not in self.nodes:

     

       29
       29
       +
                   raise ValueError(f"Node '{node_name}' does not exist in the graph.")

     

       30
       30
       +
       

     

       31
       31
       +
               if dependency_name not in self.nodes:

     

       32
       32
       +
                   raise ValueError(f"Dependency '{dependency_name}' does not exist in the graph.")

     

       33
       33
       +
       

     

       34
       34
       +
               self.nodes[node_name].dependencies.add(dependency_name)

     

       35
       35
       +
       

     

       36
       36
       +
           def get_dependencies(self, node_name):

     

       37
       37
       +
               if node_name not in self.nodes:

     

       38
       38
       +
                   raise ValueError(f"Node '{node_name}' does not exist in the graph.")

     

       39
       39
       +
       

     

       40
       40
       +
               node = self.nodes[node_name]

     

       41
       41
       +
               dependencies = set()

     

       42
       42
       +
       

     

       43
       43
       +
               def traverse_dependencies(current_node):

     

       44
       44
       +
                   for dependency in current_node.dependencies:

     

       45
       45
       +
                       dependencies.add(dependency)

     

       46
       46
       +
                       if dependency in self.nodes:

     

       47
       47
       +
                         traverse_dependencies(self.nodes[dependency])

     

       48
       48
       +
       

     

       49
       49
       +
               traverse_dependencies(node)

     

       50
       50
       +
               return dependencies

     

       51
       51
       +
       

     

       52
       52
       +
           def has_node(self, node_name):

     

       53
       53
       +
               return node_name in self.nodes

     

       54
       54
       +
       

     

       55
       55
       +
           def __str__(self):

     

       56
       56
       +
               graph_str = ""

     

       57
       57
       +
               for node_name, node in self.nodes.items():

     

       58
       58
       +
                   graph_str += f"{node_name} -> {', '.join(node.dependencies)}\n"

     

       59
       59
       +
               return graph_str

+14

pkgs/development/julia-modules/python/dedup_overrides.py

···

       1
       1
       +
       

     

       2
       2
       +
       import json

     

       3
       3
       +
       from pathlib import Path

     

       4
       4
       +
       import sys

     

       5
       5
       +
       import toml

     

       6
       6
       +
       

     

       7
       7
       +
       overrides_path = Path(sys.argv[1])

     

       8
       8
       +
       out_path = Path(sys.argv[2])

     

       9
       9
       +
       

     

       10
       10
       +
       with open(overrides_path, "r") as f:

     

       11
       11
       +
         overrides = json.loads(f.read())

     

       12
       12
       +
       

     

       13
       13
       +
       with open(out_path, "w") as f:

     

       14
       14
       +
         toml.dump(overrides, f)

+99

pkgs/development/julia-modules/python/extract_artifacts.py

···

       1
       1
       +
       

     

       2
       2
       +
       import json

     

       3
       3
       +
       from pathlib import Path

     

       4
       4
       +
       import multiprocessing

     

       5
       5
       +
       import subprocess

     

       6
       6
       +
       import sys

     

       7
       7
       +
       import toml

     

       8
       8
       +
       import yaml

     

       9
       9
       +
       

     

       10
       10
       +
       import dag

     

       11
       11
       +
       

     

       12
       12
       +
       dependencies_path = Path(sys.argv[1])

     

       13
       13
       +
       closure_yaml_path = Path(sys.argv[2])

     

       14
       14
       +
       julia_path = Path(sys.argv[3])

     

       15
       15
       +
       extract_artifacts_script = Path(sys.argv[4])

     

       16
       16
       +
       extra_libs = json.loads(sys.argv[5])

     

       17
       17
       +
       out_path = Path(sys.argv[6])

     

       18
       18
       +
       

     

       19
       19
       +
       with open(dependencies_path, "r") as f:

     

       20
       20
       +
         dependencies = yaml.safe_load(f)

     

       21
       21
       +
         dependency_uuids = dependencies.keys()

     

       22
       22
       +
       

     

       23
       23
       +
       with open(closure_yaml_path, "r") as f:

     

       24
       24
       +
         # Build up a map of UUID -> closure information

     

       25
       25
       +
         closure_yaml_list = yaml.safe_load(f) or []

     

       26
       26
       +
         closure_yaml = {}

     

       27
       27
       +
         for item in closure_yaml_list:

     

       28
       28
       +
           closure_yaml[item["uuid"]] = item

     

       29
       29
       +
       

     

       30
       30
       +
         # Build up a dependency graph of UUIDs

     

       31
       31
       +
         closure_dependencies_dag = dag.DAG()

     

       32
       32
       +
         for uuid, contents in closure_yaml.items():

     

       33
       33
       +
           if contents.get("depends_on"):

     

       34
       34
       +
             closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())

     

       35
       35
       +
       

     

       36
       36
       +
       with open(out_path, "w") as f:

     

       37
       37
       +
         f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")

     

       38
       38
       +
         f.write("rec {\n")

     

       39
       39
       +
       

     

       40
       40
       +
         def process_item(item):

     

       41
       41
       +
           uuid, src = item

     

       42
       42
       +
           lines = []

     

       43
       43
       +
           artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())

     

       44
       44
       +
           if not artifacts: return f'  uuid-{uuid} = {{}};\n'

     

       45
       45
       +
       

     

       46
       46
       +
           lines.append(f'  uuid-{uuid} = {{')

     

       47
       47
       +
       

     

       48
       48
       +
           for artifact_name, details in artifacts.items():

     

       49
       49
       +
             if len(details["download"]) == 0: continue

     

       50
       50
       +
             download = details["download"][0]

     

       51
       51
       +
             url = download["url"]

     

       52
       52
       +
             sha256 = download["sha256"]

     

       53
       53
       +
       

     

       54
       54
       +
             git_tree_sha1 = details["git-tree-sha1"]

     

       55
       55
       +
       

     

       56
       56
       +
             depends_on = set()

     

       57
       57
       +
             if closure_dependencies_dag.has_node(uuid):

     

       58
       58
       +
               depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids)

     

       59
       59
       +
       

     

       60
       60
       +
             other_libs = extra_libs.get(uuid, [])

     

       61
       61
       +
       

     

       62
       62
       +
             fixup = f"""fixupPhase = let

     

       63
       63
       +
                 libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))

     

       64
       64
       +
                                      [{" ".join(["uuid-" + x for x in depends_on])}];

     

       65
       65
       +
                 in ''

     

       66
       66
       +
                   find $out -type f -executable -exec \

     

       67
       67
       +
                     patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \;

     

       68
       68
       +
                   find $out -type f -executable -exec \

     

       69
       69
       +
                     patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \;

     

       70
       70
       +
                 ''"""

     

       71
       71
       +
       

     

       72
       72
       +
             derivation = f"""{{

     

       73
       73
       +
               name = "{artifact_name}";

     

       74
       74
       +
               src = fetchurl {{

     

       75
       75
       +
                 url = "{url}";

     

       76
       76
       +
                 sha256 = "{sha256}";

     

       77
       77
       +
               }};

     

       78
       78
       +
               sourceRoot = ".";

     

       79
       79
       +
               dontConfigure = true;

     

       80
       80
       +
               dontBuild = true;

     

       81
       81
       +
               installPhase = "cp -r . $out";

     

       82
       82
       +
               {fixup};

     

       83
       83
       +
             }}"""

     

       84
       84
       +
       

     

       85
       85
       +
             lines.append(f"""    "{artifact_name}" = {{

     

       86
       86
       +
             sha1 = "{git_tree_sha1}";

     

       87
       87
       +
             path = stdenv.mkDerivation {derivation};

     

       88
       88
       +
           }};\n""")

     

       89
       89
       +
       

     

       90
       90
       +
           lines.append('  };\n')

     

       91
       91
       +
       

     

       92
       92
       +
           return "\n".join(lines)

     

       93
       93
       +
       

     

       94
       94
       +
         with multiprocessing.Pool(10) as pool:

     

       95
       95
       +
           for s in pool.map(process_item, dependencies.items()):

     

       96
       96
       +
             f.write(s)

     

       97
       97
       +
       

     

       98
       98
       +
         f.write(f"""

     

       99
       99
       +
       }}\n""")

+24

pkgs/development/julia-modules/python/find_package_implications.py

···

       1
       1
       +
       

     

       2
       2
       +
       import json

     

       3
       3
       +
       import os

     

       4
       4
       +
       from pathlib import Path

     

       5
       5
       +
       import subprocess

     

       6
       6
       +
       import sys

     

       7
       7
       +
       import yaml

     

       8
       8
       +
       

     

       9
       9
       +
       dependencies_path = Path(sys.argv[1])

     

       10
       10
       +
       package_implications_json = sys.argv[2]

     

       11
       11
       +
       out_path = Path(sys.argv[3])

     

       12
       12
       +
       

     

       13
       13
       +
       package_implications = json.loads(package_implications_json)

     

       14
       14
       +
       with open(dependencies_path) as f:

     

       15
       15
       +
         desired_packages = yaml.safe_load(f) or []

     

       16
       16
       +
       

     

       17
       17
       +
       extra_package_names = []

     

       18
       18
       +
       for pkg in desired_packages:

     

       19
       19
       +
         if pkg["name"] in package_implications:

     

       20
       20
       +
           extra_package_names.extend(package_implications[pkg["name"]])

     

       21
       21
       +
       

     

       22
       22
       +
       if len(extra_package_names) > 0:

     

       23
       23
       +
         with open(out_path, "w") as f:

     

       24
       24
       +
           f.write("\n".join(extra_package_names))

+22

pkgs/development/julia-modules/python/format_overrides.py

···

       1
       1
       +
       

     

       2
       2
       +
       import json

     

       3
       3
       +
       from pathlib import Path

     

       4
       4
       +
       import sys

     

       5
       5
       +
       import toml

     

       6
       6
       +
       

     

       7
       7
       +
       overrides_path = Path(sys.argv[1])

     

       8
       8
       +
       out_path = Path(sys.argv[2])

     

       9
       9
       +
       

     

       10
       10
       +
       with open(overrides_path, "r") as f:

     

       11
       11
       +
         overrides = json.loads(f.read())

     

       12
       12
       +
       

     

       13
       13
       +
       result = {}

     

       14
       14
       +
       

     

       15
       15
       +
       for (uuid, artifacts) in overrides.items():

     

       16
       16
       +
         if len(artifacts) == 0: continue

     

       17
       17
       +
       

     

       18
       18
       +
         for (name, info) in artifacts.items():

     

       19
       19
       +
           result[info["sha1"]] = info["path"]

     

       20
       20
       +
       

     

       21
       21
       +
       with open(out_path, "w") as f:

     

       22
       22
       +
         toml.dump(result, f)

+98

pkgs/development/julia-modules/python/minimal_registry.py

···

       1
       1
       +
       

     

       2
       2
       +
       from collections import defaultdict

     

       3
       3
       +
       import copy

     

       4
       4
       +
       import json

     

       5
       5
       +
       import os

     

       6
       6
       +
       from pathlib import Path

     

       7
       7
       +
       import shutil

     

       8
       8
       +
       import subprocess

     

       9
       9
       +
       import sys

     

       10
       10
       +
       import tempfile

     

       11
       11
       +
       import toml

     

       12
       12
       +
       import util

     

       13
       13
       +
       import yaml

     

       14
       14
       +
       

     

       15
       15
       +
       

     

       16
       16
       +
       registry_path = Path(sys.argv[1])

     

       17
       17
       +
       desired_packages_path = Path(sys.argv[2])

     

       18
       18
       +
       package_overrides = json.loads(sys.argv[3])

     

       19
       19
       +
       dependencies_path = Path(sys.argv[4])

     

       20
       20
       +
       out_path = Path(sys.argv[5])

     

       21
       21
       +
       

     

       22
       22
       +
       with open(desired_packages_path, "r") as f:

     

       23
       23
       +
         desired_packages = yaml.safe_load(f) or []

     

       24
       24
       +
       

     

       25
       25
       +
       uuid_to_versions = defaultdict(list)

     

       26
       26
       +
       for pkg in desired_packages:

     

       27
       27
       +
           uuid_to_versions[pkg["uuid"]].append(pkg["version"])

     

       28
       28
       +
       

     

       29
       29
       +
       with open(dependencies_path, "r") as f:

     

       30
       30
       +
         uuid_to_store_path = yaml.safe_load(f)

     

       31
       31
       +
       

     

       32
       32
       +
       os.makedirs(out_path)

     

       33
       33
       +
       

     

       34
       34
       +
       registry = toml.load(registry_path / "Registry.toml")

     

       35
       35
       +
       registry["packages"] = {k: v for k, v in registry["packages"].items() if k in uuid_to_versions}

     

       36
       36
       +
       

     

       37
       37
       +
       for (uuid, versions) in uuid_to_versions.items():

     

       38
       38
       +
         if uuid in package_overrides:

     

       39
       39
       +
           info = package_overrides[uuid]

     

       40
       40
       +
       

     

       41
       41
       +
           # Make a registry entry based on the info from the package override

     

       42
       42
       +
           path = Path(info["name"][0].upper()) / Path(info["name"])

     

       43
       43
       +
           registry["packages"][uuid] = {

     

       44
       44
       +
             "name": info["name"],

     

       45
       45
       +
             "path": str(path),

     

       46
       46
       +
           }

     

       47
       47
       +
       

     

       48
       48
       +
           os.makedirs(out_path / path)

     

       49
       49
       +
       

     

       50
       50
       +
           # Read the Project.yaml from the src

     

       51
       51
       +
           project = toml.load(Path(info["src"]) / "Project.toml")

     

       52
       52
       +
       

     

       53
       53
       +
           # Generate all the registry files

     

       54
       54
       +
           with open(out_path / path / Path("Compat.toml"), "w") as f:

     

       55
       55
       +
             f.write('["%s"]\n' % info["version"])

     

       56
       56
       +
             # Write nothing in Compat.toml, because we've already resolved everything

     

       57
       57
       +
           with open(out_path / path / Path("Deps.toml"), "w") as f:

     

       58
       58
       +
             f.write('["%s"]\n' % info["version"])

     

       59
       59
       +
             toml.dump(project["deps"], f)

     

       60
       60
       +
           with open(out_path / path / Path("Versions.toml"), "w") as f:

     

       61
       61
       +
             f.write('["%s"]\n' % info["version"])

     

       62
       62
       +
             f.write('git-tree-sha1 = "%s"\n' % info["treehash"])

     

       63
       63
       +
           with open(out_path / path / Path("Package.toml"), "w") as f:

     

       64
       64
       +
             toml.dump({

     

       65
       65
       +
               "name": info["name"],

     

       66
       66
       +
               "uuid": uuid,

     

       67
       67
       +
               "repo": "file://" + info["src"],

     

       68
       68
       +
             }, f)

     

       69
       69
       +
       

     

       70
       70
       +
         elif uuid in registry["packages"]:

     

       71
       71
       +
           registry_info = registry["packages"][uuid]

     

       72
       72
       +
           name = registry_info["name"]

     

       73
       73
       +
           path = registry_info["path"]

     

       74
       74
       +
       

     

       75
       75
       +
           os.makedirs(out_path / path)

     

       76
       76
       +
       

     

       77
       77
       +
           # Copy some files to the minimal repo unchanged

     

       78
       78
       +
           for f in ["Compat.toml", "Deps.toml"]:

     

       79
       79
       +
             if (registry_path / path / f).exists():

     

       80
       80
       +
               shutil.copy2(registry_path / path / f, out_path / path)

     

       81
       81
       +
       

     

       82
       82
       +
           # Copy the Versions.toml file, trimming down to the versions we care about

     

       83
       83
       +
           all_versions = toml.load(registry_path / path / "Versions.toml")

     

       84
       84
       +
           versions_to_keep = {k: v for k, v in all_versions.items() if k in versions}

     

       85
       85
       +
           for k, v in versions_to_keep.items():

     

       86
       86
       +
             del v["nix-sha256"]

     

       87
       87
       +
           with open(out_path / path / "Versions.toml", "w") as f:

     

       88
       88
       +
             toml.dump(versions_to_keep, f)

     

       89
       89
       +
       

     

       90
       90
       +
           # Fill in the local store path for the repo

     

       91
       91
       +
           if not uuid in uuid_to_store_path: continue

     

       92
       92
       +
           package_toml = toml.load(registry_path / path / "Package.toml")

     

       93
       93
       +
           package_toml["repo"] = "file://" + uuid_to_store_path[uuid]

     

       94
       94
       +
           with open(out_path / path / "Package.toml", "w") as f:

     

       95
       95
       +
             toml.dump(package_toml, f)

     

       96
       96
       +
       

     

       97
       97
       +
       with open(out_path / "Registry.toml", "w") as f:

     

       98
       98
       +
           toml.dump(registry, f)

+67

pkgs/development/julia-modules/python/sources_nix.py

···

       1
       1
       +
       

     

       2
       2
       +
       import json

     

       3
       3
       +
       from pathlib import Path

     

       4
       4
       +
       import re

     

       5
       5
       +
       import shutil

     

       6
       6
       +
       import sys

     

       7
       7
       +
       import toml

     

       8
       8
       +
       import util

     

       9
       9
       +
       import yaml

     

       10
       10
       +
       

     

       11
       11
       +
       

     

       12
       12
       +
       registry_path = Path(sys.argv[1])

     

       13
       13
       +
       package_overrides = json.loads(sys.argv[2])

     

       14
       14
       +
       desired_packages_path = Path(sys.argv[3])

     

       15
       15
       +
       out_path = Path(sys.argv[4])

     

       16
       16
       +
       

     

       17
       17
       +
       with open(desired_packages_path, "r") as f:

     

       18
       18
       +
         desired_packages = yaml.safe_load(f) or []

     

       19
       19
       +
       

     

       20
       20
       +
       registry = toml.load(registry_path / "Registry.toml")

     

       21
       21
       +
       

     

       22
       22
       +
       def ensure_version_valid(version):

     

       23
       23
       +
         """

     

       24
       24
       +
         Ensure a version string is a valid Julia-parsable version.

     

       25
       25
       +
         It doesn't really matter what it looks like as it's just used for overrides.

     

       26
       26
       +
         """

     

       27
       27
       +
         return re.sub('[^0-9\.]','', version)

     

       28
       28
       +
       

     

       29
       29
       +
       with open(out_path, "w") as f:

     

       30
       30
       +
         f.write("{fetchgit}:\n")

     

       31
       31
       +
         f.write("{\n")

     

       32
       32
       +
         for pkg in desired_packages:

     

       33
       33
       +
           uuid = pkg["uuid"]

     

       34
       34
       +
       

     

       35
       35
       +
           if pkg["name"] in package_overrides:

     

       36
       36
       +
             treehash = util.get_commit_info(package_overrides[pkg["name"]])["tree"]

     

       37
       37
       +
             f.write(f"""  "{uuid}" = {{

     

       38
       38
       +
           src = null; # Overridden: will fill in later

     

       39
       39
       +
           name = "{pkg["name"]}";

     

       40
       40
       +
           version = "{ensure_version_valid(pkg["version"])}";

     

       41
       41
       +
           treehash = "{treehash}";

     

       42
       42
       +
         }};\n""")

     

       43
       43
       +
           elif uuid in registry["packages"]:

     

       44
       44
       +
             registry_info = registry["packages"][uuid]

     

       45
       45
       +
             path = registry_info["path"]

     

       46
       46
       +
             packageToml = toml.load(registry_path / path / "Package.toml")

     

       47
       47
       +
       

     

       48
       48
       +
             all_versions = toml.load(registry_path / path / "Versions.toml")

     

       49
       49
       +
             if not pkg["version"] in all_versions: continue

     

       50
       50
       +
             version_to_use = all_versions[pkg["version"]]

     

       51
       51
       +
       

     

       52
       52
       +
             repo = packageToml["repo"]

     

       53
       53
       +
             f.write(f"""  "{uuid}" = {{

     

       54
       54
       +
           src = fetchgit {{

     

       55
       55
       +
             url = "{repo}";

     

       56
       56
       +
             rev = "{version_to_use["git-tree-sha1"]}";

     

       57
       57
       +
             sha256 = "{version_to_use["nix-sha256"]}";

     

       58
       58
       +
           }};

     

       59
       59
       +
           name = "{pkg["name"]}";

     

       60
       60
       +
           version = "{pkg["version"]}";

     

       61
       61
       +
           treehash = "{version_to_use["git-tree-sha1"]}";

     

       62
       62
       +
         }};\n""")

     

       63
       63
       +
           else:

     

       64
       64
       +
             # print("Warning: couldn't figure out what to do with pkg in sources_nix.py", pkg)

     

       65
       65
       +
             pass

     

       66
       66
       +
       

     

       67
       67
       +
         f.write("}")

+12

pkgs/development/julia-modules/python/util.py

···

       1
       1
       +
       

     

       2
       2
       +
       import os

     

       3
       3
       +
       import subprocess

     

       4
       4
       +
       import tempfile

     

       5
       5
       +
       

     

       6
       6
       +
       def get_commit_info(repo):

     

       7
       7
       +
         with tempfile.TemporaryDirectory() as home_dir:

     

       8
       8
       +
           env_with_home = os.environ.copy()

     

       9
       9
       +
           env_with_home["HOME"] = home_dir

     

       10
       10
       +
           subprocess.check_output(["git", "config", "--global", "--add", "safe.directory", repo], env=env_with_home)

     

       11
       11
       +
           lines = subprocess.check_output(["git", "log", "--pretty=raw"], cwd=repo, env=env_with_home).decode().split("\n")

     

       12
       12
       +
           return dict([x.split() for x in lines if len(x.split()) == 2])

pkgs/development/julia-modules/registry.nix

···

       1
       1
       +
       { fetchFromGitHub }:

     

       2
       2
       +
       

     

       3
       3
       +
       fetchFromGitHub {

     

       4
       4
       +
         owner = "CodeDownIO";

     

       5
       5
       +
         repo = "General";

     

       6
       6
       +
         rev = "baf9e22ecdf97b6424a611ac4a565c6ee60d3f44";

     

       7
       7
       +
         sha256 = "1nd3x2z8r6578149pbpkx9qw2ajln1kfy7w5kjsnv56v180h0ddf";

     

       8
       8
       +
         # date = "2023-12-14T12:20:00+00:00";

     

       9
       9
       +
       }

+45

pkgs/development/julia-modules/util.nix

···

       1
       1
       +
       { git

     

       2
       2
       +
       , runCommand

     

       3
       3
       +
       }:

     

       4
       4
       +
       

     

       5
       5
       +
       {

     

       6
       6
       +
         # Add packages to a Python environment. Works if you pass something like either

     

       7
       7
       +
         # a) python3

     

       8
       8
       +
         # b) python3.withPackages (ps: [...])

     

       9
       9
       +
         # See https://github.com/NixOS/nixpkgs/pull/97467#issuecomment-689315186

     

       10
       10
       +
         addPackagesToPython = python: packages:

     

       11
       11
       +
           if python ? "env" then python.override (old: {

     

       12
       12
       +
             extraLibs = old.extraLibs ++ packages;

     

       13
       13
       +
           })

     

       14
       14
       +
           else python.withPackages (ps: packages);

     

       15
       15
       +
       

     

       16
       16
       +
         # Convert an ordinary source checkout into a repo with a single commit

     

       17
       17
       +
         repoifySimple = name: path:

     

       18
       18
       +
           runCommand ''${name}-repoified'' {buildInputs = [git];} ''

     

       19
       19
       +
             mkdir -p $out

     

       20
       20
       +
             cp -r ${path}/. $out

     

       21
       21
       +
             cd $out

     

       22
       22
       +
             chmod -R u+w .

     

       23
       23
       +
             rm -rf .git

     

       24
       24
       +
             git init

     

       25
       25
       +
             git add . -f

     

       26
       26
       +
             git config user.email "julia2nix@localhost"

     

       27
       27
       +
             git config user.name "julia2nix"

     

       28
       28
       +
             git commit -m "Dummy commit"

     

       29
       29
       +
           '';

     

       30
       30
       +
       

     

       31
       31
       +
         # Convert an dependency source info into a repo with a single commit

     

       32
       32
       +
         repoifyInfo = uuid: info:

     

       33
       33
       +
           runCommand ''julia-${info.name}-${info.version}'' {buildInputs = [git];} ''

     

       34
       34
       +
             mkdir -p $out

     

       35
       35
       +
             cp -r ${info.src}/. $out

     

       36
       36
       +
             cd $out

     

       37
       37
       +
             chmod -R u+w .

     

       38
       38
       +
             rm -rf .git

     

       39
       39
       +
             git init

     

       40
       40
       +
             git add . -f

     

       41
       41
       +
             git config user.email "julia2nix@localhost"

     

       42
       42
       +
             git config user.name "julia2nix"

     

       43
       43
       +
             git commit -m "Dummy commit"

     

       44
       44
       +
           '';

     

       45
       45
       +
       }

+7 -7

pkgs/top-level/all-packages.nix

···

       16517
       16517
        
       

     

       16518
       16518
        
         juniper = callPackage ../development/compilers/juniper { };

     

       16519
       16519
        
       

     

       16520
       16520
       +
         inherit (callPackage ../development/compilers/julia { })

     

       16521
       16521
       +
           julia_16-bin

     

       16522
       16522
       +
           julia_18-bin

     

       16523
       16523
       +
           julia_19-bin

     

       16524
       16524
       +
           julia_18

     

       16525
       16525
       +
           julia_19;

     

       16526
       16526
       +
       

     

       16520
       16527
        
         julia-lts = julia_16-bin;

     

       16521
       16528
        
         julia-stable = julia_19;

     

       16522
       16529
        
         julia = julia-stable;

     

       16523
       16523
       -
       

     

       16524
       16524
       -
         julia_16-bin = callPackage ../development/compilers/julia/1.6-bin.nix { };

     

       16525
       16525
       -
         julia_18-bin = callPackage ../development/compilers/julia/1.8-bin.nix { };

     

       16526
       16526
       -
         julia_19-bin = callPackage ../development/compilers/julia/1.9-bin.nix { };

     

       16527
       16527
       -
       

     

       16528
       16528
       -
         julia_18 = callPackage ../development/compilers/julia/1.8.nix { };

     

       16529
       16529
       -
         julia_19 = callPackage ../development/compilers/julia/1.9.nix { };

     

       16530
       16530
        
       

     

       16531
       16531
        
         julia-lts-bin = julia_16-bin;

     

       16532
       16532
        
         julia-stable-bin = julia_19-bin;