python312Packages.torch: build torch.distributed on darwin

It's disabled on MacOS by default:
https://pytorch.org/docs/stable/distributed.html#torch.distributed.is_available

When distributed module build is enabled, a compilation error happens on
fresh clang.

Upstream repo (which is a submodule in pytorch) is archived so it's
impossible to contribute the fix there. Instead, an inline patch is
included that is enough to fix the error.

Signed-off-by: Ihar Hrachyshka <ihar.hrachyshka@gmail.com>

Changed files
+59 -94
pkgs
development
python-modules
colbert-ai
compressed-tensors
docling
docling-ibm-models
finetuning-scheduler
gradio
ignite
peft
sentence-transformers
smolagents
tensordict
torch
torchsnapshot
-5
pkgs/development/python-modules/colbert-ai/default.nix
···
maintainers = with lib.maintainers; [
bachp
];
-
badPlatforms = [
-
# `import torch.distributed.tensor` fails on darwin:
-
# ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a packag
-
lib.systems.inspect.patterns.isDarwin
-
];
};
}
+9 -15
pkgs/development/python-modules/compressed-tensors/default.nix
···
pytestCheckHook
];
-
disabledTests =
-
[
-
# these try to download models from HF Hub
-
"test_get_observer_token_count"
-
"test_kv_cache_quantization"
-
"test_target_prioritization"
-
"test_load_compressed_sharded"
-
"test_save_compressed_model"
-
"test_apply_tinyllama_dynamic_activations"
-
]
-
++ lib.optionals stdenv.hostPlatform.isDarwin [
-
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
"test_composability"
-
"test_missing_and_unexpected_keys_on_compression"
-
];
+
disabledTests = [
+
# these try to download models from HF Hub
+
"test_get_observer_token_count"
+
"test_kv_cache_quantization"
+
"test_target_prioritization"
+
"test_load_compressed_sharded"
+
"test_save_compressed_model"
+
"test_apply_tinyllama_dynamic_activations"
+
];
disabledTestPaths = [
# these try to download models from HF Hub
-6
pkgs/development/python-modules/docling-ibm-models/default.nix
···
"test_tf_predictor"
];
-
disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [
-
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
"tests/test_code_formula_predictor.py"
-
"tests/test_layout_predictor.py"
-
];
-
meta = {
changelog = "https://github.com/DS4SD/docling-ibm-models/blob/${src.tag}/CHANGELOG.md";
description = "Docling IBM models";
-20
pkgs/development/python-modules/docling/default.nix
···
"test_e2e_valid_csv_conversions"
];
-
disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [
-
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
"tests/test_backend_csv.py"
-
"tests/test_backend_html.py"
-
"tests/test_backend_jats.py"
-
"tests/test_backend_msexcel.py"
-
"tests/test_backend_msword.py"
-
"tests/test_backend_pptx.py"
-
"tests/test_cli.py"
-
"tests/test_code_formula.py"
-
"tests/test_document_picture_classifier.py"
-
"tests/test_e2e_conversion.py"
-
"tests/test_e2e_ocr_conversion.py"
-
"tests/test_input_doc.py"
-
"tests/test_interfaces.py"
-
"tests/test_invalid_input.py"
-
"tests/test_legacy_format_transform.py"
-
"tests/test_options.py"
-
];
-
meta = {
description = "Get your documents ready for gen AI";
homepage = "https://github.com/DS4SD/docling";
-4
pkgs/development/python-modules/finetuning-scheduler/default.nix
···
changelog = "https://github.com/speediedan/finetuning-scheduler/blob/v${version}/CHANGELOG.md";
license = lib.licenses.asl20;
maintainers = with lib.maintainers; [ bcdarwin ];
-
badPlatforms = [
-
# "No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package" at import time:
-
lib.systems.inspect.patterns.isDarwin
-
];
};
}
+9 -14
pkgs/development/python-modules/gradio/default.nix
···
"test_updates_stored_up_to_capacity"
"test_varying_output_forms_with_generators"
];
-
disabledTestPaths =
-
[
-
# 100% touches network
-
"test/test_networking.py"
-
"client/python/test/test_client.py"
-
# makes pytest freeze 50% of the time
-
"test/test_interfaces.py"
+
disabledTestPaths = [
+
# 100% touches network
+
"test/test_networking.py"
+
"client/python/test/test_client.py"
+
# makes pytest freeze 50% of the time
+
"test/test_interfaces.py"
-
# Local network tests dependant on port availability (port 7860-7959)
-
"test/test_routes.py"
-
]
-
++ lib.optionals (stdenv.hostPlatform.isDarwin) [
-
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
"test/test_pipelines.py"
-
];
+
# Local network tests dependant on port availability (port 7860-7959)
+
"test/test_routes.py"
+
];
pytestFlagsArray = [
"-x" # abort on first failure
"-m 'not flaky'"
-2
pkgs/development/python-modules/ignite/default.nix
···
changelog = "https://github.com/pytorch/ignite/releases/tag/v${version}";
license = lib.licenses.bsd3;
maintainers = [ lib.maintainers.bcdarwin ];
-
# ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
broken = stdenv.hostPlatform.isDarwin;
};
}
-4
pkgs/development/python-modules/peft/default.nix
···
changelog = "https://github.com/huggingface/peft/releases/tag/v${version}";
license = lib.licenses.asl20;
maintainers = with lib.maintainers; [ bcdarwin ];
-
badPlatforms = [
-
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
lib.systems.inspect.patterns.isDarwin
-
];
};
}
-4
pkgs/development/python-modules/sentence-transformers/default.nix
···
changelog = "https://github.com/UKPLab/sentence-transformers/releases/tag/v${version}";
license = lib.licenses.asl20;
maintainers = with lib.maintainers; [ dit7ya ];
-
badPlatforms = [
-
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
lib.systems.inspect.patterns.isDarwin
-
];
};
}
-6
pkgs/development/python-modules/smolagents/default.nix
···
"test_new_instance"
];
-
disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [
-
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
"tests/test_final_answer.py"
-
"tests/test_types.py"
-
];
-
__darwinAllowLocalNetworking = true;
meta = {
+4 -9
pkgs/development/python-modules/tensordict/default.nix
···
"test_map_iter_interrupt_early"
];
-
disabledTestPaths =
-
[
-
# torch._dynamo.exc.Unsupported: Graph break due to unsupported builtin None.ReferenceType.__new__.
-
"test/test_compile.py"
-
]
-
++ lib.optionals stdenv.hostPlatform.isDarwin [
-
# ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
"test/test_distributed.py"
-
];
+
disabledTestPaths = [
+
# torch._dynamo.exc.Unsupported: Graph break due to unsupported builtin None.ReferenceType.__new__.
+
"test/test_compile.py"
+
];
meta = {
description = "Pytorch dedicated tensor container";
+31
pkgs/development/python-modules/torch/clang19-template-warning.patch
···
+
Submodule third_party/libnop contains modified content
+
diff --git a/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h b/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h
+
index cffbde1..b6ebfab 100644
+
--- a/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h
+
+++ b/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h
+
@@ -238,7 +238,7 @@ class Variant {
+
// resulting type.
+
template <typename... Args>
+
void Construct(Args&&... args) {
+
- index_ = value_.template Construct(std::forward<Args>(args)...);
+
+ index_ = value_.template Construct<>(std::forward<Args>(args)...);
+
}
+
void Construct(EmptyVariant) {}
+
+
@@ -255,14 +255,14 @@ class Variant {
+
// multiple element types.
+
template <typename T, typename U>
+
void Assign(TypeTag<T>, U&& value) {
+
- if (!value_.template Assign(TypeTag<T>{}, index_, std::forward<U>(value))) {
+
+ if (!value_.template Assign<>(TypeTag<T>{}, index_, std::forward<U>(value))) {
+
Destruct();
+
Construct(TypeTag<T>{}, std::forward<U>(value));
+
}
+
}
+
template <typename T>
+
void Assign(T&& value) {
+
- if (!value_.template Assign(index_, std::forward<T>(value))) {
+
+ if (!value_.template Assign<>(index_, std::forward<T>(value))) {
+
Destruct();
+
Construct(std::forward<T>(value));
+
}
+6 -1
pkgs/development/python-modules/torch/default.nix
···
};
patches =
-
lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ]
+
[ ./clang19-template-warning.patch ]
+
++ lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ]
++ lib.optionals stdenv.hostPlatform.isLinux [
# Propagate CUPTI to Kineto by overriding the search path with environment variables.
# https://github.com/pytorch/pytorch/pull/108847
···
# Explicitly enable MPS for Darwin
USE_MPS = setBool stdenv.hostPlatform.isDarwin;
+
+
# building torch.distributed on Darwin is disabled by default
+
# https://pytorch.org/docs/stable/distributed.html#torch.distributed.is_available
+
USE_DISTRIBUTED = setBool true;
cmakeFlags =
[
-4
pkgs/development/python-modules/torchsnapshot/default.nix
···
changelog = "https://github.com/pytorch/torchsnapshot/releases/tag/${version}";
license = lib.licenses.bsd3;
maintainers = with lib.maintainers; [ GaetanLepage ];
-
badPlatforms = [
-
# ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
-
lib.systems.inspect.patterns.isDarwin
-
];
};
}