pyrox.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5
6 # build-system
7 setuptools,
8
9 # dependencies
10 filelock,
11 huggingface-hub,
12 numpy,
13 protobuf,
14 packaging,
15 pyyaml,
16 regex,
17 requests,
18 tokenizers,
19 safetensors,
20 tqdm,
21
22 # optional-dependencies
23 diffusers,
24 scikit-learn,
25 tensorflow,
26 onnxconverter-common,
27 opencv4,
28 tf2onnx,
29 torch,
30 accelerate,
31 faiss,
32 datasets,
33 jax,
34 jaxlib,
35 flax,
36 optax,
37 ftfy,
38 onnxruntime,
39 onnxruntime-tools,
40 cookiecutter,
41 sagemaker,
42 fairscale,
43 optuna,
44 ray,
45 pydantic,
46 uvicorn,
47 fastapi,
48 starlette,
49 librosa,
50 phonemizer,
51 torchaudio,
52 pillow,
53 timm,
54 torchvision,
55 av,
56 sentencepiece,
57 hf-xet,
58}:
59
60buildPythonPackage rec {
61 pname = "transformers";
62 version = "4.56.2";
63 pyproject = true;
64
65 src = fetchFromGitHub {
66 owner = "huggingface";
67 repo = "transformers";
68 tag = "v${version}";
69 hash = "sha256-epq0a+QjZvXdAuwC/KjoCuAyJLewCaZq4whLSMkAkDI=";
70 };
71
72 build-system = [ setuptools ];
73
74 dependencies = [
75 filelock
76 huggingface-hub
77 numpy
78 packaging
79 pyyaml
80 regex
81 requests
82 tokenizers
83 safetensors
84 tqdm
85 ];
86
87 optional-dependencies =
88 let
89 audio = [
90 librosa
91 # pyctcdecode
92 phonemizer
93 # kenlm
94 ];
95 vision = [ pillow ];
96 in
97 {
98 agents = [
99 diffusers
100 accelerate
101 datasets
102 torch
103 sentencepiece
104 opencv4
105 pillow
106 ];
107 ja = [
108 # fugashi
109 # ipadic
110 # rhoknp
111 # sudachidict_core
112 # sudachipy
113 # unidic
114 # unidic_lite
115 ];
116 sklearn = [ scikit-learn ];
117 tf = [
118 tensorflow
119 onnxconverter-common
120 tf2onnx
121 # tensorflow-text
122 # keras-nlp
123 ];
124 torch = [
125 torch
126 accelerate
127 ];
128 retrieval = [
129 faiss
130 datasets
131 ];
132 flax = [
133 jax
134 jaxlib
135 flax
136 optax
137 ];
138 hf_xet = [
139 hf-xet
140 ];
141 tokenizers = [ tokenizers ];
142 ftfy = [ ftfy ];
143 onnxruntime = [
144 onnxruntime
145 onnxruntime-tools
146 ];
147 onnx = [
148 onnxconverter-common
149 tf2onnx
150 onnxruntime
151 onnxruntime-tools
152 ];
153 modelcreation = [ cookiecutter ];
154 sagemaker = [ sagemaker ];
155 deepspeed = [
156 # deepspeed
157 accelerate
158 ];
159 fairscale = [ fairscale ];
160 optuna = [ optuna ];
161 ray = [ ray ] ++ ray.optional-dependencies.tune;
162 # sigopt = [ sigopt ];
163 # integrations = ray ++ optuna ++ sigopt;
164 serving = [
165 pydantic
166 uvicorn
167 fastapi
168 starlette
169 ];
170 audio = audio;
171 speech = [ torchaudio ] ++ audio;
172 torch-speech = [ torchaudio ] ++ audio;
173 tf-speech = audio;
174 flax-speech = audio;
175 timm = [ timm ];
176 torch-vision = [ torchvision ] ++ vision;
177 # natten = [ natten ];
178 # codecarbon = [ codecarbon ];
179 video = [
180 av
181 ];
182 sentencepiece = [
183 sentencepiece
184 protobuf
185 ];
186 };
187
188 # Many tests require internet access.
189 doCheck = false;
190
191 pythonImportsCheck = [ "transformers" ];
192
193 meta = {
194 homepage = "https://github.com/huggingface/transformers";
195 description = "Natural Language Processing for TensorFlow 2.0 and PyTorch";
196 mainProgram = "transformers-cli";
197 changelog = "https://github.com/huggingface/transformers/releases/tag/v${version}";
198 license = lib.licenses.asl20;
199 platforms = lib.platforms.unix;
200 maintainers = with lib.maintainers; [
201 pashashocky
202 happysalada
203 ];
204 };
205}