pyrox.dev
1{
2 lib,
3 stdenv,
4 buildPythonPackage,
5 fetchPypi,
6 pythonOlder,
7 rustPlatform,
8 cargo,
9 rustc,
10 setuptools,
11 setuptools-rust,
12 libiconv,
13 requests,
14 regex,
15 blobfile,
16}:
17let
18 pname = "tiktoken";
19 version = "0.9.0";
20 src = fetchPypi {
21 inherit pname version;
22 hash = "sha256-0Cpcpqk44EkOH/lXvEjIsHjIjLg5d74WJbH9iqx5LF0=";
23 };
24 postPatch = ''
25 cp ${./Cargo.lock} Cargo.lock
26 '';
27in
28buildPythonPackage {
29 inherit
30 pname
31 version
32 src
33 postPatch
34 ;
35 pyproject = true;
36
37 disabled = pythonOlder "3.8";
38
39 build-system = [
40 setuptools
41 setuptools-rust
42 ];
43
44 cargoDeps = rustPlatform.fetchCargoVendor {
45 inherit
46 pname
47 version
48 src
49 postPatch
50 ;
51 hash = "sha256-MfTTRbSM+KgrYrWHYlJkGDc1qn3oulalDJM+huTaJ0g=";
52 };
53
54 nativeBuildInputs = [
55 rustPlatform.cargoSetupHook
56 setuptools-rust
57 cargo
58 rustc
59 ];
60
61 buildInputs = lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ];
62
63 dependencies = [
64 requests
65 regex
66 blobfile
67 ];
68
69 # almost all tests require network access
70 doCheck = false;
71
72 pythonImportsCheck = [ "tiktoken" ];
73
74 meta = with lib; {
75 description = "Fast BPE tokeniser for use with OpenAI's models";
76 homepage = "https://github.com/openai/tiktoken";
77 license = licenses.mit;
78 maintainers = with maintainers; [ happysalada ];
79 };
80}