pyrox.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5
6 # build-system
7 hatchling,
8
9 # dependencies
10 atpublic,
11 parsy,
12 python-dateutil,
13 sqlglot,
14 toolz,
15 typing-extensions,
16 tzdata,
17
18 # tests
19 pytestCheckHook,
20 black,
21 filelock,
22 hypothesis,
23 pytest-benchmark,
24 pytest-httpserver,
25 pytest-mock,
26 pytest-randomly,
27 pytest-snapshot,
28 pytest-timeout,
29 pytest-xdist,
30 writableTmpDirAsHomeHook,
31
32 # optional-dependencies
33 # - athena
34 pyathena,
35 fsspec,
36 # - bigquery
37 db-dtypes,
38 google-cloud-bigquery,
39 google-cloud-bigquery-storage,
40 pyarrow,
41 pyarrow-hotfix,
42 pydata-google-auth,
43 numpy,
44 pandas,
45 rich,
46 # - clickhouse
47 clickhouse-connect,
48 # - databricks
49 # databricks-sql-connector-core, (unpackaged)
50 # - datafusion
51 datafusion,
52 # - druid
53 pydruid,
54 # - duckdb
55 duckdb,
56 packaging,
57 # - flink
58 # - geospatial
59 geopandas,
60 shapely,
61 # - mssql
62 pyodbc,
63 # - mysql
64 pymysql,
65 # - oracle
66 oracledb,
67 # - polars
68 polars,
69 # - postgres
70 psycopg2,
71 # - pyspark
72 pyspark,
73 # - snowflake
74 snowflake-connector-python,
75 # sqlite
76 regex,
77 # - trino
78 trino-python-client,
79 # - visualization
80 graphviz,
81 # examples
82 pins,
83}:
84let
85 testBackends = [
86 "duckdb"
87 "sqlite"
88 ];
89
90 ibisTestingData = fetchFromGitHub {
91 owner = "ibis-project";
92 repo = "testing-data";
93 # https://github.com/ibis-project/ibis/blob/10.5.0/nix/overlay.nix#L94-L100
94 rev = "b26bd40cf29004372319df620c4bbe41420bb6f8";
95 hash = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU=";
96 };
97in
98
99buildPythonPackage rec {
100 pname = "ibis-framework";
101 version = "10.8.0";
102 pyproject = true;
103
104 src = fetchFromGitHub {
105 owner = "ibis-project";
106 repo = "ibis";
107 tag = version;
108 hash = "sha256-Uuqm9Exu/oK3BGBL4ViUOGArMWhVutUn1gFRj1I4vt4=";
109 };
110
111 build-system = [
112 hatchling
113 ];
114
115 dependencies = [
116 atpublic
117 parsy
118 python-dateutil
119 sqlglot
120 toolz
121 typing-extensions
122 tzdata
123 ];
124
125 nativeCheckInputs = [
126 pytestCheckHook
127 black
128 filelock
129 hypothesis
130 pytest-benchmark
131 pytest-httpserver
132 pytest-mock
133 pytest-randomly
134 pytest-snapshot
135 pytest-timeout
136 # this dependency is still needed due to use of strict markers and
137 # `pytest.mark.xdist_group` in the ibis codebase
138 pytest-xdist
139 writableTmpDirAsHomeHook
140 ]
141 ++ lib.concatMap (name: optional-dependencies.${name}) testBackends;
142
143 pytestFlags = [
144 "--benchmark-disable"
145 "-Wignore::FutureWarning"
146 ];
147
148 enabledTestMarks = testBackends ++ [ "core" ];
149
150 disabledTests = [
151 # tries to download duckdb extensions
152 "test_attach_sqlite"
153 "test_connect_extensions"
154 "test_load_extension"
155 "test_read_csv_with_types"
156 "test_read_sqlite"
157 "test_register_sqlite"
158 "test_roundtrip_xlsx"
159
160 # AssertionError: value does not match the expected value in snapshot
161 "test_union_aliasing"
162
163 # requires network connection
164 "test_s3_403_fallback"
165 "test_hugging_face"
166
167 # requires pytest 8.2+
168 "test_roundtrip_delta"
169
170 # AssertionError: value does not match the expected value in snapshot ibis/backends/tests/snapshots/test_sql/test_rewrite_context/sqlite/out.sql
171 "test_rewrite_context"
172
173 # Assertion error comparing a calculated version string with the actual (during nixpkgs-review)
174 "test_builtin_scalar_noargs"
175
176 # duckdb ParserError: syntax error at or near "AT"
177 "test_90"
178 ];
179
180 # patch out tests that check formatting with black
181 postPatch = ''
182 find ibis/tests -type f -name '*.py' -exec sed -i \
183 -e '/^ *assert_decompile_roundtrip/d' \
184 -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
185 '';
186
187 preCheck = ''
188 export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
189
190 # copy the test data to a directory
191 ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
192 '';
193
194 postCheck = ''
195 rm -r "$IBIS_TEST_DATA_DIRECTORY"
196 '';
197
198 pythonImportsCheck = [ "ibis" ] ++ map (backend: "ibis.backends.${backend}") testBackends;
199
200 optional-dependencies = {
201 athena = [
202 pyathena
203 pyarrow
204 pyarrow-hotfix
205 numpy
206 pandas
207 rich
208 packaging
209 fsspec
210 ];
211 bigquery = [
212 db-dtypes
213 google-cloud-bigquery
214 google-cloud-bigquery-storage
215 pyarrow
216 pyarrow-hotfix
217 pydata-google-auth
218 numpy
219 pandas
220 rich
221 ];
222 clickhouse = [
223 clickhouse-connect
224 pyarrow
225 pyarrow-hotfix
226 numpy
227 pandas
228 rich
229 ];
230 databricks = [
231 # databricks-sql-connector-core (unpackaged)
232 pyarrow
233 pyarrow-hotfix
234 numpy
235 pandas
236 rich
237 ];
238 datafusion = [
239 datafusion
240 pyarrow
241 pyarrow-hotfix
242 numpy
243 pandas
244 rich
245 ];
246 druid = [
247 pydruid
248 pyarrow
249 pyarrow-hotfix
250 numpy
251 pandas
252 rich
253 ];
254 duckdb = [
255 duckdb
256 pyarrow
257 pyarrow-hotfix
258 numpy
259 pandas
260 rich
261 packaging
262 ];
263 flink = [
264 pyarrow
265 pyarrow-hotfix
266 numpy
267 pandas
268 rich
269 ];
270 geospatial = [
271 geopandas
272 shapely
273 ];
274 mssql = [
275 pyodbc
276 pyarrow
277 pyarrow-hotfix
278 numpy
279 pandas
280 rich
281 ];
282 mysql = [
283 pymysql
284 pyarrow
285 pyarrow-hotfix
286 numpy
287 pandas
288 rich
289 ];
290 oracle = [
291 oracledb
292 packaging
293 pyarrow
294 pyarrow-hotfix
295 numpy
296 pandas
297 rich
298 ];
299 polars = [
300 polars
301 packaging
302 pyarrow
303 pyarrow-hotfix
304 numpy
305 pandas
306 rich
307 ];
308 postgres = [
309 psycopg2
310 pyarrow
311 pyarrow-hotfix
312 numpy
313 pandas
314 rich
315 ];
316 pyspark = [
317 pyspark
318 packaging
319 pyarrow
320 pyarrow-hotfix
321 numpy
322 pandas
323 rich
324 ];
325 snowflake = [
326 snowflake-connector-python
327 pyarrow
328 pyarrow-hotfix
329 numpy
330 pandas
331 rich
332 ];
333 sqlite = [
334 regex
335 pyarrow
336 pyarrow-hotfix
337 numpy
338 pandas
339 rich
340 ];
341 trino = [
342 trino-python-client
343 pyarrow
344 pyarrow-hotfix
345 numpy
346 pandas
347 rich
348 ];
349 visualization = [ graphviz ];
350 decompiler = [ black ];
351 examples = [ pins ] ++ pins.optional-dependencies.gcs;
352 };
353
354 meta = {
355 description = "Productivity-centric Python Big Data Framework";
356 homepage = "https://github.com/ibis-project/ibis";
357 changelog = "https://github.com/ibis-project/ibis/blob/${src.tag}/docs/release_notes.md";
358 license = lib.licenses.asl20;
359 maintainers = with lib.maintainers; [
360 cpcloud
361 sarahec
362 ];
363 };
364}