pyrox.dev
1{
2 writeShellScript,
3 writeText,
4 writers,
5 nix-prefetch-git,
6 formats,
7 lib,
8 coreutils,
9 curl,
10 xe,
11}:
12
13# Grammar list:
14# https://github.com/tree-sitter/tree-sitter/blob/master/docs/index.md
15
16let
17 # Grammars we want to fetch from the tree-sitter github orga
18 knownTreeSitterOrgGrammarRepos = [
19 "tree-sitter-javascript"
20 "tree-sitter-c"
21 "tree-sitter-json"
22 "tree-sitter-cpp"
23 "tree-sitter-ruby"
24 "tree-sitter-go"
25 "tree-sitter-c-sharp"
26 "tree-sitter-python"
27 "tree-sitter-typescript"
28 "tree-sitter-rust"
29 "tree-sitter-bash"
30 "tree-sitter-php"
31 "tree-sitter-java"
32 "tree-sitter-scala"
33 "tree-sitter-ocaml"
34 "tree-sitter-julia"
35 "tree-sitter-html"
36 "tree-sitter-haskell"
37 "tree-sitter-regex"
38 "tree-sitter-css"
39 "tree-sitter-verilog"
40 "tree-sitter-jsdoc"
41 "tree-sitter-ql"
42 "tree-sitter-ql-dbscheme"
43 "tree-sitter-embedded-template"
44 "tree-sitter-tsq"
45 "tree-sitter-toml"
46 ];
47 knownTreeSitterOrgGrammarReposJson = jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos;
48
49 # repos of the tree-sitter github orga we want to ignore (not grammars)
50 ignoredTreeSitterOrgRepos = [
51 "tree-sitter"
52 "tree-sitter-cli"
53 # this is the haskell language bindings, tree-sitter-haskell is the grammar
54 "haskell-tree-sitter"
55 # this is the ruby language bindings, tree-sitter-ruby is the grammar
56 "ruby-tree-sitter.old"
57 # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar
58 "rust-tree-sitter"
59 # this is the nodejs language bindings, tree-sitter-javascript is the grammar
60 "node-tree-sitter"
61 # this is the python language bindings, tree-sitter-python is the grammar
62 "py-tree-sitter"
63 # afl fuzzing for tree sitter
64 "afl-tree-sitter"
65 # this is the kotlin language bindings, tree-sitter-kotlin is the grammar
66 "kotlin-tree-sitter"
67 # this is the go language bindings, tree-sitter-go is the grammar
68 "go-tree-sitter"
69 # this is the java language bindings, tree-sitter-java is the grammar
70 "java-tree-sitter"
71 # archived
72 "highlight-schema"
73 # website
74 "tree-sitter.github.io"
75 # not maintained
76 "tree-sitter-razor"
77 # rust library for constructing arbitrary graph structures from source code
78 "tree-sitter-graph"
79 # abandoned
80 "tree-sitter-swift"
81 # this is the swift language bindings, tree-sitter-swift is the (abandoned) grammar
82 "swift-tree-sitter"
83 # abandoned
84 "tree-sitter-agda"
85 # abandoned
86 "tree-sitter-fluent"
87 # to unblock my update
88 "csharp-tree-sitter"
89 # (experimental) java bindings to the Tree-sitter parsing library
90 "java-tree-sitter"
91 # go bindings to the Tree-sitter parsing library
92 "go-tree-sitter"
93 # kotlin bindings to the Tree-sitter parsing library
94 "kotlin-tree-sitter"
95 # not ready to be used
96 "zig-tree-sitter"
97
98 # Non-grammar repositories
99 ".github"
100 "fuzz-action"
101 "parse-action"
102 "parser-setup-action"
103 "parser-test-action"
104 "parser-update-action"
105 "setup-action"
106 "workflows"
107 ];
108 ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos;
109
110 # Additional grammars that are not in the official github orga.
111 # If you need a grammar that already exists in the official orga,
112 # make sure to give it a different name.
113 otherGrammars = {
114 "tree-sitter-bitbake" = {
115 orga = "amaanq";
116 repo = "tree-sitter-bitbake";
117 };
118 "tree-sitter-beancount" = {
119 orga = "polarmutex";
120 repo = "tree-sitter-beancount";
121 };
122 "tree-sitter-bqn" = {
123 orga = "shnarazk";
124 repo = "tree-sitter-bqn";
125 };
126 "tree-sitter-clojure" = {
127 orga = "sogaiu";
128 repo = "tree-sitter-clojure";
129 };
130 "tree-sitter-comment" = {
131 orga = "stsewd";
132 repo = "tree-sitter-comment";
133 };
134 "tree-sitter-dart" = {
135 orga = "usernobody14";
136 repo = "tree-sitter-dart";
137 };
138 "tree-sitter-elisp" = {
139 orga = "wilfred";
140 repo = "tree-sitter-elisp";
141 };
142 "tree-sitter-just" = {
143 orga = "IndianBoy42";
144 repo = "tree-sitter-just";
145 };
146 "tree-sitter-nix" = {
147 orga = "nix-community";
148 repo = "tree-sitter-nix";
149 };
150 "tree-sitter-latex" = {
151 orga = "latex-lsp";
152 repo = "tree-sitter-latex";
153 };
154 "tree-sitter-lua" = {
155 orga = "MunifTanjim";
156 repo = "tree-sitter-lua";
157 };
158 "tree-sitter-factor" = {
159 orga = "erochest";
160 repo = "tree-sitter-factor";
161 };
162 "tree-sitter-fennel" = {
163 orga = "travonted";
164 repo = "tree-sitter-fennel";
165 };
166 "tree-sitter-make" = {
167 orga = "alemuller";
168 repo = "tree-sitter-make";
169 };
170 "tree-sitter-markdown" = {
171 orga = "MDeiml";
172 repo = "tree-sitter-markdown";
173 };
174 "tree-sitter-proto" = {
175 orga = "mitchellh";
176 repo = "tree-sitter-proto";
177 };
178 "tree-sitter-rego" = {
179 orga = "FallenAngel97";
180 repo = "tree-sitter-rego";
181 };
182 "tree-sitter-rst" = {
183 orga = "stsewd";
184 repo = "tree-sitter-rst";
185 };
186 "tree-sitter-sml" = {
187 orga = "MatthewFluet";
188 repo = "tree-sitter-sml";
189 };
190 "tree-sitter-svelte" = {
191 orga = "Himujjal";
192 repo = "tree-sitter-svelte";
193 };
194 "tree-sitter-sql" = {
195 orga = "derekstride";
196 repo = "tree-sitter-sql";
197 branch = "gh-pages";
198 };
199 "tree-sitter-talon" = {
200 orga = "wenkokke";
201 repo = "tree-sitter-talon";
202 };
203 "tree-sitter-typst" = {
204 orga = "uben0";
205 repo = "tree-sitter-typst";
206 };
207 "tree-sitter-vim" = {
208 orga = "vigoux";
209 repo = "tree-sitter-viml";
210 };
211 "tree-sitter-yaml" = {
212 orga = "tree-sitter-grammars";
213 repo = "tree-sitter-yaml";
214 };
215 "tree-sitter-zig" = {
216 orga = "maxxnino";
217 repo = "tree-sitter-zig";
218 };
219 "tree-sitter-fish" = {
220 orga = "ram02z";
221 repo = "tree-sitter-fish";
222 };
223 "tree-sitter-dot" = {
224 orga = "rydesun";
225 repo = "tree-sitter-dot";
226 };
227 "tree-sitter-norg" = {
228 orga = "nvim-neorg";
229 repo = "tree-sitter-norg";
230 };
231 "tree-sitter-norg-meta" = {
232 orga = "nvim-neorg";
233 repo = "tree-sitter-norg-meta";
234 };
235 "tree-sitter-commonlisp" = {
236 orga = "thehamsta";
237 repo = "tree-sitter-commonlisp";
238 };
239 "tree-sitter-cuda" = {
240 orga = "thehamsta";
241 repo = "tree-sitter-cuda";
242 };
243 "tree-sitter-glsl" = {
244 orga = "thehamsta";
245 repo = "tree-sitter-glsl";
246 };
247 "tree-sitter-dockerfile" = {
248 orga = "camdencheek";
249 repo = "tree-sitter-dockerfile";
250 };
251 "tree-sitter-ledger" = {
252 orga = "cbarrete";
253 repo = "tree-sitter-ledger";
254 };
255 "tree-sitter-gomod" = {
256 orga = "camdencheek";
257 repo = "tree-sitter-go-mod";
258 };
259 "tree-sitter-gowork" = {
260 orga = "omertuc";
261 repo = "tree-sitter-go-work";
262 };
263 "tree-sitter-graphql" = {
264 orga = "bkegley";
265 repo = "tree-sitter-graphql";
266 };
267 "tree-sitter-pgn" = {
268 orga = "rolandwalker";
269 repo = "tree-sitter-pgn";
270 };
271 "tree-sitter-perl" = {
272 orga = "ganezdragon";
273 repo = "tree-sitter-perl";
274 };
275 "tree-sitter-kotlin" = {
276 orga = "fwcd";
277 repo = "tree-sitter-kotlin";
278 };
279 "tree-sitter-scss" = {
280 orga = "serenadeai";
281 repo = "tree-sitter-scss";
282 };
283 "tree-sitter-erlang" = {
284 orga = "WhatsApp";
285 repo = "tree-sitter-erlang";
286 };
287 "tree-sitter-elixir" = {
288 orga = "elixir-lang";
289 repo = "tree-sitter-elixir";
290 };
291 "tree-sitter-surface" = {
292 orga = "connorlay";
293 repo = "tree-sitter-surface";
294 };
295 "tree-sitter-eex" = {
296 orga = "connorlay";
297 repo = "tree-sitter-eex";
298 };
299 "tree-sitter-heex" = {
300 orga = "connorlay";
301 repo = "tree-sitter-heex";
302 };
303 "tree-sitter-supercollider" = {
304 orga = "madskjeldgaard";
305 repo = "tree-sitter-supercollider";
306 };
307 "tree-sitter-tlaplus" = {
308 orga = "tlaplus-community";
309 repo = "tree-sitter-tlaplus";
310 };
311 "tree-sitter-glimmer" = {
312 orga = "alexlafroscia";
313 repo = "tree-sitter-glimmer";
314 };
315 "tree-sitter-pug" = {
316 orga = "zealot128";
317 repo = "tree-sitter-pug";
318 };
319 "tree-sitter-vue" = {
320 orga = "ikatyang";
321 repo = "tree-sitter-vue";
322 };
323 "tree-sitter-elm" = {
324 orga = "elm-tooling";
325 repo = "tree-sitter-elm";
326 };
327 "tree-sitter-yang" = {
328 orga = "hubro";
329 repo = "tree-sitter-yang";
330 };
331 "tree-sitter-query" = {
332 orga = "nvim-treesitter";
333 repo = "tree-sitter-query";
334 };
335 "tree-sitter-sparql" = {
336 orga = "bonabeavis";
337 repo = "tree-sitter-sparql";
338 };
339 "tree-sitter-gdscript" = {
340 orga = "prestonknopp";
341 repo = "tree-sitter-gdscript";
342 };
343 "tree-sitter-gemini" = {
344 orga = "blessanabraham";
345 repo = "tree-sitter-gemini";
346 };
347 "tree-sitter-go-template" = {
348 orga = "ngalaiko";
349 repo = "tree-sitter-go-template";
350 };
351 "tree-sitter-godot-resource" = {
352 orga = "prestonknopp";
353 repo = "tree-sitter-godot-resource";
354 };
355 "tree-sitter-turtle" = {
356 orga = "bonabeavis";
357 repo = "tree-sitter-turtle";
358 };
359 "tree-sitter-devicetree" = {
360 orga = "joelspadin";
361 repo = "tree-sitter-devicetree";
362 };
363 "tree-sitter-r" = {
364 orga = "r-lib";
365 repo = "tree-sitter-r";
366 };
367 "tree-sitter-bibtex" = {
368 orga = "latex-lsp";
369 repo = "tree-sitter-bibtex";
370 };
371 "tree-sitter-fortran" = {
372 orga = "stadelmanma";
373 repo = "tree-sitter-fortran";
374 };
375 "tree-sitter-cmake" = {
376 orga = "uyha";
377 repo = "tree-sitter-cmake";
378 };
379 "tree-sitter-janet-simple" = {
380 orga = "sogaiu";
381 repo = "tree-sitter-janet-simple";
382 };
383 "tree-sitter-json5" = {
384 orga = "joakker";
385 repo = "tree-sitter-json5";
386 };
387 "tree-sitter-pioasm" = {
388 orga = "leo60228";
389 repo = "tree-sitter-pioasm";
390 };
391 "tree-sitter-hjson" = {
392 orga = "winston0410";
393 repo = "tree-sitter-hjson";
394 };
395 "tree-sitter-llvm" = {
396 orga = "benwilliamgraham";
397 repo = "tree-sitter-llvm";
398 };
399 "tree-sitter-http" = {
400 orga = "ntbbloodbath";
401 repo = "tree-sitter-http";
402 };
403 "tree-sitter-prisma" = {
404 orga = "victorhqc";
405 repo = "tree-sitter-prisma";
406 };
407 "tree-sitter-org-nvim" = {
408 orga = "milisims";
409 repo = "tree-sitter-org";
410 };
411 "tree-sitter-hcl" = {
412 orga = "MichaHoffmann";
413 repo = "tree-sitter-hcl";
414 };
415 "tree-sitter-scheme" = {
416 orga = "6cdh";
417 repo = "tree-sitter-scheme";
418 };
419 "tree-sitter-tiger" = {
420 orga = "ambroisie";
421 repo = "tree-sitter-tiger";
422 };
423 "tree-sitter-nickel" = {
424 orga = "nickel-lang";
425 repo = "tree-sitter-nickel";
426 };
427 "tree-sitter-smithy" = {
428 orga = "indoorvivants";
429 repo = "tree-sitter-smithy";
430 };
431 "tree-sitter-jsonnet" = {
432 orga = "sourcegraph";
433 repo = "tree-sitter-jsonnet";
434 };
435 "tree-sitter-solidity" = {
436 orga = "JoranHonig";
437 repo = "tree-sitter-solidity";
438 };
439 "tree-sitter-nu" = {
440 orga = "nushell";
441 repo = "tree-sitter-nu";
442 };
443 "tree-sitter-cue" = {
444 orga = "eonpatapon";
445 repo = "tree-sitter-cue";
446 };
447 "tree-sitter-uiua" = {
448 orga = "shnarazk";
449 repo = "tree-sitter-uiua";
450 };
451 "tree-sitter-wing" = {
452 orga = "winglang";
453 repo = "tree-sitter-wing";
454 };
455 "tree-sitter-wgsl" = {
456 orga = "szebniok";
457 repo = "tree-sitter-wgsl";
458 };
459 "tree-sitter-templ" = {
460 orga = "vrischmann";
461 repo = "tree-sitter-templ";
462 };
463 "tree-sitter-gleam" = {
464 orga = "gleam-lang";
465 repo = "tree-sitter-gleam";
466 };
467 "tree-sitter-koka" = {
468 orga = "mtoohey31";
469 repo = "tree-sitter-koka";
470 };
471 "tree-sitter-earthfile" = {
472 orga = "glehmann";
473 repo = "tree-sitter-earthfile";
474 };
475 "tree-sitter-river" = {
476 orga = "grafana";
477 repo = "tree-sitter-river";
478 };
479 "tree-sitter-twig" = {
480 orga = "kaermorchen";
481 repo = "tree-sitter-twig";
482 };
483 "tree-sitter-hyprlang" = {
484 orga = "tree-sitter-grammars";
485 repo = "tree-sitter-hyprlang";
486 };
487 "tree-sitter-kdl" = {
488 orga = "tree-sitter-grammars";
489 repo = "tree-sitter-kdl";
490 };
491 "tree-sitter-tera" = {
492 orga = "uncenter";
493 repo = "tree-sitter-tera";
494 };
495 "tree-sitter-netlinx" = {
496 orga = "norgate-av";
497 repo = "tree-sitter-netlinx";
498 };
499 "tree-sitter-crystal" = {
500 orga = "crystal-lang-tools";
501 repo = "tree-sitter-crystal";
502 };
503 };
504
505 pinnedGrammars = [
506 "tree-sitter-bash"
507 "tree-sitter-bibtex"
508 "tree-sitter-c"
509 "tree-sitter-comment"
510 "tree-sitter-fortran"
511 "tree-sitter-hcl"
512 "tree-sitter-hyprlang"
513 "tree-sitter-llvm"
514 "tree-sitter-markdown"
515 "tree-sitter-query"
516 "tree-sitter-rust"
517 ];
518 pinnedGrammarsJson = jsonFile pinnedGrammars;
519
520 allGrammars =
521 let
522 treeSitterOrgaGrammars = lib.listToAttrs (
523 map (repo: {
524 name = repo;
525 value = {
526 orga = "tree-sitter";
527 inherit repo;
528 };
529 }) knownTreeSitterOrgGrammarRepos
530 );
531
532 in
533 lib.attrsets.unionOfDisjoint otherGrammars treeSitterOrgaGrammars;
534
535 jsonFile = name: val: (formats.json { }).generate name val;
536
537 # implementation of the updater
538 updateImpl =
539 passArgs "updateImpl-with-args"
540 {
541 binaries = {
542 curl = "${curl}/bin/curl";
543 nix-prefetch-git = "${nix-prefetch-git}/bin/nix-prefetch-git";
544 printf = "${coreutils}/bin/printf";
545 };
546 inherit
547 knownTreeSitterOrgGrammarRepos
548 ignoredTreeSitterOrgRepos
549 pinnedGrammars
550 ;
551 }
552 (
553 writers.writePython3 "updateImpl" {
554 flakeIgnore = [ "E501" ];
555 } ./update_impl.py
556 );
557
558 # Pass the given arguments to the command, in the ARGS environment variable.
559 # The arguments are just a json object that should be available in the script.
560 passArgs =
561 name: argAttrs: script:
562 writeShellScript name ''
563 env ARGS="$(< ${jsonFile "${name}-args" argAttrs})" \
564 ${script} "$@"
565 '';
566
567 foreachSh =
568 attrs: f: lib.concatMapStringsSep "\n" f (lib.mapAttrsToList (k: v: { name = k; } // v) attrs);
569
570 jsonNewlines = lib.concatMapStringsSep "\n" (lib.generators.toJSON { });
571
572 # Run the given script for each of the attr list.
573 # The attrs are passed to the script as a json value.
574 forEachParallel =
575 name: script: listOfAttrs:
576 writeShellScript "for-each-parallel.sh" ''
577 < ${writeText "${name}.json" (jsonNewlines listOfAttrs)} \
578 ${xe}/bin/xe -F -j5 ${script} {}
579 '';
580
581 # The output directory in the current source tree.
582 # This will depend on your local environment, but that is intentional.
583 outputDir = "${toString ./.}/grammars";
584
585 update-all-grammars = writeShellScript "update-all-grammars.sh" ''
586 set -euo pipefail
587 ${updateImpl} fetch-and-check-tree-sitter-repos '{}'
588 echo "writing files to ${outputDir}" 1>&2
589 mkdir -p "${outputDir}"
590 ${forEachParallel "repos-to-fetch"
591 (writeShellScript "fetch-repo" ''
592 ${updateImpl} fetch-repo "$1"
593 '')
594 (
595 lib.mapAttrsToList (
596 nixRepoAttrName: attrs:
597 attrs
598 // {
599 inherit
600 nixRepoAttrName
601 outputDir
602 ;
603 }
604 ) allGrammars
605 )
606 }
607 ${updateImpl} print-all-grammars-nix-file "$(< ${
608 jsonFile "all-grammars.json" {
609 allGrammars = lib.mapAttrsToList (
610 nixRepoAttrName: attrs:
611 attrs
612 // {
613 inherit nixRepoAttrName;
614 }
615 ) allGrammars;
616 inherit outputDir;
617 }
618 })"
619 '';
620
621in
622update-all-grammars