commit 62854ab9c2f2a00da5ed7106d84a58d930dfbe4c · nekomimi.pet/microcosm-rs

+236 -60

Cargo.lock

···

       112
       112
        
       

     

       113
       113
        
       [[package]]

     

       114
       114
        
       name = "anyhow"

     

       115
       115
       -
       version = "1.0.97"

     

       115
       115
       +
       version = "1.0.100"

     

       116
       116
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       117
       117
       -
       checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"

     

       117
       117
       +
       checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"

     

       118
       118
        
       

     

       119
       119
        
       [[package]]

     

       120
       120
        
       name = "arbitrary"

     
···

       192
       192
        
        "nom",

     

       193
       193
        
        "num-traits",

     

       194
       194
        
        "rusticata-macros",

     

       195
       195
       -
        "thiserror 2.0.16",

     

       195
       195
       +
        "thiserror 2.0.17",

     

       196
       196
        
        "time",

     

       197
       197
        
       ]

     

       198
       198
        
       

     
···

       644
       644
        
        "axum",

     

       645
       645
        
        "handlebars",

     

       646
       646
        
        "serde",

     

       647
       647
       -
        "thiserror 2.0.16",

     

       647
       647
       +
        "thiserror 2.0.17",

     

       648
       648
        
       ]

     

       649
       649
        
       

     

       650
       650
        
       [[package]]

     
···

       673
       673
        
       version = "0.2.0"

     

       674
       674
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       675
       675
        
       checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"

     

       676
       676
       +
       

     

       677
       677
       +
       [[package]]

     

       678
       678
       +
       name = "base256emoji"

     

       679
       679
       +
       version = "1.0.2"

     

       680
       680
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       681
       681
       +
       checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c"

     

       682
       682
       +
       dependencies = [

     

       683
       683
       +
        "const-str",

     

       684
       684
       +
        "match-lookup",

     

       685
       685
       +
       ]

     

       676
       686
        
       

     

       677
       687
        
       [[package]]

     

       678
       688
        
       name = "base64"

     
···

       839
       849
        
       checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"

     

       840
       850
        
       

     

       841
       851
        
       [[package]]

     

       852
       852
       +
       name = "byteorder-lite"

     

       853
       853
       +
       version = "0.1.0"

     

       854
       854
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       855
       855
       +
       checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"

     

       856
       856
       +
       

     

       857
       857
       +
       [[package]]

     

       842
       858
        
       name = "bytes"

     

       843
       859
        
       version = "1.10.1"

     

       844
       860
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       849
       865
        
       version = "0.6.1"

     

       850
       866
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       851
       867
        
       checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"

     

       868
       868
       +
       

     

       869
       869
       +
       [[package]]

     

       870
       870
       +
       name = "byteview"

     

       871
       871
       +
       version = "0.8.0"

     

       872
       872
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       873
       873
       +
       checksum = "1e6b0e42e210b794e14b152c6fe1a55831e30ef4a0f5dc39d73d714fb5f1906c"

     

       852
       874
        
       

     

       853
       875
        
       [[package]]

     

       854
       876
        
       name = "bzip2-sys"

     
···

       892
       914
        
       ]

     

       893
       915
        
       

     

       894
       916
        
       [[package]]

     

       917
       917
       +
       name = "cbor4ii"

     

       918
       918
       +
       version = "0.2.14"

     

       919
       919
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       920
       920
       +
       checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4"

     

       921
       921
       +
       dependencies = [

     

       922
       922
       +
        "serde",

     

       923
       923
       +
       ]

     

       924
       924
       +
       

     

       925
       925
       +
       [[package]]

     

       895
       926
        
       name = "cc"

     

       896
       927
        
       version = "1.2.18"

     

       897
       928
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       976
       1007
        
        "multihash",

     

       977
       1008
        
        "serde",

     

       978
       1009
        
        "serde_bytes",

     

       979
       979
       -
        "unsigned-varint",

     

       1010
       1010
       +
        "unsigned-varint 0.8.0",

     

       980
       1011
        
       ]

     

       981
       1012
        
       

     

       982
       1013
        
       [[package]]

     
···

       1085
       1116
        
       version = "0.9.6"

     

       1086
       1117
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1087
       1118
        
       checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"

     

       1119
       1119
       +
       

     

       1120
       1120
       +
       [[package]]

     

       1121
       1121
       +
       name = "const-str"

     

       1122
       1122
       +
       version = "0.4.3"

     

       1123
       1123
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1124
       1124
       +
       checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3"

     

       1088
       1125
        
       

     

       1089
       1126
        
       [[package]]

     

       1090
       1127
        
       name = "constellation"

     
···

       1579
       1616
        
        "slog-bunyan",

     

       1580
       1617
        
        "slog-json",

     

       1581
       1618
        
        "slog-term",

     

       1582
       1582
       -
        "thiserror 2.0.16",

     

       1619
       1619
       +
        "thiserror 2.0.17",

     

       1583
       1620
        
        "tokio",

     

       1584
       1621
        
        "tokio-rustls 0.25.0",

     

       1585
       1622
        
        "toml 0.9.7",

     
···

       1783
       1820
        
       checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"

     

       1784
       1821
        
       dependencies = [

     

       1785
       1822
        
        "byteorder",

     

       1786
       1786
       -
        "byteview",

     

       1823
       1823
       +
        "byteview 0.6.1",

     

       1787
       1824
        
        "dashmap",

     

       1788
       1825
        
        "log",

     

       1789
       1789
       -
        "lsm-tree",

     

       1826
       1826
       +
        "lsm-tree 2.10.4",

     

       1790
       1827
        
        "path-absolutize",

     

       1791
       1828
        
        "std-semaphore",

     

       1792
       1829
        
        "tempfile",

     
···

       1799
       1836
        
       source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d"

     

       1800
       1837
        
       dependencies = [

     

       1801
       1838
        
        "byteorder",

     

       1802
       1802
       -
        "byteview",

     

       1839
       1839
       +
        "byteview 0.6.1",

     

       1803
       1840
        
        "dashmap",

     

       1804
       1841
        
        "log",

     

       1805
       1805
       -
        "lsm-tree",

     

       1842
       1842
       +
        "lsm-tree 2.10.4",

     

       1806
       1843
        
        "path-absolutize",

     

       1844
       1844
       +
        "std-semaphore",

     

       1845
       1845
       +
        "tempfile",

     

       1846
       1846
       +
        "xxhash-rust",

     

       1847
       1847
       +
       ]

     

       1848
       1848
       +
       

     

       1849
       1849
       +
       [[package]]

     

       1850
       1850
       +
       name = "fjall"

     

       1851
       1851
       +
       version = "3.0.0-pre.0"

     

       1852
       1852
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1853
       1853
       +
       checksum = "467588c1f15d1cfa9e43f02a45cf55d82fa1f12a6ae961b848c520458525600c"

     

       1854
       1854
       +
       dependencies = [

     

       1855
       1855
       +
        "byteorder-lite",

     

       1856
       1856
       +
        "byteview 0.8.0",

     

       1857
       1857
       +
        "dashmap",

     

       1858
       1858
       +
        "log",

     

       1859
       1859
       +
        "lsm-tree 3.0.0-pre.0",

     

       1807
       1860
        
        "std-semaphore",

     

       1808
       1861
        
        "tempfile",

     

       1809
       1862
        
        "xxhash-rust",

     
···

       1891
       1944
        
        "mixtrics",

     

       1892
       1945
        
        "pin-project",

     

       1893
       1946
        
        "serde",

     

       1894
       1894
       -
        "thiserror 2.0.16",

     

       1947
       1947
       +
        "thiserror 2.0.17",

     

       1895
       1948
        
        "tokio",

     

       1896
       1949
        
        "tracing",

     

       1897
       1950
        
       ]

     
···

       1911
       1964
        
        "parking_lot",

     

       1912
       1965
        
        "pin-project",

     

       1913
       1966
        
        "serde",

     

       1914
       1914
       -
        "thiserror 2.0.16",

     

       1967
       1967
       +
        "thiserror 2.0.17",

     

       1915
       1968
        
        "tokio",

     

       1916
       1969
        
        "twox-hash",

     

       1917
       1970
        
       ]

     
···

       1944
       1997
        
        "parking_lot",

     

       1945
       1998
        
        "pin-project",

     

       1946
       1999
        
        "serde",

     

       1947
       1947
       -
        "thiserror 2.0.16",

     

       2000
       2000
       +
        "thiserror 2.0.17",

     

       1948
       2001
        
        "tokio",

     

       1949
       2002
        
        "tracing",

     

       1950
       2003
        
       ]

     
···

       1976
       2029
        
        "pin-project",

     

       1977
       2030
        
        "rand 0.9.1",

     

       1978
       2031
        
        "serde",

     

       1979
       1979
       -
        "thiserror 2.0.16",

     

       2032
       2032
       +
        "thiserror 2.0.17",

     

       1980
       2033
        
        "tokio",

     

       1981
       2034
        
        "tracing",

     

       1982
       2035
        
        "twox-hash",

     
···

       2220
       2273
        
        "pest_derive",

     

       2221
       2274
        
        "serde",

     

       2222
       2275
        
        "serde_json",

     

       2223
       2223
       -
        "thiserror 2.0.16",

     

       2276
       2276
       +
        "thiserror 2.0.17",

     

       2224
       2277
        
        "walkdir",

     

       2225
       2278
        
       ]

     

       2226
       2279
        
       

     
···

       2345
       2398
        
        "once_cell",

     

       2346
       2399
        
        "rand 0.9.1",

     

       2347
       2400
        
        "ring",

     

       2348
       2348
       -
        "thiserror 2.0.16",

     

       2401
       2401
       +
        "thiserror 2.0.17",

     

       2349
       2402
        
        "tinyvec",

     

       2350
       2403
        
        "tokio",

     

       2351
       2404
        
        "tracing",

     
···

       2368
       2421
        
        "rand 0.9.1",

     

       2369
       2422
        
        "resolv-conf",

     

       2370
       2423
        
        "smallvec",

     

       2371
       2371
       -
        "thiserror 2.0.16",

     

       2424
       2424
       +
        "thiserror 2.0.17",

     

       2372
       2425
        
        "tokio",

     

       2373
       2426
        
        "tracing",

     

       2374
       2427
        
       ]

     
···

       2800
       2853
        
       ]

     

       2801
       2854
        
       

     

       2802
       2855
        
       [[package]]

     

       2856
       2856
       +
       name = "iroh-car"

     

       2857
       2857
       +
       version = "0.5.1"

     

       2858
       2858
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       2859
       2859
       +
       checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a"

     

       2860
       2860
       +
       dependencies = [

     

       2861
       2861
       +
        "anyhow",

     

       2862
       2862
       +
        "cid",

     

       2863
       2863
       +
        "futures",

     

       2864
       2864
       +
        "serde",

     

       2865
       2865
       +
        "serde_ipld_dagcbor",

     

       2866
       2866
       +
        "thiserror 1.0.69",

     

       2867
       2867
       +
        "tokio",

     

       2868
       2868
       +
        "unsigned-varint 0.7.2",

     

       2869
       2869
       +
       ]

     

       2870
       2870
       +
       

     

       2871
       2871
       +
       [[package]]

     

       2803
       2872
        
       name = "is-terminal"

     

       2804
       2873
        
       version = "0.4.16"

     

       2805
       2874
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       2863
       2932
        
        "metrics",

     

       2864
       2933
        
        "serde",

     

       2865
       2934
        
        "serde_json",

     

       2866
       2866
       -
        "thiserror 2.0.16",

     

       2935
       2935
       +
        "thiserror 2.0.17",

     

       2867
       2936
        
        "tokio",

     

       2868
       2937
        
        "tokio-tungstenite 0.26.2",

     

       2869
       2938
        
        "url",

     
···

       3118
       3187
        
        "anyhow",

     

       3119
       3188
        
        "fluent-uri",

     

       3120
       3189
        
        "nom",

     

       3121
       3121
       -
        "thiserror 2.0.16",

     

       3190
       3190
       +
        "thiserror 2.0.17",

     

       3122
       3191
        
        "tinyjson",

     

       3123
       3192
        
       ]

     

       3124
       3193
        
       

     
···

       3186
       3255
        
       

     

       3187
       3256
        
       [[package]]

     

       3188
       3257
        
       name = "lsm-tree"

     

       3189
       3189
       -
       version = "2.10.2"

     

       3258
       3258
       +
       version = "2.10.4"

     

       3190
       3259
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       3191
       3191
       -
       checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c"

     

       3260
       3260
       +
       checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15"

     

       3192
       3261
        
       dependencies = [

     

       3193
       3262
        
        "byteorder",

     

       3194
       3263
        
        "crossbeam-skiplist",

     
···

       3209
       3278
        
       ]

     

       3210
       3279
        
       

     

       3211
       3280
        
       [[package]]

     

       3281
       3281
       +
       name = "lsm-tree"

     

       3282
       3282
       +
       version = "3.0.0-pre.0"

     

       3283
       3283
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       3284
       3284
       +
       checksum = "be375d45e348328e78582dffbda4f1709dd52fca27c1a81c7bf6ca134e6335f7"

     

       3285
       3285
       +
       dependencies = [

     

       3286
       3286
       +
        "byteorder-lite",

     

       3287
       3287
       +
        "byteview 0.8.0",

     

       3288
       3288
       +
        "crossbeam-skiplist",

     

       3289
       3289
       +
        "enum_dispatch",

     

       3290
       3290
       +
        "interval-heap",

     

       3291
       3291
       +
        "log",

     

       3292
       3292
       +
        "lz4_flex",

     

       3293
       3293
       +
        "quick_cache",

     

       3294
       3294
       +
        "rustc-hash 2.1.1",

     

       3295
       3295
       +
        "self_cell",

     

       3296
       3296
       +
        "sfa",

     

       3297
       3297
       +
        "tempfile",

     

       3298
       3298
       +
        "varint-rs",

     

       3299
       3299
       +
        "xxhash-rust",

     

       3300
       3300
       +
       ]

     

       3301
       3301
       +
       

     

       3302
       3302
       +
       [[package]]

     

       3212
       3303
        
       name = "lz4"

     

       3213
       3304
        
       version = "1.28.1"

     

       3214
       3305
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       3229
       3320
        
       

     

       3230
       3321
        
       [[package]]

     

       3231
       3322
        
       name = "lz4_flex"

     

       3232
       3232
       -
       version = "0.11.3"

     

       3323
       3323
       +
       version = "0.11.5"

     

       3233
       3324
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       3234
       3234
       -
       checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"

     

       3325
       3325
       +
       checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"

     

       3235
       3326
        
       

     

       3236
       3327
        
       [[package]]

     

       3237
       3328
        
       name = "mach2"

     
···

       3294
       3385
        
        "madsim",

     

       3295
       3386
        
        "spin",

     

       3296
       3387
        
        "tokio",

     

       3388
       3388
       +
       ]

     

       3389
       3389
       +
       

     

       3390
       3390
       +
       [[package]]

     

       3391
       3391
       +
       name = "match-lookup"

     

       3392
       3392
       +
       version = "0.1.1"

     

       3393
       3393
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       3394
       3394
       +
       checksum = "1265724d8cb29dbbc2b0f06fffb8bf1a8c0cf73a78eede9ba73a4a66c52a981e"

     

       3395
       3395
       +
       dependencies = [

     

       3396
       3396
       +
        "proc-macro2",

     

       3397
       3397
       +
        "quote",

     

       3398
       3398
       +
        "syn 1.0.109",

     

       3297
       3399
        
       ]

     

       3298
       3400
        
       

     

       3299
       3401
        
       [[package]]

     
···

       3384
       3486
        
        "metrics",

     

       3385
       3487
        
        "metrics-util 0.20.0",

     

       3386
       3488
        
        "quanta",

     

       3387
       3387
       -
        "thiserror 2.0.16",

     

       3489
       3489
       +
        "thiserror 2.0.17",

     

       3388
       3490
        
        "tokio",

     

       3389
       3491
        
        "tracing",

     

       3390
       3492
        
       ]

     
···

       3531
       3633
        
       

     

       3532
       3634
        
       [[package]]

     

       3533
       3635
        
       name = "multibase"

     

       3534
       3534
       -
       version = "0.9.1"

     

       3636
       3636
       +
       version = "0.9.2"

     

       3535
       3637
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       3536
       3536
       -
       checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404"

     

       3638
       3638
       +
       checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77"

     

       3537
       3639
        
       dependencies = [

     

       3538
       3640
        
        "base-x",

     

       3641
       3641
       +
        "base256emoji",

     

       3539
       3642
        
        "data-encoding",

     

       3540
       3643
        
        "data-encoding-macro",

     

       3541
       3644
        
       ]

     
···

       3548
       3651
        
       dependencies = [

     

       3549
       3652
        
        "core2",

     

       3550
       3653
        
        "serde",

     

       3551
       3551
       -
        "unsigned-varint",

     

       3654
       3654
       +
        "unsigned-varint 0.8.0",

     

       3552
       3655
        
       ]

     

       3553
       3656
        
       

     

       3554
       3657
        
       [[package]]

     
···

       3926
       4029
        
       checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323"

     

       3927
       4030
        
       dependencies = [

     

       3928
       4031
        
        "memchr",

     

       3929
       3929
       -
        "thiserror 2.0.16",

     

       4032
       4032
       +
        "thiserror 2.0.17",

     

       3930
       4033
        
        "ucd-trie",

     

       3931
       4034
        
       ]

     

       3932
       4035
        
       

     
···

       4036
       4139
        
        "rusqlite",

     

       4037
       4140
        
        "serde",

     

       4038
       4141
        
        "serde_json",

     

       4039
       4039
       -
        "thiserror 2.0.16",

     

       4142
       4142
       +
        "thiserror 2.0.17",

     

       4040
       4143
        
        "tokio",

     

       4041
       4144
        
        "tracing-subscriber",

     

       4042
       4145
        
       ]

     
···

       4079
       4182
        
        "smallvec",

     

       4080
       4183
        
        "sync_wrapper",

     

       4081
       4184
        
        "tempfile",

     

       4082
       4082
       -
        "thiserror 2.0.16",

     

       4185
       4185
       +
        "thiserror 2.0.17",

     

       4083
       4186
        
        "tokio",

     

       4084
       4187
        
        "tokio-rustls 0.26.2",

     

       4085
       4188
        
        "tokio-stream",

     
···

       4123
       4226
        
        "serde_json",

     

       4124
       4227
        
        "serde_urlencoded",

     

       4125
       4228
        
        "serde_yaml",

     

       4126
       4126
       -
        "thiserror 2.0.16",

     

       4229
       4229
       +
        "thiserror 2.0.17",

     

       4127
       4230
        
        "tokio",

     

       4128
       4231
        
       ]

     

       4129
       4232
        
       

     
···

       4142
       4245
        
        "quote",

     

       4143
       4246
        
        "regex",

     

       4144
       4247
        
        "syn 2.0.106",

     

       4145
       4145
       -
        "thiserror 2.0.16",

     

       4248
       4248
       +
        "thiserror 2.0.17",

     

       4146
       4249
        
       ]

     

       4147
       4250
        
       

     

       4148
       4251
        
       [[package]]

     
···

       4269
       4372
        
       

     

       4270
       4373
        
       [[package]]

     

       4271
       4374
        
       name = "quick_cache"

     

       4272
       4272
       -
       version = "0.6.12"

     

       4375
       4375
       +
       version = "0.6.16"

     

       4273
       4376
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       4274
       4274
       -
       checksum = "8f8ed0655cbaf18a26966142ad23b95d8ab47221c50c4f73a1db7d0d2d6e3da8"

     

       4377
       4377
       +
       checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"

     

       4275
       4378
        
       dependencies = [

     

       4276
       4379
        
        "equivalent",

     

       4277
       4380
        
        "hashbrown 0.15.2",

     
···

       4291
       4394
        
        "rustc-hash 2.1.1",

     

       4292
       4395
        
        "rustls 0.23.31",

     

       4293
       4396
        
        "socket2 0.5.9",

     

       4294
       4294
       -
        "thiserror 2.0.16",

     

       4397
       4397
       +
        "thiserror 2.0.17",

     

       4295
       4398
        
        "tokio",

     

       4296
       4399
        
        "tracing",

     

       4297
       4400
        
        "web-time",

     
···

       4312
       4415
        
        "rustls 0.23.31",

     

       4313
       4416
        
        "rustls-pki-types",

     

       4314
       4417
        
        "slab",

     

       4315
       4315
       -
        "thiserror 2.0.16",

     

       4418
       4418
       +
        "thiserror 2.0.17",

     

       4316
       4419
        
        "tinyvec",

     

       4317
       4420
        
        "tracing",

     

       4318
       4421
        
        "web-time",

     
···

       4538
       4641
        
       checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"

     

       4539
       4642
        
       

     

       4540
       4643
        
       [[package]]

     

       4644
       4644
       +
       name = "repo-stream"

     

       4645
       4645
       +
       version = "0.2.1"

     

       4646
       4646
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       4647
       4647
       +
       checksum = "727a78c392bd51b1af938e4383f2f6f46ae727cb38394136d1aebab0633faf8e"

     

       4648
       4648
       +
       dependencies = [

     

       4649
       4649
       +
        "bincode 2.0.1",

     

       4650
       4650
       +
        "futures",

     

       4651
       4651
       +
        "futures-core",

     

       4652
       4652
       +
        "ipld-core",

     

       4653
       4653
       +
        "iroh-car",

     

       4654
       4654
       +
        "log",

     

       4655
       4655
       +
        "multibase",

     

       4656
       4656
       +
        "rusqlite",

     

       4657
       4657
       +
        "serde",

     

       4658
       4658
       +
        "serde_bytes",

     

       4659
       4659
       +
        "serde_ipld_dagcbor",

     

       4660
       4660
       +
        "sha2",

     

       4661
       4661
       +
        "thiserror 2.0.17",

     

       4662
       4662
       +
        "tokio",

     

       4663
       4663
       +
       ]

     

       4664
       4664
       +
       

     

       4665
       4665
       +
       [[package]]

     

       4541
       4666
        
       name = "reqwest"

     

       4542
       4542
       -
       version = "0.12.23"

     

       4667
       4667
       +
       version = "0.12.24"

     

       4543
       4668
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       4544
       4544
       -
       checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb"

     

       4669
       4669
       +
       checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f"

     

       4545
       4670
        
       dependencies = [

     

       4546
       4671
        
        "async-compression",

     

       4547
       4672
        
        "base64 0.22.1",

     
···

       4581
       4706
        
        "url",

     

       4582
       4707
        
        "wasm-bindgen",

     

       4583
       4708
        
        "wasm-bindgen-futures",

     

       4709
       4709
       +
        "wasm-streams",

     

       4584
       4710
        
        "web-sys",

     

       4585
       4711
        
       ]

     

       4586
       4712
        
       

     
···

       4962
       5088
        
       

     

       4963
       5089
        
       [[package]]

     

       4964
       5090
        
       name = "self_cell"

     

       4965
       4965
       -
       version = "1.1.0"

     

       5091
       5091
       +
       version = "1.2.0"

     

       4966
       5092
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       4967
       4967
       -
       checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe"

     

       5093
       5093
       +
       checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"

     

       4968
       5094
        
       

     

       4969
       5095
        
       [[package]]

     

       4970
       5096
        
       name = "semver"

     
···

       4984
       5110
        
       

     

       4985
       5111
        
       [[package]]

     

       4986
       5112
        
       name = "serde_bytes"

     

       4987
       4987
       -
       version = "0.11.17"

     

       5113
       5113
       +
       version = "0.11.19"

     

       4988
       5114
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       4989
       4989
       -
       checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96"

     

       5115
       5115
       +
       checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"

     

       4990
       5116
        
       dependencies = [

     

       4991
       5117
        
        "serde",

     

       5118
       5118
       +
        "serde_core",

     

       4992
       5119
        
       ]

     

       4993
       5120
        
       

     

       4994
       5121
        
       [[package]]

     
···

       5036
       5163
        
       ]

     

       5037
       5164
        
       

     

       5038
       5165
        
       [[package]]

     

       5166
       5166
       +
       name = "serde_ipld_dagcbor"

     

       5167
       5167
       +
       version = "0.6.4"

     

       5168
       5168
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       5169
       5169
       +
       checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778"

     

       5170
       5170
       +
       dependencies = [

     

       5171
       5171
       +
        "cbor4ii",

     

       5172
       5172
       +
        "ipld-core",

     

       5173
       5173
       +
        "scopeguard",

     

       5174
       5174
       +
        "serde",

     

       5175
       5175
       +
       ]

     

       5176
       5176
       +
       

     

       5177
       5177
       +
       [[package]]

     

       5039
       5178
        
       name = "serde_json"

     

       5040
       5179
        
       version = "1.0.145"

     

       5041
       5180
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       5068
       5207
        
        "percent-encoding",

     

       5069
       5208
        
        "ryu",

     

       5070
       5209
        
        "serde",

     

       5071
       5071
       -
        "thiserror 2.0.16",

     

       5210
       5210
       +
        "thiserror 2.0.17",

     

       5072
       5211
        
       ]

     

       5073
       5212
        
       

     

       5074
       5213
        
       [[package]]

     
···

       5157
       5296
        
       ]

     

       5158
       5297
        
       

     

       5159
       5298
        
       [[package]]

     

       5299
       5299
       +
       name = "sfa"

     

       5300
       5300
       +
       version = "0.0.1"

     

       5301
       5301
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       5302
       5302
       +
       checksum = "e5f5f9dc21f55409f15103d5a7e7601b804935923c7fe4746dc806c3a422a038"

     

       5303
       5303
       +
       dependencies = [

     

       5304
       5304
       +
        "byteorder-lite",

     

       5305
       5305
       +
        "log",

     

       5306
       5306
       +
        "xxhash-rust",

     

       5307
       5307
       +
       ]

     

       5308
       5308
       +
       

     

       5309
       5309
       +
       [[package]]

     

       5160
       5310
        
       name = "sha1"

     

       5161
       5311
        
       version = "0.10.6"

     

       5162
       5312
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       5220
       5370
        
       dependencies = [

     

       5221
       5371
        
        "num-bigint",

     

       5222
       5372
        
        "num-traits",

     

       5223
       5223
       -
        "thiserror 2.0.16",

     

       5373
       5373
       +
        "thiserror 2.0.17",

     

       5224
       5374
        
        "time",

     

       5225
       5375
        
       ]

     

       5226
       5376
        
       

     
···

       5262
       5412
        
        "rustls 0.23.31",

     

       5263
       5413
        
        "serde",

     

       5264
       5414
        
        "serde_json",

     

       5265
       5265
       -
        "thiserror 2.0.16",

     

       5415
       5415
       +
        "thiserror 2.0.17",

     

       5266
       5416
        
        "time",

     

       5267
       5417
        
        "tokio",

     

       5268
       5418
        
        "tokio-util",

     
···

       5355
       5505
        
       name = "spacedust"

     

       5356
       5506
        
       version = "0.1.0"

     

       5357
       5507
        
       dependencies = [

     

       5508
       5508
       +
        "anyhow",

     

       5509
       5509
       +
        "async-channel",

     

       5358
       5510
        
        "async-trait",

     

       5359
       5511
        
        "clap",

     

       5360
       5512
        
        "ctrlc",

     

       5361
       5513
        
        "dropshot",

     

       5362
       5514
        
        "env_logger",

     

       5515
       5515
       +
        "fjall 3.0.0-pre.0",

     

       5363
       5516
        
        "futures",

     

       5364
       5517
        
        "http",

     

       5518
       5518
       +
        "ipld-core",

     

       5365
       5519
        
        "jetstream",

     

       5366
       5520
        
        "links",

     

       5367
       5521
        
        "log",

     

       5368
       5522
        
        "metrics",

     

       5369
       5523
        
        "metrics-exporter-prometheus 0.17.2",

     

       5370
       5524
        
        "rand 0.9.1",

     

       5525
       5525
       +
        "repo-stream",

     

       5526
       5526
       +
        "reqwest",

     

       5371
       5527
        
        "schemars",

     

       5372
       5528
        
        "semver",

     

       5373
       5529
        
        "serde",

     

       5530
       5530
       +
        "serde_ipld_dagcbor",

     

       5374
       5531
        
        "serde_json",

     

       5375
       5532
        
        "serde_qs",

     

       5376
       5376
       -
        "thiserror 2.0.16",

     

       5533
       5533
       +
        "thiserror 2.0.17",

     

       5377
       5534
        
        "tinyjson",

     

       5378
       5535
        
        "tokio",

     

       5379
       5536
        
        "tokio-tungstenite 0.27.0",

     
···

       5506
       5663
        
       

     

       5507
       5664
        
       [[package]]

     

       5508
       5665
        
       name = "tempfile"

     

       5509
       5509
       -
       version = "3.19.1"

     

       5666
       5666
       +
       version = "3.23.0"

     

       5510
       5667
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       5511
       5511
       -
       checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf"

     

       5668
       5668
       +
       checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"

     

       5512
       5669
        
       dependencies = [

     

       5513
       5670
        
        "fastrand",

     

       5514
       5671
        
        "getrandom 0.3.3",

     
···

       5539
       5696
        
       

     

       5540
       5697
        
       [[package]]

     

       5541
       5698
        
       name = "thiserror"

     

       5542
       5542
       -
       version = "2.0.16"

     

       5699
       5699
       +
       version = "2.0.17"

     

       5543
       5700
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       5544
       5544
       -
       checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0"

     

       5701
       5701
       +
       checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"

     

       5545
       5702
        
       dependencies = [

     

       5546
       5546
       -
        "thiserror-impl 2.0.16",

     

       5703
       5703
       +
        "thiserror-impl 2.0.17",

     

       5547
       5704
        
       ]

     

       5548
       5705
        
       

     

       5549
       5706
        
       [[package]]

     
···

       5559
       5716
        
       

     

       5560
       5717
        
       [[package]]

     

       5561
       5718
        
       name = "thiserror-impl"

     

       5562
       5562
       -
       version = "2.0.16"

     

       5719
       5719
       +
       version = "2.0.17"

     

       5563
       5720
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       5564
       5564
       -
       checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960"

     

       5721
       5721
       +
       checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"

     

       5565
       5722
        
       dependencies = [

     

       5566
       5723
        
        "proc-macro2",

     

       5567
       5724
        
        "quote",

     
···

       5993
       6150
        
        "native-tls",

     

       5994
       6151
        
        "rand 0.9.1",

     

       5995
       6152
        
        "sha1",

     

       5996
       5996
       -
        "thiserror 2.0.16",

     

       6153
       6153
       +
        "thiserror 2.0.17",

     

       5997
       6154
        
        "url",

     

       5998
       6155
        
        "utf-8",

     

       5999
       6156
        
       ]

     
···

       6011
       6168
        
        "log",

     

       6012
       6169
        
        "rand 0.9.1",

     

       6013
       6170
        
        "sha1",

     

       6014
       6014
       -
        "thiserror 2.0.16",

     

       6171
       6171
       +
        "thiserror 2.0.17",

     

       6015
       6172
        
        "utf-8",

     

       6016
       6173
        
       ]

     

       6017
       6174
        
       

     
···

       6054
       6211
        
        "http",

     

       6055
       6212
        
        "jetstream",

     

       6056
       6213
        
        "log",

     

       6057
       6057
       -
        "lsm-tree",

     

       6214
       6214
       +
        "lsm-tree 2.10.4",

     

       6058
       6215
        
        "metrics",

     

       6059
       6216
        
        "metrics-exporter-prometheus 0.17.2",

     

       6060
       6217
        
        "schemars",

     
···

       6064
       6221
        
        "serde_qs",

     

       6065
       6222
        
        "sha2",

     

       6066
       6223
        
        "tempfile",

     

       6067
       6067
       -
        "thiserror 2.0.16",

     

       6224
       6224
       +
        "thiserror 2.0.17",

     

       6068
       6225
        
        "tikv-jemallocator",

     

       6069
       6226
        
        "tokio",

     

       6070
       6227
        
        "tokio-util",

     
···

       6117
       6274
        
       

     

       6118
       6275
        
       [[package]]

     

       6119
       6276
        
       name = "unsigned-varint"

     

       6277
       6277
       +
       version = "0.7.2"

     

       6278
       6278
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       6279
       6279
       +
       checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105"

     

       6280
       6280
       +
       

     

       6281
       6281
       +
       [[package]]

     

       6282
       6282
       +
       name = "unsigned-varint"

     

       6120
       6283
        
       version = "0.8.0"

     

       6121
       6284
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       6122
       6285
        
       checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06"

     
···

       6193
       6356
        
       checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"

     

       6194
       6357
        
       dependencies = [

     

       6195
       6358
        
        "byteorder",

     

       6196
       6196
       -
        "byteview",

     

       6359
       6359
       +
        "byteview 0.6.1",

     

       6197
       6360
        
        "interval-heap",

     

       6198
       6361
        
        "log",

     

       6199
       6362
        
        "path-absolutize",

     
···

       6342
       6505
        
       ]

     

       6343
       6506
        
       

     

       6344
       6507
        
       [[package]]

     

       6508
       6508
       +
       name = "wasm-streams"

     

       6509
       6509
       +
       version = "0.4.2"

     

       6510
       6510
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       6511
       6511
       +
       checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"

     

       6512
       6512
       +
       dependencies = [

     

       6513
       6513
       +
        "futures-util",

     

       6514
       6514
       +
        "js-sys",

     

       6515
       6515
       +
        "wasm-bindgen",

     

       6516
       6516
       +
        "wasm-bindgen-futures",

     

       6517
       6517
       +
        "web-sys",

     

       6518
       6518
       +
       ]

     

       6519
       6519
       +
       

     

       6520
       6520
       +
       [[package]]

     

       6345
       6521
        
       name = "web-sys"

     

       6346
       6522
        
       version = "0.3.77"

     

       6347
       6523
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       6400
       6576
        
        "reqwest",

     

       6401
       6577
        
        "serde",

     

       6402
       6578
        
        "serde_json",

     

       6403
       6403
       -
        "thiserror 2.0.16",

     

       6579
       6579
       +
        "thiserror 2.0.17",

     

       6404
       6580
        
        "tokio",

     

       6405
       6581
        
        "tokio-util",

     

       6406
       6582
        
        "url",

     
···

       6758
       6934
        
        "nom",

     

       6759
       6935
        
        "oid-registry",

     

       6760
       6936
        
        "rusticata-macros",

     

       6761
       6761
       -
        "thiserror 2.0.16",

     

       6937
       6937
       +
        "thiserror 2.0.17",

     

       6762
       6938
        
        "time",

     

       6763
       6939
        
       ]

     

       6764
       6940

spacedust/Cargo.toml

···

       4
       4
        
       edition = "2024"

     

       5
       5
        
       

     

       6
       6
        
       [dependencies]

     

       7
       7
       +
       anyhow = "1.0.100"

     

       8
       8
       +
       async-channel = "2.5.0"

     

       7
       9
        
       async-trait = "0.1.88"

     

       8
       10
        
       clap = { version = "4.5.40", features = ["derive"] }

     

       9
       11
        
       ctrlc = "3.4.7"

     

       10
       12
        
       dropshot = "0.16.2"

     

       11
       13
        
       env_logger = "0.11.8"

     

       14
       14
       +
       fjall = "3.0.0-pre.0"

     

       12
       15
        
       futures = "0.3.31"

     

       13
       16
        
       http = "1.3.1"

     

       17
       17
       +
       ipld-core = { version = "0.4.2", features = ["serde"] }

     

       14
       18
        
       jetstream = { path = "../jetstream", features = ["metrics"] }

     

       15
       19
        
       links = { path = "../links" }

     

       16
       20
        
       log = "0.4.27"

     

       17
       21
        
       metrics = "0.24.2"

     

       18
       22
        
       metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] }

     

       19
       23
        
       rand = "0.9.1"

     

       24
       24
       +
       repo-stream = "0.2.1"

     

       25
       25
       +
       reqwest = { version = "0.12.24", features = ["json", "stream"] }

     

       20
       26
        
       schemars = "0.8.22"

     

       21
       27
        
       semver = "1.0.26"

     

       22
       28
        
       serde = { version = "1.0.219", features = ["derive"] }

     

       29
       29
       +
       serde_ipld_dagcbor = "0.6.4"

     

       23
       30
        
       serde_json = "1.0.140"

     

       24
       31
        
       serde_qs = "1.0.0-rc.3"

     

       25
       32
        
       thiserror = "2.0.12"

+24

spacedust/src/bin/import_car_file.rs

···

       1
       1
       +
       use clap::Parser;

     

       2
       2
       +
       use std::path::PathBuf;

     

       3
       3
       +
       use spacedust::storage::car;

     

       4
       4
       +
       

     

       5
       5
       +
       type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;

     

       6
       6
       +
       

     

       7
       7
       +
       #[derive(Debug, Parser)]

     

       8
       8
       +
       struct Args {

     

       9
       9
       +
           #[arg()]

     

       10
       10
       +
           file: PathBuf,

     

       11
       11
       +
       }

     

       12
       12
       +
       

     

       13
       13
       +
       #[tokio::main]

     

       14
       14
       +
       async fn main() -> Result<()> {

     

       15
       15
       +
           env_logger::init();

     

       16
       16
       +
       

     

       17
       17
       +
           let Args { file } = Args::parse();

     

       18
       18
       +
       

     

       19
       19
       +
           let reader = tokio::fs::File::open(file).await?;

     

       20
       20
       +
       

     

       21
       21
       +
           car::import(reader).await?;

     

       22
       22
       +
       

     

       23
       23
       +
           Ok(())

     

       24
       24
       +
       }

+194

spacedust/src/bin/import_scraped.rs

···

       1
       1
       +
       use clap::Parser;

     

       2
       2
       +
       use std::sync::{Arc, atomic::{AtomicUsize, Ordering}};

     

       3
       3
       +
       use std::path::PathBuf;

     

       4
       4
       +
       use tokio::{task::JoinSet, io::AsyncRead};

     

       5
       5
       +
       use repo_stream::{DriverBuilder, Driver, DiskBuilder, DiskStore, drive::NeedDisk};

     

       6
       6
       +
       

     

       7
       7
       +
       

     

       8
       8
       +
       type Result<T> = anyhow::Result<T>; //std::result::Result<T, Box<dyn std::error::Error>>;

     

       9
       9
       +
       

     

       10
       10
       +
       

     

       11
       11
       +
       #[derive(Debug, Parser)]

     

       12
       12
       +
       struct Args {

     

       13
       13
       +
           #[arg(long)]

     

       14
       14
       +
           cars_folder: PathBuf,

     

       15
       15
       +
           #[arg(long)]

     

       16
       16
       +
           mem_workers: usize,

     

       17
       17
       +
           #[arg(long)]

     

       18
       18
       +
           disk_workers: usize,

     

       19
       19
       +
           #[arg(long)]

     

       20
       20
       +
           disk_folder: PathBuf,

     

       21
       21
       +
       }

     

       22
       22
       +
       

     

       23
       23
       +
       async fn get_cars(cars_folder: PathBuf, tx: async_channel::Sender<tokio::io::BufReader<tokio::fs::File>>) -> Result<()> {

     

       24
       24
       +
           let mut dir = tokio::fs::read_dir(cars_folder).await?;

     

       25
       25
       +
           while let Some(entry) = dir.next_entry().await? {

     

       26
       26
       +
               if !entry.file_type().await?.is_file() {

     

       27
       27
       +
                   continue;

     

       28
       28
       +
               }

     

       29
       29
       +
               let reader = tokio::fs::File::open(&entry.path()).await?;

     

       30
       30
       +
               let reader = tokio::io::BufReader::new(reader);

     

       31
       31
       +
               tx.send(reader).await?;

     

       32
       32
       +
           }

     

       33
       33
       +
           Ok(())

     

       34
       34
       +
       }

     

       35
       35
       +
       

     

       36
       36
       +
       async fn drive_mem<R: AsyncRead + Unpin + Send + Sync + 'static>(

     

       37
       37
       +
           f: R,

     

       38
       38
       +
           disk_tx: &async_channel::Sender<NeedDisk<R, usize>>,

     

       39
       39
       +
       ) -> Result<Option<usize>> {

     

       40
       40
       +
           let mut n = 0;

     

       41
       41
       +
           match DriverBuilder::new()

     

       42
       42
       +
               .with_block_processor(|_| 0_usize) // don't care just counting records

     

       43
       43
       +
               .with_mem_limit_mb(32)

     

       44
       44
       +
               .load_car(f)

     

       45
       45
       +
               .await?

     

       46
       46
       +
           {

     

       47
       47
       +
               Driver::Memory(_commit, mut driver) => {

     

       48
       48
       +
                   while let Some(chunk) = driver.next_chunk(512).await? {

     

       49
       49
       +
                       n += chunk.len();

     

       50
       50
       +
                   }

     

       51
       51
       +
                   Ok(Some(n))

     

       52
       52
       +
               }

     

       53
       53
       +
               Driver::Disk(need_disk) => {

     

       54
       54
       +
                   disk_tx.send(need_disk).await?;

     

       55
       55
       +
                   Ok(None)

     

       56
       56
       +
               }

     

       57
       57
       +
           }

     

       58
       58
       +
       }

     

       59
       59
       +
       

     

       60
       60
       +
       async fn mem_worker<R: AsyncRead + Unpin + Send + Sync + 'static>(

     

       61
       61
       +
           car_rx: async_channel::Receiver<R>,

     

       62
       62
       +
           disk_tx: async_channel::Sender<NeedDisk<R, usize>>,

     

       63
       63
       +
           n: Arc<AtomicUsize>,

     

       64
       64
       +
       ) -> Result<()> {

     

       65
       65
       +
           while let Ok(f) = car_rx.recv().await {

     

       66
       66
       +
               let driven = match drive_mem(f, &disk_tx).await {

     

       67
       67
       +
                   Ok(d) => d,

     

       68
       68
       +
                   Err(e) => {

     

       69
       69
       +
                       eprintln!("failed to drive mem: {e:?}. skipping...");

     

       70
       70
       +
                       continue;

     

       71
       71
       +
                   }

     

       72
       72
       +
               };

     

       73
       73
       +
               if let Some(drove) = driven {

     

       74
       74
       +
                   n.fetch_add(drove, Ordering::Relaxed);

     

       75
       75
       +
               }

     

       76
       76
       +
           }

     

       77
       77
       +
           Ok(())

     

       78
       78
       +
       }

     

       79
       79
       +
       

     

       80
       80
       +
       async fn drive_disk<R: AsyncRead + Unpin>(

     

       81
       81
       +
           needed: NeedDisk<R, usize>,

     

       82
       82
       +
           store: DiskStore,

     

       83
       83
       +
       ) -> Result<(usize, DiskStore)> {

     

       84
       84
       +
           let (_commit, mut driver) = needed.finish_loading(store).await?;

     

       85
       85
       +
           let mut n = 0;

     

       86
       86
       +
           while let Some(chunk) = driver.next_chunk(512).await? {

     

       87
       87
       +
               n += chunk.len();

     

       88
       88
       +
           }

     

       89
       89
       +
           let store = driver.reset_store().await?;

     

       90
       90
       +
           Ok((n, store))

     

       91
       91
       +
       }

     

       92
       92
       +
       

     

       93
       93
       +
       async fn disk_worker<R: AsyncRead + Unpin>(

     

       94
       94
       +
           worker_id: usize,

     

       95
       95
       +
           disk_rx: async_channel::Receiver<NeedDisk<R, usize>>,

     

       96
       96
       +
           folder: PathBuf,

     

       97
       97
       +
           n: Arc<AtomicUsize>,

     

       98
       98
       +
           disk_workers_active: Arc<AtomicUsize>,

     

       99
       99
       +
       ) -> Result<()> {

     

       100
       100
       +
           let mut file = folder;

     

       101
       101
       +
           file.push(format!("disk-worker-{worker_id}.sqlite"));

     

       102
       102
       +
           let mut store = DiskBuilder::new()

     

       103
       103
       +
               .with_cache_size_mb(128)

     

       104
       104
       +
               .open(file.clone())

     

       105
       105
       +
               .await?;

     

       106
       106
       +
           while let Ok(needed) = disk_rx.recv().await {

     

       107
       107
       +
               let active = disk_workers_active.fetch_add(1, Ordering::AcqRel);

     

       108
       108
       +
               println!("-> disk workers active: {}", active + 1);

     

       109
       109
       +
               let drove = match drive_disk(needed, store).await {

     

       110
       110
       +
                   Ok((d, s)) => {

     

       111
       111
       +
                       store = s;

     

       112
       112
       +
                       d

     

       113
       113
       +
                   }

     

       114
       114
       +
                   Err(e) => {

     

       115
       115
       +
                       eprintln!("failed to drive disk: {e:?}. skipping...");

     

       116
       116
       +
                       store = DiskBuilder::new()

     

       117
       117
       +
                           .with_cache_size_mb(128)

     

       118
       118
       +
                           .open(file.clone())

     

       119
       119
       +
                           .await?;

     

       120
       120
       +
                       continue;

     

       121
       121
       +
                   }

     

       122
       122
       +
               };

     

       123
       123
       +
               n.fetch_add(drove, Ordering::Relaxed);

     

       124
       124
       +
               let were_active = disk_workers_active.fetch_sub(1, Ordering::AcqRel);

     

       125
       125
       +
               println!("<- disk workers active: {}", were_active - 1);

     

       126
       126
       +
           }

     

       127
       127
       +
           Ok(())

     

       128
       128
       +
       }

     

       129
       129
       +
       

     

       130
       130
       +
       

     

       131
       131
       +
       #[tokio::main]

     

       132
       132
       +
       async fn main() -> Result<()> {

     

       133
       133
       +
           env_logger::init();

     

       134
       134
       +
       

     

       135
       135
       +
           let Args { cars_folder, disk_folder, disk_workers, mem_workers } = Args::parse();

     

       136
       136
       +
       

     

       137
       137
       +
           let mut set = JoinSet::<Result<()>>::new();

     

       138
       138
       +
       

     

       139
       139
       +
       

     

       140
       140
       +
           let (cars_tx, cars_rx) = async_channel::bounded(2);

     

       141
       141
       +
           set.spawn(get_cars(cars_folder, cars_tx));

     

       142
       142
       +
       

     

       143
       143
       +
           let n: Arc<AtomicUsize> = Arc::new(0.into());

     

       144
       144
       +
           let disk_workers_active: Arc<AtomicUsize> = Arc::new(0.into());

     

       145
       145
       +
       

     

       146
       146
       +
           set.spawn({

     

       147
       147
       +
               let n = n.clone();

     

       148
       148
       +
               let mut interval = tokio::time::interval(std::time::Duration::from_secs(10));

     

       149
       149
       +
               async move {

     

       150
       150
       +
                   let mut last_n = n.load(Ordering::Relaxed);

     

       151
       151
       +
                   loop {

     

       152
       152
       +
                       interval.tick().await;

     

       153
       153
       +
                       let n = n.load(Ordering::Relaxed);

     

       154
       154
       +
                       let diff = n - last_n;

     

       155
       155
       +
                       println!("rate: {} rec/sec", diff / 10);

     

       156
       156
       +
                       if diff == 0 {

     

       157
       157
       +
                           println!("zero encountered, stopping rate calculation polling.");

     

       158
       158
       +
                           break Ok(());

     

       159
       159
       +
                       }

     

       160
       160
       +
                       last_n = n;

     

       161
       161
       +
                   }

     

       162
       162
       +
               }

     

       163
       163
       +
           });

     

       164
       164
       +
       

     

       165
       165
       +
       

     

       166
       166
       +
           let (needs_disk_tx, needs_disk_rx) = async_channel::bounded(disk_workers);

     

       167
       167
       +
       

     

       168
       168
       +
       

     

       169
       169
       +
           for _ in 0..mem_workers {

     

       170
       170
       +
               set.spawn(mem_worker(cars_rx.clone(), needs_disk_tx.clone(), n.clone()));

     

       171
       171
       +
           }

     

       172
       172
       +
           drop(cars_rx);

     

       173
       173
       +
           drop(needs_disk_tx);

     

       174
       174
       +
       

     

       175
       175
       +
           tokio::fs::create_dir_all(disk_folder.clone()).await?;

     

       176
       176
       +
           for id in 0..disk_workers {

     

       177
       177
       +
               set.spawn(disk_worker(

     

       178
       178
       +
                   id,

     

       179
       179
       +
                   needs_disk_rx.clone(),

     

       180
       180
       +
                   disk_folder.clone(),

     

       181
       181
       +
                   n.clone(),

     

       182
       182
       +
                   disk_workers_active.clone(),

     

       183
       183
       +
               ));

     

       184
       184
       +
           }

     

       185
       185
       +
           drop(needs_disk_rx);

     

       186
       186
       +
       

     

       187
       187
       +
           while let Some(res) = set.join_next().await {

     

       188
       188
       +
               println!("task from set joined: {res:?}");

     

       189
       189
       +
           }

     

       190
       190
       +
       

     

       191
       191
       +
           eprintln!("total records processed: {n:?}");

     

       192
       192
       +
       

     

       193
       193
       +
           Ok(())

     

       194
       194
       +
       }

+138

spacedust/src/bin/scrape_pds.rs

···

       1
       1
       +
       use tokio::io::AsyncWriteExt;

     

       2
       2
       +
       use clap::Parser;

     

       3
       3
       +
       use std::path::PathBuf;

     

       4
       4
       +
       use reqwest::Url;

     

       5
       5
       +
       use tokio::{sync::mpsc, time};

     

       6
       6
       +
       use serde::Deserialize;

     

       7
       7
       +
       

     

       8
       8
       +
       type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;

     

       9
       9
       +
       

     

       10
       10
       +
       use futures::StreamExt;

     

       11
       11
       +
       

     

       12
       12
       +
       #[derive(Debug, Parser)]

     

       13
       13
       +
       struct Args {

     

       14
       14
       +
           #[arg(long)]

     

       15
       15
       +
           pds: Url,

     

       16
       16
       +
           #[arg(long)]

     

       17
       17
       +
           throttle_ms: u64, // 100ms per pds?

     

       18
       18
       +
           #[arg(long)]

     

       19
       19
       +
           folder: PathBuf,

     

       20
       20
       +
       }

     

       21
       21
       +
       

     

       22
       22
       +
       async fn download_repo(

     

       23
       23
       +
           client: &reqwest::Client,

     

       24
       24
       +
           mut pds: Url,

     

       25
       25
       +
           did: String,

     

       26
       26
       +
           mut path: PathBuf,

     

       27
       27
       +
       ) -> Result<()> {

     

       28
       28
       +
           path.push(format!("{did}.car"));

     

       29
       29
       +
           let f = tokio::fs::File::create(path).await?;

     

       30
       30
       +
           let mut w = tokio::io::BufWriter::new(f);

     

       31
       31
       +
       

     

       32
       32
       +
           pds.set_path("/xrpc/com.atproto.sync.getRepo");

     

       33
       33
       +
           pds.set_query(Some(&format!("did={did}")));

     

       34
       34
       +
           let mut byte_stream = client

     

       35
       35
       +
               .get(pds)

     

       36
       36
       +
               .send()

     

       37
       37
       +
               .await?

     

       38
       38
       +
               .bytes_stream();

     

       39
       39
       +
       

     

       40
       40
       +
           while let Some(stuff) = byte_stream.next().await {

     

       41
       41
       +
               tokio::io::copy(&mut stuff?.as_ref(), &mut w).await?;

     

       42
       42
       +
           }

     

       43
       43
       +
           w.flush().await?;

     

       44
       44
       +
       

     

       45
       45
       +
           Ok(())

     

       46
       46
       +
       }

     

       47
       47
       +
       

     

       48
       48
       +
       

     

       49
       49
       +
       #[derive(Debug, Deserialize)]

     

       50
       50
       +
       struct RepoInfo {

     

       51
       51
       +
           did: String,

     

       52
       52
       +
           active: bool,

     

       53
       53
       +
       }

     

       54
       54
       +
       

     

       55
       55
       +
       #[derive(Debug, Deserialize)]

     

       56
       56
       +
       struct ListReposResponse {

     

       57
       57
       +
           cursor: Option<String>,

     

       58
       58
       +
           repos: Vec<RepoInfo>,

     

       59
       59
       +
       }

     

       60
       60
       +
       

     

       61
       61
       +
       fn get_pds_dids(client: reqwest::Client, mut pds: Url) -> mpsc::Receiver<String> {

     

       62
       62
       +
           let (tx, rx) = mpsc::channel(2);

     

       63
       63
       +
           tokio::task::spawn(async move {

     

       64
       64
       +
               pds.set_path("/xrpc/com.atproto.sync.listRepos");

     

       65
       65
       +
               let mut cursor = None;

     

       66
       66
       +
       

     

       67
       67
       +
               loop {

     

       68
       68
       +
                   if let Some(c) = cursor {

     

       69
       69
       +
                       pds.set_query(Some(&format!("cursor={c}")));

     

       70
       70
       +
                   }

     

       71
       71
       +
                   let res: ListReposResponse = client

     

       72
       72
       +
                       .get(pds.clone())

     

       73
       73
       +
                       .send()

     

       74
       74
       +
                       .await

     

       75
       75
       +
                       .expect("to send request")

     

       76
       76
       +
                       .error_for_status()

     

       77
       77
       +
                       .expect("to be ok")

     

       78
       78
       +
                       .json()

     

       79
       79
       +
                       .await

     

       80
       80
       +
                       .expect("json response");

     

       81
       81
       +
                   for repo in res.repos {

     

       82
       82
       +
                       if repo.active {

     

       83
       83
       +
                           tx.send(repo.did).await.expect("to be able to send on the channel");

     

       84
       84
       +
                       }

     

       85
       85
       +
                   }

     

       86
       86
       +
                   cursor = res.cursor;

     

       87
       87
       +
                   if cursor.is_none() {

     

       88
       88
       +
                       break;

     

       89
       89
       +
                   }

     

       90
       90
       +
               }

     

       91
       91
       +
       

     

       92
       92
       +
           });

     

       93
       93
       +
           rx

     

       94
       94
       +
       }

     

       95
       95
       +
       

     

       96
       96
       +
       

     

       97
       97
       +
       #[tokio::main]

     

       98
       98
       +
       async fn main() -> Result<()> {

     

       99
       99
       +
           env_logger::init();

     

       100
       100
       +
       

     

       101
       101
       +
           let Args { pds, throttle_ms, folder } = Args::parse();

     

       102
       102
       +
       

     

       103
       103
       +
           tokio::fs::create_dir_all(folder.clone()).await?;

     

       104
       104
       +
       

     

       105
       105
       +
           let client = reqwest::Client::builder()

     

       106
       106
       +
               .user_agent("microcosm/spacedust-testing")

     

       107
       107
       +
               .build()?;

     

       108
       108
       +
       

     

       109
       109
       +
           let mut dids = get_pds_dids(client.clone(), pds.clone());

     

       110
       110
       +
       

     

       111
       111
       +
           let mut interval = time::interval(time::Duration::from_millis(throttle_ms));

     

       112
       112
       +
           let mut oks = 0;

     

       113
       113
       +
           let mut single_fails = 0;

     

       114
       114
       +
           let mut double_fails = 0;

     

       115
       115
       +
       

     

       116
       116
       +
           while let Some(did) = dids.recv().await {

     

       117
       117
       +
               interval.tick().await;

     

       118
       118
       +
               println!("did: {did:?}");

     

       119
       119
       +
               if let Err(e) = download_repo(&client, pds.clone(), did.clone(), folder.clone()).await {

     

       120
       120
       +
                   single_fails += 1;

     

       121
       121
       +
                   eprintln!("failed to download repo for did: {did:?}: {e:?}. retrying in a moment...");

     

       122
       122
       +
                   tokio::time::sleep(time::Duration::from_secs(3)).await;

     

       123
       123
       +
                   interval.reset();

     

       124
       124
       +
                   if let Err(e) = download_repo(&client, pds.clone(), did.clone(), folder.clone()).await {

     

       125
       125
       +
                       double_fails += 1;

     

       126
       126
       +
                       eprintln!("failed again: {e:?}. moving on in a moment...");

     

       127
       127
       +
                       tokio::time::sleep(time::Duration::from_secs(1)).await;

     

       128
       128
       +
                       continue;

     

       129
       129
       +
                   }

     

       130
       130
       +
               }

     

       131
       131
       +
               oks += 1;

     

       132
       132
       +
               println!("  -> done. did: {did:?}");

     

       133
       133
       +
           }

     

       134
       134
       +
       

     

       135
       135
       +
           eprintln!("got {oks} repos. single fails: {single_fails}; doubles: {double_fails}.");

     

       136
       136
       +
       

     

       137
       137
       +
           Ok(())

     

       138
       138
       +
       }

spacedust/src/lib.rs

···

       3
       3
        
       pub mod error;

     

       4
       4
        
       pub mod removable_delay_queue;

     

       5
       5
        
       pub mod server;

     

       6
       6
       +
       pub mod storage;

     

       6
       7
        
       pub mod subscriber;

     

       7
       8
        
       

     

       8
       9
        
       use jetstream::events::CommitEvent;

spacedust/src/storage/car/drive.rs

This is a binary file and will not be displayed.

spacedust/src/storage/car/mod.rs

This is a binary file and will not be displayed.

spacedust/src/storage/car/walk.rs

This is a binary file and will not be displayed.

spacedust/src/storage/fjall/mod.rs

···

       1
       1
       +
       use crate::storage::Storage;

     

       2
       2
       +
       

     

       3
       3
       +
       struct FjallStorage {

     

       4
       4
       +
       }

     

       5
       5
       +
       

     

       6
       6
       +
       impl Storage for FjallStorage {

     

       7
       7
       +
           fn import_car() { todo!() }

     

       8
       8
       +
       }

spacedust/src/storage/mod.rs

···

       1
       1
       +
       pub mod car;

     

       2
       2
       +
       pub mod fjall;

     

       3
       3
       +
       

     

       4
       4
       +
       pub trait Storage {

     

       5
       5
       +
           fn import_car() {

     

       6
       6
       +
       

     

       7
       7
       +
           }

     

       8
       8
       +
       }