commit f957425593b22421aed5eba9988e0fe892643174 · nekomimi.pet/microcosm-rs

+60 -9

Cargo.lock

···

       129
       129
        
       checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"

     

       130
       130
        
       

     

       131
       131
        
       [[package]]

     

       132
       132
       +
       name = "arrayref"

     

       133
       133
       +
       version = "0.3.9"

     

       134
       134
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       135
       135
       +
       checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"

     

       136
       136
       +
       

     

       137
       137
       +
       [[package]]

     

       132
       138
        
       name = "arrayvec"

     

       133
       139
        
       version = "0.7.6"

     

       134
       140
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       822
       828
        
       checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"

     

       823
       829
        
       

     

       824
       830
        
       [[package]]

     

       831
       831
       +
       name = "blake3"

     

       832
       832
       +
       version = "1.8.2"

     

       833
       833
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       834
       834
       +
       checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0"

     

       835
       835
       +
       dependencies = [

     

       836
       836
       +
        "arrayref",

     

       837
       837
       +
        "arrayvec",

     

       838
       838
       +
        "cc",

     

       839
       839
       +
        "cfg-if",

     

       840
       840
       +
        "constant_time_eq",

     

       841
       841
       +
       ]

     

       842
       842
       +
       

     

       843
       843
       +
       [[package]]

     

       825
       844
        
       name = "block-buffer"

     

       826
       845
        
       version = "0.10.4"

     

       827
       846
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       923
       942
        
       ]

     

       924
       943
        
       

     

       925
       944
        
       [[package]]

     

       945
       945
       +
       name = "cbor4ii"

     

       946
       946
       +
       version = "1.2.0"

     

       947
       947
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       948
       948
       +
       checksum = "b28d2802395e3bccd95cc4ae984bff7444b6c1f5981da46a41360c42a2c7e2d9"

     

       949
       949
       +
       

     

       950
       950
       +
       [[package]]

     

       926
       951
        
       name = "cc"

     

       927
       952
        
       version = "1.2.18"

     

       928
       953
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1122
       1147
        
       version = "0.4.3"

     

       1123
       1148
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1124
       1149
        
       checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3"

     

       1150
       1150
       +
       

     

       1151
       1151
       +
       [[package]]

     

       1152
       1152
       +
       name = "constant_time_eq"

     

       1153
       1153
       +
       version = "0.3.1"

     

       1154
       1154
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1155
       1155
       +
       checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"

     

       1125
       1156
        
       

     

       1126
       1157
        
       [[package]]

     

       1127
       1158
        
       name = "constellation"

     
···

       1390
       1421
        
       ]

     

       1391
       1422
        
       

     

       1392
       1423
        
       [[package]]

     

       1424
       1424
       +
       name = "dasl"

     

       1425
       1425
       +
       version = "0.2.0"

     

       1426
       1426
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1427
       1427
       +
       checksum = "b59666035a4386b0fd272bd78da4cbc3ccb558941e97579ab00f0eb4639f2a49"

     

       1428
       1428
       +
       dependencies = [

     

       1429
       1429
       +
        "blake3",

     

       1430
       1430
       +
        "cbor4ii 1.2.0",

     

       1431
       1431
       +
        "data-encoding",

     

       1432
       1432
       +
        "data-encoding-macro",

     

       1433
       1433
       +
        "scopeguard",

     

       1434
       1434
       +
        "serde",

     

       1435
       1435
       +
        "serde_bytes",

     

       1436
       1436
       +
        "sha2",

     

       1437
       1437
       +
        "thiserror 2.0.17",

     

       1438
       1438
       +
       ]

     

       1439
       1439
       +
       

     

       1440
       1440
       +
       [[package]]

     

       1393
       1441
        
       name = "data-encoding"

     

       1394
       1394
       -
       version = "2.8.0"

     

       1442
       1442
       +
       version = "2.9.0"

     

       1395
       1443
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1396
       1396
       -
       checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010"

     

       1444
       1444
       +
       checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"

     

       1397
       1445
        
       

     

       1398
       1446
        
       [[package]]

     

       1399
       1447
        
       name = "data-encoding-macro"

     

       1400
       1400
       -
       version = "0.1.17"

     

       1448
       1448
       +
       version = "0.1.18"

     

       1401
       1449
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1402
       1402
       -
       checksum = "9f9724adfcf41f45bf652b3995837669d73c4d49a1b5ac1ff82905ac7d9b5558"

     

       1450
       1450
       +
       checksum = "47ce6c96ea0102f01122a185683611bd5ac8d99e62bc59dd12e6bda344ee673d"

     

       1403
       1451
        
       dependencies = [

     

       1404
       1452
        
        "data-encoding",

     

       1405
       1453
        
        "data-encoding-macro-internal",

     
···

       1407
       1455
        
       

     

       1408
       1456
        
       [[package]]

     

       1409
       1457
        
       name = "data-encoding-macro-internal"

     

       1410
       1410
       -
       version = "0.1.15"

     

       1458
       1458
       +
       version = "0.1.16"

     

       1411
       1459
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1412
       1412
       -
       checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f"

     

       1460
       1460
       +
       checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976"

     

       1413
       1461
        
       dependencies = [

     

       1414
       1462
        
        "data-encoding",

     

       1415
       1463
        
        "syn 2.0.106",

     
···

       3185
       3233
        
       version = "0.1.0"

     

       3186
       3234
        
       dependencies = [

     

       3187
       3235
        
        "anyhow",

     

       3236
       3236
       +
        "dasl",

     

       3188
       3237
        
        "fluent-uri",

     

       3189
       3238
        
        "nom",

     

       3239
       3239
       +
        "serde",

     

       3190
       3240
        
        "thiserror 2.0.17",

     

       3191
       3241
        
        "tinyjson",

     

       3192
       3242
        
       ]

     
···

       4642
       4692
        
       

     

       4643
       4693
        
       [[package]]

     

       4644
       4694
        
       name = "repo-stream"

     

       4645
       4645
       -
       version = "0.2.1"

     

       4695
       4695
       +
       version = "0.2.2"

     

       4646
       4696
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       4647
       4647
       -
       checksum = "727a78c392bd51b1af938e4383f2f6f46ae727cb38394136d1aebab0633faf8e"

     

       4697
       4697
       +
       checksum = "093b48e604c138949bf3d4a1a9bc1165feb1db28a73af0101c84eb703d279f43"

     

       4648
       4698
        
       dependencies = [

     

       4649
       4699
        
        "bincode 2.0.1",

     

       4650
       4700
        
        "futures",

     
···

       5168
       5218
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       5169
       5219
        
       checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778"

     

       5170
       5220
        
       dependencies = [

     

       5171
       5171
       -
        "cbor4ii",

     

       5221
       5221
       +
        "cbor4ii 0.2.14",

     

       5172
       5222
        
        "ipld-core",

     

       5173
       5223
        
        "scopeguard",

     

       5174
       5224
        
        "serde",

     
···

       5510
       5560
        
        "async-trait",

     

       5511
       5561
        
        "clap",

     

       5512
       5562
        
        "ctrlc",

     

       5563
       5563
       +
        "dasl",

     

       5513
       5564
        
        "dropshot",

     

       5514
       5565
        
        "env_logger",

     

       5515
       5566
        
        "fjall 3.0.0-pre.0",

links/Cargo.toml

···

       5
       5
        
       

     

       6
       6
        
       [dependencies]

     

       7
       7
        
       anyhow = "1.0.95"

     

       8
       8
       +
       dasl = "0.2.0"

     

       8
       9
        
       fluent-uri = "0.3.2"

     

       9
       10
        
       nom = "7.1.3"

     

       11
       11
       +
       serde = { version = "1.0.228", features = ["derive"] }

     

       10
       12
        
       thiserror = "2.0.9"

     

       11
       13
        
       tinyjson = "2.5.1"

+3 -2

links/src/lib.rs

···

       1
       1
        
       use fluent_uri::Uri;

     

       2
       2
       +
       use serde::{Deserialize, Serialize};

     

       2
       3
        
       

     

       3
       4
        
       pub mod at_uri;

     

       4
       5
        
       pub mod did;

     
···

       6
       7
        
       

     

       7
       8
        
       pub use record::collect_links;

     

       8
       9
        
       

     

       9
       9
       -
       #[derive(Debug, Clone, Ord, Eq, PartialOrd, PartialEq)]

     

       10
       10
       +
       #[derive(Debug, Clone, Ord, Eq, PartialOrd, PartialEq, Serialize, Deserialize)]

     

       10
       11
        
       pub enum Link {

     

       11
       12
        
           AtUri(String),

     

       12
       13
        
           Uri(String),

     
···

       59
       60
        
           }

     

       60
       61
        
       }

     

       61
       62
        
       

     

       62
       62
       -
       #[derive(Debug, PartialEq)]

     

       63
       63
       +
       #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]

     

       63
       64
        
       pub struct CollectedLink {

     

       64
       65
        
           pub path: String,

     

       65
       66
        
           pub target: Link,

+41

links/src/record.rs

···

       1
       1
       +
       use dasl::drisl::Value as DrislValue;

     

       1
       2
        
       use tinyjson::JsonValue;

     

       2
       3
        
       

     

       3
       4
        
       use crate::{parse_any_link, CollectedLink};

     
···

       36
       37
        
           }

     

       37
       38
        
       }

     

       38
       39
        
       

     

       40
       40
       +
       pub fn walk_drisl(path: &str, v: &DrislValue, found: &mut Vec<CollectedLink>) {

     

       41
       41
       +
           match v {

     

       42
       42
       +
               DrislValue::Map(o) => {

     

       43
       43
       +
                   for (key, child) in o {

     

       44
       44
       +
                       walk_drisl(&format!("{path}.{key}"), child, found)

     

       45
       45
       +
                   }

     

       46
       46
       +
               }

     

       47
       47
       +
               DrislValue::Array(a) => {

     

       48
       48
       +
                   for child in a {

     

       49
       49
       +
                       let child_p = match child {

     

       50
       50
       +
                           DrislValue::Map(o) => {

     

       51
       51
       +
                               if let Some(DrislValue::Text(t)) = o.get("$type") {

     

       52
       52
       +
                                   format!("{path}[{t}]")

     

       53
       53
       +
                               } else {

     

       54
       54
       +
                                   format!("{path}[]")

     

       55
       55
       +
                               }

     

       56
       56
       +
                           }

     

       57
       57
       +
                           _ => format!("{path}[]"),

     

       58
       58
       +
                       };

     

       59
       59
       +
                       walk_drisl(&child_p, child, found)

     

       60
       60
       +
                   }

     

       61
       61
       +
               }

     

       62
       62
       +
               DrislValue::Text(s) => {

     

       63
       63
       +
                   if let Some(link) = parse_any_link(s) {

     

       64
       64
       +
                       found.push(CollectedLink {

     

       65
       65
       +
                           path: path.to_string(),

     

       66
       66
       +
                           target: link,

     

       67
       67
       +
                       });

     

       68
       68
       +
                   }

     

       69
       69
       +
               }

     

       70
       70
       +
               _ => {}

     

       71
       71
       +
           }

     

       72
       72
       +
       }

     

       73
       73
       +
       

     

       39
       74
        
       pub fn collect_links(v: &JsonValue) -> Vec<CollectedLink> {

     

       40
       75
        
           let mut found = vec![];

     

       41
       76
        
           walk_record("", v, &mut found);

     

       77
       77
       +
           found

     

       78
       78
       +
       }

     

       79
       79
       +
       

     

       80
       80
       +
       pub fn collect_links_drisl(v: &DrislValue) -> Vec<CollectedLink> {

     

       81
       81
       +
           let mut found = vec![];

     

       82
       82
       +
           walk_drisl("", v, &mut found);

     

       42
       83
        
           found

     

       43
       84
        
       }

     

       44
       85

+2 -1

spacedust/Cargo.toml

···

       9
       9
        
       async-trait = "0.1.88"

     

       10
       10
        
       clap = { version = "4.5.40", features = ["derive"] }

     

       11
       11
        
       ctrlc = "3.4.7"

     

       12
       12
       +
       dasl = "0.2.0"

     

       12
       13
        
       dropshot = "0.16.2"

     

       13
       14
        
       env_logger = "0.11.8"

     

       14
       15
        
       fjall = "3.0.0-pre.0"

     
···

       21
       22
        
       metrics = "0.24.2"

     

       22
       23
        
       metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] }

     

       23
       24
        
       rand = "0.9.1"

     

       24
       24
       -
       repo-stream = "0.2.1"

     

       25
       25
       +
       repo-stream = "0.2.2"

     

       25
       26
        
       reqwest = { version = "0.12.24", features = ["json", "stream"] }

     

       26
       27
        
       schemars = "0.8.22"

     

       27
       28
        
       semver = "1.0.26"

+1 -4

spacedust/src/bin/import_car_file.rs

···

       1
       1
        
       use clap::Parser;

     

       2
       2
        
       use std::path::PathBuf;

     

       3
       3
       -
       use spacedust::storage::car;

     

       4
       3
        
       

     

       5
       4
        
       type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;

     

       6
       5
        
       

     
···

       16
       15
        
       

     

       17
       16
        
           let Args { file } = Args::parse();

     

       18
       17
        
       

     

       19
       19
       -
           let reader = tokio::fs::File::open(file).await?;

     

       20
       20
       -
       

     

       21
       21
       -
           car::import(reader).await?;

     

       18
       18
       +
           let _reader = tokio::fs::File::open(file).await?;

     

       22
       19
        
       

     

       23
       20
        
           Ok(())

     

       24
       21
        
       }

+108 -44

spacedust/src/bin/import_scraped.rs

···

       1
       1
        
       use clap::Parser;

     

       2
       2
       -
       use std::sync::{Arc, atomic::{AtomicUsize, Ordering}};

     

       2
       2
       +
       use links::CollectedLink;

     

       3
       3
       +
       use repo_stream::{

     

       4
       4
       +
           DiskBuilder, DiskStore, Driver, DriverBuilder, Processable, drive::DriverBuilderWithProcessor,

     

       5
       5
       +
           drive::NeedDisk,

     

       6
       6
       +
       };

     

       3
       7
        
       use std::path::PathBuf;

     

       4
       4
       -
       use tokio::{task::JoinSet, io::AsyncRead};

     

       5
       5
       -
       use repo_stream::{DriverBuilder, Driver, DiskBuilder, DiskStore, drive::NeedDisk};

     

       6
       6
       -
       

     

       8
       8
       +
       use std::sync::{

     

       9
       9
       +
           Arc,

     

       10
       10
       +
           atomic::{AtomicUsize, Ordering},

     

       11
       11
       +
       };

     

       12
       12
       +
       use tokio::{io::AsyncRead, task::JoinSet};

     

       7
       13
        
       

     

       8
       14
        
       type Result<T> = anyhow::Result<T>; //std::result::Result<T, Box<dyn std::error::Error>>;

     

       9
       15
        
       

     

       16
       16
       +
       #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]

     

       17
       17
       +
       struct CollectedProcessed(CollectedLink);

     

       18
       18
       +
       

     

       19
       19
       +
       impl Processable for CollectedProcessed {

     

       20
       20
       +
           fn get_size(&self) -> usize {

     

       21
       21
       +
               self.0.path.capacity() + self.0.target.as_str().len()

     

       22
       22
       +
           }

     

       23
       23
       +
       }

     

       24
       24
       +
       

     

       25
       25
       +
       #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]

     

       26
       26
       +
       struct ErrString(String);

     

       27
       27
       +
       

     

       28
       28
       +
       impl Processable for ErrString {

     

       29
       29
       +
           fn get_size(&self) -> usize {

     

       30
       30
       +
               self.0.capacity()

     

       31
       31
       +
           }

     

       32
       32
       +
       }

     

       33
       33
       +
       

     

       34
       34
       +
       type Processed = std::result::Result<Vec<CollectedProcessed>, ErrString>;

     

       35
       35
       +
       

     

       36
       36
       +
       /// hacky for now: put errors in strings 🤷‍♀️

     

       37
       37
       +
       fn process(block: Vec<u8>) -> Processed {

     

       38
       38
       +
           let value: dasl::drisl::Value = dasl::drisl::from_slice(&block)

     

       39
       39
       +
               .map_err(|e| ErrString(format!("failed to parse block with drisl: {e:?}")))?;

     

       40
       40
       +
           let links = links::record::collect_links_drisl(&value)

     

       41
       41
       +
               .into_iter()

     

       42
       42
       +
               .map(CollectedProcessed)

     

       43
       43
       +
               .collect();

     

       44
       44
       +
           Ok(links)

     

       45
       45
       +
       }

     

       10
       46
        
       

     

       11
       47
        
       #[derive(Debug, Parser)]

     

       12
       48
        
       struct Args {

     
···

       20
       56
        
           disk_folder: PathBuf,

     

       21
       57
        
       }

     

       22
       58
        
       

     

       23
       23
       -
       async fn get_cars(cars_folder: PathBuf, tx: async_channel::Sender<tokio::io::BufReader<tokio::fs::File>>) -> Result<()> {

     

       59
       59
       +
       async fn get_cars(

     

       60
       60
       +
           cars_folder: PathBuf,

     

       61
       61
       +
           tx: async_channel::Sender<tokio::io::BufReader<tokio::fs::File>>,

     

       62
       62
       +
       ) -> Result<()> {

     

       24
       63
        
           let mut dir = tokio::fs::read_dir(cars_folder).await?;

     

       25
       64
        
           while let Some(entry) = dir.next_entry().await? {

     

       26
       65
        
               if !entry.file_type().await?.is_file() {

     
···

       35
       74
        
       

     

       36
       75
        
       async fn drive_mem<R: AsyncRead + Unpin + Send + Sync + 'static>(

     

       37
       76
        
           f: R,

     

       38
       38
       -
           disk_tx: &async_channel::Sender<NeedDisk<R, usize>>,

     

       39
       39
       -
       ) -> Result<Option<usize>> {

     

       77
       77
       +
           builder: &DriverBuilderWithProcessor<Processed>,

     

       78
       78
       +
           disk_tx: &async_channel::Sender<NeedDisk<R, Processed>>,

     

       79
       79
       +
       ) -> Result<Option<(usize, usize)>> {

     

       40
       80
        
           let mut n = 0;

     

       41
       41
       -
           match DriverBuilder::new()

     

       42
       42
       -
               .with_block_processor(|_| 0_usize) // don't care just counting records

     

       43
       43
       -
               .with_mem_limit_mb(32)

     

       44
       44
       -
               .load_car(f)

     

       45
       45
       -
               .await?

     

       46
       46
       -
           {

     

       81
       81
       +
           let mut n_records = 0;

     

       82
       82
       +
           match builder.load_car(f).await? {

     

       47
       83
        
               Driver::Memory(_commit, mut driver) => {

     

       48
       84
        
                   while let Some(chunk) = driver.next_chunk(512).await? {

     

       49
       49
       -
                       n += chunk.len();

     

       85
       85
       +
                       n_records += chunk.len();

     

       86
       86
       +
                       for (_key, links) in chunk {

     

       87
       87
       +
                           match links {

     

       88
       88
       +
                               Ok(links) => n += links.len(),

     

       89
       89
       +
                               Err(e) => eprintln!("wat: {e:?}"),

     

       90
       90
       +
                           }

     

       91
       91
       +
                       }

     

       50
       92
        
                   }

     

       51
       51
       -
                   Ok(Some(n))

     

       93
       93
       +
                   Ok(Some((n, n_records)))

     

       52
       94
        
               }

     

       53
       95
        
               Driver::Disk(need_disk) => {

     

       54
       96
        
                   disk_tx.send(need_disk).await?;

     
···

       59
       101
        
       

     

       60
       102
        
       async fn mem_worker<R: AsyncRead + Unpin + Send + Sync + 'static>(

     

       61
       103
        
           car_rx: async_channel::Receiver<R>,

     

       62
       62
       -
           disk_tx: async_channel::Sender<NeedDisk<R, usize>>,

     

       104
       104
       +
           disk_tx: async_channel::Sender<NeedDisk<R, Processed>>,

     

       63
       105
        
           n: Arc<AtomicUsize>,

     

       106
       106
       +
           n_records: Arc<AtomicUsize>,

     

       64
       107
        
       ) -> Result<()> {

     

       108
       108
       +
           let builder = DriverBuilder::new()

     

       109
       109
       +
               .with_block_processor(process) // don't care just counting records

     

       110
       110
       +
               .with_mem_limit_mb(128);

     

       65
       111
        
           while let Ok(f) = car_rx.recv().await {

     

       66
       66
       -
               let driven = match drive_mem(f, &disk_tx).await {

     

       112
       112
       +
               let driven = match drive_mem(f, &builder, &disk_tx).await {

     

       67
       113
        
                   Ok(d) => d,

     

       68
       114
        
                   Err(e) => {

     

       69
       115
        
                       eprintln!("failed to drive mem: {e:?}. skipping...");

     

       70
       116
        
                       continue;

     

       71
       117
        
                   }

     

       72
       118
        
               };

     

       73
       73
       -
               if let Some(drove) = driven {

     

       119
       119
       +
               if let Some((drove, recs)) = driven {

     

       74
       120
        
                   n.fetch_add(drove, Ordering::Relaxed);

     

       121
       121
       +
                   n_records.fetch_add(recs, Ordering::Relaxed);

     

       75
       122
        
               }

     

       76
       123
        
           }

     

       77
       124
        
           Ok(())

     

       78
       125
        
       }

     

       79
       126
        
       

     

       80
       127
        
       async fn drive_disk<R: AsyncRead + Unpin>(

     

       81
       81
       -
           needed: NeedDisk<R, usize>,

     

       128
       128
       +
           needed: NeedDisk<R, Processed>,

     

       82
       129
        
           store: DiskStore,

     

       83
       83
       -
       ) -> Result<(usize, DiskStore)> {

     

       130
       130
       +
       ) -> Result<(usize, usize, DiskStore)> {

     

       84
       131
        
           let (_commit, mut driver) = needed.finish_loading(store).await?;

     

       85
       132
        
           let mut n = 0;

     

       133
       133
       +
           let mut n_records = 0;

     

       86
       134
        
           while let Some(chunk) = driver.next_chunk(512).await? {

     

       87
       87
       -
               n += chunk.len();

     

       135
       135
       +
               n_records += chunk.len();

     

       136
       136
       +
               for (_key, links) in chunk {

     

       137
       137
       +
                   match links {

     

       138
       138
       +
                       Ok(links) => n += links.len(),

     

       139
       139
       +
                       Err(e) => eprintln!("wat: {e:?}"),

     

       140
       140
       +
                   }

     

       141
       141
       +
               }

     

       88
       142
        
           }

     

       89
       143
        
           let store = driver.reset_store().await?;

     

       90
       90
       -
           Ok((n, store))

     

       144
       144
       +
           Ok((n, n_records, store))

     

       91
       145
        
       }

     

       92
       146
        
       

     

       93
       147
        
       async fn disk_worker<R: AsyncRead + Unpin>(

     

       94
       148
        
           worker_id: usize,

     

       95
       95
       -
           disk_rx: async_channel::Receiver<NeedDisk<R, usize>>,

     

       149
       149
       +
           disk_rx: async_channel::Receiver<NeedDisk<R, Processed>>,

     

       96
       150
        
           folder: PathBuf,

     

       97
       151
        
           n: Arc<AtomicUsize>,

     

       152
       152
       +
           n_records: Arc<AtomicUsize>,

     

       98
       153
        
           disk_workers_active: Arc<AtomicUsize>,

     

       99
       154
        
       ) -> Result<()> {

     

       100
       155
        
           let mut file = folder;

     

       101
       156
        
           file.push(format!("disk-worker-{worker_id}.sqlite"));

     

       102
       102
       -
           let mut store = DiskBuilder::new()

     

       103
       103
       -
               .with_cache_size_mb(128)

     

       104
       104
       -
               .open(file.clone())

     

       105
       105
       -
               .await?;

     

       157
       157
       +
           let builder = DiskBuilder::new().with_cache_size_mb(128);

     

       158
       158
       +
           let mut store = builder.open(file.clone()).await?;

     

       106
       159
        
           while let Ok(needed) = disk_rx.recv().await {

     

       107
       160
        
               let active = disk_workers_active.fetch_add(1, Ordering::AcqRel);

     

       108
       161
        
               println!("-> disk workers active: {}", active + 1);

     

       109
       109
       -
               let drove = match drive_disk(needed, store).await {

     

       110
       110
       -
                   Ok((d, s)) => {

     

       162
       162
       +
               let (drove, records) = match drive_disk(needed, store).await {

     

       163
       163
       +
                   Ok((d, r, s)) => {

     

       111
       164
        
                       store = s;

     

       112
       112
       -
                       d

     

       165
       165
       +
                       (d, r)

     

       113
       166
        
                   }

     

       114
       167
        
                   Err(e) => {

     

       115
       168
        
                       eprintln!("failed to drive disk: {e:?}. skipping...");

     

       116
       116
       -
                       store = DiskBuilder::new()

     

       117
       117
       -
                           .with_cache_size_mb(128)

     

       118
       118
       -
                           .open(file.clone())

     

       119
       119
       -
                           .await?;

     

       169
       169
       +
                       store = builder.open(file.clone()).await?;

     

       120
       170
        
                       continue;

     

       121
       171
        
                   }

     

       122
       172
        
               };

     

       123
       173
        
               n.fetch_add(drove, Ordering::Relaxed);

     

       174
       174
       +
               n_records.fetch_add(records, Ordering::Relaxed);

     

       124
       175
        
               let were_active = disk_workers_active.fetch_sub(1, Ordering::AcqRel);

     

       125
       176
        
               println!("<- disk workers active: {}", were_active - 1);

     

       126
       177
        
           }

     

       127
       178
        
           Ok(())

     

       128
       179
        
       }

     

       129
       180
        
       

     

       130
       130
       -
       

     

       131
       181
        
       #[tokio::main]

     

       132
       182
        
       async fn main() -> Result<()> {

     

       133
       183
        
           env_logger::init();

     

       134
       184
        
       

     

       135
       135
       -
           let Args { cars_folder, disk_folder, disk_workers, mem_workers } = Args::parse();

     

       185
       185
       +
           let Args {

     

       186
       186
       +
               cars_folder,

     

       187
       187
       +
               disk_folder,

     

       188
       188
       +
               disk_workers,

     

       189
       189
       +
               mem_workers,

     

       190
       190
       +
           } = Args::parse();

     

       136
       191
        
       

     

       137
       192
        
           let mut set = JoinSet::<Result<()>>::new();

     

       138
       138
       -
       

     

       139
       193
        
       

     

       140
       194
        
           let (cars_tx, cars_rx) = async_channel::bounded(2);

     

       141
       195
        
           set.spawn(get_cars(cars_folder, cars_tx));

     

       142
       196
        
       

     

       143
       197
        
           let n: Arc<AtomicUsize> = Arc::new(0.into());

     

       198
       198
       +
           let n_records: Arc<AtomicUsize> = Arc::new(0.into());

     

       144
       199
        
           let disk_workers_active: Arc<AtomicUsize> = Arc::new(0.into());

     

       145
       200
        
       

     

       146
       201
        
           set.spawn({

     

       147
       202
        
               let n = n.clone();

     

       203
       203
       +
               let n_records = n_records.clone();

     

       148
       204
        
               let mut interval = tokio::time::interval(std::time::Duration::from_secs(10));

     

       149
       205
        
               async move {

     

       150
       206
        
                   let mut last_n = n.load(Ordering::Relaxed);

     

       207
       207
       +
                   let mut last_n_records = n.load(Ordering::Relaxed);

     

       151
       208
        
                   loop {

     

       152
       209
        
                       interval.tick().await;

     

       153
       210
        
                       let n = n.load(Ordering::Relaxed);

     

       154
       154
       -
                       let diff = n - last_n;

     

       155
       155
       -
                       println!("rate: {} rec/sec", diff / 10);

     

       156
       156
       -
                       if diff == 0 {

     

       211
       211
       +
                       let n_records = n_records.load(Ordering::Relaxed);

     

       212
       212
       +
                       let diff_n = n - last_n;

     

       213
       213
       +
                       let diff_records = n_records - last_n_records;

     

       214
       214
       +
                       println!("rate: {} rec/sec; {} n/sec", diff_records / 10, diff_n / 10);

     

       215
       215
       +
                       if n_records > 0 && diff_records == 0 {

     

       157
       216
        
                           println!("zero encountered, stopping rate calculation polling.");

     

       158
       217
        
                           break Ok(());

     

       159
       218
        
                       }

     

       160
       219
        
                       last_n = n;

     

       220
       220
       +
                       last_n_records = n_records;

     

       161
       221
        
                   }

     

       162
       222
        
               }

     

       163
       223
        
           });

     

       164
       164
       -
       

     

       165
       224
        
       

     

       166
       225
        
           let (needs_disk_tx, needs_disk_rx) = async_channel::bounded(disk_workers);

     

       167
       226
        
       

     

       168
       168
       -
       

     

       169
       227
        
           for _ in 0..mem_workers {

     

       170
       170
       -
               set.spawn(mem_worker(cars_rx.clone(), needs_disk_tx.clone(), n.clone()));

     

       228
       228
       +
               set.spawn(mem_worker(

     

       229
       229
       +
                   cars_rx.clone(),

     

       230
       230
       +
                   needs_disk_tx.clone(),

     

       231
       231
       +
                   n.clone(),

     

       232
       232
       +
                   n_records.clone(),

     

       233
       233
       +
               ));

     

       171
       234
        
           }

     

       172
       235
        
           drop(cars_rx);

     

       173
       236
        
           drop(needs_disk_tx);

     
···

       179
       242
        
                   needs_disk_rx.clone(),

     

       180
       243
        
                   disk_folder.clone(),

     

       181
       244
        
                   n.clone(),

     

       245
       245
       +
                   n_records.clone(),

     

       182
       246
        
                   disk_workers_active.clone(),

     

       183
       247
        
               ));

     

       184
       248
        
           }

     
···

       188
       252
        
               println!("task from set joined: {res:?}");

     

       189
       253
        
           }

     

       190
       254
        
       

     

       191
       191
       -
           eprintln!("total records processed: {n:?}");

     

       255
       255
       +
           eprintln!("total records processed: {n_records:?}; total n: {n:?}");

     

       192
       256
        
       

     

       193
       257
        
           Ok(())

     

       194
       258
        
       }

+12 -13

spacedust/src/bin/scrape_pds.rs

···

       1
       1
       -
       use tokio::io::AsyncWriteExt;

     

       2
       1
        
       use clap::Parser;

     

       3
       3
       -
       use std::path::PathBuf;

     

       4
       2
        
       use reqwest::Url;

     

       5
       5
       -
       use tokio::{sync::mpsc, time};

     

       6
       3
        
       use serde::Deserialize;

     

       4
       4
       +
       use std::path::PathBuf;

     

       5
       5
       +
       use tokio::io::AsyncWriteExt;

     

       6
       6
       +
       use tokio::{sync::mpsc, time};

     

       7
       7
        
       

     

       8
       8
        
       type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;

     

       9
       9
        
       

     
···

       31
       31
        
       

     

       32
       32
        
           pds.set_path("/xrpc/com.atproto.sync.getRepo");

     

       33
       33
        
           pds.set_query(Some(&format!("did={did}")));

     

       34
       34
       -
           let mut byte_stream = client

     

       35
       35
       -
               .get(pds)

     

       36
       36
       -
               .send()

     

       37
       37
       -
               .await?

     

       38
       38
       -
               .bytes_stream();

     

       34
       34
       +
           let mut byte_stream = client.get(pds).send().await?.bytes_stream();

     

       39
       35
        
       

     

       40
       36
        
           while let Some(stuff) = byte_stream.next().await {

     

       41
       37
        
               tokio::io::copy(&mut stuff?.as_ref(), &mut w).await?;

     
···

       44
       40
        
       

     

       45
       41
        
           Ok(())

     

       46
       42
        
       }

     

       47
       47
       -
       

     

       48
       43
        
       

     

       49
       44
        
       #[derive(Debug, Deserialize)]

     

       50
       45
        
       struct RepoInfo {

     
···

       80
       75
        
                       .expect("json response");

     

       81
       76
        
                   for repo in res.repos {

     

       82
       77
        
                       if repo.active {

     

       83
       83
       -
                           tx.send(repo.did).await.expect("to be able to send on the channel");

     

       78
       78
       +
                           tx.send(repo.did)

     

       79
       79
       +
                               .await

     

       80
       80
       +
                               .expect("to be able to send on the channel");

     

       84
       81
        
                       }

     

       85
       82
        
                   }

     

       86
       83
        
                   cursor = res.cursor;

     
···

       88
       85
        
                       break;

     

       89
       86
        
                   }

     

       90
       87
        
               }

     

       91
       91
       -
       

     

       92
       88
        
           });

     

       93
       89
        
           rx

     

       94
       90
        
       }

     

       95
       95
       -
       

     

       96
       91
        
       

     

       97
       92
        
       #[tokio::main]

     

       98
       93
        
       async fn main() -> Result<()> {

     

       99
       94
        
           env_logger::init();

     

       100
       95
        
       

     

       101
       101
       -
           let Args { pds, throttle_ms, folder } = Args::parse();

     

       96
       96
       +
           let Args {

     

       97
       97
       +
               pds,

     

       98
       98
       +
               throttle_ms,

     

       99
       99
       +
               folder,

     

       100
       100
       +
           } = Args::parse();

     

       102
       101
        
       

     

       103
       102
        
           tokio::fs::create_dir_all(folder.clone()).await?;

     

       104
       103

spacedust/src/storage/car/mod.rs

+4 -3

spacedust/src/storage/fjall/mod.rs

···

       1
       1
        
       use crate::storage::Storage;

     

       2
       2
        
       

     

       3
       3
       -
       struct FjallStorage {

     

       4
       4
       -
       }

     

       3
       3
       +
       pub struct FjallStorage {}

     

       5
       4
        
       

     

       6
       5
        
       impl Storage for FjallStorage {

     

       7
       7
       -
           fn import_car() { todo!() }

     

       6
       6
       +
           fn import_car() {

     

       7
       7
       +
               todo!()

     

       8
       8
       +
           }

     

       8
       9
        
       }

+1 -3

spacedust/src/storage/mod.rs

···

       2
       2
        
       pub mod fjall;

     

       3
       3
        
       

     

       4
       4
        
       pub trait Storage {

     

       5
       5
       -
           fn import_car() {

     

       6
       6
       -
       

     

       7
       7
       -
           }

     

       5
       5
       +
           fn import_car() {}

     

       8
       6
        
       }