commit e798b332ddd8be3ef1a2120e6b3271418685e611 · nekomimi.pet/microcosm-rs

+7 -2

ufos/src/consumer.rs

···

       79
        
               batch_sender: Sender<LimitedBatch>,

     

       80
        
               sketch_secret: SketchSecretPrefix,

     

       81
        
           ) -> Self {

     

       82
       -
               let mut rate_limit = tokio::time::interval(std::time::Duration::from_millis(5));

     

       83
        
               rate_limit.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);

     

       84
        
               Self {

     

       85
        
                   jetstream_receiver,

     
···

       93
        
           pub async fn run(&mut self) -> anyhow::Result<()> {

     

       94
        
               // TODO: report errors *from here* probably, since this gets shipped off into a spawned task that might just vanish

     

       95
        
               loop {

     

       96
       -
                   match timeout(Duration::from_millis(9_000), self.jetstream_receiver.recv()).await {

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       97
        
                       Err(_elapsed) => self.no_events_step().await?,

     

       98
        
                       Ok(Some(event)) => self.handle_event(event).await?,

     

       99
        
                       Ok(None) => anyhow::bail!("channel closed"),

···

       79
        
               batch_sender: Sender<LimitedBatch>,

     

       80
        
               sketch_secret: SketchSecretPrefix,

     

       81
        
           ) -> Self {

     

       82
       +
               let mut rate_limit = tokio::time::interval(std::time::Duration::from_millis(3));

     

       83
        
               rate_limit.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);

     

       84
        
               Self {

     

       85
        
                   jetstream_receiver,

     
···

       93
        
           pub async fn run(&mut self) -> anyhow::Result<()> {

     

       94
        
               // TODO: report errors *from here* probably, since this gets shipped off into a spawned task that might just vanish

     

       95
        
               loop {

     

       96
       +
                   match timeout(

     

       97
       +
                       Duration::from_millis(30_000),

     

       98
       +
                       self.jetstream_receiver.recv(),

     

       99
       +
                   )

     

       100
       +
                   .await

     

       101
       +
                   {

     

       102
        
                       Err(_elapsed) => self.no_events_step().await?,

     

       103
        
                       Ok(Some(event)) => self.handle_event(event).await?,

     

       104
        
                       Ok(None) => anyhow::bail!("channel closed"),

+30 -4

ufos/src/file_consumer.rs

···

       1
        
       use crate::consumer::{Batcher, LimitedBatch, BATCH_QUEUE_SIZE};

     

       2
        
       use crate::store_types::SketchSecretPrefix;

     

       0
        
       
     

       3
        
       use anyhow::Result;

     

       4
        
       use jetstream::{error::JetstreamEventError, events::JetstreamEvent};

     

       5
        
       use std::path::PathBuf;

     
···

       9
        
           sync::mpsc::{channel, Receiver, Sender},

     

       10
        
       };

     

       11
        
       

     

       12
       -
       async fn read_jsonl(f: File, sender: Sender<JetstreamEvent>) -> Result<()> {

     

       13
        
           let mut lines = BufReader::new(f).lines();

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       14
        
           while let Some(line) = lines.next_line().await? {

     

       15
        
               match serde_json::from_str::<JetstreamEvent>(&line) {

     

       16
        
                   Ok(event) => match sender.send(event).await {

     
···

       35
        
       pub async fn consume(

     

       36
        
           p: PathBuf,

     

       37
        
           sketch_secret: SketchSecretPrefix,

     

       0
        
       
     

       38
        
       ) -> Result<Receiver<LimitedBatch>> {

     

       39
        
           let f = File::open(p).await?;

     

       40
        
           let (jsonl_sender, jsonl_receiver) = channel::<JetstreamEvent>(16);

     

       41
        
           let (batch_sender, batch_reciever) = channel::<LimitedBatch>(BATCH_QUEUE_SIZE);

     

       42
        
           let mut batcher = Batcher::new(jsonl_receiver, batch_sender, sketch_secret);

     

       43
        
           tokio::task::spawn(async move {

     

       44
       -
               let r = read_jsonl(f, jsonl_sender).await;

     

       45
       -
               log::info!("read_jsonl finished: {r:?}");

     

       46
        
           });

     

       47
        
           tokio::task::spawn(async move {

     

       48
        
               let r = batcher.run().await;

     

       49
       -
               log::info!("batcher finished: {r:?}");

     

       50
        
           });

     

       51
        
           Ok(batch_reciever)

     

       52
        
       }

···

       1
        
       use crate::consumer::{Batcher, LimitedBatch, BATCH_QUEUE_SIZE};

     

       2
        
       use crate::store_types::SketchSecretPrefix;

     

       3
       +
       use crate::Cursor;

     

       4
        
       use anyhow::Result;

     

       5
        
       use jetstream::{error::JetstreamEventError, events::JetstreamEvent};

     

       6
        
       use std::path::PathBuf;

     
···

       10
        
           sync::mpsc::{channel, Receiver, Sender},

     

       11
        
       };

     

       12
        
       

     

       13
       +
       async fn read_jsonl(f: File, sender: Sender<JetstreamEvent>, cursor: Option<Cursor>) -> Result<()> {

     

       14
        
           let mut lines = BufReader::new(f).lines();

     

       15
       +
           if let Some(db_cursor) = cursor {

     

       16
       +
               log::info!("jsonl fixture: skipping events before cursor {db_cursor:?}");

     

       17
       +
               let mut bad_lines = 0;

     

       18
       +
               let mut skipped = 0;

     

       19
       +
               while let Some(line) = lines.next_line().await? {

     

       20
       +
                   let Ok(event) = serde_json::from_str::<JetstreamEvent>(&line) else {

     

       21
       +
                       bad_lines += 1;

     

       22
       +
                       continue;

     

       23
       +
                   };

     

       24
       +
                   if event.cursor < db_cursor {

     

       25
       +
                       skipped += 1;

     

       26
       +
                       continue;

     

       27
       +
                   }

     

       28
       +
                   if event.cursor == db_cursor {

     

       29
       +
                       log::info!("jsonl fixture: found existing db cursor! skipped {skipped} old events and failed parsing {bad_lines} lines");

     

       30
       +
                       break;

     

       31
       +
                   }

     

       32
       +
                   anyhow::bail!("jsonl fixture: did not find existing db cursor, found event cursor {:?} which is newer. bailing.", event.cursor);

     

       33
       +
               }

     

       34
       +
           } else {

     

       35
       +
               log::info!("jsonl fixture: no cursor provided, sending every event");

     

       36
       +
           }

     

       37
       +
       

     

       38
       +
           log::info!("jsonl fixture: now sending events");

     

       39
        
           while let Some(line) = lines.next_line().await? {

     

       40
        
               match serde_json::from_str::<JetstreamEvent>(&line) {

     

       41
        
                   Ok(event) => match sender.send(event).await {

     
···

       60
        
       pub async fn consume(

     

       61
        
           p: PathBuf,

     

       62
        
           sketch_secret: SketchSecretPrefix,

     

       63
       +
           cursor: Option<Cursor>,

     

       64
        
       ) -> Result<Receiver<LimitedBatch>> {

     

       65
        
           let f = File::open(p).await?;

     

       66
        
           let (jsonl_sender, jsonl_receiver) = channel::<JetstreamEvent>(16);

     

       67
        
           let (batch_sender, batch_reciever) = channel::<LimitedBatch>(BATCH_QUEUE_SIZE);

     

       68
        
           let mut batcher = Batcher::new(jsonl_receiver, batch_sender, sketch_secret);

     

       69
        
           tokio::task::spawn(async move {

     

       70
       +
               let r = read_jsonl(f, jsonl_sender, cursor).await;

     

       71
       +
               log::warn!("read_jsonl finished: {r:?}");

     

       72
        
           });

     

       73
        
           tokio::task::spawn(async move {

     

       74
        
               let r = batcher.run().await;

     

       75
       +
               log::warn!("batcher finished: {r:?}");

     

       76
        
           });

     

       77
        
           Ok(batch_reciever)

     

       78
        
       }

+4 -2

ufos/src/main.rs

···

       130
        
       

     

       131
        
           let batches = if jetstream_fixture {

     

       132
        
               log::info!("starting with jestream file fixture: {jetstream:?}");

     

       133
       -
               file_consumer::consume(jetstream.into(), sketch_secret).await?

     

       134
        
           } else {

     

       135
        
               log::info!(

     

       136
        
                   "starting consumer with cursor: {cursor:?} from {:?} ago",

     
···

       163
        
           let mut last_at = std::time::SystemTime::now();

     

       164
        
           let mut last_cursor = None;

     

       165
        
           let mut last_rollup = None;

     

       0
        
       
     

       0
        
       
     

       166
        
           loop {

     

       167
       -
               tokio::time::sleep(std::time::Duration::from_secs_f64(4.)).await;

     

       168
        
               match read_store.get_consumer_info().await {

     

       169
        
                   Err(e) => log::warn!("failed to get jetstream consumer info: {e:?}"),

     

       170
        
                   Ok(ConsumerInfo::Jetstream {

···

       130
        
       

     

       131
        
           let batches = if jetstream_fixture {

     

       132
        
               log::info!("starting with jestream file fixture: {jetstream:?}");

     

       133
       +
               file_consumer::consume(jetstream.into(), sketch_secret, cursor).await?

     

       134
        
           } else {

     

       135
        
               log::info!(

     

       136
        
                   "starting consumer with cursor: {cursor:?} from {:?} ago",

     
···

       163
        
           let mut last_at = std::time::SystemTime::now();

     

       164
        
           let mut last_cursor = None;

     

       165
        
           let mut last_rollup = None;

     

       166
       +
           let mut interval = tokio::time::interval(std::time::Duration::from_secs(4));

     

       167
       +
           interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);

     

       168
        
           loop {

     

       169
       +
               interval.tick().await;

     

       170
        
               match read_store.get_consumer_info().await {

     

       171
        
                   Err(e) => log::warn!("failed to get jetstream consumer info: {e:?}"),

     

       172
        
                   Ok(ConsumerInfo::Jetstream {

+7 -7

ufos/src/storage_fjall.rs

···

       1241
        
               let mut ended_early = false;

     

       1242
        
               let mut batch = self.keyspace.batch();

     

       1243
        
               for (i, kv) in self.feeds.range(live_range).rev().enumerate() {

     

       1244
       -
                   if !full_scan && i > 1_000_000 {

     

       1245
       -
                       log::info!("stopping collection trim early: already scanned 1M elements");

     

       1246
        
                       ended_early = true;

     

       1247
        
                       break;

     

       1248
        
                   }

     
···

       1336
        
               let mut dirty_nsids = HashSet::new();

     

       1337
        
       

     

       1338
        
               let mut rollup =

     

       1339
       -
                   tokio::time::interval(Duration::from_millis(if backfill { 1 } else { 81 }));

     

       1340
       -
               rollup.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);

     

       1341
        
       

     

       1342
        
               let mut trim =

     

       1343
       -
                   tokio::time::interval(Duration::from_millis(if backfill { 3_000 } else { 6_000 }));

     

       1344
        
               trim.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);

     

       1345
        
       

     

       1346
        
               loop {

     
···

       1362
        
                               let (danglers, deleted) = self.0.trim_collection(collection, 512, false).inspect_err(|e| log::error!("trim error: {e:?}"))?;

     

       1363
        
                               total_danglers += danglers;

     

       1364
        
                               total_deleted += deleted;

     

       1365
       -
                               if total_deleted > 1_000_000 {

     

       1366
       -
                                   log::info!("trim stopped early, more than 1M records already deleted.");

     

       1367
        
                                   break;

     

       1368
        
                               }

     

       1369
        
                           }

···

       1241
        
               let mut ended_early = false;

     

       1242
        
               let mut batch = self.keyspace.batch();

     

       1243
        
               for (i, kv) in self.feeds.range(live_range).rev().enumerate() {

     

       1244
       +
                   if !full_scan && i > 10_000_000 {

     

       1245
       +
                       log::info!("stopping collection trim early: already scanned 10M elements");

     

       1246
        
                       ended_early = true;

     

       1247
        
                       break;

     

       1248
        
                   }

     
···

       1336
        
               let mut dirty_nsids = HashSet::new();

     

       1337
        
       

     

       1338
        
               let mut rollup =

     

       1339
       +
                   tokio::time::interval(Duration::from_micros(if backfill { 100 } else { 81_000 }));

     

       1340
       +
               rollup.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);

     

       1341
        
       

     

       1342
        
               let mut trim =

     

       1343
       +
                   tokio::time::interval(Duration::from_millis(if backfill { 500 } else { 6_000 }));

     

       1344
        
               trim.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);

     

       1345
        
       

     

       1346
        
               loop {

     
···

       1362
        
                               let (danglers, deleted) = self.0.trim_collection(collection, 512, false).inspect_err(|e| log::error!("trim error: {e:?}"))?;

     

       1363
        
                               total_danglers += danglers;

     

       1364
        
                               total_deleted += deleted;

     

       1365
       +
                               if total_deleted > 100_000_000 {

     

       1366
       +
                                   log::info!("trim stopped early, more than 100M records already deleted.");

     

       1367
        
                                   break;

     

       1368
        
                               }

     

       1369
        
                           }