commit 4a68dab04e79bd75dda2bff8f1ce3f9ac0d4ac70 · nekomimi.pet/microcosm-rs

+3 -2

Cargo.lock

···

       3638
        
       

     

       3639
        
       [[package]]

     

       3640
        
       name = "tokio-util"

     

       3641
       -
       version = "0.7.14"

     

       3642
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       3643
       -
       checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034"

     

       3644
        
       dependencies = [

     

       3645
        
        "bytes",

     

       3646
        
        "futures-core",

     
···

       3846
        
        "thiserror 2.0.12",

     

       3847
        
        "tikv-jemallocator",

     

       3848
        
        "tokio",

     

       0
        
       
     

       3849
        
       ]

     

       3850
        
       

     

       3851
        
       [[package]]

···

       3638
        
       

     

       3639
        
       [[package]]

     

       3640
        
       name = "tokio-util"

     

       3641
       +
       version = "0.7.15"

     

       3642
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       3643
       +
       checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df"

     

       3644
        
       dependencies = [

     

       3645
        
        "bytes",

     

       3646
        
        "futures-core",

     
···

       3846
        
        "thiserror 2.0.12",

     

       3847
        
        "tikv-jemallocator",

     

       3848
        
        "tokio",

     

       3849
       +
        "tokio-util",

     

       3850
        
       ]

     

       3851
        
       

     

       3852
        
       [[package]]

+74 -81

jetstream/src/lib.rs

···

       363
        
                       retry_attempt += 1;

     

       364
        
                       if let Ok((ws_stream, _)) = connect_async(req).await {

     

       365
        
                           let t_connected = Instant::now();

     

       366
       -
                           log::trace!("jetstream connected. starting websocket task...");

     

       367
        
                           if let Err(e) =

     

       368
        
                               websocket_task(dict, ws_stream, send_channel.clone(), &mut last_cursor)

     

       369
        
                                   .await

     
···

       374
        
                               }

     

       375
        
                               log::error!("Jetstream closed after encountering error: {e:?}");

     

       376
        
                           } else {

     

       377
       -
                               log::error!("Jetstream connection closed cleanly");

     

       378
        
                           }

     

       379
        
                           if t_connected.elapsed() > Duration::from_secs(success_threshold_s) {

     

       0
        
       
     

       380
        
                               retry_attempt = 0;

     

       381
        
                           }

     

       382
        
                       }

     

       383
        
       

     

       384
        
                       if retry_attempt >= max_retries {

     

       385
       -
                           log::error!("hit max retries, bye");

     

       386
        
                           break;

     

       387
        
                       }

     

       388
        
       

     
···

       422
        
           let mut closing_connection = false;

     

       423
        
           loop {

     

       424
        
               match socket_read.next().await {

     

       425
       -
                   Some(Ok(message)) => {

     

       426
       -
                       match message {

     

       427
       -
                           Message::Text(json) => {

     

       428
       -
                               let event: JetstreamEvent = match serde_json::from_str(&json) {

     

       429
       -
                                   Ok(ev) => ev,

     

       430
       -
                                   Err(e) => {

     

       431
       -
                                       log::warn!(

     

       432
       -
                                           "failed to parse json: {e:?} (from {})",

     

       433
       -
                                           json.get(..24).unwrap_or(&json)

     

       434
       -
                                       );

     

       435
       -
                                       continue;

     

       436
       -
                                   }

     

       437
       -
                               };

     

       438
       -
                               let event_cursor = event.cursor;

     

       439
        
       

     

       440
       -
                               if let Some(last) = last_cursor {

     

       441
       -
                                   if event_cursor <= *last {

     

       442
       -
                                       log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event.");

     

       443
       -
                                       continue;

     

       444
       -
                                   }

     

       445
        
                               }

     

       0
        
       
     

       446
        
       

     

       447
       -
                               if send_channel.send(event).await.is_err() {

     

       448
       -
                                   // We can assume that all receivers have been dropped, so we can close

     

       449
       -
                                   // the connection and exit the task.

     

       450
       -
                                   log::info!(

     

       451
        
                                       "All receivers for the Jetstream connection have been dropped, closing connection."

     

       452
        
                                   );

     

       453
       -
                                   socket_write.close().await?;

     

       454
       -
                                   return Err(JetstreamEventError::ReceiverClosedError);

     

       455
       -
                               } else if let Some(last) = last_cursor.as_mut() {

     

       456
       -
                                   *last = event_cursor;

     

       457
       -
                               }

     

       458
        
                           }

     

       459
       -
                           Message::Binary(zstd_json) => {

     

       460
       -
                               let mut cursor = IoCursor::new(zstd_json);

     

       461
       -
                               let decoder = zstd::stream::Decoder::with_prepared_dictionary(

     

       462
       -
                                   &mut cursor,

     

       463
       -
                                   &dictionary,

     

       464
       -
                               )

     

       465
       -
                               .map_err(JetstreamEventError::CompressionDictionaryError)?;

     

       466
        
       

     

       467
       -
                               let event: JetstreamEvent = match serde_json::from_reader(decoder) {

     

       468
       -
                                   Ok(ev) => ev,

     

       469
       -
                                   Err(e) => {

     

       470
       -
                                       log::warn!("failed to parse json: {e:?}");

     

       471
       -
                                       continue;

     

       472
       -
                                   }

     

       473
       -
                               };

     

       474
       -
                               let event_cursor = event.cursor;

     

       475
        
       

     

       476
       -
                               if let Some(last) = last_cursor {

     

       477
       -
                                   if event_cursor <= *last {

     

       478
       -
                                       log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event.");

     

       479
       -
                                       continue;

     

       480
       -
                                   }

     

       481
        
                               }

     

       0
        
       
     

       482
        
       

     

       483
       -
                               if send_channel.send(event).await.is_err() {

     

       484
       -
                                   // We can assume that all receivers have been dropped, so we can close

     

       485
       -
                                   // the connection and exit the task.

     

       486
       -
                                   log::info!(

     

       487
        
                                       "All receivers for the Jetstream connection have been dropped, closing connection."

     

       488
        
                                   );

     

       489
       -
                                   socket_write.close().await?;

     

       490
       -
                                   return Err(JetstreamEventError::ReceiverClosedError);

     

       491
       -
                               } else if let Some(last) = last_cursor.as_mut() {

     

       492
       -
                                   *last = event_cursor;

     

       493
       -
                               }

     

       494
        
                           }

     

       495
       -
                           Message::Ping(vec) => {

     

       496
       -
                               log::trace!("Ping recieved, responding");

     

       497
       -
                               socket_write

     

       498
       -
                                   .send(Message::Pong(vec))

     

       499
       -
                                   .await

     

       500
       -
                                   .map_err(JetstreamEventError::PingPongError)?;

     

       501
       -
                           }

     

       502
       -
                           Message::Close(close_frame) => {

     

       503
       -
                               log::trace!("Close recieved. I guess we just log here?");

     

       504
       -
                               if let Some(close_frame) = close_frame {

     

       505
       -
                                   let reason = close_frame.reason;

     

       506
       -
                                   let code = close_frame.code;

     

       507
       -
                                   log::trace!("Connection closed. Reason: {reason}, Code: {code}");

     

       508
       -
                               }

     

       509
       -
                           }

     

       510
       -
                           Message::Pong(pong) => {

     

       511
       -
                               let pong_payload = String::from_utf8(pong.to_vec())

     

       512
       -
                                   .unwrap_or("Invalid payload".to_string());

     

       513
       -
                               log::trace!("Pong recieved. Payload: {pong_payload}");

     

       514
        
                           }

     

       515
       -
                           Message::Frame(_) => (),

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       516
        
                       }

     

       517
       -
                   }

     

       0
        
       
     

       518
        
                   Some(Err(error)) => {

     

       519
        
                       log::error!("Web socket error: {error}");

     

       520
        
                       closing_connection = true;

···

       363
        
                       retry_attempt += 1;

     

       364
        
                       if let Ok((ws_stream, _)) = connect_async(req).await {

     

       365
        
                           let t_connected = Instant::now();

     

       366
       +
                           log::info!("jetstream connected. starting websocket task...");

     

       367
        
                           if let Err(e) =

     

       368
        
                               websocket_task(dict, ws_stream, send_channel.clone(), &mut last_cursor)

     

       369
        
                                   .await

     
···

       374
        
                               }

     

       375
        
                               log::error!("Jetstream closed after encountering error: {e:?}");

     

       376
        
                           } else {

     

       377
       +
                               log::warn!("Jetstream connection closed cleanly");

     

       378
        
                           }

     

       379
        
                           if t_connected.elapsed() > Duration::from_secs(success_threshold_s) {

     

       380
       +
                               log::warn!("Jetstream: more than {success_threshold_s}s since last reconnect, reconnecting immediately.");

     

       381
        
                               retry_attempt = 0;

     

       382
        
                           }

     

       383
        
                       }

     

       384
        
       

     

       385
        
                       if retry_attempt >= max_retries {

     

       386
       +
                           log::error!("jetstream: hit max retries, bye");

     

       387
        
                           break;

     

       388
        
                       }

     

       389
        
       

     
···

       423
        
           let mut closing_connection = false;

     

       424
        
           loop {

     

       425
        
               match socket_read.next().await {

     

       426
       +
                   Some(Ok(message)) => match message {

     

       427
       +
                       Message::Text(json) => {

     

       428
       +
                           let event: JetstreamEvent = match serde_json::from_str(&json) {

     

       429
       +
                               Ok(ev) => ev,

     

       430
       +
                               Err(e) => {

     

       431
       +
                                   log::warn!(

     

       432
       +
                                       "failed to parse json: {e:?} (from {})",

     

       433
       +
                                       json.get(..24).unwrap_or(&json)

     

       434
       +
                                   );

     

       435
       +
                                   continue;

     

       436
       +
                               }

     

       437
       +
                           };

     

       438
       +
                           let event_cursor = event.cursor;

     

       0
        
       
     

       439
        
       

     

       440
       +
                           if let Some(last) = last_cursor {

     

       441
       +
                               if event_cursor <= *last {

     

       442
       +
                                   log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event.");

     

       443
       +
                                   continue;

     

       0
        
       
     

       444
        
                               }

     

       445
       +
                           }

     

       446
        
       

     

       447
       +
                           if send_channel.send(event).await.is_err() {

     

       448
       +
                               log::warn!(

     

       0
        
       
     

       0
        
       
     

       449
        
                                       "All receivers for the Jetstream connection have been dropped, closing connection."

     

       450
        
                                   );

     

       451
       +
                               socket_write.close().await?;

     

       452
       +
                               return Err(JetstreamEventError::ReceiverClosedError);

     

       453
       +
                           } else if let Some(last) = last_cursor.as_mut() {

     

       454
       +
                               *last = event_cursor;

     

       0
        
       
     

       455
        
                           }

     

       456
       +
                       }

     

       457
       +
                       Message::Binary(zstd_json) => {

     

       458
       +
                           let mut cursor = IoCursor::new(zstd_json);

     

       459
       +
                           let decoder =

     

       460
       +
                               zstd::stream::Decoder::with_prepared_dictionary(&mut cursor, &dictionary)

     

       461
       +
                                   .map_err(JetstreamEventError::CompressionDictionaryError)?;

     

       0
        
       
     

       462
        
       

     

       463
       +
                           let event: JetstreamEvent = match serde_json::from_reader(decoder) {

     

       464
       +
                               Ok(ev) => ev,

     

       465
       +
                               Err(e) => {

     

       466
       +
                                   log::warn!("failed to parse json: {e:?}");

     

       467
       +
                                   continue;

     

       468
       +
                               }

     

       469
       +
                           };

     

       470
       +
                           let event_cursor = event.cursor;

     

       471
        
       

     

       472
       +
                           if let Some(last) = last_cursor {

     

       473
       +
                               if event_cursor <= *last {

     

       474
       +
                                   log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event.");

     

       475
       +
                                   continue;

     

       0
        
       
     

       476
        
                               }

     

       477
       +
                           }

     

       478
        
       

     

       479
       +
                           if send_channel.send(event).await.is_err() {

     

       480
       +
                               log::warn!(

     

       0
        
       
     

       0
        
       
     

       481
        
                                       "All receivers for the Jetstream connection have been dropped, closing connection."

     

       482
        
                                   );

     

       483
       +
                               socket_write.close().await?;

     

       484
       +
                               return Err(JetstreamEventError::ReceiverClosedError);

     

       485
       +
                           } else if let Some(last) = last_cursor.as_mut() {

     

       486
       +
                               *last = event_cursor;

     

       0
        
       
     

       487
        
                           }

     

       488
       +
                       }

     

       489
       +
                       Message::Ping(vec) => {

     

       490
       +
                           log::trace!("Ping recieved, responding");

     

       491
       +
                           socket_write

     

       492
       +
                               .send(Message::Pong(vec))

     

       493
       +
                               .await

     

       494
       +
                               .map_err(JetstreamEventError::PingPongError)?;

     

       495
       +
                       }

     

       496
       +
                       Message::Close(close_frame) => {

     

       497
       +
                           log::trace!("Close recieved. I guess we just log here?");

     

       498
       +
                           if let Some(close_frame) = close_frame {

     

       499
       +
                               let reason = close_frame.reason;

     

       500
       +
                               let code = close_frame.code;

     

       501
       +
                               log::trace!("Connection closed. Reason: {reason}, Code: {code}");

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       502
        
                           }

     

       503
       +
                       }

     

       504
       +
                       Message::Pong(pong) => {

     

       505
       +
                           let pong_payload =

     

       506
       +
                               String::from_utf8(pong.to_vec()).unwrap_or("Invalid payload".to_string());

     

       507
       +
                           log::trace!("Pong recieved. Payload: {pong_payload}");

     

       508
        
                       }

     

       509
       +
                       Message::Frame(_) => (),

     

       510
       +
                   },

     

       511
        
                   Some(Err(error)) => {

     

       512
        
                       log::error!("Web socket error: {error}");

     

       513
        
                       closing_connection = true;

ufos/Cargo.toml

···

       27
        
       sha2 = "0.10.9"

     

       28
        
       thiserror = "2.0.12"

     

       29
        
       tokio = { version = "1.44.2", features = ["full", "sync", "time"] }

     

       0
        
       
     

       30
        
       

     

       31
        
       [target.'cfg(not(target_env = "msvc"))'.dependencies]

     

       32
        
       tikv-jemallocator = "0.6.0"

···

       27
        
       sha2 = "0.10.9"

     

       28
        
       thiserror = "2.0.12"

     

       29
        
       tokio = { version = "1.44.2", features = ["full", "sync", "time"] }

     

       30
       +
       tokio-util = "0.7.15"

     

       31
        
       

     

       32
        
       [target.'cfg(not(target_env = "msvc"))'.dependencies]

     

       33
        
       tikv-jemallocator = "0.6.0"

+3 -8

ufos/src/consumer.rs

···

       68
        
           let mut batcher = Batcher::new(jetstream_receiver, batch_sender, sketch_secret);

     

       69
        
           tokio::task::spawn(async move {

     

       70
        
               let r = batcher.run().await;

     

       71
       -
               log::info!("batcher ended: {r:?}");

     

       72
        
           });

     

       73
        
           Ok(batch_reciever)

     

       74
        
       }

     
···

       93
        
           pub async fn run(&mut self) -> anyhow::Result<()> {

     

       94
        
               // TODO: report errors *from here* probably, since this gets shipped off into a spawned task that might just vanish

     

       95
        
               loop {

     

       96
       -
                   match timeout(

     

       97
       -
                       Duration::from_millis(30_000),

     

       98
       -
                       self.jetstream_receiver.recv(),

     

       99
       -
                   )

     

       100
       -
                   .await

     

       101
       -
                   {

     

       102
        
                       Err(_elapsed) => self.no_events_step().await?,

     

       103
        
                       Ok(Some(event)) => self.handle_event(event).await?,

     

       104
        
                       Ok(None) => anyhow::bail!("channel closed"),

     
···

       198
        
                   Some(Ok(t)) => format!("{:?}", t),

     

       199
        
                   Some(Err(e)) => format!("+{:?}", e.duration()),

     

       200
        
               };

     

       201
       -
               log::trace!(

     

       202
        
                   "sending batch now from {beginning}, {}, queue capacity: {}, referrer: {referrer}",

     

       203
        
                   if small { "small" } else { "full" },

     

       204
        
                   self.batch_sender.capacity(),

···

       68
        
           let mut batcher = Batcher::new(jetstream_receiver, batch_sender, sketch_secret);

     

       69
        
           tokio::task::spawn(async move {

     

       70
        
               let r = batcher.run().await;

     

       71
       +
               log::warn!("batcher ended: {r:?}");

     

       72
        
           });

     

       73
        
           Ok(batch_reciever)

     

       74
        
       }

     
···

       93
        
           pub async fn run(&mut self) -> anyhow::Result<()> {

     

       94
        
               // TODO: report errors *from here* probably, since this gets shipped off into a spawned task that might just vanish

     

       95
        
               loop {

     

       96
       +
                   match timeout(Duration::from_secs_f64(30.), self.jetstream_receiver.recv()).await {

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       97
        
                       Err(_elapsed) => self.no_events_step().await?,

     

       98
        
                       Ok(Some(event)) => self.handle_event(event).await?,

     

       99
        
                       Ok(None) => anyhow::bail!("channel closed"),

     
···

       193
        
                   Some(Ok(t)) => format!("{:?}", t),

     

       194
        
                   Some(Err(e)) => format!("+{:?}", e.duration()),

     

       195
        
               };

     

       196
       +
               log::info!(

     

       197
        
                   "sending batch now from {beginning}, {}, queue capacity: {}, referrer: {referrer}",

     

       198
        
                   if small { "small" } else { "full" },

     

       199
        
                   self.batch_sender.capacity(),

ufos/src/error.rs

···

       40
        
           JoinError(#[from] tokio::task::JoinError),

     

       41
        
           #[error("Background task already started")]

     

       42
        
           BackgroundAlreadyStarted,

     

       0
        
       
     

       0
        
       
     

       43
        
       }

···

       40
        
           JoinError(#[from] tokio::task::JoinError),

     

       41
        
           #[error("Background task already started")]

     

       42
        
           BackgroundAlreadyStarted,

     

       43
       +
           #[error("Batch sender exited")]

     

       44
       +
           BatchSenderExited,

     

       45
        
       }

+2 -2

ufos/src/main.rs

···

       100
        
           let rolling = write_store

     

       101
        
               .background_tasks(args.reroll)?

     

       102
        
               .run(args.backfill);

     

       103
       -
           let storing = write_store.receive_batches(batches);

     

       104
        
       

     

       105
        
           let stating = do_update_stuff(read_store);

     

       106
        
       

     

       107
        
           tokio::select! {

     

       108
        
               z = serving => log::warn!("serve task ended: {z:?}"),

     

       109
        
               z = rolling => log::warn!("rollup task ended: {z:?}"),

     

       110
       -
               z = storing => log::warn!("storage task ended: {z:?}"),

     

       111
        
               z = stating => log::warn!("status task ended: {z:?}"),

     

       112
        
           };

     

       113

···

       100
        
           let rolling = write_store

     

       101
        
               .background_tasks(args.reroll)?

     

       102
        
               .run(args.backfill);

     

       103
       +
           let consuming = write_store.receive_batches(batches);

     

       104
        
       

     

       105
        
           let stating = do_update_stuff(read_store);

     

       106
        
       

     

       107
        
           tokio::select! {

     

       108
        
               z = serving => log::warn!("serve task ended: {z:?}"),

     

       109
        
               z = rolling => log::warn!("rollup task ended: {z:?}"),

     

       110
       +
               z = consuming => log::warn!("consuming task ended: {z:?}"),

     

       111
        
               z = stating => log::warn!("status task ended: {z:?}"),

     

       112
        
           };

     

       113

+36 -13

ufos/src/storage.rs

···

       7
        
       use jetstream::exports::{Did, Nsid};

     

       8
        
       use std::collections::{HashMap, HashSet};

     

       9
        
       use std::path::Path;

     

       0
        
       
     

       10
        
       use tokio::sync::mpsc::Receiver;

     

       0
        
       
     

       11
        
       

     

       12
        
       pub type StorageResult<T> = Result<T, StorageError>;

     

       13
        
       

     
···

       22
        
               Self: Sized;

     

       23
        
       }

     

       24
        
       

     

       25
       -
       pub trait StoreWriter<B: StoreBackground>: Send + Sync

     

       0
        
       
     

       26
        
       where

     

       27
        
           Self: 'static,

     

       28
        
       {

     

       29
        
           fn background_tasks(&mut self, reroll: bool) -> StorageResult<B>;

     

       30
        
       

     

       31
       -
           fn receive_batches<const LIMIT: usize>(

     

       32
       -
               mut self,

     

       33
        
               mut batches: Receiver<EventBatch<LIMIT>>,

     

       34
       -
           ) -> impl std::future::Future<Output = StorageResult<()>> + Send

     

       35
       -
           where

     

       36
       -
               Self: Sized,

     

       37
       -
           {

     

       38
       -
               async {

     

       39
       -
                   tokio::task::spawn_blocking(move || {

     

       40
       -
                       while let Some(event_batch) = batches.blocking_recv() {

     

       41
       -
                           self.insert_batch(event_batch)?;

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       42
        
                       }

     

       43
       -
                       Ok::<(), StorageError>(())

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       44
        
                   })

     

       45
       -
                   .await?

     

       46
        
               }

     

       0
        
       
     

       0
        
       
     

       47
        
           }

     

       48
        
       

     

       49
        
           fn insert_batch<const LIMIT: usize>(

···

       7
        
       use jetstream::exports::{Did, Nsid};

     

       8
        
       use std::collections::{HashMap, HashSet};

     

       9
        
       use std::path::Path;

     

       10
       +
       use std::time::{Duration, SystemTime};

     

       11
        
       use tokio::sync::mpsc::Receiver;

     

       12
       +
       use tokio_util::sync::CancellationToken;

     

       13
        
       

     

       14
        
       pub type StorageResult<T> = Result<T, StorageError>;

     

       15
        
       

     
···

       24
        
               Self: Sized;

     

       25
        
       }

     

       26
        
       

     

       27
       +
       #[async_trait]

     

       28
       +
       pub trait StoreWriter<B: StoreBackground>: Clone + Send + Sync

     

       29
        
       where

     

       30
        
           Self: 'static,

     

       31
        
       {

     

       32
        
           fn background_tasks(&mut self, reroll: bool) -> StorageResult<B>;

     

       33
        
       

     

       34
       +
           async fn receive_batches<const LIMIT: usize>(

     

       35
       +
               self,

     

       36
        
               mut batches: Receiver<EventBatch<LIMIT>>,

     

       37
       +
           ) -> StorageResult<()> {

     

       38
       +
               while let Some(event_batch) = batches.recv().await {

     

       39
       +
                   let token = CancellationToken::new();

     

       40
       +
                   let cancelled = token.clone();

     

       41
       +
                   tokio::spawn(async move {

     

       42
       +
                       let started = SystemTime::now();

     

       43
       +
                       let mut concerned = false;

     

       44
       +
                       loop {

     

       45
       +
                           tokio::select! {

     

       46
       +
                               _ = tokio::time::sleep(Duration::from_secs_f64(1.)) => {

     

       47
       +
                                   log::warn!("taking a long time to insert an event batch ({:?})...", started.elapsed());

     

       48
       +
                                   concerned = true;

     

       49
       +
                               }

     

       50
       +
                               _ = cancelled.cancelled() => {

     

       51
       +
                                   if concerned {

     

       52
       +
                                       log::warn!("finally inserted slow event batch after {:?}", started.elapsed());

     

       53
       +
                                   }

     

       54
       +
                                   break

     

       55
       +
                               }

     

       56
       +
                           }

     

       57
        
                       }

     

       58
       +
                   });

     

       59
       +
                   tokio::task::spawn_blocking({

     

       60
       +
                       let mut me = self.clone();

     

       61
       +
                       move || {

     

       62
       +
                           let _guard = token.drop_guard();

     

       63
       +
                           me.insert_batch(event_batch)

     

       64
       +
                       }

     

       65
        
                   })

     

       66
       +
                   .await??;

     

       67
        
               }

     

       68
       +
       

     

       69
       +
               Err(StorageError::BatchSenderExited)

     

       70
        
           }

     

       71
        
       

     

       72
        
           fn insert_batch<const LIMIT: usize>(