commit 530ecc8a6b977c359c0a9d7c0e1f07e5e0eacecf · nekomimi.pet/microcosm-rs

+3 -1

.github/workflows/checks.yml

···

       16
       16
        
           - uses: actions/checkout@v4

     

       17
       17
        
           - name: Build lib

     

       18
       18
        
             run: cargo build --verbose

     

       19
       19
       +
           - name: Check (default features)

     

       20
       20
       +
             run: cargo check

     

       19
       21
        
           - name: Run tests

     

       20
       20
       -
             run: cargo test --verbose

     

       22
       22
       +
             run: cargo test --all-features --verbose

     

       21
       23
        
       

     

       22
       24
        
         style:

     

       23
       25
        
           runs-on: ubuntu-24.04

Cargo.lock

···

       1867
       1867
        
        "clap",

     

       1868
       1868
        
        "futures-util",

     

       1869
       1869
        
        "log",

     

       1870
       1870
       +
        "metrics",

     

       1870
       1871
        
        "serde",

     

       1871
       1872
        
        "serde_json",

     

       1872
       1873
        
        "thiserror 2.0.12",

+1 -1

Makefile

···

       2
       2
        
       all: check

     

       3
       3
        
       

     

       4
       4
        
       test:

     

       5
       5
       -
       	cargo test

     

       5
       5
       +
       	cargo test --all-features

     

       6
       6
        
       

     

       7
       7
        
       fmt:

     

       8
       8
        
       	cargo fmt --package links --package constellation --package ufos

jetstream/Cargo.toml

···

       20
       20
        
           "url",

     

       21
       21
        
       ] }

     

       22
       22
        
       futures-util = "0.3.31"

     

       23
       23
       +
       metrics = { version = "0.24.2", optional = true }

     

       23
       24
        
       url = "2.5.4"

     

       24
       25
        
       serde = { version = "1.0.215", features = ["derive"] }

     

       25
       26
        
       serde_json = { version = "1.0.140", features = ["raw_value"] }

     
···

       31
       32
        
       [dev-dependencies]

     

       32
       33
        
       anyhow = "1.0.93"

     

       33
       34
        
       clap = { version = "4.5.20", features = ["derive"] }

     

       35
       35
       +
       

     

       36
       36
       +
       [features]

     

       37
       37
       +
       default = []

     

       38
       38
       +
       metrics = ["dep:metrics"]

+1 -6

jetstream/src/error.rs

···

       36
       36
        
       /// See [websocket_task](crate::websocket_task).

     

       37
       37
        
       #[derive(Error, Debug)]

     

       38
       38
        
       pub enum JetstreamEventError {

     

       39
       39
       -
           #[error("received websocket message that could not be deserialized as JSON: {0}")]

     

       40
       40
       -
           ReceivedMalformedJSON(#[from] serde_json::Error),

     

       41
       39
        
           #[error("failed to load built-in zstd dictionary for decoding: {0}")]

     

       42
       40
        
           CompressionDictionaryError(io::Error),

     

       43
       43
       -
           #[error("failed to decode zstd-compressed message: {0}")]

     

       44
       44
       -
           CompressionDecoderError(io::Error),

     

       45
       45
       -
           #[error("all receivers were dropped but the websocket connection failed to close cleanly")]

     

       46
       46
       -
           WebSocketCloseFailure,

     

       41
       41
       +
       

     

       47
       42
        
           #[error("failed to send ping or pong: {0}")]

     

       48
       43
        
           PingPongError(#[from] tokio_tungstenite::tungstenite::Error),

     

       49
       44
        
           #[error("jetstream event receiver closed")]

+99 -4

jetstream/src/lib.rs

···

       14
       14
        
           stream::StreamExt,

     

       15
       15
        
           SinkExt,

     

       16
       16
        
       };

     

       17
       17
       +
       #[cfg(feature = "metrics")]

     

       18
       18
       +
       use metrics::{

     

       19
       19
       +
           counter,

     

       20
       20
       +
           describe_counter,

     

       21
       21
       +
           Unit,

     

       22
       22
       +
       };

     

       17
       23
        
       use tokio::{

     

       18
       24
        
           net::TcpStream,

     

       19
       25
        
           sync::mpsc::{

     
···

       299
       305
        
           }

     

       300
       306
        
       }

     

       301
       307
        
       

     

       308
       308
       +
       #[cfg(feature = "metrics")]

     

       309
       309
       +
       fn describe_metrics() {

     

       310
       310
       +
           describe_counter!(

     

       311
       311
       +
               "jetstream_connects",

     

       312
       312
       +
               Unit::Count,

     

       313
       313
       +
               "how many times we've tried to connect"

     

       314
       314
       +
           );

     

       315
       315
       +
           describe_counter!(

     

       316
       316
       +
               "jetstream_disconnects",

     

       317
       317
       +
               Unit::Count,

     

       318
       318
       +
               "how many times we've been disconnected"

     

       319
       319
       +
           );

     

       320
       320
       +
           describe_counter!(

     

       321
       321
       +
               "jetstream_total_events_received",

     

       322
       322
       +
               Unit::Count,

     

       323
       323
       +
               "total number of events received"

     

       324
       324
       +
           );

     

       325
       325
       +
           describe_counter!(

     

       326
       326
       +
               "jetstream_total_bytes_received",

     

       327
       327
       +
               Unit::Count,

     

       328
       328
       +
               "total uncompressed bytes received, not including websocket overhead"

     

       329
       329
       +
           );

     

       330
       330
       +
           describe_counter!(

     

       331
       331
       +
               "jetstream_total_event_errors",

     

       332
       332
       +
               Unit::Count,

     

       333
       333
       +
               "total errors when handling events"

     

       334
       334
       +
           );

     

       335
       335
       +
           describe_counter!(

     

       336
       336
       +
               "jetstream_total_events_sent",

     

       337
       337
       +
               Unit::Count,

     

       338
       338
       +
               "total events sent to the consumer"

     

       339
       339
       +
           );

     

       340
       340
       +
       }

     

       341
       341
       +
       

     

       302
       342
        
       impl JetstreamConnector {

     

       303
       343
        
           /// Create a Jetstream connector with a valid [JetstreamConfig].

     

       304
       344
        
           ///

     

       305
       345
        
           /// After creation, you can call [connect] to connect to the provided Jetstream instance.

     

       306
       346
        
           pub fn new(config: JetstreamConfig) -> Result<Self, ConfigValidationError> {

     

       347
       347
       +
               #[cfg(feature = "metrics")]

     

       348
       348
       +
               describe_metrics();

     

       349
       349
       +
       

     

       307
       350
        
               // We validate the configuration here so any issues are caught early.

     

       308
       351
        
               config.validate()?;

     

       309
       352
        
               Ok(JetstreamConnector { config })

     
···

       359
       402
        
                           }

     

       360
       403
        
                       };

     

       361
       404
        
       

     

       405
       405
       +
                       #[cfg(feature = "metrics")]

     

       406
       406
       +
                       if let Some(host) = req.uri().host() {

     

       407
       407
       +
                           let retry = if retry_attempt > 0 { "yes" } else { "no" };

     

       408
       408
       +
                           counter!("jetstream_connects", "host" => host.to_string(), "retry" => retry)

     

       409
       409
       +
                               .increment(1);

     

       410
       410
       +
                       }

     

       411
       411
       +
       

     

       362
       412
        
                       let mut last_cursor = connect_cursor;

     

       363
       413
        
                       retry_attempt += 1;

     

       364
       414
        
                       if let Ok((ws_stream, _)) = connect_async(req).await {

     
···

       368
       418
        
                               websocket_task(dict, ws_stream, send_channel.clone(), &mut last_cursor)

     

       369
       419
        
                                   .await

     

       370
       420
        
                           {

     

       371
       371
       -
                               if let JetstreamEventError::ReceiverClosedError = e {

     

       372
       372
       -
                                   log::error!("Jetstream receiver channel closed. Exiting consumer.");

     

       373
       373
       -
                                   return;

     

       421
       421
       +
                               match e {

     

       422
       422
       +
                                   JetstreamEventError::ReceiverClosedError => {

     

       423
       423
       +
                                       #[cfg(feature="metrics")]

     

       424
       424
       +
                                       counter!("jetstream_disconnects", "reason" => "channel", "fatal" => "yes").increment(1);

     

       425
       425
       +
                                       log::error!("Jetstream receiver channel closed. Exiting consumer.");

     

       426
       426
       +
                                       return;

     

       427
       427
       +
                                   }

     

       428
       428
       +
                                   JetstreamEventError::CompressionDictionaryError(_) => {

     

       429
       429
       +
                                       #[cfg(feature="metrics")]

     

       430
       430
       +
                                       counter!("jetstream_disconnects", "reason" => "zstd", "fatal" => "no").increment(1);

     

       431
       431
       +
                                   }

     

       432
       432
       +
                                   JetstreamEventError::PingPongError(_) => {

     

       433
       433
       +
                                       #[cfg(feature="metrics")]

     

       434
       434
       +
                                       counter!("jetstream_disconnects", "reason" => "pingpong", "fatal" => "no").increment(1);

     

       435
       435
       +
                                   }

     

       374
       436
        
                               }

     

       375
       375
       -
                               log::error!("Jetstream closed after encountering error: {e:?}");

     

       437
       437
       +
                               log::warn!("Jetstream closed after encountering error: {e:?}");

     

       376
       438
        
                           } else {

     

       439
       439
       +
                               #[cfg(feature = "metrics")]

     

       440
       440
       +
                               counter!("jetstream_disconnects", "reason" => "close", "fatal" => "no")

     

       441
       441
       +
                                   .increment(1);

     

       377
       442
        
                               log::warn!("Jetstream connection closed cleanly");

     

       378
       443
        
                           }

     

       379
       444
        
                           if t_connected.elapsed() > Duration::from_secs(success_threshold_s) {

     
···

       425
       490
        
               match socket_read.next().await {

     

       426
       491
        
                   Some(Ok(message)) => match message {

     

       427
       492
        
                       Message::Text(json) => {

     

       493
       493
       +
                           #[cfg(feature = "metrics")]

     

       494
       494
       +
                           {

     

       495
       495
       +
                               counter!("jetstream_total_events_received", "compressed" => "false")

     

       496
       496
       +
                                   .increment(1);

     

       497
       497
       +
                               counter!("jetstream_total_bytes_received", "compressed" => "false")

     

       498
       498
       +
                                   .increment(json.len() as u64);

     

       499
       499
       +
                           }

     

       428
       500
        
                           let event: JetstreamEvent = match serde_json::from_str(&json) {

     

       429
       501
        
                               Ok(ev) => ev,

     

       430
       502
        
                               Err(e) => {

     

       503
       503
       +
                                   #[cfg(feature = "metrics")]

     

       504
       504
       +
                                   counter!("jetstream_total_event_errors", "reason" => "deserialize")

     

       505
       505
       +
                                       .increment(1);

     

       431
       506
        
                                   log::warn!(

     

       432
       507
        
                                       "failed to parse json: {e:?} (from {})",

     

       433
       508
        
                                       json.get(..24).unwrap_or(&json)

     
···

       439
       514
        
       

     

       440
       515
        
                           if let Some(last) = last_cursor {

     

       441
       516
        
                               if event_cursor <= *last {

     

       517
       517
       +
                                   #[cfg(feature = "metrics")]

     

       518
       518
       +
                                   counter!("jetstream_total_event_errors", "reason" => "old")

     

       519
       519
       +
                                       .increment(1);

     

       442
       520
        
                                   log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event.");

     

       443
       521
        
                                   continue;

     

       444
       522
        
                               }

     
···

       453
       531
        
                           } else if let Some(last) = last_cursor.as_mut() {

     

       454
       532
        
                               *last = event_cursor;

     

       455
       533
        
                           }

     

       534
       534
       +
                           #[cfg(feature = "metrics")]

     

       535
       535
       +
                           counter!("jetstream_total_events_sent").increment(1);

     

       456
       536
        
                       }

     

       457
       537
        
                       Message::Binary(zstd_json) => {

     

       538
       538
       +
                           #[cfg(feature = "metrics")]

     

       539
       539
       +
                           {

     

       540
       540
       +
                               counter!("jetstream_total_events_received", "compressed" => "true")

     

       541
       541
       +
                                   .increment(1);

     

       542
       542
       +
                               counter!("jetstream_total_bytes_received", "compressed" => "true")

     

       543
       543
       +
                                   .increment(zstd_json.len() as u64);

     

       544
       544
       +
                           }

     

       458
       545
        
                           let mut cursor = IoCursor::new(zstd_json);

     

       459
       546
        
                           let decoder =

     

       460
       547
        
                               zstd::stream::Decoder::with_prepared_dictionary(&mut cursor, &dictionary)

     
···

       463
       550
        
                           let event: JetstreamEvent = match serde_json::from_reader(decoder) {

     

       464
       551
        
                               Ok(ev) => ev,

     

       465
       552
        
                               Err(e) => {

     

       553
       553
       +
                                   #[cfg(feature = "metrics")]

     

       554
       554
       +
                                   counter!("jetstream_total_event_errors", "reason" => "deserialize")

     

       555
       555
       +
                                       .increment(1);

     

       466
       556
        
                                   log::warn!("failed to parse json: {e:?}");

     

       467
       557
        
                                   continue;

     

       468
       558
        
                               }

     
···

       471
       561
        
       

     

       472
       562
        
                           if let Some(last) = last_cursor {

     

       473
       563
        
                               if event_cursor <= *last {

     

       564
       564
       +
                                   #[cfg(feature = "metrics")]

     

       565
       565
       +
                                   counter!("jetstream_total_event_errors", "reason" => "old")

     

       566
       566
       +
                                       .increment(1);

     

       474
       567
        
                                   log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event.");

     

       475
       568
        
                                   continue;

     

       476
       569
        
                               }

     
···

       485
       578
        
                           } else if let Some(last) = last_cursor.as_mut() {

     

       486
       579
        
                               *last = event_cursor;

     

       487
       580
        
                           }

     

       581
       581
       +
                           #[cfg(feature = "metrics")]

     

       582
       582
       +
                           counter!("jetstream_total_events_sent").increment(1);

     

       488
       583
        
                       }

     

       489
       584
        
                       Message::Ping(vec) => {

     

       490
       585
        
                           log::trace!("Ping recieved, responding");

+1 -1

ufos/Cargo.toml

···

       16
       16
        
       fjall = { version = "2.8.0", features = ["lz4"] }

     

       17
       17
        
       getrandom = "0.3.3"

     

       18
       18
        
       http = "1.3.1"

     

       19
       19
       -
       jetstream = { path = "../jetstream" }

     

       19
       19
       +
       jetstream = { path = "../jetstream", features = ["metrics"] }

     

       20
       20
        
       log = "0.4.26"

     

       21
       21
        
       lsm-tree = "2.6.6"

     

       22
       22
        
       metrics = "0.24.2"