commit 7cd979b3fd62263475e2448e34b0014392f3e781 · ptr.pet/nsid-tracker

+16

server/Cargo.lock

···

       424
       424
        
       checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"

     

       425
       425
        
       

     

       426
       426
        
       [[package]]

     

       427
       427
       +
       name = "either"

     

       428
       428
       +
       version = "1.15.0"

     

       429
       429
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       430
       430
       +
       checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"

     

       431
       431
       +
       

     

       432
       432
       +
       [[package]]

     

       427
       433
        
       name = "enum_dispatch"

     

       428
       434
        
       version = "0.3.13"

     

       429
       435
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       744
       750
        
        "bitflags",

     

       745
       751
        
        "cfg-if",

     

       746
       752
        
        "libc",

     

       753
       753
       +
       ]

     

       754
       754
       +
       

     

       755
       755
       +
       [[package]]

     

       756
       756
       +
       name = "itertools"

     

       757
       757
       +
       version = "0.14.0"

     

       758
       758
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       759
       759
       +
       checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"

     

       760
       760
       +
       dependencies = [

     

       761
       761
       +
        "either",

     

       747
       762
        
       ]

     

       748
       763
        
       

     

       749
       764
        
       [[package]]

     
···

       1480
       1495
        
        "axum-tws",

     

       1481
       1496
        
        "fjall",

     

       1482
       1497
        
        "futures-util",

     

       1498
       1498
       +
        "itertools",

     

       1483
       1499
        
        "ordered-varint",

     

       1484
       1500
        
        "quanta",

     

       1485
       1501
        
        "rkyv",

server/Cargo.toml

···

       26
       26
        
       threadpool = "1.8.1"

     

       27
       27
        
       snmalloc-rs = "0.3.8"

     

       28
       28
        
       quanta = "0.12.6"

     

       29
       29
       +
       itertools = "0.14.0"

+35 -24

server/src/db/mod.rs

···

       10
       10
        
       };

     

       11
       11
        
       

     

       12
       12
        
       use fjall::{Config, Keyspace, Partition, PartitionCreateOptions, Slice};

     

       13
       13
       +
       use itertools::Itertools;

     

       13
       14
        
       use ordered_varint::Variable;

     

       14
       15
        
       use rkyv::{Archive, Deserialize, Serialize, rancor::Error};

     

       15
       16
        
       use smol_str::SmolStr;

     
···

       269
       270
        
           #[inline(always)]

     

       270
       271
        
           fn run_in_nsid_tree<T>(

     

       271
       272
        
               &self,

     

       272
       272
       -
               nsid: SmolStr,

     

       273
       273
       +
               nsid: &SmolStr,

     

       273
       274
        
               f: impl FnOnce(&LexiconHandle) -> AppResult<T>,

     

       274
       275
        
           ) -> AppResult<T> {

     

       275
       276
        
               f(self

     

       276
       277
        
                   .hits

     

       277
       278
        
                   .entry(nsid.clone())

     

       278
       278
       -
                   .or_insert_with(move || Arc::new(LexiconHandle::new(&self.inner, &nsid)))

     

       279
       279
       +
                   .or_insert_with(|| Arc::new(LexiconHandle::new(&self.inner, &nsid)))

     

       279
       280
        
                   .get())

     

       280
       281
        
           }

     

       281
       282
        
       

     

       282
       282
       -
           pub fn record_event(&self, e: EventRecord) -> AppResult<()> {

     

       283
       283
       -
               let EventRecord {

     

       284
       284
       -
                   nsid,

     

       285
       285
       -
                   timestamp,

     

       286
       286
       -
                   deleted,

     

       287
       287
       -
               } = e.clone();

     

       283
       283
       +
           pub fn ingest_events(&self, events: impl Iterator<Item = EventRecord>) -> AppResult<()> {

     

       284
       284
       +
               for (key, chunk) in events.chunk_by(|event| event.nsid.clone()).into_iter() {

     

       285
       285
       +
                   let mut counts = self.get_count(&key)?;

     

       286
       286
       +
                   self.run_in_nsid_tree(&key, move |tree| {

     

       287
       287
       +
                       for event in chunk {

     

       288
       288
       +
                           let EventRecord {

     

       289
       289
       +
                               timestamp, deleted, ..

     

       290
       290
       +
                           } = event.clone();

     

       288
       291
        
       

     

       289
       289
       -
               // insert event

     

       290
       290
       -
               self.run_in_nsid_tree(nsid.clone(), move |tree| Ok(tree.insert(e)))?;

     

       291
       291
       -
               // increment count

     

       292
       292
       -
               let mut counts = self.get_count(&nsid)?;

     

       293
       293
       -
               counts.last_seen = timestamp;

     

       294
       294
       -
               if deleted {

     

       295
       295
       -
                   counts.deleted_count += 1;

     

       296
       296
       -
               } else {

     

       297
       297
       -
                   counts.count += 1;

     

       298
       298
       -
               }

     

       299
       299
       -
               self.insert_count(&nsid, counts.clone())?;

     

       300
       300
       -
               if self.event_broadcaster.receiver_count() > 0 {

     

       301
       301
       -
                   let _ = self.event_broadcaster.send((SmolStr::new(&nsid), counts));

     

       292
       292
       +
                           tree.insert(event);

     

       293
       293
       +
       

     

       294
       294
       +
                           // increment count

     

       295
       295
       +
                           counts.last_seen = timestamp;

     

       296
       296
       +
                           if deleted {

     

       297
       297
       +
                               counts.deleted_count += 1;

     

       298
       298
       +
                           } else {

     

       299
       299
       +
                               counts.count += 1;

     

       300
       300
       +
                           }

     

       301
       301
       +
       

     

       302
       302
       +
                           self.eps.observe();

     

       303
       303
       +
                       }

     

       304
       304
       +
                       Ok(())

     

       305
       305
       +
                   })?;

     

       306
       306
       +
                   self.insert_count(&key, &counts)?;

     

       307
       307
       +
                   if self.event_broadcaster.receiver_count() > 0 {

     

       308
       308
       +
                       let _ = self.event_broadcaster.send((key, counts));

     

       309
       309
       +
                   }

     

       302
       310
        
               }

     

       303
       303
       -
               self.eps.observe();

     

       304
       311
        
               Ok(())

     

       305
       312
        
           }

     

       306
       313
        
       

     

       314
       314
       +
           pub fn record_event(&self, e: EventRecord) -> AppResult<()> {

     

       315
       315
       +
               self.ingest_events(std::iter::once(e))

     

       316
       316
       +
           }

     

       317
       317
       +
       

     

       307
       318
        
           #[inline(always)]

     

       308
       308
       -
           fn insert_count(&self, nsid: &str, counts: NsidCounts) -> AppResult<()> {

     

       319
       319
       +
           fn insert_count(&self, nsid: &str, counts: &NsidCounts) -> AppResult<()> {

     

       309
       320
        
               self.counts

     

       310
       321
        
                   .insert(

     

       311
       322
        
                       nsid,

     

       312
       312
       -
                       unsafe { rkyv::to_bytes::<Error>(&counts).unwrap_unchecked() }.as_slice(),

     

       323
       323
       +
                       unsafe { rkyv::to_bytes::<Error>(counts).unwrap_unchecked() }.as_slice(),

     

       313
       324
        
                   )

     

       314
       325
        
                   .map_err(AppError::from)

     

       315
       326
        
           }