commit 2a7d6005770b0a71e8c4bd3cb21cc5ff11b4fee9 · ptr.pet/nsid-tracker

+35

server/Cargo.lock

···

       203
       203
        
       ]

     

       204
       204
        
       

     

       205
       205
        
       [[package]]

     

       206
       206
       +
       name = "branches"

     

       207
       207
       +
       version = "0.2.2"

     

       208
       208
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       209
       209
       +
       checksum = "a918aa7a861caeba57e502465c30e3a0d74ae02ee0b9db2933602fdb6a3a90e5"

     

       210
       210
       +
       dependencies = [

     

       211
       211
       +
        "rustc_version",

     

       212
       212
       +
       ]

     

       213
       213
       +
       

     

       214
       214
       +
       [[package]]

     

       206
       215
        
       name = "brotli"

     

       207
       216
        
       version = "8.0.1"

     

       208
       217
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1177
       1186
        
       ]

     

       1178
       1187
        
       

     

       1179
       1188
        
       [[package]]

     

       1189
       1189
       +
       name = "rclite"

     

       1190
       1190
       +
       version = "0.2.7"

     

       1191
       1191
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1192
       1192
       +
       checksum = "2f528dfeba924f5fc67bb84a17fe043451d1b392758016ce2d9e9116649b0f35"

     

       1193
       1193
       +
       dependencies = [

     

       1194
       1194
       +
        "branches",

     

       1195
       1195
       +
       ]

     

       1196
       1196
       +
       

     

       1197
       1197
       +
       [[package]]

     

       1180
       1198
        
       name = "redox_syscall"

     

       1181
       1199
        
       version = "0.5.15"

     

       1182
       1200
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1295
       1313
        
       checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"

     

       1296
       1314
        
       

     

       1297
       1315
        
       [[package]]

     

       1316
       1316
       +
       name = "rustc_version"

     

       1317
       1317
       +
       version = "0.4.1"

     

       1318
       1318
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1319
       1319
       +
       checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"

     

       1320
       1320
       +
       dependencies = [

     

       1321
       1321
       +
        "semver",

     

       1322
       1322
       +
       ]

     

       1323
       1323
       +
       

     

       1324
       1324
       +
       [[package]]

     

       1298
       1325
        
       name = "rustix"

     

       1299
       1326
        
       version = "1.0.8"

     

       1300
       1327
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1462
       1489
        
       checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"

     

       1463
       1490
        
       

     

       1464
       1491
        
       [[package]]

     

       1492
       1492
       +
       name = "semver"

     

       1493
       1493
       +
       version = "1.0.26"

     

       1494
       1494
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1495
       1495
       +
       checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"

     

       1496
       1496
       +
       

     

       1497
       1497
       +
       [[package]]

     

       1465
       1498
        
       name = "serde"

     

       1466
       1499
        
       version = "1.0.219"

     

       1467
       1500
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1528
       1561
        
        "futures-util",

     

       1529
       1562
        
        "itertools",

     

       1530
       1563
        
        "ordered-varint",

     

       1564
       1564
       +
        "parking_lot",

     

       1531
       1565
        
        "quanta",

     

       1532
       1566
        
        "rayon",

     

       1567
       1567
       +
        "rclite",

     

       1533
       1568
        
        "rkyv",

     

       1534
       1569
        
        "rustls",

     

       1535
       1570
        
        "scc",

+3 -1

server/Cargo.toml

···

       8
       8
        
       async-trait = "0.1"

     

       9
       9
        
       tracing-subscriber = {version = "0.3", features = ["env-filter"]}

     

       10
       10
        
       tracing = "0.1"

     

       11
       11
       -
       tokio = { version = "1", features = ["full"] }

     

       11
       11
       +
       tokio = { version = "1", features = ["full", "parking_lot"] }

     

       12
       12
        
       tokio-util = { version = "0.7", features = ["tracing"] }

     

       13
       13
        
       rustls = { version = "0.23", default-features = false, features = ["log", "ring", "std"] }

     

       14
       14
        
       tokio-websockets = { version = "0.12", features = ["client", "rustls-platform-verifier", "getrandom", "ring"] }

     
···

       29
       29
        
       itertools = "0.14.0"

     

       30
       30
        
       byteview = "0.6.1"

     

       31
       31
        
       rayon = "1.10.0"

     

       32
       32
       +
       parking_lot = { version = "0.12", features = ["send_guard", "hardware-lock-elision"] }

     

       33
       33
       +
       rclite = "0.2.7"

+1 -1

server/src/api.rs

···

       3
       3
        
           fmt::Display,

     

       4
       4
        
           net::SocketAddr,

     

       5
       5
        
           ops::{Bound, Deref, RangeBounds},

     

       6
       6
       -
           sync::Arc,

     

       7
       6
        
           time::Duration,

     

       8
       7
        
       };

     

       9
       8
        
       

     
···

       16
       15
        
           routing::get,

     

       17
       16
        
       };

     

       18
       17
        
       use axum_tws::{Message, WebSocketUpgrade};

     

       18
       18
       +
       use rclite::Arc;

     

       19
       19
        
       use serde::{Deserialize, Serialize};

     

       20
       20
        
       use smol_str::SmolStr;

     

       21
       21
        
       use tokio_util::sync::CancellationToken;

server/src/db/block.rs

···

       119
       119
        
               })

     

       120
       120
        
           }

     

       121
       121
        
       

     

       122
       122
       +
           pub fn item_count(&self) -> usize {

     

       123
       123
       +
               self.expected

     

       124
       124
       +
           }

     

       125
       125
       +
       

     

       122
       126
        
           pub fn decode(&mut self) -> io::Result<Option<Item<T>>> {

     

       123
       127
        
               if self.items_read == 0 {

     

       124
       128
        
                   // read the first timestamp

+22 -22

server/src/db/mod.rs

···

       2
       2
        
           io::Cursor,

     

       3
       3
        
           ops::{Bound, Deref, RangeBounds},

     

       4
       4
        
           path::Path,

     

       5
       5
       -
           sync::{

     

       6
       6
       -
               Arc,

     

       7
       7
       -
               atomic::{AtomicU64, AtomicUsize, Ordering as AtomicOrdering},

     

       8
       8
       -
           },

     

       5
       5
       +
           sync::atomic::{AtomicU64, AtomicUsize, Ordering as AtomicOrdering},

     

       9
       6
        
           time::Duration,

     

       10
       7
        
       };

     

       11
       8
        
       

     

       12
       9
        
       use byteview::ByteView;

     

       13
       10
        
       use fjall::{Config, Keyspace, Partition, PartitionCreateOptions, Slice};

     

       14
       11
        
       use itertools::{Either, Itertools};

     

       12
       12
       +
       use parking_lot::Mutex;

     

       15
       13
        
       use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};

     

       14
       14
       +
       use rclite::Arc;

     

       16
       15
        
       use rkyv::{Archive, Deserialize, Serialize, rancor::Error};

     

       17
       16
        
       use smol_str::SmolStr;

     

       18
       17
        
       use tokio::sync::broadcast;

     
···

       85
       84
        
       pub struct LexiconHandle {

     

       86
       85
        
           tree: Partition,

     

       87
       86
        
           nsid: SmolStr,

     

       88
       88
       -
           buf: Arc<scc::Queue<EventRecord>>,

     

       89
       89
       -
           // this is stored here since scc::Queue does not have O(1) length

     

       90
       90
       -
           buf_len: AtomicUsize,   // seqcst

     

       87
       87
       +
           buf: Arc<Mutex<Vec<EventRecord>>>,

     

       91
       88
        
           last_insert: AtomicU64, // relaxed

     

       92
       89
        
           eps: DefaultRateTracker,

     

       93
       90
        
       }

     
···

       99
       96
        
                   tree: keyspace.open_partition(nsid, opts).unwrap(),

     

       100
       97
        
                   nsid: nsid.into(),

     

       101
       98
        
                   buf: Default::default(),

     

       102
       102
       -
                   buf_len: AtomicUsize::new(0),

     

       103
       99
        
                   last_insert: AtomicU64::new(0),

     

       104
       100
        
                   eps: RateTracker::new(Duration::from_secs(10)),

     

       105
       101
        
               }

     

       106
       102
        
           }

     

       107
       103
        
       

     

       108
       104
        
           fn item_count(&self) -> usize {

     

       109
       109
       -
               self.buf_len.load(AtomicOrdering::SeqCst)

     

       105
       105
       +
               self.buf.lock().len()

     

       110
       106
        
           }

     

       111
       107
        
       

     

       112
       108
        
           fn since_last_activity(&self) -> u64 {

     
···

       118
       114
        
           }

     

       119
       115
        
       

     

       120
       116
        
           fn insert(&self, event: EventRecord) {

     

       121
       121
       -
               self.buf.push(event);

     

       122
       122
       -
               self.buf_len.fetch_add(1, AtomicOrdering::SeqCst);

     

       117
       117
       +
               self.buf.lock().push(event);

     

       123
       118
        
               self.last_insert.store(CLOCK.raw(), AtomicOrdering::Relaxed);

     

       124
       119
        
               self.eps.observe();

     

       125
       120
        
           }

     

       126
       121
        
       

     

       122
       122
       +
           fn compact(&self) {}

     

       123
       123
       +
       

     

       127
       124
        
           fn encode_block(&self, item_count: usize) -> AppResult<Block> {

     

       128
       125
        
               let mut writer = ItemEncoder::new(

     

       129
       126
        
                   WritableByteView::with_size(ItemEncoder::encoded_len(item_count)),

     
···

       132
       129
        
               let mut start_timestamp = None;

     

       133
       130
        
               let mut end_timestamp = None;

     

       134
       131
        
               let mut written = 0_usize;

     

       135
       135
       -
               while let Some(event) = self.buf.pop() {

     

       132
       132
       +
               for event in self.buf.lock().drain(..) {

     

       136
       133
        
                   let item = Item::new(

     

       137
       134
        
                       event.timestamp,

     

       138
       135
        
                       &NsidHit {

     
···

       157
       154
        
                   .into());

     

       158
       155
        
               }

     

       159
       156
        
               if let (Some(start_timestamp), Some(end_timestamp)) = (start_timestamp, end_timestamp) {

     

       160
       160
       -
                   self.buf_len.store(0, AtomicOrdering::SeqCst);

     

       161
       157
        
                   let value = writer.finish()?;

     

       162
       158
        
                   let key = varints_unsigned_encoded([start_timestamp, end_timestamp]);

     

       163
       159
        
                   return Ok(Block {

     
···

       208
       204
        
               })

     

       209
       205
        
           }

     

       210
       206
        
       

     

       207
       207
       +
           #[inline(always)]

     

       211
       208
        
           pub fn shutting_down(&self) -> impl Future<Output = ()> {

     

       212
       209
        
               self.cancel_token.cancelled()

     

       213
       210
        
           }

     

       214
       211
        
       

     

       212
       212
       +
           #[inline(always)]

     

       215
       213
        
           pub fn is_shutting_down(&self) -> bool {

     

       216
       214
        
               self.cancel_token.is_cancelled()

     

       215
       215
       +
           }

     

       216
       216
       +
       

     

       217
       217
       +
           #[inline(always)]

     

       218
       218
       +
           pub fn eps(&self) -> usize {

     

       219
       219
       +
               self.eps.rate() as usize

     

       220
       220
       +
           }

     

       221
       221
       +
       

     

       222
       222
       +
           #[inline(always)]

     

       223
       223
       +
           pub fn new_listener(&self) -> broadcast::Receiver<(SmolStr, NsidCounts)> {

     

       224
       224
       +
               self.event_broadcaster.subscribe()

     

       217
       225
        
           }

     

       218
       226
        
       

     

       219
       227
        
           pub fn sync(&self, all: bool) -> AppResult<()> {

     
···

       284
       292
        
               Ok(())

     

       285
       293
        
           }

     

       286
       294
        
       

     

       287
       287
       -
           #[inline(always)]

     

       288
       288
       -
           pub fn eps(&self) -> usize {

     

       289
       289
       -
               self.eps.rate() as usize

     

       290
       290
       -
           }

     

       291
       291
       -
       

     

       292
       292
       -
           #[inline(always)]

     

       293
       293
       -
           pub fn new_listener(&self) -> broadcast::Receiver<(SmolStr, NsidCounts)> {

     

       294
       294
       -
               self.event_broadcaster.subscribe()

     

       295
       295
       -
           }

     

       295
       295
       +
           pub fn compact(&self) {}

     

       296
       296
        
       

     

       297
       297
        
           #[inline(always)]

     

       298
       298
        
           fn maybe_run_in_nsid_tree<T>(

+2 -1

server/src/main.rs

···

       1
       1
       -
       use std::{ops::Deref, sync::Arc};

     

       1
       1
       +
       use std::ops::Deref;

     

       2
       2
        
       

     

       3
       3
       +
       use rclite::Arc;

     

       3
       4
        
       use smol_str::ToSmolStr;

     

       4
       5
        
       use tokio_util::sync::CancellationToken;

     

       5
       6
        
       use tracing::Level;