commit 6001b7f80de7520f72ba6b3aaa609b6087d0a299 · ptr.pet/nsid-tracker

+1

server/Cargo.lock

···

       1580
       1580
        
        "tower-http",

     

       1581
       1581
        
        "tracing",

     

       1582
       1582
        
        "tracing-subscriber",

     

       1583
       1583
       +
        "zstd",

     

       1583
       1584
        
       ]

     

       1584
       1585
        
       

     

       1585
       1586
        
       [[package]]

+1

server/Cargo.toml

···

       30
       30
        
       rayon = "1.10.0"

     

       31
       31
        
       parking_lot = { version = "0.12", features = ["send_guard", "hardware-lock-elision"] }

     

       32
       32
        
       rclite = "0.2.7"

     

       33
       33
       +
       zstd = "0.13.3"

     

       33
       34
        
       

     

       34
       35
        
       [target.'cfg(target_env = "msvc")'.dependencies]

     

       35
       36
        
       snmalloc-rs = "0.3.8"

+19

server/src/db/mod.rs

···

       380
       380
        
               })

     

       381
       381
        
           }

     

       382
       382
        
       

     

       383
       383
       +
           // train zstd dict with 100 blocks from every lexicon

     

       384
       384
       +
           pub fn train_zstd_dict(&self) -> AppResult<Vec<u8>> {

     

       385
       385
       +
               let samples = self

     

       386
       386
       +
                   .get_nsids()

     

       387
       387
       +
                   .filter_map(|nsid| self.get_handle(&nsid))

     

       388
       388
       +
                   .map(|handle| {

     

       389
       389
       +
                       handle

     

       390
       390
       +
                           .iter()

     

       391
       391
       +
                           .rev()

     

       392
       392
       +
                           .map(|res| {

     

       393
       393
       +
                               res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))

     

       394
       394
       +
                                   .map(|(_, value)| Cursor::new(value))

     

       395
       395
       +
                           })

     

       396
       396
       +
                           .take(1000)

     

       397
       397
       +
                   })

     

       398
       398
       +
                   .flatten();

     

       399
       399
       +
               zstd::dict::from_sample_iterator(samples, 1024 * 128).map_err(AppError::from)

     

       400
       400
       +
           }

     

       401
       401
       +
       

     

       383
       402
        
           pub fn get_hits(

     

       384
       403
        
               &self,

     

       385
       404
        
               nsid: &str,

+10

server/src/main.rs

···

       53
       53
        
                   debug();

     

       54
       54
        
                   return;

     

       55
       55
        
               }

     

       56
       56
       +
               Some("traindict") => {

     

       57
       57
       +
                   train_zstd_dict();

     

       58
       58
       +
                   return;

     

       59
       59
       +
               }

     

       56
       60
        
               Some(x) => {

     

       57
       61
        
                   tracing::error!("unknown command: {}", x);

     

       58
       62
        
                   return;

     
···

       205
       209
        
           ingest_events.join().expect("failed to join ingest events");

     

       206
       210
        
           db_task.await.expect("cant join db task");

     

       207
       211
        
           db.sync(true).expect("cant sync db");

     

       212
       212
       +
       }

     

       213
       213
       +
       

     

       214
       214
       +
       fn train_zstd_dict() {

     

       215
       215
       +
           let db = Db::new(DbConfig::default(), CancellationToken::new()).expect("couldnt create db");

     

       216
       216
       +
           let dict_data = db.train_zstd_dict().expect("cant train zstd dict");

     

       217
       217
       +
           std::fs::write("zstd_dict", dict_data).expect("cant save zstd dict")

     

       208
       218
        
       }

     

       209
       219
        
       

     

       210
       220
        
       fn debug() {