···

       
1580
       
1580
       
 
       
 "tower-http",

     

       
1581
       
1581
       
 
       
 "tracing",

     

       
1582
       
1582
       
 
       
 "tracing-subscriber",

     

       
1583
       
1583
       
-
       
 "zstd",

     

       
1584
       
1583
       
 
       
]

     

       
1585
       
1584
       
 
       


     

       
1586
       
1585
       
 
       
[[package]]

     

···

       
30
       
30
       
 
       
rayon = "1.10.0"

     

       
31
       
31
       
 
       
parking_lot = { version = "0.12", features = ["send_guard", "hardware-lock-elision"] }

     

       
32
       
32
       
 
       
rclite = "0.2.7"

     

       
33
       
33
       
-
       
zstd = "0.13.3"

     

       
34
       
33
       
 
       


     

       
35
       
34
       
 
       
[target.'cfg(target_env = "msvc")'.dependencies]

     

       
36
       
35
       
 
       
snmalloc-rs = "0.3.8"

     

···

       
380
       
380
       
 
       
        })

     

       
381
       
381
       
 
       
    }

     

       
382
       
382
       
 
       


     

       
383
       
383
       
-
       
    // train zstd dict with 100 blocks from every lexicon

     

       
384
       
384
       
-
       
    pub fn train_zstd_dict(&self) -> AppResult<Vec<u8>> {

     

       
385
       
385
       
-
       
        let samples = self

     

       
386
       
386
       
-
       
            .get_nsids()

     

       
387
       
387
       
-
       
            .filter_map(|nsid| self.get_handle(&nsid))

     

       
388
       
388
       
-
       
            .map(|handle| {

     

       
389
       
389
       
-
       
                handle

     

       
390
       
390
       
-
       
                    .iter()

     

       
391
       
391
       
-
       
                    .rev()

     

       
392
       
392
       
-
       
                    .map(|res| {

     

       
393
       
393
       
-
       
                        res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))

     

       
394
       
394
       
-
       
                            .map(|(_, value)| Cursor::new(value))

     

       
395
       
395
       
-
       
                    })

     

       
396
       
396
       
-
       
                    .take(1000)

     

       
397
       
397
       
-
       
            })

     

       
398
       
398
       
-
       
            .flatten();

     

       
399
       
399
       
-
       
        zstd::dict::from_sample_iterator(samples, 1024 * 128).map_err(AppError::from)

     

       
400
       
400
       
-
       
    }

     

       
401
       
401
       
-
       


     

       
402
       
383
       
 
       
    pub fn get_hits(

     

       
403
       
384
       
 
       
        &self,

     

       
404
       
385
       
 
       
        nsid: &str,

     

···

       
53
       
53
       
 
       
            debug();

     

       
54
       
54
       
 
       
            return;

     

       
55
       
55
       
 
       
        }

     

       
56
       
56
       
-
       
        Some("traindict") => {

     

       
57
       
57
       
-
       
            train_zstd_dict();

     

       
58
       
58
       
-
       
            return;

     

       
59
       
59
       
-
       
        }

     

       
60
       
56
       
 
       
        Some(x) => {

     

       
61
       
57
       
 
       
            tracing::error!("unknown command: {}", x);

     

       
62
       
58
       
 
       
            return;

     
···

       
209
       
205
       
 
       
    ingest_events.join().expect("failed to join ingest events");

     

       
210
       
206
       
 
       
    db_task.await.expect("cant join db task");

     

       
211
       
207
       
 
       
    db.sync(true).expect("cant sync db");

     

       
212
       
212
       
-
       
}

     

       
213
       
213
       
-
       


     

       
214
       
214
       
-
       
fn train_zstd_dict() {

     

       
215
       
215
       
-
       
    let db = Db::new(DbConfig::default(), CancellationToken::new()).expect("couldnt create db");

     

       
216
       
216
       
-
       
    let dict_data = db.train_zstd_dict().expect("cant train zstd dict");

     

       
217
       
217
       
-
       
    std::fs::write("zstd_dict", dict_data).expect("cant save zstd dict")

     

       
218
       
208
       
 
       
}

     

       
219
       
209
       
 
       


     

       
220
       
210
       
 
       
fn debug() {