···

       
177
       
 
       
    let from = params.to.map(Bound::Included).unwrap_or(Bound::Unbounded);

     

       
178
       
 
       
    let to = params.from.map(Bound::Included).unwrap_or(Bound::Unbounded);

     

       
179
       
 
       
    let maybe_hits = db

     

       
180
       
-
       
        .get_hits(&params.nsid, HitsRange { from, to })

     

       
181
       
 
       
        .take(MAX_HITS);

     

       
182
       
 
       
    let mut hits = Vec::with_capacity(maybe_hits.size_hint().0);

     

       
183
       
 
       


     

···

       
177
       
 
       
    let from = params.to.map(Bound::Included).unwrap_or(Bound::Unbounded);

     

       
178
       
 
       
    let to = params.from.map(Bound::Included).unwrap_or(Bound::Unbounded);

     

       
179
       
 
       
    let maybe_hits = db

     

       
180
       
+
       
        .get_hits(&params.nsid, HitsRange { from, to }, MAX_HITS)

     

       
181
       
 
       
        .take(MAX_HITS);

     

       
182
       
 
       
    let mut hits = Vec::with_capacity(maybe_hits.size_hint().0);

     

       
183
       
 
       


     

···

       
383
       
 
       
        &self,

     

       
384
       
 
       
        nsid: &str,

     

       
385
       
 
       
        range: impl RangeBounds<u64> + std::fmt::Debug,

     

       
0
       
 
       

     

       
386
       
 
       
    ) -> impl Iterator<Item = AppResult<handle::Item>> {

     

       
387
       
 
       
        let start_limit = match range.start_bound().cloned() {

     

       
388
       
 
       
            Bound::Included(start) => start,

     
···

       
401
       
 
       
        };

     

       
402
       
 
       


     

       
403
       
 
       
        // let mut ts = CLOCK.now();

     

       
0
       
 
       

     

       
404
       
 
       
        let map_block = move |(key, val)| {

     

       
405
       
 
       
            let mut key_reader = Cursor::new(key);

     

       
406
       
 
       
            let start_timestamp = key_reader.read_varint::<u64>()?;

     

       
407
       
 
       
            // let end_timestamp = key_reader.read_varint::<u64>()?;

     

       
408
       
 
       
            if start_timestamp < start_limit {

     

       
409
       
 
       
                // tracing::info!(

     

       
410
       
-
       
                //     "skipped block with timestamps {start_timestamp}..{end_timestamp} because {start_limit} is greater"

     

       
411
       
 
       
                // );

     

       
412
       
 
       
                return Ok(None);

     

       
413
       
-
       
            } else {

     

       
414
       
-
       
                // tracing::info!("using block with timestamp {start_timestamp}..{end_timestamp}");

     

       
415
       
 
       
            }

     

       
416
       
 
       
            let decoder = handle::ItemDecoder::new(Cursor::new(val), start_timestamp)?;

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
417
       
 
       
            // tracing::info!(

     

       
418
       
 
       
            //     "took {}ns to get block with size {}",

     

       
419
       
 
       
            //     ts.elapsed().as_nanos(),

     

···

       
383
       
 
       
        &self,

     

       
384
       
 
       
        nsid: &str,

     

       
385
       
 
       
        range: impl RangeBounds<u64> + std::fmt::Debug,

     

       
386
       
+
       
        max_items: usize,

     

       
387
       
 
       
    ) -> impl Iterator<Item = AppResult<handle::Item>> {

     

       
388
       
 
       
        let start_limit = match range.start_bound().cloned() {

     

       
389
       
 
       
            Bound::Included(start) => start,

     
···

       
402
       
 
       
        };

     

       
403
       
 
       


     

       
404
       
 
       
        // let mut ts = CLOCK.now();

     

       
405
       
+
       
        let mut current_item_count = 0;

     

       
406
       
 
       
        let map_block = move |(key, val)| {

     

       
407
       
 
       
            let mut key_reader = Cursor::new(key);

     

       
408
       
 
       
            let start_timestamp = key_reader.read_varint::<u64>()?;

     

       
409
       
 
       
            // let end_timestamp = key_reader.read_varint::<u64>()?;

     

       
410
       
 
       
            if start_timestamp < start_limit {

     

       
411
       
 
       
                // tracing::info!(

     

       
412
       
+
       
                //     "stopped at block with timestamps {start_timestamp}..{end_timestamp} because {start_limit} is greater"

     

       
413
       
 
       
                // );

     

       
414
       
 
       
                return Ok(None);

     

       
0
       
 
       

     

       
0
       
 
       

     

       
415
       
 
       
            }

     

       
416
       
 
       
            let decoder = handle::ItemDecoder::new(Cursor::new(val), start_timestamp)?;

     

       
417
       
+
       
            current_item_count += decoder.item_count();

     

       
418
       
+
       
            if current_item_count > max_items {

     

       
419
       
+
       
                return Ok(None);

     

       
420
       
+
       
            }

     

       
421
       
 
       
            // tracing::info!(

     

       
422
       
 
       
            //     "took {}ns to get block with size {}",

     

       
423
       
 
       
            //     ts.elapsed().as_nanos(),

     

···

       
1
       
-
       
use std::{ops::Deref, time::Duration, u64};

     

       
2
       
 
       


     

       
3
       
 
       
use itertools::Itertools;

     

       
4
       
 
       
use rclite::Arc;

     
···

       
300
       
 
       
        threads.push(std::thread::spawn(move || {

     

       
301
       
 
       
            tracing::info!("{}: migrating...", nsid.deref());

     

       
302
       
 
       
            let mut count = 0_u64;

     

       
303
       
-
       
            for hits in from.get_hits(&nsid, ..).chunks(100000).into_iter() {

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
304
       
 
       
                to.ingest_events(hits.map(|hit| {

     

       
305
       
 
       
                    count += 1;

     

       
306
       
 
       
                    let hit = hit.expect("cant decode hit");

     

···

       
1
       
+
       
use std::{ops::Deref, time::Duration, u64, usize};

     

       
2
       
 
       


     

       
3
       
 
       
use itertools::Itertools;

     

       
4
       
 
       
use rclite::Arc;

     
···

       
300
       
 
       
        threads.push(std::thread::spawn(move || {

     

       
301
       
 
       
            tracing::info!("{}: migrating...", nsid.deref());

     

       
302
       
 
       
            let mut count = 0_u64;

     

       
303
       
+
       
            for hits in from

     

       
304
       
+
       
                .get_hits(&nsid, .., usize::MAX)

     

       
305
       
+
       
                .chunks(100000)

     

       
306
       
+
       
                .into_iter()

     

       
307
       
+
       
            {

     

       
308
       
 
       
                to.ingest_events(hits.map(|hit| {

     

       
309
       
 
       
                    count += 1;

     

       
310
       
 
       
                    let hit = hit.expect("cant decode hit");