commit a050efab334343bef8f5b618fcb2a8eea12b6da1 · ptr.pet/nsid-tracker

+37 -16
server/src/db/mod.rs
···

       402
       402
        
               };

     

       403
       403
        
       

     

       404
       404
        
               // let mut ts = CLOCK.now();

     

       405
       405
       -
               let mut current_item_count = 0;

     

       406
       406
       -
               let map_block = move |(key, val)| {

     

       405
       405
       +
               let map_block = move |(res, current_item_count)| -> AppResult<(Option<_>, usize)> {

     

       407
       406
        
                   if current_item_count >= max_items {

     

       408
       408
       -
                       return Ok(None);

     

       407
       407
       +
                       return Ok((None, current_item_count));

     

       409
       408
        
                   }

     

       409
       409
       +
                   let (key, val) = res?;

     

       410
       410
        
                   let mut key_reader = Cursor::new(key);

     

       411
       411
        
                   let start_timestamp = key_reader.read_varint::<u64>()?;

     

       412
       412
        
                   // let end_timestamp = key_reader.read_varint::<u64>()?;

     
···

       414
       414
        
                       // tracing::info!(

     

       415
       415
        
                       //     "stopped at block with timestamps {start_timestamp}..{end_timestamp} because {start_limit} is greater"

     

       416
       416
        
                       // );

     

       417
       417
       -
                       return Ok(None);

     

       417
       417
       +
                       return Ok((None, current_item_count));

     

       418
       418
        
                   }

     

       419
       419
        
                   let decoder = handle::ItemDecoder::new(Cursor::new(val), start_timestamp)?;

     

       420
       420
       -
                   current_item_count += decoder.item_count();

     

       420
       420
       +
                   let current_item_count = current_item_count + decoder.item_count();

     

       421
       421
        
                   // tracing::info!(

     

       422
       422
        
                   //     "took {}ns to get block with size {}",

     

       423
       423
        
                   //     ts.elapsed().as_nanos(),

     

       424
       424
        
                   //     decoder.item_count()

     

       425
       425
        
                   // );

     

       426
       426
        
                   // ts = CLOCK.now();

     

       427
       427
       -
                   Ok(Some(

     

       428
       428
       -
                       decoder

     

       429
       429
       -
                           .take_while(move |item| {

     

       430
       430
       -
                               item.as_ref().map_or(true, |item| {

     

       431
       431
       -
                                   item.timestamp <= end_limit && item.timestamp >= start_limit

     

       427
       427
       +
                   Ok((

     

       428
       428
       +
                       Some(

     

       429
       429
       +
                           decoder

     

       430
       430
       +
                               .take_while(move |item| {

     

       431
       431
       +
                                   item.as_ref().map_or(true, |item| {

     

       432
       432
       +
                                       item.timestamp <= end_limit && item.timestamp >= start_limit

     

       433
       433
       +
                                   })

     

       432
       434
        
                               })

     

       433
       433
       -
                           })

     

       434
       434
       -
                           .map(|res| res.map_err(AppError::from)),

     

       435
       435
       +
                               .map(|res| res.map_err(AppError::from)),

     

       436
       436
       +
                       ),

     

       437
       437
       +
                       current_item_count,

     

       435
       438
        
                   ))

     

       436
       439
        
               };

     

       437
       440
        
       

     

       438
       438
       -
               let blocks = handle

     

       441
       441
       +
               let (blocks, counted) = handle

     

       439
       442
        
                   .range(..end_key)

     

       443
       443
       +
                   .map(|res| res.map_err(AppError::from))

     

       440
       444
        
                   .rev()

     

       441
       441
       -
                   .map_while(move |res| res.map_err(AppError::from).and_then(map_block).transpose())

     

       442
       442
       -
                   .collect_vec();

     

       445
       445
       +
                   .fold_while(

     

       446
       446
       +
                       (Vec::with_capacity(20), 0),

     

       447
       447
       +
                       |(mut blocks, current_item_count), res| {

     

       448
       448
       +
                           use itertools::FoldWhile::*;

     

       449
       449
       +
       

     

       450
       450
       +
                           match map_block((res, current_item_count)) {

     

       451
       451
       +
                               Ok((Some(block), current_item_count)) => {

     

       452
       452
       +
                                   blocks.push(Ok(block));

     

       453
       453
       +
                                   Continue((blocks, current_item_count))

     

       454
       454
       +
                               }

     

       455
       455
       +
                               Ok((None, current_item_count)) => Done((blocks, current_item_count)),

     

       456
       456
       +
                               Err(err) => {

     

       457
       457
       +
                                   blocks.push(Err(err));

     

       458
       458
       +
                                   Done((blocks, current_item_count))

     

       459
       459
       +
                               }

     

       460
       460
       +
                           }

     

       461
       461
       +
                       },

     

       462
       462
       +
                   )

     

       463
       463
       +
                   .into_inner();

     

       443
       464
        
       

     

       444
       465
        
               tracing::info!(

     

       445
       445
       -
                   "got blocks with size {}, item count {current_item_count}",

     

       466
       466
       +
                   "got blocks with size {}, item count {counted}",

     

       446
       467
        
                   blocks.len()

     

       447
       468
        
               );

     

       448
       469