commit a9dd3c0c2a1ce3951ff280169e3bed245c1d863d · nekomimi.pet/microcosm-rs

+3 -2

ufos/src/consumer.rs

···

       121
       121
        
                       created_collection = true;

     

       122
       122
        
                       Default::default()

     

       123
       123
        
                   });

     

       124
       124
       -
               collection.push_front(record);

     

       125
       125
       -
               collection.truncate(MAX_BATCHED_RECORDS);

     

       124
       124
       +
               collection.total_seen += 1;

     

       125
       125
       +
               collection.samples.push_front(record);

     

       126
       126
       +
               collection.samples.truncate(MAX_BATCHED_RECORDS);

     

       126
       127
        
       

     

       127
       128
        
               if created_collection {

     

       128
       129
        
                   self.current_batch.records.len() >= MAX_BATCHED_COLLECTIONS // full if we have collections to the max

+7 -1

ufos/src/lib.rs

···

       12
       12
        
           pub record: serde_json::Value,

     

       13
       13
        
       }

     

       14
       14
        
       

     

       15
       15
       +
       #[derive(Debug, Default)]

     

       16
       16
       +
       pub struct CollectionSamples {

     

       17
       17
       +
           pub total_seen: usize,

     

       18
       18
       +
           pub samples: VecDeque<SetRecord>,

     

       19
       19
       +
       }

     

       20
       20
       +
       

     

       15
       21
        
       #[derive(Debug)]

     

       16
       22
        
       pub struct DeleteRecord {

     

       17
       23
        
           pub did: Did,

     
···

       21
       27
        
       

     

       22
       28
        
       #[derive(Debug, Default)]

     

       23
       29
        
       pub struct EventBatch {

     

       24
       24
       -
           pub records: HashMap<Nsid, VecDeque<SetRecord>>,

     

       30
       30
       +
           pub records: HashMap<Nsid, CollectionSamples>,

     

       25
       31
        
           pub record_deletes: Vec<DeleteRecord>,

     

       26
       32
        
           pub account_removes: Vec<Did>,

     

       27
       33
        
       }

+3 -2

ufos/src/store.rs

···

       20
       20
        
               record_deletes,

     

       21
       21
        
               account_removes,

     

       22
       22
        
           } = batch;

     

       23
       23
       -
           let total_records: usize = records.values().map(|v| v.len()).sum();

     

       23
       23
       +
           let total_records: usize = records.values().map(|v| v.total_seen).sum();

     

       24
       24
       +
           let total_samples: usize = records.values().map(|v| v.samples.len()).sum();

     

       24
       25
        
           println!(

     

       25
       25
       -
               "got batch with {total_records} records in {} collections, {} record deletes, {} account removes",

     

       26
       26
       +
               "got batch of {total_samples: >3} samples from {total_records: >3} records in {: >2} collections, {: >2} record deletes, {} account removes",

     

       26
       27
        
               records.len(),

     

       27
       28
        
               record_deletes.len(),

     

       28
       29
        
               account_removes.len()