commit 0880516e8fcdf49685fac5718886a66609a061f4 · nekomimi.pet/microcosm-rs

+29 -3

ufos/src/lib.rs

···

       10
        
       

     

       11
        
       use crate::db_types::{EncodingError, EncodingResult};

     

       12
        
       use crate::error::BatchInsertError;

     

       13
       -
       use crate::store_types::SketchSecretPrefix;

     

       14
        
       use cardinality_estimator_safe::{Element, Sketch};

     

       15
        
       use error::FirehoseEventError;

     

       16
        
       use jetstream::events::{CommitEvent, CommitOp, Cursor};

     
···

       281
        
       pub struct NsidCount {

     

       282
        
           nsid: String,

     

       283
        
           creates: u64,

     

       284
       -
           // TODO: add updates and deletes

     

       0
        
       
     

       285
        
           dids_estimate: u64,

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       286
        
       }

     

       287
        
       

     

       288
        
       #[derive(Debug, PartialEq, Serialize, JsonSchema)]

     

       289
        
       pub struct PrefixCount {

     

       290
        
           prefix: String,

     

       291
        
           creates: u64,

     

       292
       -
           // TODO: add updates and deletes

     

       0
        
       
     

       293
        
           dids_estimate: u64,

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       294
        
       }

     

       295
        
       

     

       296
        
       #[derive(Debug, PartialEq, Serialize, JsonSchema)]

···

       10
        
       

     

       11
        
       use crate::db_types::{EncodingError, EncodingResult};

     

       12
        
       use crate::error::BatchInsertError;

     

       13
       +
       use crate::store_types::{CountsValue, SketchSecretPrefix};

     

       14
        
       use cardinality_estimator_safe::{Element, Sketch};

     

       15
        
       use error::FirehoseEventError;

     

       16
        
       use jetstream::events::{CommitEvent, CommitOp, Cursor};

     
···

       281
        
       pub struct NsidCount {

     

       282
        
           nsid: String,

     

       283
        
           creates: u64,

     

       284
       +
           updates: u64,

     

       285
       +
           deletes: u64,

     

       286
        
           dids_estimate: u64,

     

       287
       +
       }

     

       288
       +
       impl NsidCount {

     

       289
       +
           pub fn new(nsid: &Nsid, counts: &CountsValue) -> Self {

     

       290
       +
               let crud = counts.counts();

     

       291
       +
               Self {

     

       292
       +
                   nsid: nsid.to_string(),

     

       293
       +
                   creates: crud.creates,

     

       294
       +
                   updates: crud.updates,

     

       295
       +
                   deletes: crud.deletes,

     

       296
       +
                   dids_estimate: counts.dids().estimate() as u64,

     

       297
       +
               }

     

       298
       +
           }

     

       299
        
       }

     

       300
        
       

     

       301
        
       #[derive(Debug, PartialEq, Serialize, JsonSchema)]

     

       302
        
       pub struct PrefixCount {

     

       303
        
           prefix: String,

     

       304
        
           creates: u64,

     

       305
       +
           updates: u64,

     

       306
       +
           deletes: u64,

     

       307
        
           dids_estimate: u64,

     

       308
       +
       }

     

       309
       +
       impl PrefixCount {

     

       310
       +
           pub fn new(prefix: &str, counts: &CountsValue) -> Self {

     

       311
       +
               let crud = counts.counts();

     

       312
       +
               Self {

     

       313
       +
                   prefix: prefix.to_string(),

     

       314
       +
                   creates: crud.creates,

     

       315
       +
                   updates: crud.updates,

     

       316
       +
                   deletes: crud.deletes,

     

       317
       +
                   dids_estimate: counts.dids().estimate() as u64,

     

       318
       +
               }

     

       319
       +
           }

     

       320
        
       }

     

       321
        
       

     

       322
        
       #[derive(Debug, PartialEq, Serialize, JsonSchema)]

+27 -35

ufos/src/storage_fjall.rs

···

       507
        
                           merged.merge(&counts);

     

       508
        
                       }

     

       509
        
                   }

     

       510
       -
                   out.push(NsidCount {

     

       511
       -
                       nsid: nsid.to_string(),

     

       512
       -
                       creates: merged.counts().creates,

     

       513
       -
                       dids_estimate: merged.dids().estimate() as u64,

     

       514
       -
                   });

     

       515
        
               }

     

       516
        
       

     

       517
        
               let next_cursor = current_nsid.map(|s| s.to_db_bytes()).transpose()?;

     
···

       617
        
                   .into_iter()

     

       618
        
                   .rev()

     

       619
        
                   .take(limit)

     

       620
       -
                   .map(|(nsid, cv)| NsidCount {

     

       621
       -
                       nsid: nsid.to_string(),

     

       622
       -
                       creates: cv.counts().creates,

     

       623
       -
                       dids_estimate: cv.dids().estimate() as u64,

     

       624
       -
                   })

     

       625
        
                   .collect();

     

       626
        
               Ok(counts)

     

       627
        
           }

     
···

       727
        
               let mut prefix_count = CountsValue::default();

     

       728
        
               #[derive(Debug, Clone, PartialEq)]

     

       729
        
               enum Child {

     

       730
       -
                   FullNsid(String),

     

       731
        
                   ChildPrefix(String),

     

       732
        
               }

     

       733
        
               impl Child {

     

       734
        
                   fn from_prefix(nsid: &Nsid, prefix: &NsidPrefix) -> Option<Self> {

     

       735
        
                       if prefix.is_group_of(nsid) {

     

       736
       -
                           return Some(Child::FullNsid(nsid.to_string()));

     

       737
        
                       }

     

       738
        
                       let suffix = nsid.as_str().strip_prefix(&format!("{}.", prefix.0))?;

     

       739
        
                       let (segment, _) = suffix.split_once('.').unwrap();

     
···

       742
        
                   }

     

       743
        
                   fn is_before(&self, other: &Child) -> bool {

     

       744
        
                       match (self, other) {

     

       745
       -
                           (Child::FullNsid(s), Child::ChildPrefix(o)) if s == o => true,

     

       746
       -
                           (Child::ChildPrefix(s), Child::FullNsid(o)) if s == o => false,

     

       747
       -
                           (Child::FullNsid(s), Child::FullNsid(o)) => s < o,

     

       748
        
                           (Child::ChildPrefix(s), Child::ChildPrefix(o)) => s < o,

     

       749
       -
                           (Child::FullNsid(s), Child::ChildPrefix(o)) => s < o,

     

       750
       -
                           (Child::ChildPrefix(s), Child::FullNsid(o)) => s < o,

     

       751
        
                       }

     

       752
        
                   }

     

       753
        
                   fn into_inner(self) -> String {

     

       754
        
                       match self {

     

       755
       -
                           Child::FullNsid(s) => s,

     

       756
        
                           Child::ChildPrefix(s) => s,

     

       757
        
                       }

     

       758
        
                   }

     
···

       791
        
                       }

     

       792
        
                   }

     

       793
        
                   items.push(match child {

     

       794
       -
                       Child::FullNsid(nsid) => PrefixChild::Collection(NsidCount {

     

       795
       -
                           nsid,

     

       796
       -
                           creates: merged.counts().creates,

     

       797
       -
                           dids_estimate: merged.dids().estimate() as u64,

     

       798
       -
                       }),

     

       799
       -
                       Child::ChildPrefix(prefix) => PrefixChild::Prefix(PrefixCount {

     

       800
       -
                           prefix,

     

       801
       -
                           creates: merged.counts().creates,

     

       802
       -
                           dids_estimate: merged.dids().estimate() as u64,

     

       803
       -
                       }),

     

       804
        
                   });

     

       805
        
               }

     

       806
        
       

     
···

       991
        
               for kv in self.rollups.range((start, end)) {

     

       992
        
                   let (key_bytes, val_bytes) = kv?;

     

       993
        
                   let key = db_complete::<AllTimeRollupKey>(&key_bytes)?;

     

       994
       -
                   let nsid = key.collection().as_str().to_string();

     

       995
        
                   for term in &terms {

     

       996
        
                       if nsid.contains(term) {

     

       997
        
                           let counts = db_complete::<CountsValue>(&val_bytes)?;

     

       998
       -
                           matches.push(NsidCount {

     

       999
       -
                               nsid: nsid.clone(),

     

       1000
       -
                               creates: counts.counts().creates,

     

       1001
       -
                               dids_estimate: counts.dids().estimate() as u64,

     

       1002
       -
                           });

     

       1003
        
                           break;

     

       1004
        
                       }

     

       1005
        
                   }

     
···

       2649
        
                   vec![PrefixChild::Collection(NsidCount {

     

       2650
        
                       nsid: "a.a.a".to_string(),

     

       2651
        
                       creates: 1,

     

       0
        
       
     

       0
        
       
     

       2652
        
                       dids_estimate: 1

     

       2653
        
                   }),]

     

       2654
        
               );

     
···

       2695
        
                   vec![PrefixChild::Prefix(PrefixCount {

     

       2696
        
                       prefix: "a.a.a".to_string(),

     

       2697
        
                       creates: 1,

     

       2698
       -
                       dids_estimate: 1

     

       0
        
       
     

       0
        
       
     

       2699
        
                   }),]

     

       2700
        
               );

     

       2701
        
               assert_eq!(cursor, None);

     
···

       2750
        
                   vec![PrefixChild::Prefix(PrefixCount {

     

       2751
        
                       prefix: "a.a.a".to_string(),

     

       2752
        
                       creates: 2,

     

       0
        
       
     

       0
        
       
     

       2753
        
                       dids_estimate: 1

     

       2754
        
                   }),]

     

       2755
        
               );

     
···

       2818
        
                       PrefixChild::Collection(NsidCount {

     

       2819
        
                           nsid: "a.a.a.a".to_string(),

     

       2820
        
                           creates: 1,

     

       0
        
       
     

       0
        
       
     

       2821
        
                           dids_estimate: 1

     

       2822
        
                       }),

     

       2823
        
                       PrefixChild::Prefix(PrefixCount {

     

       2824
        
                           prefix: "a.a.a.a".to_string(),

     

       2825
        
                           creates: 1,

     

       0
        
       
     

       0
        
       
     

       2826
        
                           dids_estimate: 1

     

       2827
        
                       }),

     

       2828
        
                   ]

···

       507
        
                           merged.merge(&counts);

     

       508
        
                       }

     

       509
        
                   }

     

       510
       +
                   out.push(NsidCount::new(&nsid, &merged));

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       511
        
               }

     

       512
        
       

     

       513
        
               let next_cursor = current_nsid.map(|s| s.to_db_bytes()).transpose()?;

     
···

       613
        
                   .into_iter()

     

       614
        
                   .rev()

     

       615
        
                   .take(limit)

     

       616
       +
                   .map(|(nsid, cv)| NsidCount::new(&nsid, &cv))

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       617
        
                   .collect();

     

       618
        
               Ok(counts)

     

       619
        
           }

     
···

       719
        
               let mut prefix_count = CountsValue::default();

     

       720
        
               #[derive(Debug, Clone, PartialEq)]

     

       721
        
               enum Child {

     

       722
       +
                   FullNsid(Nsid),

     

       723
        
                   ChildPrefix(String),

     

       724
        
               }

     

       725
        
               impl Child {

     

       726
        
                   fn from_prefix(nsid: &Nsid, prefix: &NsidPrefix) -> Option<Self> {

     

       727
        
                       if prefix.is_group_of(nsid) {

     

       728
       +
                           return Some(Child::FullNsid(nsid.clone()));

     

       729
        
                       }

     

       730
        
                       let suffix = nsid.as_str().strip_prefix(&format!("{}.", prefix.0))?;

     

       731
        
                       let (segment, _) = suffix.split_once('.').unwrap();

     
···

       734
        
                   }

     

       735
        
                   fn is_before(&self, other: &Child) -> bool {

     

       736
        
                       match (self, other) {

     

       737
       +
                           (Child::FullNsid(s), Child::ChildPrefix(o)) if s.as_str() == o => true,

     

       738
       +
                           (Child::ChildPrefix(s), Child::FullNsid(o)) if s == o.as_str() => false,

     

       739
       +
                           (Child::FullNsid(s), Child::FullNsid(o)) => s.as_str() < o.as_str(),

     

       740
        
                           (Child::ChildPrefix(s), Child::ChildPrefix(o)) => s < o,

     

       741
       +
                           (Child::FullNsid(s), Child::ChildPrefix(o)) => s.to_string() < *o,

     

       742
       +
                           (Child::ChildPrefix(s), Child::FullNsid(o)) => *s < o.to_string(),

     

       743
        
                       }

     

       744
        
                   }

     

       745
        
                   fn into_inner(self) -> String {

     

       746
        
                       match self {

     

       747
       +
                           Child::FullNsid(s) => s.to_string(),

     

       748
        
                           Child::ChildPrefix(s) => s,

     

       749
        
                       }

     

       750
        
                   }

     
···

       783
        
                       }

     

       784
        
                   }

     

       785
        
                   items.push(match child {

     

       786
       +
                       Child::FullNsid(nsid) => PrefixChild::Collection(NsidCount::new(&nsid, &merged)),

     

       787
       +
                       Child::ChildPrefix(prefix) => {

     

       788
       +
                           PrefixChild::Prefix(PrefixCount::new(&prefix, &merged))

     

       789
       +
                       }

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       790
        
                   });

     

       791
        
               }

     

       792
        
       

     
···

       977
        
               for kv in self.rollups.range((start, end)) {

     

       978
        
                   let (key_bytes, val_bytes) = kv?;

     

       979
        
                   let key = db_complete::<AllTimeRollupKey>(&key_bytes)?;

     

       980
       +
                   let nsid = key.collection();

     

       981
        
                   for term in &terms {

     

       982
        
                       if nsid.contains(term) {

     

       983
        
                           let counts = db_complete::<CountsValue>(&val_bytes)?;

     

       984
       +
                           matches.push(NsidCount::new(nsid, &counts));

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       985
        
                           break;

     

       986
        
                       }

     

       987
        
                   }

     
···

       2631
        
                   vec![PrefixChild::Collection(NsidCount {

     

       2632
        
                       nsid: "a.a.a".to_string(),

     

       2633
        
                       creates: 1,

     

       2634
       +
                       updates: 0,

     

       2635
       +
                       deletes: 0,

     

       2636
        
                       dids_estimate: 1

     

       2637
        
                   }),]

     

       2638
        
               );

     
···

       2679
        
                   vec![PrefixChild::Prefix(PrefixCount {

     

       2680
        
                       prefix: "a.a.a".to_string(),

     

       2681
        
                       creates: 1,

     

       2682
       +
                       updates: 0,

     

       2683
       +
                       deletes: 0,

     

       2684
       +
                       dids_estimate: 1,

     

       2685
        
                   }),]

     

       2686
        
               );

     

       2687
        
               assert_eq!(cursor, None);

     
···

       2736
        
                   vec![PrefixChild::Prefix(PrefixCount {

     

       2737
        
                       prefix: "a.a.a".to_string(),

     

       2738
        
                       creates: 2,

     

       2739
       +
                       updates: 0,

     

       2740
       +
                       deletes: 0,

     

       2741
        
                       dids_estimate: 1

     

       2742
        
                   }),]

     

       2743
        
               );

     
···

       2806
        
                       PrefixChild::Collection(NsidCount {

     

       2807
        
                           nsid: "a.a.a.a".to_string(),

     

       2808
        
                           creates: 1,

     

       2809
       +
                           updates: 0,

     

       2810
       +
                           deletes: 0,

     

       2811
        
                           dids_estimate: 1

     

       2812
        
                       }),

     

       2813
        
                       PrefixChild::Prefix(PrefixCount {

     

       2814
        
                           prefix: "a.a.a.a".to_string(),

     

       2815
        
                           creates: 1,

     

       2816
       +
                           updates: 0,

     

       2817
       +
                           deletes: 0,

     

       2818
        
                           dids_estimate: 1

     

       2819
        
                       }),

     

       2820
        
                   ]