commit a32a20f03fc9e0d886f84a086d0f2b632d33906e · nekomimi.pet/microcosm-rs

+1 -1

ufos/fuzz/fuzz_targets/counts_value.rs

···

       19
        
               assert_eq!(serialized.len(), n);

     

       20
        
               let (and_back, n_again) = CountsValue::from_db_bytes(&serialized).unwrap();

     

       21
        
               assert_eq!(n_again, n);

     

       22
       -
               assert_eq!(and_back.records(), counts_value.records());

     

       23
        
               assert_eq!(and_back.dids().estimate(), counts_value.dids().estimate());

     

       24
        
           }

     

       25
        
       });

···

       19
        
               assert_eq!(serialized.len(), n);

     

       20
        
               let (and_back, n_again) = CountsValue::from_db_bytes(&serialized).unwrap();

     

       21
        
               assert_eq!(n_again, n);

     

       22
       +
               assert_eq!(and_back.counts(), counts_value.counts());

     

       23
        
               assert_eq!(and_back.dids().estimate(), counts_value.dids().estimate());

     

       24
        
           }

     

       25
        
       });

ufos/src/db_types.rs

···

       104
        
           }

     

       105
        
       }

     

       106
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       107
        
       impl<P: DbBytes + std::fmt::Debug, S: DbBytes + std::fmt::Debug> fmt::Debug for DbConcat<P, S> {

     

       108
        
           fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

     

       109
        
               write!(f, "DbConcat<{:?} || {:?}>", self.prefix, self.suffix)

···

       104
        
           }

     

       105
        
       }

     

       106
        
       

     

       107
       +
       impl<P: DbBytes + Default, S: DbBytes + Default> Default for DbConcat<P, S> {

     

       108
       +
           fn default() -> Self {

     

       109
       +
               Self {

     

       110
       +
                   prefix: Default::default(),

     

       111
       +
                   suffix: Default::default(),

     

       112
       +
               }

     

       113
       +
           }

     

       114
       +
       }

     

       115
       +
       

     

       116
        
       impl<P: DbBytes + std::fmt::Debug, S: DbBytes + std::fmt::Debug> fmt::Debug for DbConcat<P, S> {

     

       117
        
           fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

     

       118
        
               write!(f, "DbConcat<{:?} || {:?}>", self.prefix, self.suffix)

+71 -25

ufos/src/lib.rs

···

       27
        
       

     

       28
        
       #[derive(Debug, Default, Clone)]

     

       29
        
       pub struct CollectionCommits<const LIMIT: usize> {

     

       30
       -
           pub total_seen: usize,

     

       0
        
       
     

       0
        
       
     

       31
        
           pub dids_estimate: Sketch<14>,

     

       32
        
           pub commits: Vec<UFOsCommit>,

     

       33
        
           head: usize,

     

       34
       -
           non_creates: usize,

     

       35
        
       }

     

       36
        
       

     

       37
        
       impl<const LIMIT: usize> CollectionCommits<LIMIT> {

     
···

       41
        
                   self.head = 0;

     

       42
        
               }

     

       43
        
           }

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       44
        
           pub fn truncating_insert(

     

       45
        
               &mut self,

     

       46
        
               commit: UFOsCommit,

     

       47
        
               sketch_secret: &SketchSecretPrefix,

     

       48
        
           ) -> Result<(), BatchInsertError> {

     

       49
       -
               if self.non_creates == LIMIT {

     

       0
        
       
     

       50
        
                   return Err(BatchInsertError::BatchFull(commit));

     

       51
        
               }

     

       52
       -
               let did = commit.did.clone();

     

       53
       -
               let is_create = commit.action.is_create();

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       54
        
               if self.commits.len() < LIMIT {

     

       0
        
       
     

       55
        
                   self.commits.push(commit);

     

       56
       -
                   if self.commits.capacity() > LIMIT {

     

       57
       -
                       self.commits.shrink_to(LIMIT); // save mem?????? maybe??

     

       58
       -
                   }

     

       59
        
               } else {

     

       0
        
       
     

       60
        
                   let head_started_at = self.head;

     

       61
        
                   loop {

     

       62
        
                       let candidate = self

     
···

       74
        
                   }

     

       75
        
               }

     

       76
        
       

     

       77
       -
               if is_create {

     

       78
       -
                   self.total_seen += 1;

     

       79
       -
                   self.dids_estimate.insert(did_element(sketch_secret, &did));

     

       80
       -
               } else {

     

       81
       -
                   self.non_creates += 1;

     

       82
       -
               }

     

       83
       -
       

     

       84
        
               Ok(())

     

       85
        
           }

     

       86
        
       }

     
···

       179
        
                   .truncating_insert(commit, sketch_secret)?;

     

       180
        
               Ok(())

     

       181
        
           }

     

       182
       -
           pub fn total_records(&self) -> usize {

     

       183
       -
               self.commits_by_nsid.values().map(|v| v.commits.len()).sum()

     

       184
       -
           }

     

       185
       -
           pub fn total_seen(&self) -> usize {

     

       186
       -
               self.commits_by_nsid.values().map(|v| v.total_seen).sum()

     

       187
       -
           }

     

       188
        
           pub fn total_collections(&self) -> usize {

     

       189
        
               self.commits_by_nsid.len()

     

       190
        
           }

     
···

       237
        
       #[derive(Debug, Serialize, JsonSchema)]

     

       238
        
       pub struct NsidCount {

     

       239
        
           nsid: String,

     

       240
       -
           records: u64,

     

       241
        
           dids_estimate: u64,

     

       242
        
       }

     

       243
        
       

     

       244
        
       #[derive(Debug, Serialize, JsonSchema)]

     

       245
        
       pub struct JustCount {

     

       246
       -
           records: u64,

     

       247
        
           dids_estimate: u64,

     

       248
        
       }

     

       249
        
       

     
···

       309
        
                   &[0u8; 16],

     

       310
        
               )?;

     

       311
        
       

     

       312
       -
               assert_eq!(commits.total_seen, 3);

     

       313
        
               assert_eq!(commits.dids_estimate.estimate(), 1);

     

       314
        
               assert_eq!(commits.commits.len(), 2);

     

       315
        
       

     
···

       333
        
           }

     

       334
        
       

     

       335
        
           #[test]

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       336
        
           fn test_truncating_insert_does_not_truncate_deletes() -> anyhow::Result<()> {

     

       337
        
               let mut commits: CollectionCommits<2> = Default::default();

     

       338
        
       

     
···

       375
        
                   &[0u8; 16],

     

       376
        
               )?;

     

       377
        
       

     

       378
       -
               assert_eq!(commits.total_seen, 2);

     

       0
        
       
     

       379
        
               assert_eq!(commits.dids_estimate.estimate(), 1);

     

       380
        
               assert_eq!(commits.commits.len(), 2);

     

       381

···

       27
        
       

     

       28
        
       #[derive(Debug, Default, Clone)]

     

       29
        
       pub struct CollectionCommits<const LIMIT: usize> {

     

       30
       +
           pub creates: usize,

     

       31
       +
           pub updates: usize,

     

       32
       +
           pub deletes: usize,

     

       33
        
           pub dids_estimate: Sketch<14>,

     

       34
        
           pub commits: Vec<UFOsCommit>,

     

       35
        
           head: usize,

     

       0
        
       
     

       36
        
       }

     

       37
        
       

     

       38
        
       impl<const LIMIT: usize> CollectionCommits<LIMIT> {

     
···

       42
        
                   self.head = 0;

     

       43
        
               }

     

       44
        
           }

     

       45
       +
           /// lossy-ish commit insertion

     

       46
       +
           ///

     

       47
       +
           /// - new commits are *always* added to the batch or else rejected as full.

     

       48
       +
           /// - when LIMIT is reached, new commits can displace existing `creates`.

     

       49
       +
           ///   `update`s and `delete`s are *never* displaced.

     

       50
       +
           /// - if all batched `creates` have been displaced, the batch is full.

     

       51
       +
           ///

     

       52
       +
           /// in general it's rare for commits to be displaced except for very high-

     

       53
       +
           /// volume collections such as `app.bsky.feed.like`.

     

       54
       +
           ///

     

       55
       +
           /// it could be nice in the future to retain all batched commits and just

     

       56
       +
           /// drop new `creates` after a limit instead.

     

       57
        
           pub fn truncating_insert(

     

       58
        
               &mut self,

     

       59
        
               commit: UFOsCommit,

     

       60
        
               sketch_secret: &SketchSecretPrefix,

     

       61
        
           ) -> Result<(), BatchInsertError> {

     

       62
       +
               if (self.updates + self.deletes) == LIMIT {

     

       63
       +
                   // nothing can be displaced (only `create`s may be displaced)

     

       64
        
                   return Err(BatchInsertError::BatchFull(commit));

     

       65
        
               }

     

       66
       +
       

     

       67
       +
               // every kind of commit counts as "user activity"

     

       68
       +
               self.dids_estimate

     

       69
       +
                   .insert(did_element(sketch_secret, &commit.did));

     

       70
       +
       

     

       71
       +
               match commit.action {

     

       72
       +
                   CommitAction::Put(PutAction {

     

       73
       +
                       is_update: false, ..

     

       74
       +
                   }) => {

     

       75
       +
                       self.creates += 1;

     

       76
       +
                   }

     

       77
       +
                   CommitAction::Put(PutAction {

     

       78
       +
                       is_update: true, ..

     

       79
       +
                   }) => {

     

       80
       +
                       self.updates += 1;

     

       81
       +
                   }

     

       82
       +
                   CommitAction::Cut => {

     

       83
       +
                       self.deletes += 1;

     

       84
       +
                   }

     

       85
       +
               }

     

       86
       +
       

     

       87
        
               if self.commits.len() < LIMIT {

     

       88
       +
                   // normal insert: there's space left to put a new commit at the end

     

       89
        
                   self.commits.push(commit);

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       90
        
               } else {

     

       91
       +
                   // displacement insert: find an old `create` we can displace

     

       92
        
                   let head_started_at = self.head;

     

       93
        
                   loop {

     

       94
        
                       let candidate = self

     
···

       106
        
                   }

     

       107
        
               }

     

       108
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       109
        
               Ok(())

     

       110
        
           }

     

       111
        
       }

     
···

       204
        
                   .truncating_insert(commit, sketch_secret)?;

     

       205
        
               Ok(())

     

       206
        
           }

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       207
        
           pub fn total_collections(&self) -> usize {

     

       208
        
               self.commits_by_nsid.len()

     

       209
        
           }

     
···

       256
        
       #[derive(Debug, Serialize, JsonSchema)]

     

       257
        
       pub struct NsidCount {

     

       258
        
           nsid: String,

     

       259
       +
           creates: u64,

     

       260
        
           dids_estimate: u64,

     

       261
        
       }

     

       262
        
       

     

       263
        
       #[derive(Debug, Serialize, JsonSchema)]

     

       264
        
       pub struct JustCount {

     

       265
       +
           creates: u64,

     

       266
        
           dids_estimate: u64,

     

       267
        
       }

     

       268
        
       

     
···

       328
        
                   &[0u8; 16],

     

       329
        
               )?;

     

       330
        
       

     

       331
       +
               assert_eq!(commits.creates, 3);

     

       332
        
               assert_eq!(commits.dids_estimate.estimate(), 1);

     

       333
        
               assert_eq!(commits.commits.len(), 2);

     

       334
        
       

     
···

       352
        
           }

     

       353
        
       

     

       354
        
           #[test]

     

       355
       +
           fn test_truncating_insert_counts_updates() -> anyhow::Result<()> {

     

       356
       +
               let mut commits: CollectionCommits<2> = Default::default();

     

       357
       +
       

     

       358
       +
               commits.truncating_insert(

     

       359
       +
                   UFOsCommit {

     

       360
       +
                       cursor: Cursor::from_raw_u64(100),

     

       361
       +
                       did: Did::new("did:plc:whatever".to_string()).unwrap(),

     

       362
       +
                       rkey: RecordKey::new("rkey-asdf-a".to_string()).unwrap(),

     

       363
       +
                       rev: "rev-asdf".to_string(),

     

       364
       +
                       action: CommitAction::Put(PutAction {

     

       365
       +
                           record: RawValue::from_string("{}".to_string())?,

     

       366
       +
                           is_update: true,

     

       367
       +
                       }),

     

       368
       +
                   },

     

       369
       +
                   &[0u8; 16],

     

       370
       +
               )?;

     

       371
       +
       

     

       372
       +
               assert_eq!(commits.creates, 0);

     

       373
       +
               assert_eq!(commits.updates, 1);

     

       374
       +
               assert_eq!(commits.deletes, 0);

     

       375
       +
               assert_eq!(commits.dids_estimate.estimate(), 1);

     

       376
       +
               assert_eq!(commits.commits.len(), 1);

     

       377
       +
               Ok(())

     

       378
       +
           }

     

       379
       +
       

     

       380
       +
           #[test]

     

       381
        
           fn test_truncating_insert_does_not_truncate_deletes() -> anyhow::Result<()> {

     

       382
        
               let mut commits: CollectionCommits<2> = Default::default();

     

       383
        
       

     
···

       420
        
                   &[0u8; 16],

     

       421
        
               )?;

     

       422
        
       

     

       423
       +
               assert_eq!(commits.creates, 2);

     

       424
       +
               assert_eq!(commits.deletes, 1);

     

       425
        
               assert_eq!(commits.dids_estimate.estimate(), 1);

     

       426
        
               assert_eq!(commits.commits.len(), 2);

     

       427

+5 -3

ufos/src/server.rs

···

       109
        
               consumer,

     

       110
        
           })

     

       111
        
       }

     

       0
        
       
     

       0
        
       
     

       112
        
       fn to_multiple_nsids(s: &str) -> Result<HashSet<Nsid>, String> {

     

       113
        
           let mut out = HashSet::new();

     

       114
        
           for collection in s.split(',') {

     
···

       197
        
       }

     

       198
        
       #[derive(Debug, Serialize, JsonSchema)]

     

       199
        
       struct TotalCounts {

     

       200
       -
           total_records: u64,

     

       201
        
           dids_estimate: u64,

     

       202
        
       }

     

       203
        
       /// Get total records seen by collection

     
···

       218
        
           let mut seen_by_collection = HashMap::with_capacity(collections.len());

     

       219
        
       

     

       220
        
           for collection in &collections {

     

       221
       -
               let (total_records, dids_estimate) = storage

     

       222
        
                   .get_counts_by_collection(collection)

     

       223
        
                   .await

     

       224
        
                   .map_err(|e| HttpError::for_internal_error(format!("boooo: {e:?}")))?;

     
···

       226
        
               seen_by_collection.insert(

     

       227
        
                   collection.to_string(),

     

       228
        
                   TotalCounts {

     

       229
       -
                       total_records,

     

       230
        
                       dids_estimate,

     

       231
        
                   },

     

       232
        
               );

···

       109
        
               consumer,

     

       110
        
           })

     

       111
        
       }

     

       112
       +
       

     

       113
       +
       // TODO: replace with normal (🙃) multi-qs value somehow

     

       114
        
       fn to_multiple_nsids(s: &str) -> Result<HashSet<Nsid>, String> {

     

       115
        
           let mut out = HashSet::new();

     

       116
        
           for collection in s.split(',') {

     
···

       199
        
       }

     

       200
        
       #[derive(Debug, Serialize, JsonSchema)]

     

       201
        
       struct TotalCounts {

     

       202
       +
           total_creates: u64,

     

       203
        
           dids_estimate: u64,

     

       204
        
       }

     

       205
        
       /// Get total records seen by collection

     
···

       220
        
           let mut seen_by_collection = HashMap::with_capacity(collections.len());

     

       221
        
       

     

       222
        
           for collection in &collections {

     

       223
       +
               let (total_creates, dids_estimate) = storage

     

       224
        
                   .get_counts_by_collection(collection)

     

       225
        
                   .await

     

       226
        
                   .map_err(|e| HttpError::for_internal_error(format!("boooo: {e:?}")))?;

     
···

       228
        
               seen_by_collection.insert(

     

       229
        
                   collection.to_string(),

     

       230
        
                   TotalCounts {

     

       231
       +
                       total_creates,

     

       232
        
                       dids_estimate,

     

       233
        
                   },

     

       234
        
               );

+66 -62

ufos/src/storage_fjall.rs

···

       2
        
       use crate::error::StorageError;

     

       3
        
       use crate::storage::{StorageResult, StorageWhatever, StoreBackground, StoreReader, StoreWriter};

     

       4
        
       use crate::store_types::{

     

       5
       -
           AllTimeDidsKey, AllTimeRecordsKey, AllTimeRollupKey, CountsValue, CursorBucket,

     

       6
        
           DeleteAccountQueueKey, DeleteAccountQueueVal, HourTruncatedCursor, HourlyDidsKey,

     

       7
        
           HourlyRecordsKey, HourlyRollupKey, HourlyRollupStaticPrefix, JetstreamCursorKey,

     

       8
        
           JetstreamCursorValue, JetstreamEndpointKey, JetstreamEndpointValue, LiveCountsKey,

     
···

       507
        
                   }

     

       508
        
                   out.push(NsidCount {

     

       509
        
                       nsid: nsid.to_string(),

     

       510
       -
                       records: merged.records(),

     

       511
        
                       dids_estimate: merged.dids().estimate() as u64,

     

       512
        
                   });

     

       513
        
               }

     
···

       607
        
               }

     

       608
        
               let mut ranked: Vec<(Nsid, CountsValue)> = ranked.into_iter().collect();

     

       609
        
               match order {

     

       610
       -
                   OrderCollectionsBy::RecordsCreated => ranked.sort_by_key(|(_, c)| c.records()),

     

       611
        
                   OrderCollectionsBy::DidsEstimate => ranked.sort_by_key(|(_, c)| c.dids().estimate()),

     

       612
        
                   OrderCollectionsBy::Lexi { .. } => unreachable!(),

     

       613
        
               }

     
···

       617
        
                   .take(limit)

     

       618
        
                   .map(|(nsid, cv)| NsidCount {

     

       619
        
                       nsid: nsid.to_string(),

     

       620
       -
                       records: cv.records(),

     

       621
        
                       dids_estimate: cv.dids().estimate() as u64,

     

       622
        
                   })

     

       623
        
                   .collect();

     
···

       746
        
                   }

     

       747
        
               }

     

       748
        
               Ok((

     

       749
       -
                   total_counts.records(),

     

       750
        
                   total_counts.dids().estimate() as u64,

     

       751
        
               ))

     

       752
        
           }

     
···

       973
        
                       .unwrap_or_default();

     

       974
        
       

     

       975
        
                   // now that we have values, we can know the exising ranks

     

       976
       -
                   let before_records_count = rolled.records();

     

       977
        
                   let before_dids_estimate = rolled.dids().estimate() as u64;

     

       978
        
       

     

       979
        
                   // update the rollup

     

       980
        
                   rolled.merge(&counts);

     

       981
        
       

     

       982
       -
                   // replace rank entries

     

       983
       -
                   let (old_records, new_records, dids) = match rollup {

     

       984
       -
                       Rollup::Hourly(hourly_cursor) => {

     

       985
       -
                           let old_records =

     

       986
       -
                               HourlyRecordsKey::new(hourly_cursor, before_records_count.into(), &nsid);

     

       987
       -
                           let new_records = old_records.with_rank(rolled.records().into());

     

       988
       -
                           let new_estimate = rolled.dids().estimate() as u64;

     

       989
       -
                           let dids = if new_estimate == before_dids_estimate {

     

       990
       -
                               None

     

       991
       -
                           } else {

     

       992
       -
                               let old_dids =

     

       993
       -
                                   HourlyDidsKey::new(hourly_cursor, before_dids_estimate.into(), &nsid);

     

       994
       -
                               let new_dids = old_dids.with_rank(new_estimate.into());

     

       995
       -
                               Some((old_dids.to_db_bytes()?, new_dids.to_db_bytes()?))

     

       996
       -
                           };

     

       997
       -
                           (old_records.to_db_bytes()?, new_records.to_db_bytes()?, dids)

     

       998
       -
                       }

     

       999
       -
                       Rollup::Weekly(weekly_cursor) => {

     

       1000
       -
                           let old_records =

     

       1001
       -
                               WeeklyRecordsKey::new(weekly_cursor, before_records_count.into(), &nsid);

     

       1002
       -
                           let new_records = old_records.with_rank(rolled.records().into());

     

       1003
       -
                           let new_estimate = rolled.dids().estimate() as u64;

     

       1004
       -
                           let dids = if new_estimate == before_dids_estimate {

     

       1005
       -
                               None

     

       1006
       -
                           } else {

     

       1007
       -
                               let old_dids =

     

       1008
       -
                                   WeeklyDidsKey::new(weekly_cursor, before_dids_estimate.into(), &nsid);

     

       1009
       -
                               let new_dids = old_dids.with_rank(new_estimate.into());

     

       1010
       -
                               Some((old_dids.to_db_bytes()?, new_dids.to_db_bytes()?))

     

       1011
       -
                           };

     

       1012
       -
                           (old_records.to_db_bytes()?, new_records.to_db_bytes()?, dids)

     

       1013
       -
                       }

     

       1014
       -
                       Rollup::AllTime => {

     

       1015
       -
                           let old_records = AllTimeRecordsKey::new(before_records_count.into(), &nsid);

     

       1016
       -
                           let new_records = old_records.with_rank(rolled.records().into());

     

       1017
       -
                           let new_estimate = rolled.dids().estimate() as u64;

     

       1018
       -
                           let dids = if new_estimate == before_dids_estimate {

     

       1019
       -
                               None

     

       1020
       -
                           } else {

     

       1021
       -
                               let old_dids = AllTimeDidsKey::new(before_dids_estimate.into(), &nsid);

     

       1022
       -
                               let new_dids = old_dids.with_rank(new_estimate.into());

     

       1023
       -
                               Some((old_dids.to_db_bytes()?, new_dids.to_db_bytes()?))

     

       1024
       -
                           };

     

       1025
       -
                           (old_records.to_db_bytes()?, new_records.to_db_bytes()?, dids)

     

       1026
       -
                       }

     

       1027
       -
                   };

     

       1028
        
       

     

       1029
       -
                   // replace the ranks

     

       1030
       -
                   batch.remove(&self.rollups, &old_records);

     

       1031
       -
                   batch.insert(&self.rollups, &new_records, "");

     

       1032
       -
                   if let Some((old_dids, new_dids)) = dids {

     

       1033
       -
                       batch.remove(&self.rollups, &old_dids);

     

       1034
       -
                       batch.insert(&self.rollups, &new_dids, "");

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       1035
        
                   }

     

       1036
        
       

     

       1037
       -
                   // replace the rollup

     

       1038
        
                   batch.insert(&self.rollups, &rollup_key_bytes, &rolled.to_db_bytes()?);

     

       1039
        
               }

     

       1040
        
       

     
···

       1114
        
                       }

     

       1115
        
                   }

     

       1116
        
                   let live_counts_key: LiveCountsKey = (latest, &nsid).into();

     

       1117
       -
                   let counts_value = CountsValue::new(commits.total_seen as u64, commits.dids_estimate);

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       1118
        
                   batch.insert(

     

       1119
        
                       &self.rollups,

     

       1120
        
                       &live_counts_key.to_db_bytes()?,

     
···

       1838
        
               );

     

       1839
        
               write.insert_batch(batch.batch)?;

     

       1840
        
       

     

       1841
       -
               let (records, dids) = read.get_counts_by_collection(&collection)?;

     

       1842
       -
               assert_eq!(records, 1);

     

       1843
        
               assert_eq!(dids, 1);

     

       1844
        
       

     

       1845
        
               let records = read.get_records_by_collections([collection].into(), 2, false)?;

···

       2
        
       use crate::error::StorageError;

     

       3
        
       use crate::storage::{StorageResult, StorageWhatever, StoreBackground, StoreReader, StoreWriter};

     

       4
        
       use crate::store_types::{

     

       5
       +
           AllTimeDidsKey, AllTimeRecordsKey, AllTimeRollupKey, CommitCounts, CountsValue, CursorBucket,

     

       6
        
           DeleteAccountQueueKey, DeleteAccountQueueVal, HourTruncatedCursor, HourlyDidsKey,

     

       7
        
           HourlyRecordsKey, HourlyRollupKey, HourlyRollupStaticPrefix, JetstreamCursorKey,

     

       8
        
           JetstreamCursorValue, JetstreamEndpointKey, JetstreamEndpointValue, LiveCountsKey,

     
···

       507
        
                   }

     

       508
        
                   out.push(NsidCount {

     

       509
        
                       nsid: nsid.to_string(),

     

       510
       +
                       creates: merged.counts().creates,

     

       511
        
                       dids_estimate: merged.dids().estimate() as u64,

     

       512
        
                   });

     

       513
        
               }

     
···

       607
        
               }

     

       608
        
               let mut ranked: Vec<(Nsid, CountsValue)> = ranked.into_iter().collect();

     

       609
        
               match order {

     

       610
       +
                   OrderCollectionsBy::RecordsCreated => ranked.sort_by_key(|(_, c)| c.counts().creates),

     

       611
        
                   OrderCollectionsBy::DidsEstimate => ranked.sort_by_key(|(_, c)| c.dids().estimate()),

     

       612
        
                   OrderCollectionsBy::Lexi { .. } => unreachable!(),

     

       613
        
               }

     
···

       617
        
                   .take(limit)

     

       618
        
                   .map(|(nsid, cv)| NsidCount {

     

       619
        
                       nsid: nsid.to_string(),

     

       620
       +
                       creates: cv.counts().creates,

     

       621
        
                       dids_estimate: cv.dids().estimate() as u64,

     

       622
        
                   })

     

       623
        
                   .collect();

     
···

       746
        
                   }

     

       747
        
               }

     

       748
        
               Ok((

     

       749
       +
                   total_counts.counts().creates,

     

       750
        
                   total_counts.dids().estimate() as u64,

     

       751
        
               ))

     

       752
        
           }

     
···

       973
        
                       .unwrap_or_default();

     

       974
        
       

     

       975
        
                   // now that we have values, we can know the exising ranks

     

       976
       +
                   let before_creates_count = rolled.counts().creates;

     

       977
        
                   let before_dids_estimate = rolled.dids().estimate() as u64;

     

       978
        
       

     

       979
        
                   // update the rollup

     

       980
        
                   rolled.merge(&counts);

     

       981
        
       

     

       982
       +
                   // new ranks

     

       983
       +
                   let new_creates_count = rolled.counts().creates;

     

       984
       +
                   let new_dids_estimate = rolled.dids().estimate() as u64;

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       985
        
       

     

       986
       +
                   // update create-ranked secondary index if rank changed

     

       987
       +
                   if new_creates_count != before_creates_count {

     

       988
       +
                       let (old_k, new_k) = match rollup {

     

       989
       +
                           Rollup::Hourly(cursor) => (

     

       990
       +
                               HourlyRecordsKey::new(cursor, before_creates_count.into(), &nsid)

     

       991
       +
                                   .to_db_bytes()?,

     

       992
       +
                               HourlyRecordsKey::new(cursor, new_creates_count.into(), &nsid)

     

       993
       +
                                   .to_db_bytes()?,

     

       994
       +
                           ),

     

       995
       +
                           Rollup::Weekly(cursor) => (

     

       996
       +
                               WeeklyRecordsKey::new(cursor, before_creates_count.into(), &nsid)

     

       997
       +
                                   .to_db_bytes()?,

     

       998
       +
                               WeeklyRecordsKey::new(cursor, new_creates_count.into(), &nsid)

     

       999
       +
                                   .to_db_bytes()?,

     

       1000
       +
                           ),

     

       1001
       +
                           Rollup::AllTime => (

     

       1002
       +
                               AllTimeRecordsKey::new(before_creates_count.into(), &nsid).to_db_bytes()?,

     

       1003
       +
                               AllTimeRecordsKey::new(new_creates_count.into(), &nsid).to_db_bytes()?,

     

       1004
       +
                           ),

     

       1005
       +
                       };

     

       1006
       +
                       batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better

     

       1007
       +
                       batch.insert(&self.rollups, &new_k, "");

     

       1008
       +
                   }

     

       1009
       +
       

     

       1010
       +
                   // update dids-ranked secondary index if rank changed

     

       1011
       +
                   if new_dids_estimate != before_dids_estimate {

     

       1012
       +
                       let (old_k, new_k) = match rollup {

     

       1013
       +
                           Rollup::Hourly(cursor) => (

     

       1014
       +
                               HourlyDidsKey::new(cursor, before_dids_estimate.into(), &nsid)

     

       1015
       +
                                   .to_db_bytes()?,

     

       1016
       +
                               HourlyDidsKey::new(cursor, new_dids_estimate.into(), &nsid)

     

       1017
       +
                                   .to_db_bytes()?,

     

       1018
       +
                           ),

     

       1019
       +
                           Rollup::Weekly(cursor) => (

     

       1020
       +
                               WeeklyDidsKey::new(cursor, before_dids_estimate.into(), &nsid)

     

       1021
       +
                                   .to_db_bytes()?,

     

       1022
       +
                               WeeklyDidsKey::new(cursor, new_dids_estimate.into(), &nsid)

     

       1023
       +
                                   .to_db_bytes()?,

     

       1024
       +
                           ),

     

       1025
       +
                           Rollup::AllTime => (

     

       1026
       +
                               AllTimeDidsKey::new(before_dids_estimate.into(), &nsid).to_db_bytes()?,

     

       1027
       +
                               AllTimeDidsKey::new(new_dids_estimate.into(), &nsid).to_db_bytes()?,

     

       1028
       +
                           ),

     

       1029
       +
                       };

     

       1030
       +
                       batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better

     

       1031
       +
                       batch.insert(&self.rollups, &new_k, "");

     

       1032
        
                   }

     

       1033
        
       

     

       1034
       +
                   // replace the main counts rollup

     

       1035
        
                   batch.insert(&self.rollups, &rollup_key_bytes, &rolled.to_db_bytes()?);

     

       1036
        
               }

     

       1037
        
       

     
···

       1111
        
                       }

     

       1112
        
                   }

     

       1113
        
                   let live_counts_key: LiveCountsKey = (latest, &nsid).into();

     

       1114
       +
                   let counts_value = CountsValue::new(

     

       1115
       +
                       CommitCounts {

     

       1116
       +
                           creates: commits.creates as u64,

     

       1117
       +
                           updates: commits.updates as u64,

     

       1118
       +
                           deletes: commits.deletes as u64,

     

       1119
       +
                       },

     

       1120
       +
                       commits.dids_estimate,

     

       1121
       +
                   );

     

       1122
        
                   batch.insert(

     

       1123
        
                       &self.rollups,

     

       1124
        
                       &live_counts_key.to_db_bytes()?,

     
···

       1842
        
               );

     

       1843
        
               write.insert_batch(batch.batch)?;

     

       1844
        
       

     

       1845
       +
               let (creates, dids) = read.get_counts_by_collection(&collection)?;

     

       1846
       +
               assert_eq!(creates, 1);

     

       1847
        
               assert_eq!(dids, 1);

     

       1848
        
       

     

       1849
        
               let records = read.get_records_by_collections([collection].into(), 2, false)?;

+12 -5

ufos/src/storage_mem.rs

···

       5
        
       use crate::error::StorageError;

     

       6
        
       use crate::storage::{StorageResult, StorageWhatever, StoreBackground, StoreReader, StoreWriter};

     

       7
        
       use crate::store_types::{

     

       8
       -
           AllTimeRollupKey, CountsValue, DeleteAccountQueueKey, DeleteAccountQueueVal,

     

       9
        
           HourTruncatedCursor, HourlyRollupKey, JetstreamCursorKey, JetstreamCursorValue,

     

       10
        
           JetstreamEndpointKey, JetstreamEndpointValue, LiveCountsKey, NewRollupCursorKey,

     

       11
        
           NewRollupCursorValue, NsidRecordFeedKey, NsidRecordFeedVal, RecordLocationKey,

     
···

       483
        
                   }

     

       484
        
               }

     

       485
        
               Ok((

     

       486
       -
                   total_counts.records(),

     

       487
        
                   total_counts.dids().estimate() as u64,

     

       488
        
               ))

     

       489
        
           }

     
···

       724
        
                   assert_eq!(n, tripppin.len());

     

       725
        
                   assert_eq!(counts.prefix, and_back.prefix);

     

       726
        
                   assert_eq!(counts.dids().estimate(), and_back.dids().estimate());

     

       727
       -
                   if counts.records() > 20000000 {

     

       728
        
                       panic!("COUNTS maybe wtf? {counts:?}")

     

       729
        
                   }

     

       730
        
                   // assert_eq!(rolled, and_back);

     
···

       737
        
                   assert_eq!(n, tripppin.len());

     

       738
        
                   assert_eq!(rolled.prefix, and_back.prefix);

     

       739
        
                   assert_eq!(rolled.dids().estimate(), and_back.dids().estimate());

     

       740
       -
                   if rolled.records() > 20000000 {

     

       741
        
                       panic!("maybe wtf? {rolled:?}")

     

       742
        
                   }

     

       743
        
                   // assert_eq!(rolled, and_back);

     
···

       804
        
                       }

     

       805
        
                   }

     

       806
        
                   let live_counts_key: LiveCountsKey = (latest, &nsid).into();

     

       807
       -
                   let counts_value = CountsValue::new(commits.total_seen as u64, commits.dids_estimate);

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       808
        
                   batch.insert(

     

       809
        
                       &self.rollups,

     

       810
        
                       &live_counts_key.to_db_bytes()?,

···

       5
        
       use crate::error::StorageError;

     

       6
        
       use crate::storage::{StorageResult, StorageWhatever, StoreBackground, StoreReader, StoreWriter};

     

       7
        
       use crate::store_types::{

     

       8
       +
           AllTimeRollupKey, CommitCounts, CountsValue, DeleteAccountQueueKey, DeleteAccountQueueVal,

     

       9
        
           HourTruncatedCursor, HourlyRollupKey, JetstreamCursorKey, JetstreamCursorValue,

     

       10
        
           JetstreamEndpointKey, JetstreamEndpointValue, LiveCountsKey, NewRollupCursorKey,

     

       11
        
           NewRollupCursorValue, NsidRecordFeedKey, NsidRecordFeedVal, RecordLocationKey,

     
···

       483
        
                   }

     

       484
        
               }

     

       485
        
               Ok((

     

       486
       +
                   total_counts.counts().creates,

     

       487
        
                   total_counts.dids().estimate() as u64,

     

       488
        
               ))

     

       489
        
           }

     
···

       724
        
                   assert_eq!(n, tripppin.len());

     

       725
        
                   assert_eq!(counts.prefix, and_back.prefix);

     

       726
        
                   assert_eq!(counts.dids().estimate(), and_back.dids().estimate());

     

       727
       +
                   if counts.counts().creates > 20000000 {

     

       728
        
                       panic!("COUNTS maybe wtf? {counts:?}")

     

       729
        
                   }

     

       730
        
                   // assert_eq!(rolled, and_back);

     
···

       737
        
                   assert_eq!(n, tripppin.len());

     

       738
        
                   assert_eq!(rolled.prefix, and_back.prefix);

     

       739
        
                   assert_eq!(rolled.dids().estimate(), and_back.dids().estimate());

     

       740
       +
                   if rolled.counts().creates > 20000000 {

     

       741
        
                       panic!("maybe wtf? {rolled:?}")

     

       742
        
                   }

     

       743
        
                   // assert_eq!(rolled, and_back);

     
···

       804
        
                       }

     

       805
        
                   }

     

       806
        
                   let live_counts_key: LiveCountsKey = (latest, &nsid).into();

     

       807
       +
                   let counts_value = CountsValue::new(

     

       808
       +
                       CommitCounts {

     

       809
       +
                           creates: commits.creates as u64,

     

       810
       +
                           updates: commits.updates as u64,

     

       811
       +
                           deletes: commits.deletes as u64,

     

       812
       +
                       },

     

       813
       +
                       commits.dids_estimate,

     

       814
       +
                   );

     

       815
        
                   batch.insert(

     

       816
        
                       &self.rollups,

     

       817
        
                       &live_counts_key.to_db_bytes()?,

+39 -23

ufos/src/store_types.rs

···

       209
        
               )

     

       210
        
           }

     

       211
        
       }

     

       212
       -
       #[derive(Debug, PartialEq, Decode, Encode)]

     

       213
       -
       pub struct TotalRecordsValue(pub u64);

     

       214
       -
       impl UseBincodePlz for TotalRecordsValue {}

     

       215
        
       

     

       216
       -
       #[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)]

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       217
        
       pub struct EstimatedDidsValue(pub Sketch<14>);

     

       218
        
       impl SerdeBytes for EstimatedDidsValue {}

     

       219
        
       impl DbBytes for EstimatedDidsValue {

     
···

       236
        
           }

     

       237
        
       }

     

       238
        
       

     

       239
       -
       pub type CountsValue = DbConcat<TotalRecordsValue, EstimatedDidsValue>;

     

       240
        
       impl CountsValue {

     

       241
       -
           pub fn new(total: u64, dids: Sketch<14>) -> Self {

     

       242
        
               Self {

     

       243
       -
                   prefix: TotalRecordsValue(total),

     

       244
        
                   suffix: EstimatedDidsValue(dids),

     

       245
        
               }

     

       246
        
           }

     

       247
       -
           pub fn records(&self) -> u64 {

     

       248
       -
               self.prefix.0

     

       249
        
           }

     

       250
        
           pub fn dids(&self) -> &Sketch<14> {

     

       251
        
               &self.suffix.0

     

       252
        
           }

     

       253
        
           pub fn merge(&mut self, other: &Self) {

     

       254
       -
               self.prefix.0 += other.records();

     

       255
       -
               self.suffix.0.merge(other.dids());

     

       256
       -
           }

     

       257
       -
       }

     

       258
       -
       impl Default for CountsValue {

     

       259
       -
           fn default() -> Self {

     

       260
       -
               Self {

     

       261
       -
                   prefix: TotalRecordsValue(0),

     

       262
       -
                   suffix: EstimatedDidsValue(Sketch::<14>::default()),

     

       263
       -
               }

     

       264
        
           }

     

       265
        
       }

     

       266
        
       impl From<&CountsValue> for JustCount {

     

       267
        
           fn from(cv: &CountsValue) -> Self {

     

       268
        
               Self {

     

       269
       -
                   records: cv.records(),

     

       270
        
                   dids_estimate: cv.dids().estimate() as u64,

     

       271
        
               }

     

       272
        
           }

     
···

       608
        
       #[cfg(test)]

     

       609
        
       mod test {

     

       610
        
           use super::{

     

       611
       -
               CountsValue, Cursor, CursorBucket, Did, EncodingError, HourTruncatedCursor,

     

       612
        
               HourlyRollupKey, Nsid, Sketch, HOUR_IN_MICROS, WEEK_IN_MICROS,

     

       613
        
           };

     

       614
        
           use crate::db_types::DbBytes;

     
···

       642
        
                       Did::new(format!("did:plc:inze6wrmsm7pjl7yta3oig7{i}")).unwrap(),

     

       643
        
                   ));

     

       644
        
               }

     

       645
       -
               let original = CountsValue::new(123, estimator.clone());

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       646
        
               let serialized = original.to_db_bytes()?;

     

       647
        
               let (restored, bytes_consumed) = CountsValue::from_db_bytes(&serialized)?;

     

       648
        
               assert_eq!(restored, original);

     
···

       653
        
                       Did::new(format!("did:plc:inze6wrmsm7pjl7yta3oig{i}")).unwrap(),

     

       654
        
                   ));

     

       655
        
               }

     

       656
       -
               let original = CountsValue::new(123, estimator);

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       657
        
               let serialized = original.to_db_bytes()?;

     

       658
        
               let (restored, bytes_consumed) = CountsValue::from_db_bytes(&serialized)?;

     

       659
        
               assert_eq!(restored, original);

···

       209
        
               )

     

       210
        
           }

     

       211
        
       }

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       212
        
       

     

       213
       +
       #[derive(Debug, Clone, Copy, Default, PartialEq, Decode, Encode)]

     

       214
       +
       pub struct CommitCounts {

     

       215
       +
           pub creates: u64,

     

       216
       +
           pub updates: u64,

     

       217
       +
           pub deletes: u64,

     

       218
       +
       }

     

       219
       +
       impl CommitCounts {

     

       220
       +
           pub fn merge(&mut self, other: &Self) {

     

       221
       +
               self.creates += other.creates;

     

       222
       +
               self.updates += other.updates;

     

       223
       +
               self.deletes += other.deletes;

     

       224
       +
           }

     

       225
       +
       }

     

       226
       +
       impl UseBincodePlz for CommitCounts {}

     

       227
       +
       

     

       228
       +
       #[derive(Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)]

     

       229
        
       pub struct EstimatedDidsValue(pub Sketch<14>);

     

       230
        
       impl SerdeBytes for EstimatedDidsValue {}

     

       231
        
       impl DbBytes for EstimatedDidsValue {

     
···

       248
        
           }

     

       249
        
       }

     

       250
        
       

     

       251
       +
       pub type CountsValue = DbConcat<CommitCounts, EstimatedDidsValue>;

     

       252
        
       impl CountsValue {

     

       253
       +
           pub fn new(counts: CommitCounts, dids: Sketch<14>) -> Self {

     

       254
        
               Self {

     

       255
       +
                   prefix: counts,

     

       256
        
                   suffix: EstimatedDidsValue(dids),

     

       257
        
               }

     

       258
        
           }

     

       259
       +
           pub fn counts(&self) -> CommitCounts {

     

       260
       +
               self.prefix

     

       261
        
           }

     

       262
        
           pub fn dids(&self) -> &Sketch<14> {

     

       263
        
               &self.suffix.0

     

       264
        
           }

     

       265
        
           pub fn merge(&mut self, other: &Self) {

     

       266
       +
               self.prefix.merge(&other.prefix);

     

       267
       +
               self.suffix.0.merge(&other.suffix.0);

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       268
        
           }

     

       269
        
       }

     

       270
        
       impl From<&CountsValue> for JustCount {

     

       271
        
           fn from(cv: &CountsValue) -> Self {

     

       272
        
               Self {

     

       273
       +
                   creates: cv.counts().creates,

     

       274
        
                   dids_estimate: cv.dids().estimate() as u64,

     

       275
        
               }

     

       276
        
           }

     
···

       612
        
       #[cfg(test)]

     

       613
        
       mod test {

     

       614
        
           use super::{

     

       615
       +
               CommitCounts, CountsValue, Cursor, CursorBucket, Did, EncodingError, HourTruncatedCursor,

     

       616
        
               HourlyRollupKey, Nsid, Sketch, HOUR_IN_MICROS, WEEK_IN_MICROS,

     

       617
        
           };

     

       618
        
           use crate::db_types::DbBytes;

     
···

       646
        
                       Did::new(format!("did:plc:inze6wrmsm7pjl7yta3oig7{i}")).unwrap(),

     

       647
        
                   ));

     

       648
        
               }

     

       649
       +
               let original = CountsValue::new(

     

       650
       +
                   CommitCounts {

     

       651
       +
                       creates: 123,

     

       652
       +
                       ..Default::default()

     

       653
       +
                   },

     

       654
       +
                   estimator.clone(),

     

       655
       +
               );

     

       656
        
               let serialized = original.to_db_bytes()?;

     

       657
        
               let (restored, bytes_consumed) = CountsValue::from_db_bytes(&serialized)?;

     

       658
        
               assert_eq!(restored, original);

     
···

       663
        
                       Did::new(format!("did:plc:inze6wrmsm7pjl7yta3oig{i}")).unwrap(),

     

       664
        
                   ));

     

       665
        
               }

     

       666
       +
               let original = CountsValue::new(

     

       667
       +
                   CommitCounts {

     

       668
       +
                       creates: 123,

     

       669
       +
                       ..Default::default()

     

       670
       +
                   },

     

       671
       +
                   estimator,

     

       672
       +
               );

     

       673
        
               let serialized = original.to_db_bytes()?;

     

       674
        
               let (restored, bytes_consumed) = CountsValue::from_db_bytes(&serialized)?;

     

       675
        
               assert_eq!(restored, original);