···
1
-
use super::{ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, PagedOrderedCollection, StorageStats};
2
+
ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, PagedOrderedCollection,
use crate::{CountsByCount, Did, RecordId};
use anyhow::{bail, Result};
use bincode::Options as BincodeOptions;
···
MultiThreaded, Options, PrefixRange, ReadOptions, WriteBatch,
use serde::{Deserialize, Serialize};
14
-
use std::collections::{HashMap, HashSet};
17
+
use std::collections::{BTreeMap, HashMap, HashSet};
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
···
impl LinkReader for RocksStorage {
fn get_many_to_many_counts(
834
-
_path_to_other: &str,
836
-
_after: Option<String>,
837
-
_filter_dids: &HashSet<Did>,
838
-
_filter_to_targets: &HashSet<String>,
837
+
path_to_other: &str,
839
+
after: Option<String>,
840
+
filter_dids: &HashSet<Did>,
841
+
filter_to_targets: &HashSet<String>,
) -> Result<PagedOrderedCollection<(String, u64, u64), String>> {
843
+
let collection = Collection(collection.to_string());
844
+
let path = RPath(path.to_string());
846
+
let target_key = TargetKey(Target(target.to_string()), collection.clone(), path.clone());
848
+
// unfortunately the cursor is a, uh, stringified number.
849
+
// this was easier for the memstore (plain target, not target id), and
850
+
// making it generic is a bit awful.
851
+
// so... parse the number out of a string here :(
852
+
// TODO: this should bubble up to a BAD_REQUEST response
853
+
let after = after.map(|s| s.parse::<u64>().map(TargetId)).transpose()?;
855
+
let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else {
856
+
return Ok(Default::default());
859
+
let filter_did_ids: HashMap<DidId, bool> = filter_dids
861
+
.filter_map(|did| self.did_id_table.get_id_val(&self.db, did).transpose())
862
+
.collect::<Result<Vec<DidIdValue>>>()?
864
+
.map(|DidIdValue(id, active)| (id, active))
867
+
let filter_to_target_ids = filter_to_targets
869
+
.filter_map(|target| {
870
+
self.target_id_table
873
+
&TargetKey(Target(target.to_string()), collection.clone(), path.clone()),
877
+
.collect::<Result<HashSet<TargetId>>>()?;
879
+
let linkers = self.get_target_linkers(&target_id)?;
881
+
let mut grouped_counts: BTreeMap<TargetId, (u64, HashSet<DidId>)> = BTreeMap::new();
883
+
for (did_id, rkey) in linkers.0 {
884
+
if did_id.is_empty() {
888
+
if !filter_did_ids.is_empty() && filter_did_ids.get(&did_id) != Some(&true) {
892
+
let record_link_key = RecordLinkKey(did_id, collection.clone(), rkey);
893
+
let Some(targets) = self.get_record_link_targets(&record_link_key)? else {
897
+
let Some(fwd_target) = targets
900
+
.filter_map(|RecordLinkTarget(rpath, target_id)| {
901
+
if rpath.0 == path_to_other
902
+
&& (filter_to_target_ids.is_empty()
903
+
|| filter_to_target_ids.contains(&target_id))
916
+
// small relief: we page over target ids, so we can already bail
917
+
// reprocessing previous pages here
918
+
if after.as_ref().map(|a| fwd_target <= *a).unwrap_or(false) {
922
+
// aand we can skip target ids that must be on future pages
923
+
// (this check continues after the did-lookup, which we have to do)
924
+
let page_is_full = grouped_counts.len() as u64 >= limit;
926
+
let current_max = grouped_counts.keys().rev().next().unwrap(); // limit should be non-zero bleh
927
+
if fwd_target > *current_max {
932
+
// bit painful: 2-step lookup to make sure this did is active
933
+
let Some(did) = self.did_id_table.get_val_from_id(&self.db, did_id.0)? else {
934
+
eprintln!("failed to look up did from did_id {did_id:?}");
937
+
let Some(DidIdValue(_, active)) = self.did_id_table.get_id_val(&self.db, &did)? else {
938
+
eprintln!("failed to look up did_value from did_id {did_id:?}: {did:?}: data consistency bug?");
945
+
// page-management, continued
946
+
// if we have a full page, and we're inserting a *new* key less than
947
+
// the current max, then we can evict the current max
948
+
let mut should_evict = false;
949
+
let entry = grouped_counts.entry(fwd_target.clone()).or_insert_with(|| {
950
+
// this is a *new* key, so kick the max if we're full
951
+
should_evict = page_is_full;
955
+
entry.1.insert(did_id.clone());
958
+
grouped_counts.pop_last();
962
+
let mut items: Vec<(String, u64, u64)> = Vec::with_capacity(grouped_counts.len());
963
+
for (target_id, (n, dids)) in grouped_counts {
964
+
let Some(target) = self.target_id_table.get_val_from_id(&self.db, target_id)? else {
965
+
eprintln!("failed to look up target from target_id {target_id:?}");
968
+
items.push((target, n, dids.len() as u64));
971
+
let next = if grouped_counts.len() as u64 >= limit {
972
+
// yeah.... it's a number saved as a string......sorry
977
+
.map(|k| format!("{}", k.0))
982
+
Ok(PagedOrderedCollection { items, next })
fn get_count(&self, target: &str, collection: &str, path: &str) -> Result<u64> {
···
1159
-
#[derive(Debug, Clone, Serialize, Deserialize)]
1301
+
#[derive(Debug, Clone, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, Hash)]
struct TargetId(u64); // key
1162
-
#[derive(Debug, Clone, Serialize, Deserialize)]
1304
+
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub struct Target(pub String); // the actual target/uri
// targets (uris, dids, etc.): the reverse index
nekomimi.pet
bad-example.com