···
-
use crate::db_types::{db_complete, DbBytes, DbStaticStr, StaticStr};
use crate::error::StorageError;
use crate::storage::{StorageResult, StorageWhatever, StoreBackground, StoreReader, StoreWriter};
use crate::store_types::{
···
NewRollupCursorKey, NewRollupCursorValue, NsidRecordFeedKey, NsidRecordFeedVal,
RecordLocationKey, RecordLocationMeta, RecordLocationVal, RecordRawValue, SketchSecretKey,
SketchSecretPrefix, TakeoffKey, TakeoffValue, TrimCollectionCursorKey, WeekTruncatedCursor,
-
WeeklyDidsKey, WeeklyRecordsKey, WeeklyRollupKey, WithCollection,
CommitAction, ConsumerInfo, Did, EventBatch, Nsid, NsidCount, OrderCollectionsBy, UFOsRecord,
···
fn get_storage_stats(&self) -> StorageResult<serde_json::Value> {
···
-
order: OrderCollectionsBy,
-
since: Option<HourTruncatedCursor>,
-
until: Option<HourTruncatedCursor>,
) -> StorageResult<(Vec<NsidCount>, Option<Vec<u8>>)> {
-
let snapshot = self.rollups.snapshot();
-
let buckets = if let (None, None) = (since, until) {
-
vec![CursorBucket::AllTime]
-
let mut lower = self.get_earliest_hour(Some(&snapshot))?;
-
if let Some(specified) = since {
-
let upper = until.unwrap_or_else(|| Cursor::at(SystemTime::now()).into());
-
CursorBucket::buckets_spanning(lower, upper)
let mut iters: Vec<Peekable<NsidCounter>> = Vec::with_capacity(buckets.len());
-
OrderCollectionsBy::Lexi { cursor } => {
-
let cursor_nsid = cursor.as_deref().map(db_complete::<Nsid>).transpose()?;
-
for bucket in &buckets {
-
let it: NsidCounter = match bucket {
-
CursorBucket::Hour(t) => {
-
let start = cursor_nsid
-
.map(|nsid| HourlyRollupKey::after_nsid(*t, nsid))
-
.unwrap_or_else(|| HourlyRollupKey::start(*t))?;
-
let end = HourlyRollupKey::end(*t)?;
-
get_lexi_iter::<HourlyRollupKey>(&snapshot, start, end)?
-
CursorBucket::Week(t) => {
-
let start = cursor_nsid
-
.map(|nsid| WeeklyRollupKey::after_nsid(*t, nsid))
-
.unwrap_or_else(|| WeeklyRollupKey::start(*t))?;
-
let end = WeeklyRollupKey::end(*t)?;
-
get_lexi_iter::<WeeklyRollupKey>(&snapshot, start, end)?
-
CursorBucket::AllTime => {
-
let start = cursor_nsid
-
.map(AllTimeRollupKey::after_nsid)
-
.unwrap_or_else(AllTimeRollupKey::start)?;
-
let end = AllTimeRollupKey::end()?;
-
get_lexi_iter::<AllTimeRollupKey>(&snapshot, start, end)?
-
iters.push(it.peekable());
-
OrderCollectionsBy::RecordsCreated => todo!(),
-
OrderCollectionsBy::DidsEstimate => todo!(),
let mut out = Vec::new();
···
-
fn get_top_collections_by_count(
-
since: Option<HourTruncatedCursor>,
-
until: Option<HourTruncatedCursor>,
) -> StorageResult<Vec<NsidCount>> {
-
Ok(if since.is_none() && until.is_none() {
-
let snapshot = self.rollups.snapshot();
-
let mut out = Vec::with_capacity(limit);
-
let prefix = AllTimeRecordsKey::from_prefix_to_db_bytes(&Default::default())?;
-
for kv in snapshot.prefix(prefix).rev().take(limit) {
-
let (key_bytes, _) = kv?;
-
let key = db_complete::<AllTimeRecordsKey>(&key_bytes)?;
-
let rollup_key = AllTimeRollupKey::new(key.collection());
-
let db_count_bytes = snapshot.get(rollup_key.to_db_bytes()?)?.expect(
-
"integrity: all-time rank rollup must have corresponding all-time count rollup",
-
let db_counts = db_complete::<CountsValue>(&db_count_bytes)?;
-
assert_eq!(db_counts.records(), key.count());
-
nsid: key.collection().to_string(),
-
records: db_counts.records(),
-
dids_estimate: db_counts.dids().estimate() as u64,
-
fn get_top_collections_by_dids(
since: Option<HourTruncatedCursor>,
until: Option<HourTruncatedCursor>,
-
) -> StorageResult<Vec<NsidCount>> {
-
Ok(if since.is_none() && until.is_none() {
-
let snapshot = self.rollups.snapshot();
-
let mut out = Vec::with_capacity(limit);
-
let prefix = AllTimeDidsKey::from_prefix_to_db_bytes(&Default::default())?;
-
for kv in snapshot.prefix(prefix).rev().take(limit) {
-
let (key_bytes, _) = kv?;
-
let key = db_complete::<AllTimeDidsKey>(&key_bytes)?;
-
let rollup_key = AllTimeRollupKey::new(key.collection());
-
let db_count_bytes = snapshot.get(rollup_key.to_db_bytes()?)?.expect(
-
"integrity: all-time rank rollup must have corresponding all-time count rollup",
-
let db_counts = db_complete::<CountsValue>(&db_count_bytes)?;
-
assert_eq!(db_counts.dids().estimate() as u64, key.count());
-
nsid: key.collection().to_string(),
-
records: db_counts.records(),
-
dids_estimate: db_counts.dids().estimate() as u64,
fn get_counts_by_collection(&self, collection: &Nsid) -> StorageResult<(u64, u64)> {
···
tokio::task::spawn_blocking(move || {
FjallReader::get_collections(&s, limit, order, since, until)
-
async fn get_top_collections_by_count(
-
since: Option<HourTruncatedCursor>,
-
until: Option<HourTruncatedCursor>,
-
) -> StorageResult<Vec<NsidCount>> {
-
tokio::task::spawn_blocking(move || {
-
FjallReader::get_top_collections_by_count(&s, limit, since, until)
-
async fn get_top_collections_by_dids(
-
since: Option<HourTruncatedCursor>,
-
until: Option<HourTruncatedCursor>,
-
) -> StorageResult<Vec<NsidCount>> {
-
tokio::task::spawn_blocking(move || {
-
FjallReader::get_top_collections_by_dids(&s, limit, since, until)
···
+
use crate::db_types::{db_complete, DbBytes, DbStaticStr, EncodingResult, StaticStr};
use crate::error::StorageError;
use crate::storage::{StorageResult, StorageWhatever, StoreBackground, StoreReader, StoreWriter};
use crate::store_types::{
···
NewRollupCursorKey, NewRollupCursorValue, NsidRecordFeedKey, NsidRecordFeedVal,
RecordLocationKey, RecordLocationMeta, RecordLocationVal, RecordRawValue, SketchSecretKey,
SketchSecretPrefix, TakeoffKey, TakeoffValue, TrimCollectionCursorKey, WeekTruncatedCursor,
+
WeeklyDidsKey, WeeklyRecordsKey, WeeklyRollupKey, WithCollection, WithRank,
CommitAction, ConsumerInfo, Did, EventBatch, Nsid, NsidCount, OrderCollectionsBy, UFOsRecord,
···
+
type GetRollupKey = Arc<dyn Fn(&Nsid) -> EncodingResult<Vec<u8>>>;
+
fn get_lookup_iter<T: WithCollection + WithRank + DbBytes + 'static>(
+
snapshot: lsm_tree::Snapshot,
+
get_rollup_key: GetRollupKey,
+
) -> StorageResult<NsidCounter> {
+
Ok(Box::new(snapshot.range((start, end)).rev().map(
+
let (k_bytes, _) = kv?;
+
let key = db_complete::<T>(&k_bytes)?;
+
let nsid = key.collection().clone();
+
let get_counts: GetCounts = Box::new({
+
let nsid = nsid.clone();
+
let snapshot = snapshot.clone();
+
let get_rollup_key = get_rollup_key.clone();
+
let db_count_bytes = snapshot.get(get_rollup_key(&nsid)?)?.expect(
+
"integrity: all-time rank rollup must have corresponding all-time count rollup",
+
Ok(db_complete::<CountsValue>(&db_count_bytes)?)
fn get_storage_stats(&self) -> StorageResult<serde_json::Value> {
···
+
fn get_lexi_collections(
+
cursor: Option<Vec<u8>>,
+
buckets: Vec<CursorBucket>,
) -> StorageResult<(Vec<NsidCount>, Option<Vec<u8>>)> {
+
let cursor_nsid = cursor.as_deref().map(db_complete::<Nsid>).transpose()?;
let mut iters: Vec<Peekable<NsidCounter>> = Vec::with_capacity(buckets.len());
+
for bucket in &buckets {
+
let it: NsidCounter = match bucket {
+
CursorBucket::Hour(t) => {
+
let start = cursor_nsid
+
.map(|nsid| HourlyRollupKey::after_nsid(*t, nsid))
+
.unwrap_or_else(|| HourlyRollupKey::start(*t))?;
+
let end = HourlyRollupKey::end(*t)?;
+
get_lexi_iter::<HourlyRollupKey>(&snapshot, start, end)?
+
CursorBucket::Week(t) => {
+
let start = cursor_nsid
+
.map(|nsid| WeeklyRollupKey::after_nsid(*t, nsid))
+
.unwrap_or_else(|| WeeklyRollupKey::start(*t))?;
+
let end = WeeklyRollupKey::end(*t)?;
+
get_lexi_iter::<WeeklyRollupKey>(&snapshot, start, end)?
+
CursorBucket::AllTime => {
+
let start = cursor_nsid
+
.map(AllTimeRollupKey::after_nsid)
+
.unwrap_or_else(AllTimeRollupKey::start)?;
+
let end = AllTimeRollupKey::end()?;
+
get_lexi_iter::<AllTimeRollupKey>(&snapshot, start, end)?
+
iters.push(it.peekable());
let mut out = Vec::new();
···
+
fn get_ordered_collections(
+
order: OrderCollectionsBy,
+
buckets: Vec<CursorBucket>,
) -> StorageResult<Vec<NsidCount>> {
+
let mut iters: Vec<NsidCounter> = Vec::with_capacity(buckets.len());
+
for bucket in buckets {
+
let it: NsidCounter = match (&order, bucket) {
+
(OrderCollectionsBy::RecordsCreated, CursorBucket::Hour(t)) => {
+
get_lookup_iter::<HourlyRecordsKey>(
+
HourlyRecordsKey::start(t)?,
+
HourlyRecordsKey::end(t)?,
+
move |collection| HourlyRollupKey::new(t, collection).to_db_bytes()
+
(OrderCollectionsBy::DidsEstimate, CursorBucket::Hour(t)) => {
+
get_lookup_iter::<HourlyDidsKey>(
+
HourlyDidsKey::start(t)?,
+
HourlyDidsKey::end(t)?,
+
move |collection| HourlyRollupKey::new(t, collection).to_db_bytes()
+
(OrderCollectionsBy::RecordsCreated, CursorBucket::Week(t)) => {
+
get_lookup_iter::<WeeklyRecordsKey>(
+
WeeklyRecordsKey::start(t)?,
+
WeeklyRecordsKey::end(t)?,
+
move |collection| WeeklyRollupKey::new(t, collection).to_db_bytes()
+
(OrderCollectionsBy::DidsEstimate, CursorBucket::Week(t)) => {
+
get_lookup_iter::<WeeklyDidsKey>(
+
WeeklyDidsKey::start(t)?,
+
WeeklyDidsKey::end(t)?,
+
move |collection| WeeklyRollupKey::new(t, collection).to_db_bytes()
+
(OrderCollectionsBy::RecordsCreated, CursorBucket::AllTime) => {
+
get_lookup_iter::<AllTimeRecordsKey>(
+
AllTimeRecordsKey::start()?,
+
AllTimeRecordsKey::end()?,
+
Arc::new(|collection| AllTimeRollupKey::new(collection).to_db_bytes()),
+
(OrderCollectionsBy::DidsEstimate, CursorBucket::AllTime) => {
+
get_lookup_iter::<AllTimeDidsKey>(
+
AllTimeDidsKey::start()?,
+
AllTimeDidsKey::end()?,
+
Arc::new(|collection| AllTimeRollupKey::new(collection).to_db_bytes()),
+
(OrderCollectionsBy::Lexi { .. }, _) => unreachable!(),
+
// overfetch by taking a bit more than the limit
+
// sort by requested order, take limit, discard all remaining
+
// this isn't guaranteed to be correct, but it will hopefully be close most of the time:
+
// - it's possible that some NSIDs might score low during some time-buckets, and miss being merged
+
// - overfetching hopefully helps a bit by catching nsids near the threshold more often, but. yeah.
+
// this thing is heavy, there's probably a better way
+
let mut ranked: HashMap<Nsid, CountsValue> = HashMap::with_capacity(limit * 2);
+
for pair in iter.take((limit as f64 * 1.3).ceil() as usize) {
+
let (nsid, get_counts) = pair?;
+
let counts = get_counts()?;
+
ranked.entry(nsid).or_default().merge(&counts);
+
let mut ranked: Vec<(Nsid, CountsValue)> = ranked.into_iter().collect();
+
OrderCollectionsBy::RecordsCreated => ranked.sort_by_key(|(_, c)| c.records()),
+
OrderCollectionsBy::DidsEstimate => ranked.sort_by_key(|(_, c)| c.dids().estimate()),
+
OrderCollectionsBy::Lexi { .. } => unreachable!(),
+
.map(|(nsid, cv)| NsidCount {
+
nsid: nsid.to_string(),
+
dids_estimate: cv.dids().estimate() as u64,
+
order: OrderCollectionsBy,
since: Option<HourTruncatedCursor>,
until: Option<HourTruncatedCursor>,
+
) -> StorageResult<(Vec<NsidCount>, Option<Vec<u8>>)> {
+
let snapshot = self.rollups.snapshot();
+
let buckets = if let (None, None) = (since, until) {
+
vec![CursorBucket::AllTime]
+
let mut lower = self.get_earliest_hour(Some(&snapshot))?;
+
if let Some(specified) = since {
+
let upper = until.unwrap_or_else(|| Cursor::at(SystemTime::now()).into());
+
CursorBucket::buckets_spanning(lower, upper)
+
OrderCollectionsBy::Lexi { cursor } => {
+
self.get_lexi_collections(snapshot, limit, cursor, buckets)
+
self.get_ordered_collections(snapshot, limit, order, buckets)?,
fn get_counts_by_collection(&self, collection: &Nsid) -> StorageResult<(u64, u64)> {
···
tokio::task::spawn_blocking(move || {
FjallReader::get_collections(&s, limit, order, since, until)
nekomimi.pet
bad-example.com