···
72
-
// counts is nsid -> NsidCounts
73
-
// hits is tree per nsid: timestamp -> NsidHit
76
-
hits: scc::HashIndex<SmolStr, Partition>,
78
-
event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>,
83
-
pub fn new(path: impl AsRef<Path>) -> AppResult<Self> {
84
-
tracing::info!("opening db...");
85
-
let ks = Config::new(path)
86
-
.cache_size(8 * 1024 * 1024) // from talna
89
-
hits: Default::default(),
90
-
counts: ks.open_partition(
92
-
PartitionCreateOptions::default().compression(fjall::CompressionType::None),
95
-
event_broadcaster: broadcast::channel(1000).0,
96
-
eps: Rate::new(Duration::from_secs(1)),
100
-
pub fn eps(&self) -> usize {
101
-
self.eps.rate(&()) as usize
104
-
pub fn new_listener(&self) -> broadcast::Receiver<(SmolStr, NsidCounts)> {
105
-
self.event_broadcaster.subscribe()
109
-
fn get_part_opts() -> PartitionCreateOptions {
110
-
PartitionCreateOptions::default()
111
-
.compression(fjall::CompressionType::Miniz(9))
112
-
.compaction_strategy(fjall::compaction::Strategy::Fifo(fjall::compaction::Fifo {
113
-
limit: 5 * 1024 * 1024 * 1024, // 5 gb
114
-
ttl_seconds: Some(60 * 60 * 24 * 30), // 30 days
119
-
fn maybe_run_in_nsid_tree<T>(&self, nsid: &str, f: impl FnOnce(&Partition) -> T) -> Option<T> {
120
-
let _guard = scc::ebr::Guard::new();
121
-
let handle = match self.hits.peek(nsid, &_guard) {
122
-
Some(handle) => handle.clone(),
124
-
if self.inner.partition_exists(nsid) {
127
-
.open_partition(nsid, Self::get_part_opts())
128
-
.expect("cant open partition");
129
-
let _ = self.hits.insert(SmolStr::new(nsid), handle.clone());
140
-
fn run_in_nsid_tree<T>(
143
-
f: impl FnOnce(&Partition) -> AppResult<T>,
144
-
) -> AppResult<T> {
147
-
.entry(SmolStr::new(nsid))
148
-
.or_insert_with(|| {
149
-
let opts = Self::get_part_opts();
150
-
self.inner.open_partition(nsid, opts).unwrap()
155
-
pub fn record_event(&self, e: EventRecord) -> AppResult<()> {
162
-
self.insert_event(&nsid, timestamp, deleted)?;
164
-
let mut counts = self.get_count(&nsid)?;
165
-
counts.last_seen = timestamp;
167
-
counts.deleted_count += 1;
171
-
self.insert_count(&nsid, counts.clone())?;
172
-
if self.event_broadcaster.receiver_count() > 0 {
173
-
let _ = self.event_broadcaster.send((SmolStr::new(&nsid), counts));
175
-
self.eps.observe(&(), 1);
180
-
fn insert_event(&self, nsid: &str, timestamp: u64, deleted: bool) -> AppResult<()> {
181
-
self.run_in_nsid_tree(nsid, |tree| {
183
-
timestamp.to_be_bytes(),
184
-
unsafe { rkyv::to_bytes::<Error>(&NsidHit { deleted }).unwrap_unchecked() }
187
-
.map_err(AppError::from)
192
-
fn insert_count(&self, nsid: &str, counts: NsidCounts) -> AppResult<()> {
196
-
unsafe { rkyv::to_bytes::<Error>(&counts).unwrap_unchecked() }.as_slice(),
198
-
.map_err(AppError::from)
201
-
pub fn get_count(&self, nsid: &str) -> AppResult<NsidCounts> {
202
-
let Some(raw) = self.counts.get(nsid)? else {
203
-
return Ok(NsidCounts::default());
205
-
Ok(unsafe { rkyv::from_bytes_unchecked::<_, Error>(&raw).unwrap_unchecked() })
208
-
pub fn get_counts(&self) -> impl Iterator<Item = AppResult<(SmolStr, NsidCounts)>> {
209
-
self.counts.iter().map(|res| {
210
-
res.map_err(AppError::from).map(|(key, val)| {
212
-
SmolStr::new(unsafe { str::from_utf8_unchecked(&key) }),
213
-
unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },
219
-
pub fn get_nsids(&self) -> impl Iterator<Item = impl Deref<Target = str> + 'static> {
223
-
.filter(|k| k.deref() != "_counts")
229
-
range: impl RangeBounds<u64>,
230
-
) -> BoxedIter<AppResult<(u64, NsidHit)>> {
231
-
let start = range.start_bound().cloned().map(u64::to_be_bytes);
232
-
let end = range.end_bound().cloned().map(u64::to_be_bytes);
234
-
self.maybe_run_in_nsid_tree(nsid, |tree| -> BoxedIter<AppResult<(u64, NsidHit)>> {
235
-
Box::new(tree.range(TimestampRangeOld { start, end }).map(|res| {
236
-
res.map_err(AppError::from).map(|(key, val)| {
238
-
u64::from_be_bytes(key.as_ref().try_into().unwrap()),
239
-
unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },
244
-
.unwrap_or_else(|| Box::new(std::iter::empty()))
247
-
pub fn tracking_since(&self) -> AppResult<u64> {
248
-
// HACK: we should actually store when we started tracking but im lazy
249
-
// should be accurate enough
250
-
self.maybe_run_in_nsid_tree("app.bsky.feed.like", |tree| {
251
-
let Some((timestamp_raw, _)) = tree.first_key_value()? else {
254
-
Ok(u64::from_be_bytes(
255
-
timestamp_raw.as_ref().try_into().unwrap(),
type ItemDecoder = block::ItemDecoder<Cursor<Slice>, NsidHit>;
type ItemEncoder = block::ItemEncoder<Vec<u8>, NsidHit>;
type Item = block::Item<NsidHit>;
···
event_broadcaster: broadcast::channel(1000).0,
eps: Rate::new(Duration::from_secs(1)),
379
-
max_block_size: 100_000,
189
+
max_block_size: 500_000,
max_last_activity: Duration::from_secs(10),
···
598
-
type TimestampReprOld = [u8; 8];
600
-
struct TimestampRangeOld {
601
-
start: Bound<TimestampReprOld>,
602
-
end: Bound<TimestampReprOld>,
605
-
impl RangeBounds<TimestampReprOld> for TimestampRangeOld {
607
-
fn start_bound(&self) -> Bound<&TimestampReprOld> {
608
-
self.start.as_ref()
612
-
fn end_bound(&self) -> Bound<&TimestampReprOld> {
ptr.pet