···
ops::{Bound, Deref, RangeBounds},
-
use fjall::{Config, Keyspace, Partition, PartitionCreateOptions};
use itertools::{Either, Itertools};
-
use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};
use rkyv::{Archive, Deserialize, Serialize, rancor::Error};
use smol_str::{SmolStr, ToSmolStr};
···
use tokio_util::sync::CancellationToken;
-
db::handle::{ItemDecoder, LexiconHandle},
error::{AppError, AppResult},
jetstream::JetstreamEvent,
utils::{RateTracker, ReadVariableExt, varints_unsigned_encoded},
···
pub ks_config: fjall::Config,
pub min_block_size: usize,
pub max_block_size: usize,
pub max_last_activity: u64,
···
ks_config: fjall::Config::default(),
max_last_activity: Duration::from_secs(10).as_nanos() as u64,
···
event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>,
cancel_token: CancellationToken,
pub fn new(cfg: DbConfig, cancel_token: CancellationToken) -> AppResult<Self> {
tracing::info!("opening db...");
let ks = cfg.ks_config.clone().open()?;
hits: Default::default(),
···
event_broadcaster: broadcast::channel(1000).0,
eps: RateTracker::new(Duration::from_secs(1)),
···
.map(|(i, items, handle)| {
-
let block = LexiconHandle::encode_block_from_items(items, count)?;
"{}: encoded block with {} items",
···
Some(handle) => handle.clone(),
if self.ks.partition_exists(nsid.as_ref()) {
-
let handle = Arc::new(LexiconHandle::new(&self.ks, nsid.as_ref()));
let _ = self.hits.insert(SmolStr::new(nsid), handle.clone());
···
fn ensure_handle(&self, nsid: &SmolStr) -> impl Deref<Target = Arc<LexiconHandle>> + use<'_> {
-
.or_insert_with(|| Arc::new(LexiconHandle::new(&self.ks, &nsid)))
pub fn ingest_events(&self, events: impl Iterator<Item = EventRecord>) -> AppResult<()> {
···
let block_lens = handle.iter().rev().try_fold(Vec::new(), |mut acc, item| {
let (key, value) = item?;
-
let mut timestamps = Cursor::new(key);
-
let start_timestamp = timestamps.read_varint()?;
-
let decoder = ItemDecoder::new(Cursor::new(value), start_timestamp)?;
acc.push(decoder.item_count());
···
-
// train zstd dict with 100 blocks from every lexicon
pub fn train_zstd_dict(&self) -> AppResult<Vec<u8>> {
···
res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
-
.map(|(_, value)| Cursor::new(value))
···
return Either::Right(std::iter::empty());
-
let map_block = move |(key, val)| {
-
let mut key_reader = Cursor::new(key);
-
let start_timestamp = key_reader.read_varint::<u64>()?;
-
if start_timestamp < start_limit {
-
let items = handle::ItemDecoder::new(Cursor::new(val), start_timestamp)?
.take_while(move |item| {
item.as_ref().map_or(true, |item| {
item.timestamp <= end_limit && item.timestamp >= start_limit
···
ops::{Bound, Deref, RangeBounds},
+
use fjall::{Keyspace, Partition, PartitionCreateOptions};
use itertools::{Either, Itertools};
+
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use rkyv::{Archive, Deserialize, Serialize, rancor::Error};
use smol_str::{SmolStr, ToSmolStr};
···
use tokio_util::sync::CancellationToken;
+
db::handle::{Compression, LexiconHandle},
error::{AppError, AppResult},
jetstream::JetstreamEvent,
utils::{RateTracker, ReadVariableExt, varints_unsigned_encoded},
···
pub ks_config: fjall::Config,
+
#[cfg(feature = "compress")]
+
pub dict_path: std::path::PathBuf,
pub min_block_size: usize,
pub max_block_size: usize,
pub max_last_activity: u64,
···
ks_config: fjall::Config::default(),
+
#[cfg(feature = "compress")]
+
dict_path: "zstd_dict".parse().unwrap(),
max_last_activity: Duration::from_secs(10).as_nanos() as u64,
···
event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>,
cancel_token: CancellationToken,
+
compression: Compression,
pub fn new(cfg: DbConfig, cancel_token: CancellationToken) -> AppResult<Self> {
tracing::info!("opening db...");
let ks = cfg.ks_config.clone().open()?;
+
let _compression = Compression::None;
+
#[cfg(feature = "compress")]
+
let dict = std::fs::File::open(&cfg.dict_path).ok().and_then(|mut f| {
+
let meta = f.metadata().ok()?;
+
byteview::ByteView::from_reader(&mut f, meta.len() as usize).ok()
+
#[cfg(feature = "compress")]
+
let _compression = match dict {
+
"using zstd compression with dict from {}",
+
cfg.dict_path.to_string_lossy()
+
Compression::Zstd(dict)
+
None => Compression::None,
hits: Default::default(),
···
event_broadcaster: broadcast::channel(1000).0,
eps: RateTracker::new(Duration::from_secs(1)),
+
compression: _compression,
···
.map(|(i, items, handle)| {
+
let block = handle.encode_block_from_items(items, count)?;
"{}: encoded block with {} items",
···
Some(handle) => handle.clone(),
if self.ks.partition_exists(nsid.as_ref()) {
+
let handle = Arc::new(LexiconHandle::new(
+
self.compression.clone(),
let _ = self.hits.insert(SmolStr::new(nsid), handle.clone());
···
fn ensure_handle(&self, nsid: &SmolStr) -> impl Deref<Target = Arc<LexiconHandle>> + use<'_> {
+
self.hits.entry(nsid.clone()).or_insert_with(|| {
+
Arc::new(LexiconHandle::new(
+
self.compression.clone(),
pub fn ingest_events(&self, events: impl Iterator<Item = EventRecord>) -> AppResult<()> {
···
let block_lens = handle.iter().rev().try_fold(Vec::new(), |mut acc, item| {
let (key, value) = item?;
+
let decoder = handle.get_decoder_for(key, value)?;
acc.push(decoder.item_count());
···
+
// train zstd dict with 1000 blocks from every lexicon
+
#[cfg(feature = "compress")]
pub fn train_zstd_dict(&self) -> AppResult<Vec<u8>> {
···
res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
+
.and_then(|(_, value)| Ok(Cursor::new(handle.get_raw_block(value)?)))
···
return Either::Right(std::iter::empty());
+
let map_block = |(key, val)| {
+
let decoder = handle.get_decoder_for(key, val)?;
.take_while(move |item| {
item.as_ref().map_or(true, |item| {
item.timestamp <= end_limit && item.timestamp >= start_limit
ptr.pet