···
17
+
#[cfg(feature = "compress")]
18
+
use zstd::bulk::{Compressor as ZstdCompressor, Decompressor as ZstdDecompressor};
db::{EventRecord, NsidHit, block},
utils::{CLOCK, DefaultRateTracker, RateTracker, ReadVariableExt, varints_unsigned_encoded},
23
-
pub type ItemDecoder = block::ItemDecoder<Cursor<Slice>, NsidHit>;
24
-
pub type ItemEncoder = block::ItemEncoder<Vec<u8>, NsidHit>;
26
+
#[cfg(feature = "compress")]
28
+
static COMPRESSOR: std::cell::RefCell<Option<ZstdCompressor<'static>>> = std::cell::RefCell::new(None);
29
+
static DECOMPRESSOR: std::cell::RefCell<Option<ZstdDecompressor<'static>>> = std::cell::RefCell::new(None);
32
+
type ItemDecoder = block::ItemDecoder<Cursor<Vec<u8>>, NsidHit>;
33
+
type ItemEncoder = block::ItemEncoder<Vec<u8>, NsidHit>;
pub type Item = block::Item<NsidHit>;
37
+
pub enum Compression {
39
+
#[cfg(feature = "compress")]
44
+
#[cfg(feature = "compress")]
45
+
fn get_dict(&self) -> Option<&ByteView> {
47
+
Compression::None => None,
48
+
Compression::Zstd(dict) => Some(dict),
···
buf: Arc<Mutex<Vec<EventRecord>>>,
last_insert: AtomicU64, // relaxed
65
+
compress: Compression,
impl Debug for LexiconHandle {
···
58
-
pub fn new(keyspace: &Keyspace, nsid: &str) -> Self {
85
+
pub fn new(keyspace: &Keyspace, nsid: &str, compress: Compression) -> Self {
let opts = PartitionCreateOptions::default()
61
-
.compression(fjall::CompressionType::Miniz(9));
88
+
.compression(fjall::CompressionType::Lz4);
tree: keyspace.open_partition(nsid, opts).unwrap(),
last_insert: AtomicU64::new(0),
eps: RateTracker::new(Duration::from_secs(10)),
99
+
#[cfg(feature = "compress")]
100
+
fn with_compressor<T>(&self, mut f: impl FnMut(&mut ZstdCompressor<'static>) -> T) -> T {
101
+
COMPRESSOR.with_borrow_mut(|compressor| {
102
+
if compressor.is_none() {
103
+
*compressor = Some({
104
+
let mut c = ZstdCompressor::new(9).expect("cant construct zstd compressor");
105
+
c.include_checksum(false).unwrap();
106
+
if let Some(dict) = self.compress.get_dict() {
107
+
c.set_dictionary(9, dict).expect("cant set dict");
112
+
// SAFETY: this is safe because we just initialized the compressor
113
+
f(unsafe { compressor.as_mut().unwrap_unchecked() })
117
+
#[cfg(feature = "compress")]
118
+
pub fn compress(&self, data: impl AsRef<[u8]>) -> std::io::Result<Vec<u8>> {
119
+
self.with_compressor(|compressor| compressor.compress(data.as_ref()))
122
+
#[cfg(feature = "compress")]
123
+
fn with_decompressor<T>(&self, mut f: impl FnMut(&mut ZstdDecompressor<'static>) -> T) -> T {
124
+
DECOMPRESSOR.with_borrow_mut(|decompressor| {
125
+
if decompressor.is_none() {
126
+
*decompressor = Some({
127
+
let mut d = ZstdDecompressor::new().expect("cant construct zstd decompressor");
128
+
if let Some(dict) = self.compress.get_dict() {
129
+
d.set_dictionary(dict).expect("cant set dict");
134
+
// SAFETY: this is safe because we just initialized the decompressor
135
+
f(unsafe { decompressor.as_mut().unwrap_unchecked() })
139
+
#[cfg(feature = "compress")]
140
+
pub fn decompress(&self, data: impl AsRef<[u8]>) -> std::io::Result<Vec<u8>> {
141
+
self.with_decompressor(|decompressor| {
142
+
decompressor.decompress(data.as_ref(), 1024 * 1024 * 20)
pub fn nsid(&self) -> &SmolStr {
···
let start_blocks_size = blocks_to_compact.len();
126
-
let keys_to_delete = blocks_to_compact.iter().map(|(key, _)| key);
201
+
let keys_to_delete = blocks_to_compact
203
+
.map(|(key, _)| key)
.try_fold(Vec::new(), |mut acc, (key, value)| {
131
-
let mut timestamps = Cursor::new(key);
132
-
let start_timestamp = timestamps.read_varint()?;
133
-
let decoder = block::ItemDecoder::new(Cursor::new(value), start_timestamp)?;
210
+
let decoder = self.get_decoder_for(key, value)?;
let mut items = decoder.collect::<Result<Vec<_>, _>>()?;
···
152
-
Self::encode_block_from_items(chunk, count)
229
+
self.encode_block_from_items(chunk, count)
.collect::<Result<Vec<_>, _>>()?;
let end_blocks_size = new_blocks.len();
···
pub fn encode_block_from_items(
items: impl IntoIterator<Item = Item>,
···
if let (Some(start_timestamp), Some(end_timestamp)) = (start_timestamp, end_timestamp) {
207
-
let value = writer.finish()?;
285
+
let data = self.put_raw_block(writer.finish()?)?;
let key = varints_unsigned_encoded([start_timestamp, end_timestamp]);
287
+
return Ok(Block { written, key, data });
Err(std::io::Error::new(std::io::ErrorKind::WriteZero, "no items are in queue").into())
···
307
+
pub fn get_raw_block(&self, value: Slice) -> std::io::Result<Vec<u8>> {
308
+
match &self.compress {
309
+
Compression::None => Ok(value.as_ref().into()),
310
+
#[cfg(feature = "compress")]
311
+
Compression::Zstd(_) => self.decompress(value),
315
+
pub fn put_raw_block(&self, value: Vec<u8>) -> std::io::Result<Vec<u8>> {
316
+
match &self.compress {
317
+
Compression::None => Ok(value),
318
+
#[cfg(feature = "compress")]
319
+
Compression::Zstd(_) => self.compress(value),
323
+
pub fn get_decoder_for(&self, key: Slice, value: Slice) -> AppResult<ItemDecoder> {
324
+
let mut timestamps = Cursor::new(key);
325
+
let start_timestamp = timestamps.read_varint()?;
326
+
let decoder = ItemDecoder::new(Cursor::new(self.get_raw_block(value)?), start_timestamp)?;
ptr.pet