···
17
-
#[cfg(feature = "compress")]
18
-
use zstd::bulk::{Compressor as ZstdCompressor, Decompressor as ZstdDecompressor};
db::{EventRecord, NsidHit, block},
utils::{CLOCK, DefaultRateTracker, RateTracker, ReadVariableExt, varints_unsigned_encoded},
26
-
#[cfg(feature = "compress")]
28
-
static COMPRESSOR: std::cell::RefCell<Option<ZstdCompressor<'static>>> = std::cell::RefCell::new(None);
29
-
static DECOMPRESSOR: std::cell::RefCell<Option<ZstdDecompressor<'static>>> = std::cell::RefCell::new(None);
32
-
type ItemDecoder = block::ItemDecoder<Cursor<Vec<u8>>, NsidHit>;
33
-
type ItemEncoder = block::ItemEncoder<Vec<u8>, NsidHit>;
23
+
pub type ItemDecoder = block::ItemDecoder<Cursor<Slice>, NsidHit>;
24
+
pub type ItemEncoder = block::ItemEncoder<Vec<u8>, NsidHit>;
pub type Item = block::Item<NsidHit>;
37
-
pub enum Compression {
39
-
#[cfg(feature = "compress")]
44
-
#[cfg(feature = "compress")]
45
-
fn get_dict(&self) -> Option<&ByteView> {
47
-
Compression::None => None,
48
-
Compression::Zstd(dict) => Some(dict),
···
buf: Arc<Mutex<Vec<EventRecord>>>,
last_insert: AtomicU64, // relaxed
65
-
compress: Compression,
impl Debug for LexiconHandle {
···
85
-
pub fn new(keyspace: &Keyspace, nsid: &str, compress: Compression) -> Self {
58
+
pub fn new(keyspace: &Keyspace, nsid: &str) -> Self {
let opts = PartitionCreateOptions::default()
88
-
.compression(fjall::CompressionType::Lz4);
61
+
.compression(fjall::CompressionType::Miniz(9));
tree: keyspace.open_partition(nsid, opts).unwrap(),
last_insert: AtomicU64::new(0),
eps: RateTracker::new(Duration::from_secs(10)),
99
-
#[cfg(feature = "compress")]
100
-
fn with_compressor<T>(&self, mut f: impl FnMut(&mut ZstdCompressor<'static>) -> T) -> T {
101
-
COMPRESSOR.with_borrow_mut(|compressor| {
102
-
if compressor.is_none() {
103
-
*compressor = Some({
104
-
let mut c = ZstdCompressor::new(9).expect("cant construct zstd compressor");
105
-
c.include_checksum(false).unwrap();
106
-
if let Some(dict) = self.compress.get_dict() {
107
-
c.set_dictionary(9, dict).expect("cant set dict");
112
-
// SAFETY: this is safe because we just initialized the compressor
113
-
f(unsafe { compressor.as_mut().unwrap_unchecked() })
117
-
#[cfg(feature = "compress")]
118
-
pub fn compress(&self, data: impl AsRef<[u8]>) -> std::io::Result<Vec<u8>> {
119
-
self.with_compressor(|compressor| compressor.compress(data.as_ref()))
122
-
#[cfg(feature = "compress")]
123
-
fn with_decompressor<T>(&self, mut f: impl FnMut(&mut ZstdDecompressor<'static>) -> T) -> T {
124
-
DECOMPRESSOR.with_borrow_mut(|decompressor| {
125
-
if decompressor.is_none() {
126
-
*decompressor = Some({
127
-
let mut d = ZstdDecompressor::new().expect("cant construct zstd decompressor");
128
-
if let Some(dict) = self.compress.get_dict() {
129
-
d.set_dictionary(dict).expect("cant set dict");
134
-
// SAFETY: this is safe because we just initialized the decompressor
135
-
f(unsafe { decompressor.as_mut().unwrap_unchecked() })
139
-
#[cfg(feature = "compress")]
140
-
pub fn decompress(&self, data: impl AsRef<[u8]>) -> std::io::Result<Vec<u8>> {
141
-
self.with_decompressor(|decompressor| {
142
-
decompressor.decompress(data.as_ref(), 1024 * 1024 * 20)
pub fn nsid(&self) -> &SmolStr {
···
let start_blocks_size = blocks_to_compact.len();
201
-
let keys_to_delete = blocks_to_compact
203
-
.map(|(key, _)| key)
126
+
let keys_to_delete = blocks_to_compact.iter().map(|(key, _)| key);
.try_fold(Vec::new(), |mut acc, (key, value)| {
210
-
let decoder = self.get_decoder_for(key, value)?;
131
+
let mut timestamps = Cursor::new(key);
132
+
let start_timestamp = timestamps.read_varint()?;
133
+
let decoder = block::ItemDecoder::new(Cursor::new(value), start_timestamp)?;
let mut items = decoder.collect::<Result<Vec<_>, _>>()?;
···
229
-
self.encode_block_from_items(chunk, count)
152
+
Self::encode_block_from_items(chunk, count)
.collect::<Result<Vec<_>, _>>()?;
let end_blocks_size = new_blocks.len();
···
pub fn encode_block_from_items(
items: impl IntoIterator<Item = Item>,
···
if let (Some(start_timestamp), Some(end_timestamp)) = (start_timestamp, end_timestamp) {
285
-
let data = self.put_raw_block(writer.finish()?)?;
207
+
let value = writer.finish()?;
let key = varints_unsigned_encoded([start_timestamp, end_timestamp]);
287
-
return Ok(Block { written, key, data });
Err(std::io::Error::new(std::io::ErrorKind::WriteZero, "no items are in queue").into())
···
307
-
pub fn get_raw_block(&self, value: Slice) -> std::io::Result<Vec<u8>> {
308
-
match &self.compress {
309
-
Compression::None => Ok(value.as_ref().into()),
310
-
#[cfg(feature = "compress")]
311
-
Compression::Zstd(_) => self.decompress(value),
315
-
pub fn put_raw_block(&self, value: Vec<u8>) -> std::io::Result<Vec<u8>> {
316
-
match &self.compress {
317
-
Compression::None => Ok(value),
318
-
#[cfg(feature = "compress")]
319
-
Compression::Zstd(_) => self.compress(value),
323
-
pub fn get_decoder_for(&self, key: Slice, value: Slice) -> AppResult<ItemDecoder> {
324
-
let mut timestamps = Cursor::new(key);
325
-
let start_timestamp = timestamps.read_varint()?;
326
-
let decoder = ItemDecoder::new(Cursor::new(self.get_raw_block(value)?), start_timestamp)?;
ptr.pet