···
/// this is only meant for tests
impl StorageWhatever<FjallReader, FjallWriter, FjallBackground, FjallConfig> for FjallStorage {
···
) -> StorageResult<(FjallReader, FjallWriter, Option<Cursor>, SketchSecretPrefix)> {
let config = Config::new(path);
···
-
for (partition, name) in [
-
let size0 = partition.disk_space();
-
log::info!("beggining major compaction for {name} (original size: {size0})");
-
let t0 = Instant::now();
-
partition.major_compact().expect("compact better work 😬");
-
let sizef = partition.disk_space();
-
let dsize = (sizef as i64) - (size0 as i64);
-
log::info!("completed compaction for {name} in {dt:?} (new size: {sizef}, {dsize})");
let reader = FjallReader {
···
Ok((cursors_advanced, dirty_nsids))
impl StoreWriter<FjallBackground> for FjallWriter {
···
tempfile::tempdir().unwrap(),
"offline test (no real jetstream endpoint)".to_string(),
-
FjallConfig { temp: true },
···
/// this is only meant for tests
+
/// do major compaction on startup
+
/// default is false. probably a good thing unless it's too slow.
+
pub major_compact: bool,
impl StorageWhatever<FjallReader, FjallWriter, FjallBackground, FjallConfig> for FjallStorage {
···
) -> StorageResult<(FjallReader, FjallWriter, Option<Cursor>, SketchSecretPrefix)> {
let config = Config::new(path);
···
+
if config.major_compact {
+
for (partition, name) in [
+
let size0 = partition.disk_space();
+
log::info!("beggining major compaction for {name} (original size: {size0})");
+
let t0 = Instant::now();
+
partition.major_compact().expect("compact better work 😬");
+
let sizef = partition.disk_space();
+
let dsize = (sizef as i64) - (size0 as i64);
+
"completed compaction for {name} in {dt:?} (new size: {sizef}, {dsize})"
+
log::info!("skipping major compaction on startup");
let reader = FjallReader {
···
Ok((cursors_advanced, dirty_nsids))
+
pub fn records_brute_gc_danger(&self) -> StorageResult<(usize, usize)> {
+
let (mut removed, mut retained) = (0, 0);
+
let mut to_retain = HashSet::<Vec<u8>>::new();
+
// - Per-collection list of record references ordered by jetstream cursor
+
// - key: nullstr || u64 (collection nsid null-terminated, jetstream cursor)
+
// - val: nullstr || nullstr || nullstr (did, rkey, rev. rev is mostly a sanity-check for now.)
+
// Partition: 'records'
+
// - Actual records by their atproto location
+
// - key: nullstr || nullstr || nullstr (did, collection, rkey)
+
// - val: u64 || bool || nullstr || rawval (js_cursor, is_update, rev, actual record)
+
log::warn!("loading *all* record keys from feed into memory (yikes)");
+
let t0 = Instant::now();
+
for (i, kv) in self.feeds.iter().enumerate() {
+
if i > 0 && (i % 100000 == 0) {
+
let (key_bytes, val_bytes) = kv?;
+
let key = db_complete::<NsidRecordFeedKey>(&key_bytes)?;
+
let val = db_complete::<NsidRecordFeedVal>(&val_bytes)?;
+
let record_key: RecordLocationKey = (&key, &val).into();
+
to_retain.insert(record_key.to_db_bytes()?);
+
"loaded. wow. took {:?}, found {} keys",
+
log::warn!("warmup OVER, iterating some billions of record keys now");
+
let t0 = Instant::now();
+
for (i, k) in self.records.keys().enumerate() {
+
if to_retain.contains(&*key_bytes) {
+
self.records.remove(key_bytes)?;
+
if i > 0 && (i % 10_000_000) == 0 {
+
log::info!("{i}: {retained} retained, {removed} removed.");
+
log::warn!("whew! that took {:?}", t0.elapsed());
+
Ok((removed, retained))
impl StoreWriter<FjallBackground> for FjallWriter {
···
tempfile::tempdir().unwrap(),
"offline test (no real jetstream endpoint)".to_string(),
nekomimi.pet
bad-example.com