nekomimi.pet
bad-example.com
+205
-110
Cargo.lock
+205
-110
Cargo.lock
··································································-source = "git+https://github.com/fjall-rs/fjall.git?branch=fix%2Flockless-ranges#d2102006958b0b30bdde0f7315b9b22539bb5f89"··········································-source = "git+https://github.com/fjall-rs/lsm-tree.git?branch=fix%2Flockless-ranges#c1684bdf57488a6195942fde5ea0c756dc0b6035"·····································································
+1
-3
Cargo.toml
+1
-3
Cargo.toml
+1
-1
constellation/templates/base.html.j2
+1
-1
constellation/templates/base.html.j2
···-<h1><a href="/">This</a> is a <a href="https://github.com/at-ucosm/links/tree/main/constellation">constellation 🌌</a> API server from <a href="https://github.com/at-microcosm">microcosm</a> ✨</h1>+<h1><a href="/">This</a> is a <a href="https://github.com/at-microcosm/links/tree/main/constellation">constellation 🌌</a> API server from <a href="https://github.com/at-microcosm">microcosm</a> ✨</h1>
+2
-2
jetstream/Cargo.toml
+2
-2
jetstream/Cargo.toml
······
+22
-5
jetstream/examples/arbitrary_record.rs
+22
-5
jetstream/examples/arbitrary_record.rs
·········
+20
-31
jetstream/examples/basic.rs
+20
-31
jetstream/examples/basic.rs
············
+205
jetstream/src/events.rs
+205
jetstream/src/events.rs
···+"record": {"$type":"app.bsky.feed.post","createdAt":"2025-04-01T16:58:06.154Z","langs":["en"],"text":"I wish apirl 1st would stop existing lol"}+r#"{"$type":"app.bsky.feed.post","createdAt":"2025-04-01T16:58:06.154Z","langs":["en"],"text":"I wish apirl 1st would stop existing lol"}"#+let json = r#"{"did":"did:plc:ai3dzf35cth7s3st7n7jsd7r","time_us":1743526687419798,"kind":"commit","commit":{"rev":"3llrdsginou2i","operation":"create","collection":"app.bsky.feed.post","rkey":"3llrdsglqdc2s","record":{"$type":"app.bsky.feed.post","createdAt":"2025-04-01T16:58:06.154Z","langs":["en"],"text":"I wish apirl 1st would stop existing lol"},"cid":"bafyreidofvwoqvd2cnzbun6dkzgfucxh57tirf3ohhde7lsvh4fu3jehgy"}}"#;+r#"{"$type":"app.bsky.feed.post","createdAt":"2025-04-01T16:58:06.154Z","langs":["en"],"text":"I wish apirl 1st would stop existing lol"}"#
-40
jetstream/src/events/account.rs
-40
jetstream/src/events/account.rs
···
-55
jetstream/src/events/commit.rs
-55
jetstream/src/events/commit.rs
···-/// An event representing a repo commit, which can be a `create`, `update`, or `delete` operation.-/// Basic commit specific info bundled with every event, also the only data included with a `delete`
-28
jetstream/src/events/identity.rs
-28
jetstream/src/events/identity.rs
···
-138
jetstream/src/events/mod.rs
-138
jetstream/src/events/mod.rs
···
+21
-40
jetstream/src/lib.rs
+21
-40
jetstream/src/lib.rs
······/// A wrapper connector type for working with a WebSocket connection to a Jetstream instance to···············/// A [JetstreamReceiver] is returned which can be used to respond to events. When all instances··················
+6
-6
ufos/Cargo.toml
+6
-6
ufos/Cargo.toml
······
+39
ufos/fuzz/Cargo.toml
+39
ufos/fuzz/Cargo.toml
···
+20
ufos/fuzz/fuzz_targets/cardinality_estimator.rs
+20
ufos/fuzz/fuzz_targets/cardinality_estimator.rs
···
+25
ufos/fuzz/fuzz_targets/counts_value.rs
+25
ufos/fuzz/fuzz_targets/counts_value.rs
···
+24
ufos/fuzz/fuzz_targets/estimated_dids_value.rs
+24
ufos/fuzz/fuzz_targets/estimated_dids_value.rs
···
+15
ufos/readme.md
+15
ufos/readme.md
···+got bit by https://github.com/cloudflare/cardinality-estimator/pull/12, so now we have a fuzz target.
+92
-140
ufos/src/consumer.rs
+92
-140
ufos/src/consumer.rs
······-use crate::{CreateRecord, DeleteAccount, DeleteRecord, EventBatch, ModifyRecord, UpdateRecord};-const MAX_BATCHED_RECORDS: usize = 128; // *non-blocking* limit. drops oldest batched record per collection once reached.-const MAX_BATCHED_MODIFIES: usize = 512; // hard limit, total updates and deletes across all collections.-const MAX_ACCOUNT_REMOVES: usize = 512; // hard limit, total account deletions. actually the least frequent event, but tiny.-const MAX_BATCHED_COLLECTIONS: usize = 64; // hard limit, MAX_BATCHED_RECORDS applies per collection-const MAX_BATCH_SPAN_SECS: f64 = 60.; // hard limit of duration from oldest to latest event cursor within a batch, in seconds.+pub const MAX_BATCHED_RECORDS: usize = 128; // *non-blocking* limit. drops oldest batched record per collection once reached.+pub const MAX_ACCOUNT_REMOVES: usize = 1024; // hard limit, extremely unlikely to reach, but just in case+pub const MAX_BATCHED_COLLECTIONS: usize = 64; // hard limit, MAX_BATCHED_RECORDS applies per-collection+pub const MAX_BATCH_SPAN_SECS: f64 = 60.; // hard limit, pause consumer if we're unable to send by now+pub const SEND_TIMEOUT_S: f64 = 15.; // if the channel is blocked longer than this, something is probably up-const BATCH_QUEUE_SIZE: usize = 512; // 4096 got OOM'd. update: 1024 also got OOM'd during L0 compaction blocking+pub fn new(jetstream_receiver: JetstreamReceiver, batch_sender: Sender<LimitedBatch>) -> Self {······// if the queue is empty and we have enough, send immediately. otherewise, let the current batch fill up.-anyhow::bail!("Could not send batch, likely because the receiver closed or dropped: {send_err:?}");+async fn handle_commit(&mut self, commit: UFOsCommit, collection: Nsid) -> anyhow::Result<()> {
+51
-5
ufos/src/db_types.rs
+51
-5
ufos/src/db_types.rs
·····················
+41
ufos/src/error.rs
+41
ufos/src/error.rs
···
+32
ufos/src/file_consumer.rs
+32
ufos/src/file_consumer.rs
···
+429
-33
ufos/src/lib.rs
+429
-33
ufos/src/lib.rs
···
+86
-33
ufos/src/main.rs
+86
-33
ufos/src/main.rs
···············
+91
-92
ufos/src/server.rs
+91
-92
ufos/src/server.rs
·····················-async fn get_top_collections(ctx: RequestContext<Context>) -> OkCorsResponse<HashMap<String, u64>> {+async fn get_top_collections(ctx: RequestContext<Context>) -> OkCorsResponse<TopCollections> {·········
+49
ufos/src/storage.rs
+49
ufos/src/storage.rs
···
+1795
ufos/src/storage_fjall.rs
+1795
ufos/src/storage_fjall.rs
···+/// - Jetstream server endpoint (persisted because the cursor can't be used on another instance without data loss)+/// - val: nullstr || nullstr || nullstr (did, rkey, rev. rev is mostly a sanity-check for now.)+/// TODO: account privacy preferences. Might wait for the protocol-level (PDS-level?) stuff to land. Will probably do lazy fetching + caching on read.+log::info!("trim_collection ({collection:?}) removed {dangling_feed_keys_cleaned} dangling feed entries and {records_deleted} records");+fn get_static_neu<K: StaticStr, V: DbBytes>(global: &PartitionHandle) -> StorageResult<Option<V>> {+// "batch of {: >3} samples from {: >4} records in {: >2} collections from ~{: >4} DIDs, {} acct removes, cursor {: <12?}",
+1841
ufos/src/storage_mem.rs
+1841
ufos/src/storage_mem.rs
···+/// - Jetstream server endpoint (persisted because the cursor can't be used on another instance without data loss)+/// - val: nullstr || nullstr || nullstr (did, rkey, rev. rev is mostly a sanity-check for now.)+/// TODO: account privacy preferences. Might wait for the protocol-level (PDS-level?) stuff to land. Will probably do lazy fetching + caching on read.+log::info!("trim_collection ({collection:?}) removed {dangling_feed_keys_cleaned} dangling feed entries and {records_deleted} records");+fn get_static_neu<K: StaticStr, V: DbBytes>(global: &MemPartion) -> StorageResult<Option<V>> {+fn insert_static_neu<K: StaticStr>(global: &MemPartion, value: impl DbBytes) -> StorageResult<()> {
-802
ufos/src/store.rs
-802
ufos/src/store.rs
···-Batch as FjallBatch, CompressionType, Config, Keyspace, PartitionCreateOptions, PartitionHandle,-* ["js_endpoint"] => &str, // checked on startup because jetstream instance cursors are not interchangeable-* UpdateRecord(did, collection, rkey, new_record) // delete + put, but don't delete if cursor is newer-* ["seen_by_js_cursor_collection"|js_cursor|collection] => u64 // batched total, gets cleaned up by rollup-* ["total_by_collection"|collection] => [u64, js_cursor] // rollup; live total requires scanning seen_by_collection after js_cursor-* ["hour_by_collection"|hour(u64)|collection] => u64 // rollup from seen_by_js_cursor_collection-* ["by_id"|did|collection|rkey|js_cursor] => [] // required to support deletes; did first prefix for account deletes.-* TODO: account privacy preferences. Might wait for the protocol-level (PDS-level?) stuff to land. Will probably do lazy-anyhow::bail!("stored js_endpoint {stored:?} differs from provided {endpoint:?}, refusing to start.");-let last = event_batch.last_jetstream_cursor.clone(); // TODO: get this from the data. track last in consumer. compute or track first.-println!("{batch_summary}, slept {slept_for: <12?}, wrote {wrote_for: <11?}, queue: {queue_size}");-// TODO: lock so that only one rw loop can possibly be run. or even better, take a mutable resource thing to enforce at compile time.-log::trace!("rw: skipping batch commit since apparently no items were added (this is normal, skipping is new)");-log::info!("rw: committing rw batch with {batched_rw_items} items (items != total inserts/deletes)...");-// 5. if we didn't end iteration yet, start deleting records (and their forward links) until we get to the end-// ... we can probably do even better with cursor ranges too, since we'll have a cursor range from rollup and it's in the by_collection key-pub async fn get_jetstream_endpoint(&self) -> anyhow::Result<Option<JetstreamEndpointValue>> {-fn get_static<K: StaticStr, V: DbBytes>(partition: &PartitionHandle) -> anyhow::Result<Option<V>> {-// put the cursor of the actual deletion event in to prevent prefix iter from touching newer docs-return Ok((items_added, false)); // there might be more records but we've done enough for this batch-// ["by_id"|did|collection|rkey|js_cursor] => [] // required to support deletes; did first prefix for account deletes.-"batch of {total_samples: >3} samples from {total_records: >4} records in {: >2} collections, {: >3} modifies, {} acct removes, cursor {: <12?}",
+312
-212
ufos/src/store_types.rs
+312
-212
ufos/src/store_types.rs
·········+pub type HourlyRollupKey = DbConcat<DbConcat<HourlyRollupStaticPrefix, HourTruncatedCursor>, Nsid>;+pub type WeeklyRollupKey = DbConcat<DbConcat<WeeklyRollupStaticPrefix, WeekTruncatedCursor>, Nsid>;