nekomimi.pet
bad-example.com
+15
.github/FUNDING.yml
+15
.github/FUNDING.yml
···
+21
.github/workflows/build.yml
+21
.github/workflows/build.yml
···+run: cargo build --bin reflector --release && mv target/release/reflector target/release/reflector_amd64
+4
-2
.github/workflows/checks.yml
+4
-2
.github/workflows/checks.yml
······+run: cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot --package pocket -- --check
+2580
-184
Cargo.lock
+2580
-184
Cargo.lock
·····················-source = "git+https://github.com/uniphil/atrium?branch=fix%2Fnsid-allow-nonleading-name-digits#c4364f318d337bbc3e3e3aaf97c9f971e95f5f7e"+source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c"+"atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+"atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",···-source = "git+https://github.com/uniphil/atrium?branch=fix%2Fnsid-allow-nonleading-name-digits#c4364f318d337bbc3e3e3aaf97c9f971e95f5f7e"+source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c"···+source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c"+"atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+"atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+"atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c"+"atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+"atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+"atrium-identity 0.1.5 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+"atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",-source = "git+https://github.com/uniphil/atrium?branch=fix%2Fnsid-allow-nonleading-name-digits#c4364f318d337bbc3e3e3aaf97c9f971e95f5f7e"+source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c"············································································································+source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d"·························································+"atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",·········+source = "git+https://github.com/fatfingers23/jwt-compact.git#aed088b8ff5ad44ef2785c453f6a4b7916728b1c"······················································································································································+"atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+"atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+"atrium-identity 0.1.5 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",+"atrium-oauth 0.1.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)",·············································································································································
+6
Cargo.toml
+6
Cargo.toml
+9
-2
Makefile
+9
-2
Makefile
···
+1
-1
cozy-setup (move to another repo).md
legacy/cozy-setup (move to another repo).md
+1
-1
cozy-setup (move to another repo).md
legacy/cozy-setup (move to another repo).md
+6
-1
jetstream/Cargo.toml
+6
-1
jetstream/Cargo.toml
···-atrium-api = { git = "https://github.com/uniphil/atrium", branch = "fix/nsid-allow-nonleading-name-digits", default-features = false, features = [+atrium-api = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace", default-features = false, features = [······
+2
-6
jetstream/src/error.rs
+2
-6
jetstream/src/error.rs
···
+11
jetstream/src/events.rs
+11
jetstream/src/events.rs
···
+128
-8
jetstream/src/lib.rs
+128
-8
jetstream/src/lib.rs
···························log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event.");·········log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event.");···
+35
legacy/old-readme-details.md
+35
legacy/old-readme-details.md
···+- Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD+- Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs+- Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts and respecting users data choices+All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more.+- **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet.+_note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._+using the same "link source" concept as [constellation](./constellation/), offer webhook notifications for new references created to records+A rust crate (not published on crates.io yet) for optimistically parsing links out of arbitrary atproto PDS records, and potentially canonicalizing them
+123
legacy/original-notes.md
+123
legacy/original-notes.md
···+as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document.+it's pretty messy so embrace the mess: atproto wants to be part of the web, so this library will also extract URLs and other URIs if you want it to. all the links.+the atproto firehose that bluesky sprays at you will contain raw _contents_ from peoples' pdses. these are isolated, decontextualized updates. it's very easy to build some kinds of interesting downstream apps off of this feed.+but bringing almost kind of _context_ into your project requires a big step up in complexity and potentially cost: you're entering "appview" territory. _how many likes does a post have? who follows this account?_+you own your atproto data: it's kept in your personal data repository (PDS) and noone else can write to it. when someone likes your post, they create a "like" record in their _own_ pds, and that like belongs to _them_, not to you/your post.+in the firehose you'll see a `app.bsky.feed.post` record created, with no details about who has liked it. then you'll see separate `app.bsky.feed.like` records show up for each like that comes in on that post, with no context about the post except a random-looking reference to it. storing these in order to do so is up to you!+everything is links, and they're a mess, but they all kinda work the same, so maybe some tooling can bring down that big step in complexity from firehose raw-content apps -> apps requiring any social context.+i think that making these low-level services as easy to use as jetstream could open up pathways for building more atproto apps that operate at full scale with interesting features for reasonable effort at low cost to operate.+- med-level: pass a &str of record in json form and get a list of parsed links + json paths back. (todo: should also handle dag-cbor prob?)+- high-ish level: pass the json record and maybe apply some pre-loaded rules based on known lexicons to get the best result.+for now, a link is only considered if it matches for the entire value of the record's field -- links embedded in text content are not included. note that urls in bluesky posts _will_ still be extracted, since they are broken out into facets.+every at-uri has at least two equivalent forms, one with a `DID`, and one with an account handle. the at-uri spec [illustrates this by example](https://atproto.com/specs/at-uri-scheme):+some applications, like a reverse link index, may wish to canonicalize at-uris to a single form. the `DID`-form is stable as an account changes its handle and probably the right choice to canonicalize to, but maybe some apps would actually perfer to canonicalise to handles?+links might be relative, in which case they might need to be made absolute before being useful. is that a concern for this library, or up to the user? (seems like we might not have context here to determine its absolute)+- using `tinyjson` because it's nice -- maybe should switch to serde_json to share deps with atrium?+- would use atrium for parsing at-uris, but it's not in there. there's a did-only version in the non-lib commands.rs. its identifier parser is strict to did + handle, which makes sense, but for our purposes we might want to allow unknown methods too?
+196
legacy/ufos ops (move to micro-ops).md
+196
legacy/ufos ops (move to micro-ops).md
···+0 1/6 * * * rsync -ahP --delete "/mnt/ufos-db/.snapshots/$(sudo snapper --csvout -c ufos-db list --columns number | tail -n1)/snapshot/" backup/ufos/+format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $upstream_cache_status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""+format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $upstream_cache_status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""+format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $upstream_cache_status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
+15
links/src/lib.rs
+15
links/src/lib.rs
···
+1
pocket/.gitignore
+1
pocket/.gitignore
···
+19
pocket/Cargo.toml
+19
pocket/Cargo.toml
···+jwt-compact = { git = "https://github.com/fatfingers23/jwt-compact.git", features = ["es256k"] }
+17
pocket/api-description.md
+17
pocket/api-description.md
···+This API leverages atproto service proxying to offer a bit of per-user per-app non-public data storage.+Perfect for things like application preferences that might be better left out of the public PDS data.+The intent is to use oauth scopes to isolate storage on a per-application basis, and to allow easy data migration from a community hosted instance to your own if you end up needing that.+> Pocket is currently in a **v0, pre-release state**. There is one production instance and you can use it! Expect short downtimes for restarts as development progresses and occaisional data loss until it's stable.+ATProto might end up adding a similar feature to [PDSs](https://atproto.com/guides/glossary#pds-personal-data-server). If/when that happens, you should use it instead of this!
+7
pocket/src/lib.rs
+7
pocket/src/lib.rs
+34
pocket/src/main.rs
+34
pocket/src/main.rs
···
+265
pocket/src/server.rs
+265
pocket/src/server.rs
···
+50
pocket/src/storage.rs
+50
pocket/src/storage.rs
···
+143
pocket/src/token.rs
+143
pocket/src/token.rs
···+// danger! unfortunately we need to decode the DID from the jwt body before we have a public key to verify the jwt with
+67
pocket/static/index.html
+67
pocket/static/index.html
···+<meta name="description" content="API Documentation for Pocket, a simple user-preference storage system for atproto" />
+8
quasar/Cargo.toml
+8
quasar/Cargo.toml
+3
quasar/readme.md
+3
quasar/readme.md
+57
-129
readme.md
+57
-129
readme.md
···+[](https://bsky.app/profile/microcosm.blue)+[](https://discord.gg/tcDfe4PGVB)+[](https://github.com/sponsors/uniphil/)+[](https://ko-fi.com/bad_example)+Welcome! Documentation is under active development. If you like reading API docs, you'll probably hit the ground running!+Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://discord.gg/tcDfe4PGVB), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere.-- Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD-- Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs-- Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts and respecting users data choices+> This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.sh/@microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place!-All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more.-- **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet.-_note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._-using the same "link source" concept as [constellation](./constellation/), offer webhook notifications for new references created to records-A rust crate (not published on crates.io yet) for optimistically parsing links out of arbitrary atproto PDS records, and potentially canonicalizing them-as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document.-it's pretty messy so embrace the mess: atproto wants to be part of the web, so this library will also extract URLs and other URIs if you want it to. all the links.-the atproto firehose that bluesky sprays at you will contain raw _contents_ from peoples' pdses. these are isolated, decontextualized updates. it's very easy to build some kinds of interesting downstream apps off of this feed.-but bringing almost kind of _context_ into your project requires a big step up in complexity and potentially cost: you're entering "appview" territory. _how many likes does a post have? who follows this account?_-you own your atproto data: it's kept in your personal data repository (PDS) and noone else can write to it. when someone likes your post, they create a "like" record in their _own_ pds, and that like belongs to _them_, not to you/your post.-in the firehose you'll see a `app.bsky.feed.post` record created, with no details about who has liked it. then you'll see separate `app.bsky.feed.like` records show up for each like that comes in on that post, with no context about the post except a random-looking reference to it. storing these in order to do so is up to you!-everything is links, and they're a mess, but they all kinda work the same, so maybe some tooling can bring down that big step in complexity from firehose raw-content apps -> apps requiring any social context.-i think that making these low-level services as easy to use as jetstream could open up pathways for building more atproto apps that operate at full scale with interesting features for reasonable effort at low cost to operate.+A global atproto interactions backlink index as a simple JSON API. Works with every lexicon, runs on a raspberry pi, consumes less than 2GiB of disk per day. Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts while respecting users' data choices.+- Status: used in production. APIs will change but backwards compatibility will be maintained as long as needed.+A global atproto interactions firehose. Extracts all at-uris, DIDs, and URLs from every lexicon in the firehose, and exposes them over a websocket modelled after [jetstream](github.com/bluesky-social/jetstream).-- med-level: pass a &str of record in json form and get a list of parsed links + json paths back. (todo: should also handle dag-cbor prob?)+- Status: v0: the basics work and the APIs are in place! missing cursor replay, forward link storage, and delete event link hydration.-- high-ish level: pass the json record and maybe apply some pre-loaded rules based on known lexicons to get the best result.-for now, a link is only considered if it matches for the entire value of the record's field -- links embedded in text content are not included. note that urls in bluesky posts _will_ still be extracted, since they are broken out into facets.+- [Spacedust notifications](https://notifications.microcosm.blue/): web push notifications for _every_ atproto app+- [Zero-Bluesky real-time interaction-updating post embed](https://bsky.bad-example.com/zero-bluesky-realtime-embed/)+A fast, eager, production-grade edge cache for atproto records and identities. Pre-caches all records from the firehose and maintains a longer-term cache of requested records on disk.-every at-uri has at least two equivalent forms, one with a `DID`, and one with an account handle. the at-uri spec [illustrates this by example](https://atproto.com/specs/at-uri-scheme):-some applications, like a reverse link index, may wish to canonicalize at-uris to a single form. the `DID`-form is stable as an account changes its handle and probably the right choice to canonicalize to, but maybe some apps would actually perfer to canonicalise to handles?+Timeseries stats and sample records for every [collection](https://atproto.com/guides/glossary#collection) ever seen in the atproto firehose. Unique users are counted in hyperloglog sketches enabling arbitrary cardinality aggregation across time buckets and/or NSIDs.+- Status: Used in production. It has APIs and they work! Needs improvement on indexing; needs more indexes and some more APIs to the data exposed.+> See also: [UFOs atproto explorer](https://ufos.microcosm.blue/) built on UFOs API. ([source](github.com/at-microcosm/spacedust-utils))-links might be relative, in which case they might need to be made absolute before being useful. is that a concern for this library, or up to the user? (seems like we might not have context here to determine its absolute)+- Status: used in multiple apps in production, but not yet published to crates.io; some rework planned-- using `tinyjson` because it's nice -- maybe should switch to serde_json to share deps with atrium?+> See also: [Rocketman](https://github.com/teal-fm/cadet/tree/main/rocketman), another excellent rust jetstream client which shares some lineage and _is_ published on crates.io.-- would use atrium for parsing at-uris, but it's not in there. there's a did-only version in the non-lib commands.rs. its identifier parser is strict to did + handle, which makes sense, but for our purposes we might want to allow unknown methods too?+An identity bridge for microcosm demos, that kinda worked. Fixing its problems is about equivalent to reinventing a lot of OIDC, so it's being retired.+> `who-am-i` is still in use for the Spacedust Notifications demo, but that will hopefully be migrated to use atproto oauth directly instead.
+12
reflector/Cargo.toml
+12
reflector/Cargo.toml
···
+9
reflector/readme.md
+9
reflector/readme.md
···+receiving requests from multiple subdomains is left as a problem for the reverse proxy to solve, since acme wildcard certificates (ie. letsencrypt) require the most complicated and involved challenge type (DNS).+caddy [has good support for](https://caddyserver.com/docs/caddyfile/patterns#wildcard-certificates) configuring the wildcard DNS challenge with various DNS providers, and also supports [on-demand](https://caddyserver.com/docs/automatic-https#using-on-demand-tls) provisioning via the simpler methods.+if you only need a small fixed number of subdomains, you can also use certbot or otherwise individually configure them in your reverse proxy.
+113
reflector/src/main.rs
+113
reflector/src/main.rs
···
+1
slingshot/.gitignore
+1
slingshot/.gitignore
···
+31
slingshot/Cargo.toml
+31
slingshot/Cargo.toml
···+atrium-api = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace", default-features = false }+atrium-common = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" }+atrium-identity = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" }+atrium-oauth = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" }
+93
slingshot/api-description.md
+93
slingshot/api-description.md
···+_A [gravitational slingshot](https://en.wikipedia.org/wiki/Gravity_assist) makes use of the gravity and relative movements of celestial bodies to accelerate a spacecraft and change its trajectory._+Applications in [ATProtocol](https://atproto.com/) store data in users' own [PDS](https://atproto.com/guides/self-hosting) (Personal Data Server), which are distributed across thousands of independently-run servers all over the world. Trying to access this data poses challenges for client applications:+Large projects like [Bluesky](https://bsky.app/) control their performance and reliability by syncing all app-relevant data from PDSs into first-party databases. But for new apps, building out this additional data infrastructure adds significant effort and complexity up front.+**Slingshot is a fast, eager, production-grade cache of data in the [ATmosphere](https://atproto.com/)**, offering performance and reliability without custom infrastructure.+> Slingshot is currently in a **v0, pre-release state**. There is one production instance and you can use it! Expect short downtimes for restarts as development progresses and lower cache hit-rates as the internal storage caches are adjusted and reset.+The core APIs will not change, since they are standard third-party `com.atproto` query APIs from ATProtocol.+Slingshot subscribes to the global [Firehose](https://atproto.com/specs/sync#firehose) of data updates. It keeps a short-term rolling indexed window of *all* data, and automatically promotes content likely to be requested to its longer-term main cache. _(automatic promotion is still a work in progress)_+When there is a cache miss, Slingshot can often still accelerate record fetching, since it keeps a large cache of resolved identities: it can usually request from the correct PDS without extra lookups.+The fireshose includes **update** and **delete** events, which Slingshot uses to ensure stale and deleted data is removed within a very short window. Additonally, identity and account-level events can trigger rapid cleanup of data for deactivated and deleted accounts. _(some of this is still a work in progress)_+The "AT" in ATProtocol [stands for _Authenticated Transfer_](https://atproto.com/guides/glossary#at-protocol): all data is cryptographically signed, which makes it possible to broadcast data through third parties and trust that it's real _without_ having to directly contact the originating server.+Two core standard query APIs are supported to balance convenience and trust. They both fetch [records](https://atproto.com/guides/glossary#record):+### [`com.atproto.repo.getRecord`](#tag/comatproto-queries/get/xrpc/com.atproto.repo.getRecord)+### [`com.atproto.sync.getRecord`](#tag/comatproto-queries/get/xrpc/com.atproto.sync.getRecord)+- [`DAG-CBOR`](https://atproto.com/specs/data-model)-encoded response requires extra libraries to decode, but+Clients can proxy atproto queries through their own PDS with [Service Proxying](https://atproto.com/specs/xrpc#service-proxying), and this is supported by Slingshot. The Slingshot instance must be started the `--domain` argument specified.+Where `<your pds>` is the user's own PDS host, and `<slingshot domain>` is the domain that the slingshot instance is deployed at (eg. `slingshot.microcosm.blue`). See the [Service Proxying](https://atproto.com/specs/xrpc#service-proxying) docs for more.+> Service proxying is supported but completely optional. All APIs are directly accessible over the public internet, and GeoDNS helps route users to the closest instance to them for the lowest possible latency. (_note: deploying multiple slingshot instances with GeoDNS is still TODO_)+- Slingshot also offers variants of the `getRecord` endpoints that accept a full `at-uri` as a parameter, to save clients from needing to parse and validate all parts of a record location.+- Bi-directionally verifying identity endpoints, so you can directly exchange atproto [`handle`](https://atproto.com/guides/glossary#handle)s for [`DID`](https://atproto.com/guides/glossary#did-decentralized-id)s without extra steps, plus a convenient [Mini-Doc](#tag/slingshot-specific-queries/get/xrpc/com.bad-example.identity.resolveMiniDoc) verified identity summary.+[Microcosm](https://www.microcosm.blue/) is a collection of services and independent community-run infrastructure for ATProtocol.+Slingshot excels when combined with _shallow indexing_ services, which offer fast queries of global data relationships but with only references to the data records. Microcosm has a few!+- [🌌 Constellation](https://constellation.microcosm.blue/), a global backlink index (all social interactions in atproto are links!)+> All microcosm projects are [open source](https://tangled.sh/@bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)!
+7
slingshot/readme.md
+7
slingshot/readme.md
+80
slingshot/src/consumer.rs
+80
slingshot/src/consumer.rs
···
+93
slingshot/src/error.rs
+93
slingshot/src/error.rs
···
+22
slingshot/src/firehose_cache.rs
+22
slingshot/src/firehose_cache.rs
···+.with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records
+32
slingshot/src/healthcheck.rs
+32
slingshot/src/healthcheck.rs
···
+525
slingshot/src/identity.rs
+525
slingshot/src/identity.rs
···+/// 3. DID -> handle resolution: for bidirectional handle validation and in case we want to offer this+use atrium_oauth::DefaultHttpClient; // it's probably not worth bringing all of atrium_oauth for this but+/// just a lock to ensure only one refresher (queue consumer) is running (to be improved with a better refresher)+.with_file_size(2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records
+14
slingshot/src/lib.rs
+14
slingshot/src/lib.rs
···
+194
slingshot/src/main.rs
+194
slingshot/src/main.rs
···+/// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value:+.set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here.+.set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage)
+155
slingshot/src/record.rs
+155
slingshot/src/record.rs
···+// TODO: throttle outgoing requests by host probably, generally guard against outgoing requests+.map_err(RecordError::StatusError)? // TODO atproto error handling (think about handling not found)
+778
slingshot/src/server.rs
+778
slingshot/src/server.rs
···+/// more convenient [request parameters](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.repo.getUriRecord)+/// or [response formats](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.identity.resolveMiniDoc).+/// > See also the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-repo-get-record)+/// Ergonomic complement to [`com.atproto.repo.getRecord`](https://docs.bsky.app/docs/api/com-atproto-repo-get-record)+/// > See the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-handle)+/// Like [com.atproto.identity.resolveIdentity](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-identity)+// but these are both not specified to do bidirectional validation, which is what we want to offer+// we could do horrible things and implement resolveIdentity with only a stripped-down fake did doc+// but this will *definitely* cause problems because eg. we're not currently storing pubkeys and+/// - slingshot *may* implement more generous per-user rate-limits for proxied requests in the future+async fn run<L>(listener: L, app: Route, shutdown: CancellationToken) -> Result<(), ServerError>
slingshot/static/favicon.ico
slingshot/static/favicon.ico
This is a binary file and will not be displayed.
+67
slingshot/static/index.html
+67
slingshot/static/index.html
···+<meta name="description" content="API Documentation for Slingshot, a firehose-listening atproto edge record and identity cache." />
+29
spacedust/Cargo.toml
+29
spacedust/Cargo.toml
···
+116
spacedust/src/consumer.rs
+116
spacedust/src/consumer.rs
···+let _ = b.send(message.clone()); // only errors if no subscribers are connected, which is just fine.
+22
spacedust/src/delay.rs
+22
spacedust/src/delay.rs
···
+55
spacedust/src/error.rs
+55
spacedust/src/error.rs
···
+104
spacedust/src/lib.rs
+104
spacedust/src/lib.rs
···+origin: "live", // TODO: indicate when we're locally replaying jetstream on reconnect?? maybe not.
+144
spacedust/src/main.rs
+144
spacedust/src/main.rs
···+/// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value:+.set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here.+.set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage)
+125
spacedust/src/removable_delay_queue.rs
+125
spacedust/src/removable_delay_queue.rs
···
+339
spacedust/src/server.rs
+339
spacedust/src/server.rs
···+async fn instrument_handler<T, H, R>(ctx: &RequestContext<T>, handler: H) -> Result<R, HttpError>+histogram!("server_handler_latency", "endpoint" => endpoint).record(latency.as_micros() as f64);+/// Typically [a little less than 1%](https://bsky.app/profile/bad-example.com/post/3ls32wctsrs2l)
+164
spacedust/src/subscriber.rs
+164
spacedust/src/subscriber.rs
···
spacedust/static/favicon.ico
spacedust/static/favicon.ico
This is a binary file and will not be displayed.
+54
spacedust/static/index.html
+54
spacedust/static/index.html
···+<meta name="description" content="API Documentation for Spacedust, a configurable ATProto notifications firehose" />
+5
-3
ufos/Cargo.toml
+5
-3
ufos/Cargo.toml
···+metrics-exporter-prometheus = { version = "0.17.0", default-features = false, features = ["http-listener"] }
+38
-5
ufos/src/consumer.rs
+38
-5
ufos/src/consumer.rs
·········async fn send_current_batch_now(&mut self, small: bool, referrer: &str) -> anyhow::Result<()> {+"sending batch now from {beginning}, {size_label}, queue capacity: {queue_cap}, referrer: {referrer}",
+2
-2
ufos/src/db_types.rs
+2
-2
ufos/src/db_types.rs
······
+6
-3
ufos/src/index_html.rs
+6
-3
ufos/src/index_html.rs
···<meta name="description" content="API Documentation for UFOs: Samples and stats for all atproto lexicons." />···
+29
-3
ufos/src/lib.rs
+29
-3
ufos/src/lib.rs
······
+85
-11
ufos/src/main.rs
+85
-11
ufos/src/main.rs
·········+.set_bucket_count(std::num::NonZero::new(10).unwrap()) // count * duration = 10 mins. stuff doesn't happen that fast here.+.set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage)······let nice_dt_two_maybes = |earlier: Option<Cursor>, later: Option<Cursor>| match (earlier, later)···"cursor: {} behind (→{}, {cursor_rate}x, {cursor_avg}x avg). rollup: {} behind (→{}, {rollup_rate}x, {rollup_avg}x avg).",latest_cursor.map(|c| c.elapsed().map(nice_duration).unwrap_or("++".to_string())).unwrap_or("?".to_string()),
+1
-1
ufos/src/server/collections_query.rs
+1
-1
ufos/src/server/collections_query.rs
···
+310
-181
ufos/src/server/mod.rs
+310
-181
ufos/src/server/mod.rs
···+async fn instrument_handler<T, H, R>(ctx: &RequestContext<T>, handler: H) -> Result<R, HttpError>+histogram!("server_handler_latency", "endpoint" => endpoint).record(latency.as_micros() as f64);············-let min_time_ago = SystemTime::now() - Duration::from_secs(86_400 * 3); // we want at least 3 days of data+let min_time_ago = SystemTime::now() - Duration::from_secs(86_400 * 3); // we want at least 3 days of data··················
+26
-5
ufos/src/storage.rs
+26
-5
ufos/src/storage.rs
···············
+182
-54
ufos/src/storage_fjall.rs
+182
-54
ufos/src/storage_fjall.rs
·············································-batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better+// remove_weak is allowed here because the secondary ranking index only ever inserts once at a key···-batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better+// remove_weak is allowed here because the secondary ranking index only ever inserts once at a key··················+log::trace!("finished trimming {n} nsids in {dt:?}: {total_danglers} dangling and {total_deleted} total removed.");+counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64);+log::warn!("weird trim case: more danglers than deleted? metric will be missing for dangling=false. deleted={total_deleted} danglers={total_danglers}");-log::info!("finished trimming {n} nsids in {:?}: {total_danglers} dangling and {total_deleted} total removed.", t0.elapsed());············
+37
who-am-i/Cargo.toml
+37
who-am-i/Cargo.toml
···
+37
who-am-i/demo/index.html
+37
who-am-i/demo/index.html
···+<iframe src="http://127.0.0.1:9997/prompt" id="whoami" style="border: none" height="160" width="320"></iframe>
+4
who-am-i/demo/serve
+4
who-am-i/demo/serve
+66
who-am-i/readme.md
+66
who-am-i/readme.md
···+for now the deployment is restricted to microcosm -- expanding it for wider use likely requires solving a number of challenges that oauth exists for.+- you drop an iframe and a short few lines of JS on your web page, and get a nice-ish atproto login prompt.+- if the user has ever authorized this service before (and within some expiration), they will be presented with an in-frame one-click option to proceed.+- clickjacking: if this were allowed on arbitrary domains, malicious sites could trick users into proving their atproto identity.+- all the other problems oauth exists to solve: it's a little tricky to hook around the oauth flow so there are probably some annoying attacks.+- auth in front of auth: it's just a bit awkward to run an auth service that acts as an intermediary for a more-real auth behind it, but that's worse, less secure, and doesn't conform to any standards.+sometimes you want to make a thing that people can use with an atproto identity, and you might not want to let them put in any else's identity. apps that operate on public data like skircle, cred.blue, and the microcosm spacedust notifications demo don't require any special permission to operate for any user, and that's sometimes fine, but sometimes creepy/stalker-y/etc.+to avoid building a small torment nexus for a microcosm demo (while also not wanting to get deep into oauth or operate a demo-specific auth backend), i made this little service to just get a verified identity.+since the requirements (read-only, just verifying identity) seem modest, i was hoping that a fairly simple implementation could be Good Enough, but in the time that i was willing to spend on it, the simple version without major obvious weaknesses i was hoping for didn't emerge.+it's still nice to have an explicit opt-in on a per-demo basis for microcosm so it will be used for that. it's allow-listed for the microcosm domain however (so not deployed on any adversarial hosting pages), so it's simultaenously overkill and restrictive.+i will get back to oauth eventually and hopefully roll out a microcosm service to make it easy for clients (and demos), but there are a few more things in the pipeline to get to first.+provide a pubkey-signed JWT of the identity (just the DID as `sub` probably). (**you probably SHOULD NOT USE THIS in any serious environment**)+atrium-oauth uses reqwest with default tls config that requires openssl which `cross` doesn't have a good time getting the os deps for.+fortunately, simply *enabling* a differnent tls feature for reqwest actually stops the default problematic one from causing problems, so we have a `reqwest` direct dependency with a feature enabled, even though it's never imported into actual code,
+89
who-am-i/src/expiring_task_map.rs
+89
who-am-i/src/expiring_task_map.rs
···
+93
who-am-i/src/jwt.rs
+93
who-am-i/src/jwt.rs
···+// https://github.com/atrium-rs/atrium/blob/b48810f84d83d037ee89b79b8566df9e0f2a6dae/atrium-oauth/src/keyset.rs#L41
+9
who-am-i/src/lib.rs
+9
who-am-i/src/lib.rs
+120
who-am-i/src/main.rs
+120
who-am-i/src/main.rs
···
+268
who-am-i/src/oauth.rs
+268
who-am-i/src/oauth.rs
···+state: Option<String>, // TODO: we _should_ use state to associate the auth request but how to do that with atrium is unclear+pub async fn complete(&self, params: OAuthCallbackParams) -> Result<Did, OAuthCompleteError> {
+525
who-am-i/src/server.rs
+525
who-am-i/src/server.rs
···
who-am-i/static/favicon.ico
who-am-i/static/favicon.ico
This is a binary file and will not be displayed.
+195
who-am-i/static/style.css
+195
who-am-i/static/style.css
···
+20
who-am-i/templates/auth-fail.hbs
+20
who-am-i/templates/auth-fail.hbs
···
+17
who-am-i/templates/base-base.hbs
+17
who-am-i/templates/base-base.hbs
···
+28
who-am-i/templates/base-framed.hbs
+28
who-am-i/templates/base-framed.hbs
···
+26
who-am-i/templates/base-full.hbs
+26
who-am-i/templates/base-full.hbs
···
+122
who-am-i/templates/hello.hbs
+122
who-am-i/templates/hello.hbs
···+{{#*inline "description"}}A little identity-verifying auth service for microcosm demos{{/inline}}+<p>Only <strong>read access to your public data</strong> is required to connect: connecting does not grant any ability to modify your account or data.</p>+<form id="form-action" action="/auth" target="_blank" method="GET" class="action {{#if did}}hidden{{/if}}">
+18
who-am-i/templates/prompt-error.hbs
+18
who-am-i/templates/prompt-error.hbs
···
+187
who-am-i/templates/prompt.hbs
+187
who-am-i/templates/prompt.hbs
···+<form id="form-action" action="/auth" method="GET" target="_blank" class="action {{#if did}}hidden{{/if}}">+// so if you have two flows going, it grants for both (or the first responder?) if you grant for either.