Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm

Web: stats and summary

+17
Cargo.lock
···
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
[[package]]
name = "askama"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
···
"metrics",
"metrics-exporter-prometheus",
"metrics-process",
"rocksdb",
"serde",
"serde_with",
···
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-traits"
···
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
[[package]]
+
name = "arrayvec"
+
version = "0.7.6"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+
+
[[package]]
name = "askama"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
···
"metrics",
"metrics-exporter-prometheus",
"metrics-process",
+
"num-format",
"rocksdb",
"serde",
"serde_with",
···
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+
+
[[package]]
+
name = "num-format"
+
version = "0.4.4"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3"
+
dependencies = [
+
"arrayvec",
+
"itoa",
+
]
[[package]]
name = "num-traits"
+1
constellation/Cargo.toml
···
metrics = "0.24.1"
metrics-exporter-prometheus = { version = "0.16.1", default-features = false, features = ["http-listener"] }
metrics-process = "2.4.0"
rocksdb = { version = "0.23.0", optional = true }
serde = { version = "1.0.215", features = ["derive"] }
serde_with = { version = "3.12.0", features = ["hex"] }
···
metrics = "0.24.1"
metrics-exporter-prometheus = { version = "0.16.1", default-features = false, features = ["http-listener"] }
metrics-process = "2.4.0"
+
num-format = "0.4.4"
rocksdb = { version = "0.23.0", optional = true }
serde = { version = "1.0.215", features = ["derive"] }
serde_with = { version = "3.12.0", features = ["hex"] }
+5
constellation/src/server/filters.rs
···
use links::{parse_any_link, Link};
pub fn to_browseable(s: &str) -> askama::Result<Option<String>> {
Ok({
···
}
})
}
···
use links::{parse_any_link, Link};
+
use num_format::{Locale, ToFormattedString};
pub fn to_browseable(s: &str) -> askama::Result<Option<String>> {
Ok({
···
}
})
}
+
+
pub fn human_number(n: &u64) -> askama::Result<String> {
+
Ok(n.to_formatted_string(&Locale::en))
+
}
+27 -6
constellation/src/server/mod.rs
···
use serde::{Deserialize, Serialize};
use serde_with::serde_as;
use std::collections::HashMap;
use tokio::net::{TcpListener, ToSocketAddrs};
use tokio::task::block_in_place;
use tokio_util::sync::CancellationToken;
-
use crate::storage::LinkReader;
use constellation::{CountsByCount, Did, RecordId};
mod acceptable;
···
const DEFAULT_CURSOR_LIMIT: u64 = 16;
const DEFAULT_CURSOR_LIMIT_MAX: u64 = 100;
pub async fn serve<S, A>(store: S, addr: A, stay_alive: CancellationToken) -> anyhow::Result<()>
where
S: LinkReader,
A: ToSocketAddrs,
{
let app = Router::new()
-
.route("/", get(hello))
.route(
"/links/count",
get({
···
#[template(path = "hello.html.j2")]
struct HelloReponse {
help: &'static str,
}
-
async fn hello(accept: ExtractAccept) -> impl IntoResponse {
-
acceptable(accept, HelloReponse {
-
help: "open this URL in a web browser (or request with Accept: text/html) for information about this API."
-
})
}
#[derive(Clone, Deserialize)]
···
use serde::{Deserialize, Serialize};
use serde_with::serde_as;
use std::collections::HashMap;
+
use std::time::{UNIX_EPOCH, Duration};
use tokio::net::{TcpListener, ToSocketAddrs};
use tokio::task::block_in_place;
use tokio_util::sync::CancellationToken;
+
use crate::storage::{LinkReader, StorageStats};
use constellation::{CountsByCount, Did, RecordId};
mod acceptable;
···
const DEFAULT_CURSOR_LIMIT: u64 = 16;
const DEFAULT_CURSOR_LIMIT_MAX: u64 = 100;
+
const INDEX_BEGAN_AT_TS: u64 = 1738083600; // TODO: not this
+
+
pub async fn serve<S, A>(store: S, addr: A, stay_alive: CancellationToken) -> anyhow::Result<()>
where
S: LinkReader,
A: ToSocketAddrs,
{
let app = Router::new()
+
.route(
+
"/",
+
get({
+
let store = store.clone();
+
move |accept| async { block_in_place(|| hello(accept, store)) }
+
}),
+
)
.route(
"/links/count",
get({
···
#[template(path = "hello.html.j2")]
struct HelloReponse {
help: &'static str,
+
days_indexed: u64,
+
stats: StorageStats,
}
+
fn hello(accept: ExtractAccept, store: impl LinkReader) -> Result<impl IntoResponse, http::StatusCode> {
+
let stats = store
+
.get_stats()
+
.map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?;
+
let days_indexed = (UNIX_EPOCH + Duration::from_secs(INDEX_BEGAN_AT_TS))
+
.elapsed()
+
.map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?
+
.as_secs() / 86400;
+
Ok(acceptable(accept, HelloReponse {
+
help: "open this URL in a web browser (or request with Accept: text/html) for information about this API.",
+
days_indexed,
+
stats,
+
}))
}
#[derive(Clone, Deserialize)]
+2 -1
constellation/src/storage/mod.rs
···
use anyhow::Result;
use constellation::{ActionableEvent, CountsByCount, Did, RecordId};
use std::collections::HashMap;
pub mod mem_store;
pub use mem_store::MemStorage;
···
pub next: Option<u64>,
}
-
#[derive(Debug, PartialEq)]
pub struct StorageStats {
/// estimate of how many accounts we've seen create links. the _subjects_ of any links are not represented here.
/// for example: new user A follows users B and C. this count will only increment by one, for A.
···
use anyhow::Result;
use constellation::{ActionableEvent, CountsByCount, Did, RecordId};
use std::collections::HashMap;
+
use serde::{Deserialize, Serialize};
pub mod mem_store;
pub use mem_store::MemStorage;
···
pub next: Option<u64>,
}
+
#[derive(Debug, Deserialize, Serialize, PartialEq)]
pub struct StorageStats {
/// estimate of how many accounts we've seen create links. the _subjects_ of any links are not represented here.
/// for example: new user A follows users B and C. this count will only increment by one, for A.
+6 -1
constellation/templates/base.html.j2
···
padding: 0.5em 0.3em;
max-width: 100%;
}
details {
margin: 2em 0 3em;
}
···
</style>
</head>
<body class="{% block body_classes %}{% endblock %}">
-
<h1><a href="/">This</a> is a <a href="https://github.com/at-ucosm/links/tree/main/constellation">constellation 🌌</a> server from <a href="https://github.com/at-ucosm">microcosm</a> ✨</h1>
{% block content %}{% endblock %}
<footer>
···
padding: 0.5em 0.3em;
max-width: 100%;
}
+
.stat {
+
color: #f90;
+
font-size: 1.618rem;
+
font-weight: bold;
+
}
details {
margin: 2em 0 3em;
}
···
</style>
</head>
<body class="{% block body_classes %}{% endblock %}">
+
<h1><a href="/">This</a> is a <a href="https://github.com/at-ucosm/links/tree/main/constellation">constellation 🌌</a> API server from <a href="https://github.com/at-ucosm">microcosm</a> ✨</h1>
{% block content %}{% endblock %}
<footer>
+1 -1
constellation/templates/dids-count.html.j2
···
{% endif %}
</h2>
-
<p><strong><code>{{ total }}</code></strong> total linking DIDs from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p>
<ul>
<li>See these dids at <code>/links/distinct-dids</code>: <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode() }}">/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}</a></li>
···
{% endif %}
</h2>
+
<p><strong><code>{{ total|human_number }}</code></strong> total linking DIDs from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p>
<ul>
<li>See these dids at <code>/links/distinct-dids</code>: <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode() }}">/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}</a></li>
+1 -1
constellation/templates/dids.html.j2
···
{% endif %}
</h2>
-
<p><strong>{{ total }} dids</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p>
<ul>
<li>See linking records to this target at <code>/links</code>: <a href="/links?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li>
···
{% endif %}
</h2>
+
<p><strong>{{ total|human_number }} dids</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p>
<ul>
<li>See linking records to this target at <code>/links</code>: <a href="/links?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li>
+1 -1
constellation/templates/explore-links.html.j2
···
{%- for (collection, collection_links) in links -%}
<strong>{{ collection }}</strong>
{%- for (path, counts) in collection_links %}
-
{{ path }}: <a href="/links?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.records }} links</a> from <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.distinct_dids }} distinct DIDs</a></li>
{%- endfor %}
{% else -%}
···
{%- for (collection, collection_links) in links -%}
<strong>{{ collection }}</strong>
{%- for (path, counts) in collection_links %}
+
{{ path }}: <a href="/links?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.records|human_number }} links</a> from <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.distinct_dids|human_number }} distinct DIDs</a></li>
{%- endfor %}
{% else -%}
+16 -4
constellation/templates/hello.html.j2
···
{% block body_classes %}home{% endblock %}
{% block content %}
-
<p>Every interaction in Bluesky and atproto at large tends to appear as a <em>link</em> from a new repository record to <em>somewhere</em>: liking a post creates a record with a link to the post, blocking a spammer creates a record with reference to their DID.</p>
-
<p>This service attempts to aggregate all of these links, globally, from all content coming throught the firehose. It provides generic API endpoints to answer questions like <strong>how many likes does a post have</strong> and <strong>who follows a user</strong>.</p>
-
<p>It is very much a <strong>work in progress</strong>. The database has not been backfilled, so any interactions occurring before its last reset will be missing.</p>
-
<h2>Endpoints</h2>
<h3 class="route"><code>GET /links</code></h3>
···
{% block body_classes %}home{% endblock %}
{% block content %}
+
<p>Constellation is a self-hosted JSON API to an atproto-wide index of PDS record back-links, so you can query social interactions in real time. It can answer questions like:</p>
+
<ul>
+
<li><a href="/links/count/distinct-dids?target={{ "at://did:plc:44ybard66vv44zksje25o7dz/app.bsky.feed.post/3lhhz7k2yqk2h"|urlencode }}&collection=app.bsky.feed.like&path=.subject.uri">How many people liked a liked a bluesky post?</a></li>
+
<li><a href="/links/distinct-dids?target=did:plc:oky5czdrnfjpqslsw2a5iclo&collection=app.bsky.graph.follow&path=.subject">Who are all the bluesky followers of an identity?</a></li>
+
<li><a href="/links?target=at://did:plc:nlromb2qyyl6rszaluwhfy6j/fyi.unravel.frontpage.post/3lhd2ivyc422n&collection=fyi.unravel.frontpage.comment&path=.post.uri">What are all the replies to a Frontpage submission?</a></li>
+
<li><a href="/links/all?target=did:plc:vc7f4oafdgxsihk4cry2xpze">What are <em>all</em> the sources of links to an identity?</a></li>
+
<li>and more</li>
+
</ul>
+
<p>
+
This server has indexed <span class="stat">{{ stats.linking_records|human_number }}</span> links between <span class="stat">{{ stats.targetables|human_number }}</span> targets and sources from <span class="stat">{{ stats.dids|human_number }}</span> identities over <span class="stat">{{ days_indexed|human_number }}</span> days.<br/>
+
<small>(indexing new records in real time, backfill still TODO)</small>
+
</p>
+
<p>The API is currently <strong>unstable</strong>. But feel free to use it! If you want to be nice, put your project name and bsky username (or email) in your user-agent header for api requests.</p>
+
+
+
<h2>API Endpoints</h2>
<h3 class="route"><code>GET /links</code></h3>
+1 -1
constellation/templates/links.html.j2
···
{% endif %}
</h2>
-
<p><strong>{{ total }} links</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p>
<ul>
<li>See distinct linking DIDs at <code>/links/distinct-dids</code>: <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links/distinct-dids?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li>
···
{% endif %}
</h2>
+
<p><strong>{{ total|human_number }} links</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p>
<ul>
<li>See distinct linking DIDs at <code>/links/distinct-dids</code>: <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links/distinct-dids?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li>