commit 487385753b342647c9abe059274e96979fd0f59d · nekomimi.pet/microcosm-rs

+32 -32

Cargo.lock

···

       494
       494
        
       checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"

     

       495
       495
        
       

     

       496
       496
        
       [[package]]

     

       497
       497
       +
       name = "constellation"

     

       498
       498
       +
       version = "0.1.0"

     

       499
       499
       +
       dependencies = [

     

       500
       500
       +
        "anyhow",

     

       501
       501
       +
        "askama",

     

       502
       502
       +
        "axum",

     

       503
       503
       +
        "axum-extra",

     

       504
       504
       +
        "axum-metrics",

     

       505
       505
       +
        "bincode",

     

       506
       506
       +
        "clap",

     

       507
       507
       +
        "ctrlc",

     

       508
       508
       +
        "flume",

     

       509
       509
       +
        "fs4",

     

       510
       510
       +
        "headers-accept",

     

       511
       511
       +
        "links",

     

       512
       512
       +
        "mediatype",

     

       513
       513
       +
        "metrics",

     

       514
       514
       +
        "metrics-exporter-prometheus",

     

       515
       515
       +
        "metrics-process",

     

       516
       516
       +
        "rocksdb",

     

       517
       517
       +
        "serde",

     

       518
       518
       +
        "serde_with",

     

       519
       519
       +
        "tempfile",

     

       520
       520
       +
        "tinyjson",

     

       521
       521
       +
        "tokio",

     

       522
       522
       +
        "tokio-util",

     

       523
       523
       +
        "tower-http",

     

       524
       524
       +
        "tungstenite",

     

       525
       525
       +
        "zstd",

     

       526
       526
       +
       ]

     

       527
       527
       +
       

     

       528
       528
       +
       [[package]]

     

       497
       529
        
       name = "core-foundation"

     

       498
       530
        
       version = "0.9.4"

     

       499
       531
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1130
       1162
        
        "cc",

     

       1131
       1163
        
        "pkg-config",

     

       1132
       1164
        
        "vcpkg",

     

       1133
       1133
       -
       ]

     

       1134
       1134
       -
       

     

       1135
       1135
       -
       [[package]]

     

       1136
       1136
       -
       name = "link_aggregator"

     

       1137
       1137
       -
       version = "0.1.0"

     

       1138
       1138
       -
       dependencies = [

     

       1139
       1139
       -
        "anyhow",

     

       1140
       1140
       -
        "askama",

     

       1141
       1141
       -
        "axum",

     

       1142
       1142
       -
        "axum-extra",

     

       1143
       1143
       -
        "axum-metrics",

     

       1144
       1144
       -
        "bincode",

     

       1145
       1145
       -
        "clap",

     

       1146
       1146
       -
        "ctrlc",

     

       1147
       1147
       -
        "flume",

     

       1148
       1148
       -
        "fs4",

     

       1149
       1149
       -
        "headers-accept",

     

       1150
       1150
       -
        "links",

     

       1151
       1151
       -
        "mediatype",

     

       1152
       1152
       -
        "metrics",

     

       1153
       1153
       -
        "metrics-exporter-prometheus",

     

       1154
       1154
       -
        "metrics-process",

     

       1155
       1155
       -
        "rocksdb",

     

       1156
       1156
       -
        "serde",

     

       1157
       1157
       -
        "serde_with",

     

       1158
       1158
       -
        "tempfile",

     

       1159
       1159
       -
        "tinyjson",

     

       1160
       1160
       -
        "tokio",

     

       1161
       1161
       -
        "tokio-util",

     

       1162
       1162
       -
        "tower-http",

     

       1163
       1163
       -
        "tungstenite",

     

       1164
       1164
       -
        "zstd",

     

       1165
       1165
        
       ]

     

       1166
       1166
        
       

     

       1167
       1167
        
       [[package]]

+1 -1

Cargo.toml

···

       2
       2
        
       resolver = "2"

     

       3
       3
        
       members = [

     

       4
       4
        
           "links",

     

       5
       5
       -
           "link_aggregator",

     

       5
       5
       +
           "constellation",

     

       6
       6
        
       ]

+1 -1

link_aggregator/Cargo.toml constellation/Cargo.toml

···

       1
       1
        
       [package]

     

       2
       2
       -
       name = "link_aggregator"

     

       2
       2
       +
       name = "constellation"

     

       3
       3
        
       version = "0.1.0"

     

       4
       4
        
       edition = "2021"

     

       5
       5

+1 -1

link_aggregator/readme.md constellation/readme.md

···

       1
       1
       -
       # link_aggregator

     

       1
       1
       +
       # constellation 🌌

     

       2
       2
        
       

     

       3
       3
        
       ## endpoints

     

       4
       4

link_aggregator/src/consumer/jetstream.rs constellation/src/consumer/jetstream.rs

link_aggregator/src/consumer/jsonl_file.rs constellation/src/consumer/jsonl_file.rs

+1 -1

link_aggregator/src/consumer/mod.rs constellation/src/consumer/mod.rs

···

       3
       3
        
       

     

       4
       4
        
       use crate::storage::LinkStorage;

     

       5
       5
        
       use anyhow::Result;

     

       6
       6
       +
       use constellation::{ActionableEvent, RecordId};

     

       6
       7
        
       use jetstream::consume_jetstream;

     

       7
       8
        
       use jsonl_file::consume_jsonl_file;

     

       8
       8
       -
       use link_aggregator::{ActionableEvent, RecordId};

     

       9
       9
        
       use links::collect_links;

     

       10
       10
        
       use metrics::{counter, describe_counter, describe_histogram, histogram, Unit};

     

       11
       11
        
       use std::path::PathBuf;

link_aggregator/src/lib.rs constellation/src/lib.rs

link_aggregator/src/main.rs constellation/src/main.rs

link_aggregator/src/server/acceptable.rs constellation/src/server/acceptable.rs

link_aggregator/src/server/filters.rs constellation/src/server/filters.rs

+1 -1

link_aggregator/src/server/mod.rs constellation/src/server/mod.rs

···

       9
       9
        
       use tokio_util::sync::CancellationToken;

     

       10
       10
        
       

     

       11
       11
        
       use crate::storage::LinkReader;

     

       12
       12
       -
       use link_aggregator::RecordId;

     

       12
       12
       +
       use constellation::RecordId;

     

       13
       13
        
       

     

       14
       14
        
       mod acceptable;

     

       15
       15
        
       mod filters;

+1 -1

link_aggregator/src/storage/mem_store.rs constellation/src/storage/mem_store.rs

···

       1
       1
        
       use super::{LinkReader, LinkStorage, PagedAppendingCollection, StorageStats};

     

       2
       2
        
       use anyhow::Result;

     

       3
       3
       -
       use link_aggregator::{ActionableEvent, Did, RecordId};

     

       3
       3
       +
       use constellation::{ActionableEvent, Did, RecordId};

     

       4
       4
        
       use links::CollectedLink;

     

       5
       5
        
       use std::collections::HashMap;

     

       6
       6
        
       use std::sync::{Arc, Mutex};

+1 -1

link_aggregator/src/storage/mod.rs constellation/src/storage/mod.rs

···

       1
       1
        
       use anyhow::Result;

     

       2
       2
       -
       use link_aggregator::{ActionableEvent, RecordId};

     

       2
       2
       +
       use constellation::{ActionableEvent, RecordId};

     

       3
       3
        
       use std::collections::HashMap;

     

       4
       4
        
       

     

       5
       5
        
       pub mod mem_store;

+1 -1

link_aggregator/src/storage/rocks_store.rs constellation/src/storage/rocks_store.rs

···

       1
       1
        
       use super::{ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, StorageStats};

     

       2
       2
        
       use anyhow::{bail, Result};

     

       3
       3
        
       use bincode::Options as BincodeOptions;

     

       4
       4
       -
       use link_aggregator::{Did, RecordId};

     

       4
       4
       +
       use constellation::{Did, RecordId};

     

       5
       5
        
       use links::CollectedLink;

     

       6
       6
        
       use rocksdb::{

     

       7
       7
        
           AsColumnFamilyRef, ColumnFamilyDescriptor, DBWithThreadMode, IteratorMode, MergeOperands,

+1 -1

link_aggregator/templates/base.html.j2 constellation/templates/base.html.j2

···

       39
       39
        
           </style>

     

       40
       40
        
         </head>

     

       41
       41
        
         <body class="{% block body_classes %}{% endblock %}">

     

       42
       42
       -
           <h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/link_aggregator">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1>

     

       42
       42
       +
           <h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/constellation">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1>

     

       43
       43
        
           {% block content %}{% endblock %}

     

       44
       44
        
       

     

       45
       45
        
           <footer>

link_aggregator/templates/hello.html.j2 constellation/templates/hello.html.j2

link_aggregator/templates/links-all-count.html.j2 constellation/templates/links-all-count.html.j2

link_aggregator/templates/links-count.html.j2 constellation/templates/links-count.html.j2

link_aggregator/templates/links.html.j2 constellation/templates/links.html.j2

link_aggregator/templates/try-it-macros.html.j2 constellation/templates/try-it-macros.html.j2

link_aggregator/zstd/dictionary constellation/zstd/dictionary

+43 -6

readme.md

···

       1
       1
       -
       µcosm links

     

       2
       2
       -
       ===========

     

       1
       1
       +
       microcosm: links

     

       2
       2
       +
       ================

     

       3
       3
       +
       

     

       4
       4
       +
       this repo contains libraries and apps for working with cross-record references in at-protocol.

     

       5
       5
       +
       

     

       6
       6
       +
       

     

       7
       7
       +
       App: [Constellation](./tree/main/microcosm/)

     

       8
       8
       +
       --------------------------------------------

     

       9
       9
       +
       

     

       10
       10
       +
       A global atproto backlink index ✨

     

       11
       11
       +
       

     

       12
       12
       +
       - Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD

     

       13
       13
       +
       - Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs

     

       14
       14
       +
       - Simple JSON API

     

       15
       15
       +
       

     

       16
       16
       +
       All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more.

     

       17
       17
       +
       

     

       18
       18
       +
       - **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet.

     

       19
       19
       +
       - source: [./microcosm/](./tree/main/microcosm/)

     

       20
       20
       +
       - public instance: [links.bsky.bad-example.com](https://links.bsky.bad-example.com/)

     

       21
       21
       +
       

     

       22
       22
       +
       _note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._

     

       23
       23
       +
       

     

       24
       24
       +
       

     

       25
       25
       +
       App: Spacedust

     

       26
       26
       +
       --------------

     

       3
       27
        
       

     

       4
       4
       -
       optimistically extract links from arbitrary atproto records, optionally resolving canonical representations and possibly validating StrongRefs.

     

       28
       28
       +
       A notification subscription service 💫

     

       5
       29
        
       

     

       30
       30
       +
       using the same "link source" concept as [constellation](./tree/main/microcosm), offer webhook notifications for new references created to records

     

       6
       31
        
       

     

       7
       7
       -
       status

     

       8
       8
       -
       ------

     

       32
       32
       +
       - **status**: in design

     

       9
       33
        
       

     

       10
       10
       -
       not at all ready (yet)

     

       34
       34
       +
       

     

       35
       35
       +
       Library: [links](./tree/main/links/)

     

       36
       36
       +
       ------------------------------------

     

       37
       37
       +
       

     

       38
       38
       +
       A rust crate (not published on crates.io yet) for optimistically parsing links out of arbitrary atproto PDS records, and potentially canonicalizing them

     

       39
       39
       +
       

     

       40
       40
       +
       - **status**: unstable, might remain an internal lib for constellation (and spacedust, soon)

     

       41
       41
       +
       

     

       42
       42
       +
       

     

       11
       43
        
       

     

       12
       44
        
       ---

     

       45
       45
       +
       

     

       46
       46
       +
       

     

       47
       47
       +
       old notes follow, ignore

     

       48
       48
       +
       ------------------------

     

       49
       49
       +
       

     

       13
       50
        
       

     

       14
       51
        
       as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document.

     

       15
       52