A URL shortener service that uses ATProto to allow self hosting and ensuring the user owns their data

Compare changes

Choose any two refs to compare.

+4
.dockerignore
···
···
+
at-shorter
+
database.db
+
.env
+
example.env
+19
Dockerfile
···
···
+
FROM golang:alpine AS builder
+
+
WORKDIR /app
+
+
COPY . .
+
RUN go mod download
+
+
COPY . .
+
+
RUN CGO_ENABLED=0 go build -o at-shorter ./cmd/atshorter/main.go
+
+
FROM alpine:latest
+
+
RUN apk --no-cache add ca-certificates
+
+
WORKDIR /app/
+
COPY --from=builder /app/at-shorter .
+
+
ENTRYPOINT ["./at-shorter"]
+5 -3
README.md
···
### How it works
-
When a user logs in with their ATProto account, they will be able to create a new short URL. This action creates a record in their PDS containing the URL to redirect to and in the future could contain other peices of metadata. The key bit here is that the record in their PDS is created with an RKey, which in this application is a TID ((timestamp identifier)[https://atproto.com/specs/tid]) which is unique and that key becomes the "short URL" ID. Combine that with the host that created the ID and you get a short URL. The users DID, key and URL are then stored in a local database to the service.
As long as the service is running at that host, the short URL will be available to be redirected. When the short URL is hit, the service will take the key part of the URL, look it up in the database to find the real URL and then redirect to that URL.
···
### Self hosting
-
Another neat part of this architecture is that if a user wants to self host this, they can and the links they create will only work for their service. For example.
-
If I create a short URL `https://my-short-service/a/abcd` on my service and then someone creates `https://a-different-short-service/a/1234` those links will only work with the domain that created them. I wouldn't be able to take the key `abcd` and use it with the `https://a-different-short-service` domain because that isn't the domain that created the link. This means that even though every service is consuming all of the short URL ATProto records, there will never be a case where someone could hijack an ID to redirect to somewhere else without the user that created and owning the link from doing so (because updating a record in a PDS requires the users auth).
···
### How it works
+
When a user logs in with their ATProto account, they will be able to create a new short URL. This action creates a record in their PDS containing the URL to redirect to and in the future could contain other peices of metadata. The key bit here is that the record in their PDS is created with an RKey, which in this application is a TID ([timestamp identifier](https://atproto.com/specs/tid)) which is unique and that key becomes the "short URL" ID. Combine that with the host that created the ID and you get a short URL. The users DID, key and URL are then stored in a local database to the service.
As long as the service is running at that host, the short URL will be available to be redirected. When the short URL is hit, the service will take the key part of the URL, look it up in the database to find the real URL and then redirect to that URL.
···
### Self hosting
+
Another neat part of this architecture is that it allows the user to self host the service. Traditional short URL services turn out to be a lot of hassle. Read this really interesting [thread on Bluesky](https://bsky.app/profile/gbl08ma.com/post/3m2pft7a3dc23) about it from the creator of [tny.im](tny.im). Turns out hosting one for others to use isn't a great move these days, so by having one that you can host yourself and only you can use, could be a really handy tool. You create the short URLs and pass them out to people to use.
+
The way the service works, is that whe you start it up, you configure it to be for your DID (ATProto identifier) so that only you can log into the service to create short URLs. Then when the Jetstream consumer runs, it only listens to events for your DID which means you only get your short URL records in the local database.
+
+
Remember!!! All of the links you create are stored in your PDS which is public. So anyone that wants to see what links you've created and go to them, will be able to do that.
+5
auth_handlers.go
···
return
}
next(w, r)
}
}
···
return
}
+
if did.String() != s.usersDID {
+
http.Error(w, "not authorized", http.StatusUnauthorized)
+
return
+
}
+
next(w, r)
}
}
+24 -7
cmd/atshorter/main.go
···
defaultServerAddr = "wss://jetstream.atproto.tools/subscribe"
httpClientTimeoutDuration = time.Second * 5
transportIdleConnTimeoutDuration = time.Second * 90
)
func main() {
-
err := godotenv.Load(".env")
if err != nil {
if !os.IsNotExist(err) {
log.Fatal("Error loading .env file")
···
return
}
dbFilename := path.Join(dbMountPath, "database.db")
db, err := database.New(dbFilename)
if err != nil {
···
defer db.Close()
var config oauth.ClientConfig
-
bind := ":8080"
scopes := []string{
"atproto",
"repo:com.atshorter.shorturl?action=create",
···
"repo:com.atshorter.shorturl?action=delete",
}
if host == "" {
config = oauth.NewLocalhostConfig(
-
fmt.Sprintf("http://127.0.0.1%s/oauth-callback", bind),
scopes,
)
slog.Info("configuring localhost OAuth client", "CallbackURL", config.CallbackURL)
···
},
}
-
server, err := atshorter.NewServer(host, 8080, db, oauthClient, httpClient)
if err != nil {
slog.Error("create new server", "error", err)
return
···
_ = server.Stop(context.Background())
}()
-
go consumeLoop(ctx, db)
server.Run()
}
-
func consumeLoop(ctx context.Context, db *database.DB) {
jsServerAddr := os.Getenv("JS_SERVER_ADDR")
if jsServerAddr == "" {
jsServerAddr = defaultServerAddr
}
-
consumer := atshorter.NewConsumer(jsServerAddr, slog.Default(), db)
err := retry.Do(func() error {
err := consumer.Consume(ctx)
···
defaultServerAddr = "wss://jetstream.atproto.tools/subscribe"
httpClientTimeoutDuration = time.Second * 5
transportIdleConnTimeoutDuration = time.Second * 90
+
defaultPort = "8080"
)
func main() {
+
envLocation := os.Getenv("ENV_LOCATION")
+
if envLocation == "" {
+
envLocation = ".env"
+
}
+
+
err := godotenv.Load(envLocation)
if err != nil {
if !os.IsNotExist(err) {
log.Fatal("Error loading .env file")
···
return
}
+
usersDID := os.Getenv("DID")
+
if usersDID == "" {
+
slog.Error("DID env not set")
+
return
+
}
+
dbFilename := path.Join(dbMountPath, "database.db")
db, err := database.New(dbFilename)
if err != nil {
···
defer db.Close()
var config oauth.ClientConfig
+
port := os.Getenv("PORT")
+
if port == "" {
+
port = defaultPort
+
}
+
scopes := []string{
"atproto",
"repo:com.atshorter.shorturl?action=create",
···
"repo:com.atshorter.shorturl?action=delete",
}
if host == "" {
+
host = fmt.Sprintf("http://127.0.0.1:%s", port)
config = oauth.NewLocalhostConfig(
+
fmt.Sprintf("%s/oauth-callback", host),
scopes,
)
slog.Info("configuring localhost OAuth client", "CallbackURL", config.CallbackURL)
···
},
}
+
server, err := atshorter.NewServer(host, port, db, oauthClient, httpClient, usersDID)
if err != nil {
slog.Error("create new server", "error", err)
return
···
_ = server.Stop(context.Background())
}()
+
go consumeLoop(ctx, db, usersDID)
server.Run()
}
+
func consumeLoop(ctx context.Context, db *database.DB, did string) {
jsServerAddr := os.Getenv("JS_SERVER_ADDR")
if jsServerAddr == "" {
jsServerAddr = defaultServerAddr
}
+
consumer := atshorter.NewConsumer(jsServerAddr, slog.Default(), db, did)
err := retry.Do(func() error {
err := consumer.Consume(ctx)
+4 -6
consumer.go
···
logger *slog.Logger
}
-
func NewConsumer(jsAddr string, logger *slog.Logger, store HandlerStore) *consumer {
cfg := client.DefaultClientConfig()
if jsAddr != "" {
cfg.WebsocketURL = jsAddr
···
cfg.WantedCollections = []string{
"com.atshorter.shorturl",
}
-
cfg.WantedDids = []string{} // TODO: possibly when self hosting, limit this to just a select few?
return &consumer{
cfg: cfg,
···
}
type HandlerStore interface {
-
CreateURL(id, url, did string, createdAt int64) error
DeleteURL(id, did string) error
}
···
return nil
}
-
// TODO: if origin isn't this instance, ignore
-
-
err := h.store.CreateURL(event.Commit.RKey, record.URL, event.Did, record.CreatedAt.UnixMilli())
if err != nil {
// TODO: proper error handling in case this fails, we want to try again
slog.Error("failed to store short URL", "error", err)
···
logger *slog.Logger
}
+
func NewConsumer(jsAddr string, logger *slog.Logger, store HandlerStore, did string) *consumer {
cfg := client.DefaultClientConfig()
if jsAddr != "" {
cfg.WebsocketURL = jsAddr
···
cfg.WantedCollections = []string{
"com.atshorter.shorturl",
}
+
cfg.WantedDids = []string{did}
return &consumer{
cfg: cfg,
···
}
type HandlerStore interface {
+
CreateURL(id, url, did, originHost string, createdAt int64) error
DeleteURL(id, did string) error
}
···
return nil
}
+
err := h.store.CreateURL(event.Commit.RKey, record.URL, event.Did, record.Origin, record.CreatedAt.UnixMilli())
if err != nil {
// TODO: proper error handling in case this fails, we want to try again
slog.Error("failed to store short URL", "error", err)
+8 -7
database/urls.go
···
"id" TEXT NOT NULL PRIMARY KEY,
"url" TEXT NOT NULL,
"did" TEXT NOT NULL,
"createdAt" integer
);`
···
return nil
}
-
func (d *DB) CreateURL(id, url, did string, createdAt int64) error {
-
sql := `INSERT INTO urls (id, url, did, createdAt) VALUES (?, ?, ?, ?) ON CONFLICT(id) DO NOTHING;`
-
_, err := d.db.Exec(sql, id, url, did, createdAt)
if err != nil {
// TODO: catch already exists
return fmt.Errorf("exec insert url: %w", err)
···
}
func (d *DB) GetURLs(did string) ([]atshorter.ShortURL, error) {
-
sql := "SELECT id, url, did FROM urls WHERE did = ?;"
rows, err := d.db.Query(sql, did)
if err != nil {
return nil, fmt.Errorf("run query to get URLS': %w", err)
···
var results []atshorter.ShortURL
for rows.Next() {
var shortURL atshorter.ShortURL
-
if err := rows.Scan(&shortURL.ID, &shortURL.URL, &shortURL.Did); err != nil {
return nil, fmt.Errorf("scan row: %w", err)
}
···
}
func (d *DB) GetURLByID(id string) (atshorter.ShortURL, error) {
-
sql := "SELECT id, url, did FROM urls WHERE id = ?;"
rows, err := d.db.Query(sql, id)
if err != nil {
return atshorter.ShortURL{}, fmt.Errorf("run query to get URL by id': %w", err)
···
var result atshorter.ShortURL
for rows.Next() {
-
if err := rows.Scan(&result.ID, &result.URL, &result.Did); err != nil {
return atshorter.ShortURL{}, fmt.Errorf("scan row: %w", err)
}
return result, nil
···
"id" TEXT NOT NULL PRIMARY KEY,
"url" TEXT NOT NULL,
"did" TEXT NOT NULL,
+
"originHost" TEXT NOT NULL,
"createdAt" integer
);`
···
return nil
}
+
func (d *DB) CreateURL(id, url, did, originHost string, createdAt int64) error {
+
sql := `INSERT INTO urls (id, url, did, originHost, createdAt) VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO NOTHING;`
+
_, err := d.db.Exec(sql, id, url, did, originHost, createdAt)
if err != nil {
// TODO: catch already exists
return fmt.Errorf("exec insert url: %w", err)
···
}
func (d *DB) GetURLs(did string) ([]atshorter.ShortURL, error) {
+
sql := "SELECT id, url, did, originHost FROM urls WHERE did = ?;"
rows, err := d.db.Query(sql, did)
if err != nil {
return nil, fmt.Errorf("run query to get URLS': %w", err)
···
var results []atshorter.ShortURL
for rows.Next() {
var shortURL atshorter.ShortURL
+
if err := rows.Scan(&shortURL.ID, &shortURL.URL, &shortURL.Did, &shortURL.OriginHost); err != nil {
return nil, fmt.Errorf("scan row: %w", err)
}
···
}
func (d *DB) GetURLByID(id string) (atshorter.ShortURL, error) {
+
sql := "SELECT id, url, did, originHost FROM urls WHERE id = ?;"
rows, err := d.db.Query(sql, id)
if err != nil {
return atshorter.ShortURL{}, fmt.Errorf("run query to get URL by id': %w", err)
···
var result atshorter.ShortURL
for rows.Next() {
+
if err := rows.Scan(&result.ID, &result.URL, &result.Did, &result.OriginHost); err != nil {
return atshorter.ShortURL{}, fmt.Errorf("scan row: %w", err)
}
return result, nil
+11
docker-compose.yaml
···
···
+
services:
+
knot:
+
container_name: at-shorter
+
image: willdot/at-shorter:latest
+
environment:
+
ENV_LOCATION: "/data/at-shorter.env"
+
volumes:
+
- ./data:/app/data
+
ports:
+
- "3005:3005"
+
restart: always
+2
example.env
···
HOST="the host of the service such as https://my-url-shortner.com"
DATABASE_PATH="./"
JS_SERVER_ADDR="set to a different Jetstream instance"
···
HOST="the host of the service such as https://my-url-shortner.com"
DATABASE_PATH="./"
JS_SERVER_ADDR="set to a different Jetstream instance"
+
PORT="3002"
+
DID="Enter your account DID here"
+2 -2
html/home.html
···
<body>
<div id="header">
<h1>AT-Shorter</h1>
-
<p>Create you own short URLs</p>
</div>
<div class="container">
<div class="card">
···
{{range .UsersShortURLs}}
<tr>
<td>
-
<a href="http://127.0.0.1:8080/a/{{.ID}}">{{.ID}}</a>
</td>
<td>
<a href="{{ .URL }}">{{.URL}}</a>
···
<body>
<div id="header">
<h1>AT-Shorter</h1>
+
<p>Create your own short URLs</p>
</div>
<div class="container">
<div class="card">
···
{{range .UsersShortURLs}}
<tr>
<td>
+
<a href="{{.OriginHost}}/a/{{.ID}}">{{.ID}}</a>
</td>
<td>
<a href="{{ .URL }}">{{.URL}}</a>
+14 -7
server.go
···
import (
"context"
_ "embed"
"encoding/json"
"fmt"
···
var ErrorNotFound = fmt.Errorf("not found")
type Store interface {
-
CreateURL(id, url, did string, createdAt int64) error
GetURLs(did string) ([]ShortURL, error)
GetURLByID(id string) (ShortURL, error)
DeleteURL(id, did string) error
···
type Server struct {
host string
httpserver *http.Server
sessionStore *sessions.CookieStore
templates []*template.Template
···
mu sync.Mutex
}
-
func NewServer(host string, port int, store Store, oauthClient *oauth.ClientApp, httpClient *http.Client) (*Server, error) {
sessionStore := sessions.NewCookieStore([]byte(os.Getenv("SESSION_KEY")))
-
homeTemplate, err := template.ParseFiles("./html/home.html")
if err != nil {
-
return nil, fmt.Errorf("parsing home template: %w", err)
}
-
loginTemplate, err := template.ParseFiles("./html/login.html")
if err != nil {
return nil, fmt.Errorf("parsing login template: %w", err)
}
···
srv := &Server{
host: host,
oauthClient: oauthClient,
sessionStore: sessionStore,
templates: templates,
···
mux.HandleFunc("GET /oauth-client-metadata.json", srv.serveClientMetadata)
mux.HandleFunc("GET /oauth-callback", srv.handleOauthCallback)
-
addr := fmt.Sprintf("0.0.0.0:%d", port)
srv.httpserver = &http.Server{
Addr: addr,
Handler: mux,
···
metadata.ClientName = &clientName
metadata.ClientURI = &s.host
if s.oauthClient.Config.IsConfidential() {
-
jwksURI := fmt.Sprintf("%s/jwks.json", r.Host)
metadata.JWKSURI = &jwksURI
}
···
import (
"context"
+
"embed"
_ "embed"
"encoding/json"
"fmt"
···
var ErrorNotFound = fmt.Errorf("not found")
type Store interface {
+
CreateURL(id, url, did, originHost string, createdAt int64) error
GetURLs(did string) ([]ShortURL, error)
GetURLByID(id string) (ShortURL, error)
DeleteURL(id, did string) error
···
type Server struct {
host string
+
usersDID string
httpserver *http.Server
sessionStore *sessions.CookieStore
templates []*template.Template
···
mu sync.Mutex
}
+
//go:embed html
+
var htmlFolder embed.FS
+
+
func NewServer(host string, port string, store Store, oauthClient *oauth.ClientApp, httpClient *http.Client, usersDID string) (*Server, error) {
sessionStore := sessions.NewCookieStore([]byte(os.Getenv("SESSION_KEY")))
+
homeTemplate, err := template.ParseFS(htmlFolder, "html/home.html")
if err != nil {
+
return nil, fmt.Errorf("error parsing templates: %w", err)
}
+
+
loginTemplate, err := template.ParseFS(htmlFolder, "html/login.html")
if err != nil {
return nil, fmt.Errorf("parsing login template: %w", err)
}
···
srv := &Server{
host: host,
+
usersDID: usersDID,
oauthClient: oauthClient,
sessionStore: sessionStore,
templates: templates,
···
mux.HandleFunc("GET /oauth-client-metadata.json", srv.serveClientMetadata)
mux.HandleFunc("GET /oauth-callback", srv.handleOauthCallback)
+
addr := fmt.Sprintf("0.0.0.0:%s", port)
srv.httpserver = &http.Server{
Addr: addr,
Handler: mux,
···
metadata.ClientName = &clientName
metadata.ClientURI = &s.host
if s.oauthClient.Config.IsConfidential() {
+
jwksURI := fmt.Sprintf("%s/jwks.json", s.host)
metadata.JWKSURI = &jwksURI
}
+12 -22
short_url_handler.go
···
}
type ShortURL struct {
-
ID string
-
URL string
-
Did string
}
func (s *Server) HandleRedirect(w http.ResponseWriter, r *http.Request) {
···
return
}
-
record, err := s.getUrlRecord(r.Context(), shortURL.Did, shortURL.ID)
-
if err != nil {
-
slog.Error("getting URL record from PDS", "error", err, "did", shortURL.Did, "id", shortURL.ID)
-
http.Error(w, "error verifying short URl link", http.StatusInternalServerError)
-
return
-
}
-
-
// TODO: use the host from the record to check that it was created using this host - otherwise it's a short URL
-
// created by another hosted instance of this service
-
-
slog.Info("got record from PDS", "record", record)
-
http.Redirect(w, r, shortURL.URL, http.StatusSeeOther)
return
}
···
tmpl.Execute(w, data)
return
}
-
data.UsersShortURLs = usersURLs
tmpl.Execute(w, data)
···
createdAt := time.Now()
api := session.APIClient()
bodyReq := map[string]any{
"repo": api.AccountDID.String(),
"collection": "com.atshorter.shorturl",
"rkey": rkey,
-
"record": map[string]any{
-
"url": url,
-
"createdAt": createdAt,
-
"orgin": "atshorter.com", // TODO: this needs to be pulled from the host env
-
},
}
err = api.Post(r.Context(), "com.atproto.repo.createRecord", bodyReq, nil)
if err != nil {
···
return
}
-
err = s.store.CreateURL(rkey, url, did.String(), createdAt.UnixMilli())
if err != nil {
slog.Error("store in local database", "error", err)
}
···
}
type ShortURL struct {
+
ID string
+
URL string
+
Did string
+
OriginHost string
}
func (s *Server) HandleRedirect(w http.ResponseWriter, r *http.Request) {
···
return
}
http.Redirect(w, r, shortURL.URL, http.StatusSeeOther)
return
}
···
tmpl.Execute(w, data)
return
}
data.UsersShortURLs = usersURLs
tmpl.Execute(w, data)
···
createdAt := time.Now()
api := session.APIClient()
+
record := ShortURLRecord{
+
URL: url,
+
CreatedAt: createdAt,
+
Origin: s.host,
+
}
+
bodyReq := map[string]any{
"repo": api.AccountDID.String(),
"collection": "com.atshorter.shorturl",
"rkey": rkey,
+
"record": record,
}
err = api.Post(r.Context(), "com.atproto.repo.createRecord", bodyReq, nil)
if err != nil {
···
return
}
+
err = s.store.CreateURL(rkey, url, did.String(), s.host, createdAt.UnixMilli())
if err != nil {
slog.Error("store in local database", "error", err)
}