commit c2d0d07ccd94d0cbbb55c2757c1381be34559f6c · nekomimi.pet/wisp.place-monorepo

+3 -7

apps/hosting-service/src/lib/utils.ts

···

       327
       327
        
         // Expand subfs nodes before caching

     

       328
       328
        
         const expandedRoot = await expandSubfsNodes(record.root, pdsEndpoint);

     

       329
       329
        
       

     

       330
       330
       -
         // Verify all subfs nodes were expanded (defensive check)

     

       330
       330
       +
         // Verify all subfs nodes were expanded

     

       331
       331
        
         const remainingSubfs = extractSubfsUris(expandedRoot);

     

       332
       332
        
         if (remainingSubfs.length > 0) {

     

       333
       333
        
           console.warn(`[Cache] Warning: ${remainingSubfs.length} subfs nodes remain unexpanded after expansion`, remainingSubfs);

     

       334
       334
        
         }

     

       335
       335
        
       

     

       336
       336
       -
         // ===== VALIDATE LIMITS BEFORE DOWNLOADING ANY BLOBS =====

     

       337
       337
       -
       

     

       338
       338
       -
         // 1. Validate file count limit

     

       336
       336
       +
         // Validate file count limit

     

       339
       337
        
         const fileCount = countFilesInDirectory(expandedRoot);

     

       340
       338
        
         if (fileCount > MAX_FILE_COUNT) {

     

       341
       339
        
           throw new Error(`Site exceeds file count limit: ${fileCount} files (max ${MAX_FILE_COUNT})`);

     

       342
       340
        
         }

     

       343
       341
        
         console.log(`[Cache] File count validation passed: ${fileCount} files (limit: ${MAX_FILE_COUNT})`);

     

       344
       342
        
       

     

       345
       345
       -
         // 2. Validate total size from blob metadata in manifest (before downloading)

     

       343
       343
       +
         // Validate total size from blob metadata

     

       346
       344
        
         const totalBlobSize = calculateTotalBlobSize(expandedRoot);

     

       347
       345
        
         if (totalBlobSize > MAX_SITE_SIZE) {

     

       348
       346
        
           throw new Error(`Site exceeds size limit: ${(totalBlobSize / 1024 / 1024).toFixed(2)}MB (max ${(MAX_SITE_SIZE / 1024 / 1024).toFixed(0)}MB)`);

     

       349
       347
        
         }

     

       350
       348
        
         console.log(`[Cache] Size validation passed: ${(totalBlobSize / 1024 / 1024).toFixed(2)}MB (limit: ${(MAX_SITE_SIZE / 1024 / 1024).toFixed(0)}MB)`);

     

       351
       349
        
       

     

       352
       352
       -
         // All validations passed, proceed with caching

     

       353
       350
        
         // Get existing cache metadata to check for incremental updates

     

       354
       351
        
         const existingMetadata = await getCacheMetadata(did, rkey);

     

       355
       352
        
         const existingFileCids = existingMetadata?.fileCids || {};

     
···

       564
       561
        
       

     

       565
       562
        
         console.log(`[Cache] Fetching blob for file: ${filePath}, CID: ${cid}`);

     

       566
       563
        
       

     

       567
       567
       -
         // Allow up to MAX_BLOB_SIZE per file blob, with 5 minute timeout

     

       568
       564
        
         let content = await safeFetchBlob(blobUrl, { maxSize: MAX_BLOB_SIZE, timeout: 300000 });

     

       569
       565
        
       

     

       570
       566
        
         // If content is base64-encoded, decode it back to raw binary (gzipped or not)

+116 -401

claude.md

···

       1
       1
       -
       # Wisp.place - Codebase Overview

     

       1
       1
       +
       The project is wisp.place. It is a static site hoster built on top of the AT Protocol. The overall basis of the project is that users upload site assets to their PDS as blobs, and creates a manifest record listing every blob as well as site name. The hosting service then catches events relating to the site (create, read, upload, delete) and handles them appropriately. 

     

       2
       2
        
       

     

       3
       3
       -
       **Project URL**: https://wisp.place

     

       3
       3
       +
       The lexicons look like this:

     

       4
       4
       +
       ```typescript

     

       5
       5
       +
       //place.wisp.fs

     

       6
       6
       +
       interface Main {

     

       7
       7
       +
         $type: 'place.wisp.fs'

     

       8
       8
       +
         site: string

     

       9
       9
       +
         root: Directory

     

       10
       10
       +
         fileCount?: number

     

       11
       11
       +
         createdAt: string

     

       12
       12
       +
       }

     

       4
       13
        
       

     

       5
       5
       -
       A decentralized static site hosting service built on the AT Protocol (Bluesky). Users can host static websites directly in their AT Protocol accounts, keeping full control and ownership while benefiting from fast CDN distribution.

     

       14
       14
       +
       interface File {

     

       15
       15
       +
         $type?: 'place.wisp.fs#file'

     

       16
       16
       +
         type: 'file'

     

       17
       17
       +
         blob: BlobRef

     

       18
       18
       +
         encoding?: 'gzip'

     

       19
       19
       +
         mimeType?: string

     

       20
       20
       +
         base64?: boolean

     

       21
       21
       +
       }

     

       6
       22
        
       

     

       7
       7
       -
       ---

     

       23
       23
       +
       interface Directory {

     

       24
       24
       +
         $type?: 'place.wisp.fs#directory'

     

       25
       25
       +
         type: 'directory'

     

       26
       26
       +
         entries: Entry[]

     

       27
       27
       +
       }

     

       8
       28
        
       

     

       9
       9
       -
       ## 🏗️ Architecture Overview

     

       29
       29
       +
       interface Entry {

     

       30
       30
       +
         $type?: 'place.wisp.fs#entry'

     

       31
       31
       +
         name: string

     

       32
       32
       +
         node: $Typed<File> | $Typed<Directory> | $Typed<Subfs> | { $type: string }

     

       33
       33
       +
       }

     

       10
       34
        
       

     

       11
       11
       -
       ### Multi-Part System

     

       12
       12
       -
       1. **Main Backend** (`/src`) - OAuth, site management, custom domains

     

       13
       13
       -
       2. **Hosting Service** (`/hosting-service`) - Microservice that serves cached sites

     

       14
       14
       -
       3. **CLI Tool** (`/cli`) - Rust CLI for direct site uploads to PDS

     

       15
       15
       -
       4. **Frontend** (`/public`) - React UI for onboarding, editor, admin

     

       16
       16
       -
       

     

       17
       17
       -
       ### Tech Stack

     

       18
       18
       -
       - **Backend**: Elysia (Bun) + TypeScript + PostgreSQL

     

       19
       19
       -
       - **Frontend**: React 19 + Tailwind CSS 4 + Radix UI

     

       20
       20
       -
       - **CLI**: Rust with Jacquard (AT Protocol library)

     

       21
       21
       -
       - **Database**: PostgreSQL for session/domain/site caching

     

       22
       22
       -
       - **AT Protocol**: OAuth 2.0 + custom lexicons for storage

     

       35
       35
       +
       interface Subfs {

     

       36
       36
       +
         $type?: 'place.wisp.fs#subfs'

     

       37
       37
       +
         type: 'subfs'

     

       38
       38
       +
         subject: string  // AT-URI pointing to a place.wisp.subfs record 

     

       39
       39
       +
         flat?: boolean

     

       40
       40
       +
       }

     

       23
       41
        
       

     

       24
       24
       -
       ---

     

       42
       42
       +
       //place.wisp.subfs

     

       43
       43
       +
       interface Main {

     

       44
       44
       +
         $type: 'place.wisp.subfs'

     

       45
       45
       +
         root: Directory

     

       46
       46
       +
         fileCount?: number

     

       47
       47
       +
         createdAt: string

     

       48
       48
       +
       }

     

       25
       49
        
       

     

       26
       26
       -
       ## 📂 Directory Structure

     

       50
       50
       +
       interface File {

     

       51
       51
       +
         $type?: 'place.wisp.subfs#file'

     

       52
       52
       +
         type: 'file'

     

       53
       53
       +
         blob: BlobRef

     

       54
       54
       +
         encoding?: 'gzip'

     

       55
       55
       +
         mimeType?: string

     

       56
       56
       +
         base64?: boolean

     

       57
       57
       +
       }

     

       27
       58
        
       

     

       28
       28
       -
       ### `/src` - Main Backend Server

     

       29
       29
       -
       **Purpose**: Core server handling OAuth, site management, custom domains, admin features

     

       59
       59
       +
       interface Directory {

     

       60
       60
       +
         $type?: 'place.wisp.subfs#directory'

     

       61
       61
       +
         type: 'directory'

     

       62
       62
       +
         entries: Entry[]

     

       63
       63
       +
       }

     

       30
       64
        
       

     

       31
       31
       -
       **Key Routes**:

     

       32
       32
       -
       - `/api/auth/*` - OAuth signin/callback/logout/status

     

       33
       33
       -
       - `/api/domain/*` - Custom domain management (BYOD)

     

       34
       34
       -
       - `/wisp/*` - Site upload and management

     

       35
       35
       -
       - `/api/user/*` - User info and site listing

     

       36
       36
       -
       - `/api/admin/*` - Admin console (logs, metrics, DNS verification)

     

       65
       65
       +
       interface Entry {

     

       66
       66
       +
         $type?: 'place.wisp.subfs#entry'

     

       67
       67
       +
         name: string

     

       68
       68
       +
         node: $Typed<File> | $Typed<Directory> | $Typed<Subfs> | { $type: string }

     

       69
       69
       +
       }

     

       37
       70
        
       

     

       38
       38
       -
       **Key Files**:

     

       39
       39
       -
       - `index.ts` - Express-like Elysia app setup with middleware (CORS, CSP, security headers)

     

       40
       40
       -
       - `lib/oauth-client.ts` - OAuth client setup with session/state persistence

     

       41
       41
       -
       - `lib/db.ts` - PostgreSQL schema and queries for all tables

     

       42
       42
       -
       - `lib/wisp-auth.ts` - Cookie-based authentication middleware

     

       43
       43
       -
       - `lib/wisp-utils.ts` - File compression (gzip), manifest creation, blob handling

     

       44
       44
       -
       - `lib/sync-sites.ts` - Syncs user's place.wisp.fs records from PDS to database cache

     

       45
       45
       -
       - `lib/dns-verify.ts` - DNS verification for custom domains (TXT + CNAME)

     

       46
       46
       -
       - `lib/dns-verification-worker.ts` - Background worker that checks domain verification every 10 minutes

     

       47
       47
       -
       - `lib/admin-auth.ts` - Simple username/password admin authentication

     

       48
       48
       -
       - `lib/observability.ts` - Logging, error tracking, metrics collection

     

       49
       49
       -
       - `routes/auth.ts` - OAuth flow handlers

     

       50
       50
       -
       - `routes/wisp.ts` - File upload and site creation (/wisp/upload-files)

     

       51
       51
       -
       - `routes/domain.ts` - Domain claiming/verification API

     

       52
       52
       -
       - `routes/user.ts` - User status/info/sites listing

     

       53
       53
       -
       - `routes/site.ts` - Site metadata and file retrieval

     

       54
       54
       -
       - `routes/admin.ts` - Admin dashboard API (logs, system health, manual DNS trigger)

     

       71
       71
       +
       interface Subfs {

     

       72
       72
       +
         $type?: 'place.wisp.subfs#subfs'

     

       73
       73
       +
         type: 'subfs'

     

       74
       74
       +
         subject: string  // AT-URI pointing to another place.wisp.subfs record

     

       75
       75
       +
       }

     

       55
       76
        
       

     

       56
       56
       -
       ### `/lexicons` & `src/lexicons/`

     

       57
       57
       -
       **Purpose**: AT Protocol Lexicon definitions for custom data types

     

       77
       77
       +
       //place.wisp.settings

     

       78
       78
       +
       interface Main {

     

       79
       79
       +
         $type: 'place.wisp.settings'

     

       80
       80
       +
         directoryListing: boolean

     

       81
       81
       +
         spaMode?: string

     

       82
       82
       +
         custom404?: string

     

       83
       83
       +
         indexFiles?: string[]

     

       84
       84
       +
         cleanUrls: boolean

     

       85
       85
       +
         headers?: CustomHeader[]

     

       86
       86
       +
       }

     

       58
       87
        
       

     

       59
       59
       -
       **Key File**: `fs.json` - Defines `place.wisp.fs` record format

     

       60
       60
       -
       - **structure**: Virtual filesystem manifest with tree structure

     

       61
       61
       -
       - **site**: string identifier

     

       62
       62
       -
       - **root**: directory object containing entries

     

       63
       63
       -
       - **file**: blob reference + metadata (encoding, mimeType, base64 flag)

     

       64
       64
       -
       - **directory**: array of entries (recursive)

     

       65
       65
       -
       - **entry**: name + node (file or directory)

     

       66
       66
       -
       

     

       67
       67
       -
       **Important**: Files are gzip-compressed and base64-encoded before upload to bypass PDS content sniffing

     

       68
       68
       -
       

     

       69
       69
       -
       ### `/hosting-service`

     

       70
       70
       -
       **Purpose**: Lightweight microservice that serves cached sites from disk

     

       71
       71
       -
       

     

       72
       72
       -
       **Architecture**:

     

       73
       73
       -
       - Routes by domain lookup in PostgreSQL

     

       74
       74
       -
       - Caches site content locally on first access or firehose event

     

       75
       75
       -
       - Listens to AT Protocol firehose for new site records

     

       76
       76
       -
       - Automatically downloads and caches files from PDS

     

       77
       77
       -
       - SSRF-protected fetch (timeout, size limits, private IP blocking)

     

       78
       78
       -
       

     

       79
       79
       -
       **Routes**:

     

       80
       80
       -
       1. Custom domains (`/*`) → lookup custom_domains table

     

       81
       81
       -
       2. Wisp subdomains (`/*.wisp.place/*`) → lookup domains table

     

       82
       82
       -
       3. DNS hash routing (`/hash.dns.wisp.place/*`) → lookup custom_domains by hash

     

       83
       83
       -
       4. Direct serving (`/s.wisp.place/:identifier/:site/*`) → fetch from PDS if not cached

     

       84
       84
       -
       

     

       85
       85
       -
       **HTML Path Rewriting**: Absolute paths in HTML (`/style.css`) automatically rewritten to relative (`/:identifier/:site/style.css`)

     

       86
       86
       -
       

     

       87
       87
       -
       ### `/cli`

     

       88
       88
       -
       **Purpose**: Rust CLI tool for direct site uploads using app password or OAuth

     

       89
       89
       -
       

     

       90
       90
       -
       **Flow**:

     

       91
       91
       -
       1. Authenticate with handle + app password or OAuth

     

       92
       92
       -
       2. Walk directory tree, compress files

     

       93
       93
       -
       3. Upload blobs to PDS via agent

     

       94
       94
       -
       4. Create place.wisp.fs record with manifest

     

       95
       95
       -
       5. Store site in database cache

     

       96
       96
       -
       

     

       97
       97
       -
       **Auth Methods**:

     

       98
       98
       -
       - `--password` flag for app password auth

     

       99
       99
       -
       - OAuth loopback server for browser-based auth

     

       100
       100
       -
       - Supports both (password preferred if provided)

     

       101
       101
       -
       

     

       102
       102
       -
       ---

     

       103
       103
       -
       

     

       104
       104
       -
       ## 🔐 Key Concepts

     

       105
       105
       -
       

     

       106
       106
       -
       ### Custom Domains (BYOD - Bring Your Own Domain)

     

       107
       107
       -
       **Process**:

     

       108
       108
       -
       1. User claims custom domain via API

     

       109
       109
       -
       2. System generates hash (SHA256(domain + secret))

     

       110
       110
       -
       3. User adds DNS records:

     

       111
       111
       -
          - TXT at `_wisp.example.com` = their DID

     

       112
       112
       -
          - CNAME at `example.com` = `{hash}.dns.wisp.place`

     

       113
       113
       -
       4. Background worker checks verification every 10 minutes

     

       114
       114
       -
       5. Once verified, custom domain routes to their hosted sites

     

       115
       115
       -
       

     

       116
       116
       -
       **Tables**: `custom_domains` (id, domain, did, rkey, verified, last_verified_at)

     

       117
       117
       -
       

     

       118
       118
       -
       ### Wisp Subdomains

     

       119
       119
       -
       **Process**:

     

       120
       120
       -
       1. Handle claimed on first signup (e.g., alice → alice.wisp.place)

     

       121
       121
       -
       2. Stored in `domains` table mapping domain → DID

     

       122
       122
       -
       3. Served by hosting service

     

       123
       123
       -
       

     

       124
       124
       -
       ### Site Storage

     

       125
       125
       -
       **Locations**:

     

       126
       126
       -
       - **Authoritative**: PDS (AT Protocol repo) as `place.wisp.fs` record

     

       127
       127
       -
       - **Cache**: PostgreSQL `sites` table (rkey, did, site_name, created_at)

     

       128
       128
       -
       - **File Cache**: Hosting service caches downloaded files on disk

     

       129
       129
       -
       

     

       130
       130
       -
       **Limits**:

     

       131
       131
       -
       - MAX_SITE_SIZE: 300MB total

     

       132
       132
       -
       - MAX_FILE_SIZE: 100MB per file

     

       133
       133
       -
       - MAX_FILE_COUNT: 2000 files

     

       134
       134
       -
       

     

       135
       135
       -
       ### File Compression Strategy

     

       136
       136
       -
       **Why**: Bypass PDS content sniffing issues (was treating HTML as images)

     

       137
       137
       -
       

     

       138
       138
       -
       **Process**:

     

       139
       139
       -
       1. All files gzip-compressed (level 9)

     

       140
       140
       -
       2. Compressed content base64-encoded

     

       141
       141
       -
       3. Uploaded as `application/octet-stream` MIME type

     

       142
       142
       -
       4. Blob metadata stores original MIME type + encoding flag

     

       143
       143
       -
       5. Hosting service decompresses on serve

     

       144
       144
       -
       

     

       145
       145
       -
       ---

     

       146
       146
       -
       

     

       147
       147
       -
       ## 🔄 Data Flow

     

       148
       148
       -
       

     

       149
       149
       -
       ### User Registration → Site Upload

     

       150
       150
       -
       ```

     

       151
       151
       -
       1. OAuth signin → state/session stored in DB

     

       152
       152
       -
       2. Cookie set with DID

     

       153
       153
       -
       3. Sync sites from PDS to cache DB

     

       154
       154
       -
       4. If no sites/domain → redirect to onboarding

     

       155
       155
       -
       5. User creates site → POST /wisp/upload-files

     

       156
       156
       -
       6. Files compressed, uploaded as blobs

     

       157
       157
       -
       7. place.wisp.fs record created

     

       158
       158
       -
       8. Site cached in DB

     

       159
       159
       -
       9. Hosting service notified via firehose

     

       88
       88
       +
       interface CustomHeader {

     

       89
       89
       +
         $type?: 'place.wisp.settings#customHeader'

     

       90
       90
       +
         name: string

     

       91
       91
       +
         value: string

     

       92
       92
       +
         path?: string  // Optional glob pattern

     

       93
       93
       +
       }

     

       160
       94
        
       ```

     

       161
       95
        
       

     

       162
       162
       -
       ### Custom Domain Setup

     

       163
       163
       -
       ```

     

       164
       164
       -
       1. User claims domain (DB check + allocation)

     

       165
       165
       -
       2. System generates hash

     

       166
       166
       -
       3. User adds DNS records (_wisp.domain TXT + CNAME)

     

       167
       167
       -
       4. Background worker verifies every 10 min

     

       168
       168
       -
       5. Hosting service routes based on verification status

     

       169
       169
       -
       ```

     

       96
       96
       +
       The main differences between place.wisp.fs and place.wisp.subfs:

     

       97
       97
       +
         - place.wisp.fs has a required site field

     

       98
       98
       +
         - place.wisp.fs#subfs has an optional flat field that place.wisp.subfs#subfs doesn't have

     

       170
       99
        
       

     

       171
       171
       -
       ### Site Access

     

       172
       172
       -
       ```

     

       173
       173
       -
       Hosting Service:

     

       174
       174
       -
       1. Request arrives at custom domain or *.wisp.place

     

       175
       175
       -
       2. Domain lookup in PostgreSQL

     

       176
       176
       -
       3. Check cache for site files

     

       177
       177
       -
       4. If not cached:

     

       178
       178
       -
          - Fetch from PDS using DID + rkey

     

       179
       179
       -
          - Decompress files

     

       180
       180
       -
          - Save to disk cache

     

       181
       181
       -
       5. Serve files (with HTML path rewriting)

     

       182
       182
       -
       ```

     

       100
       100
       +
       The project is a monorepo. The package handler it uses for the typescript side is Bun. For the Rust cli, it is cargo.

     

       183
       101
        
       

     

       184
       184
       -
       ---

     

       102
       102
       +
       ### Typescript Bun Workspace Layout

     

       185
       103
        
       

     

       186
       186
       -
       ## 🛠️ Important Implementation Details

     

       104
       104
       +
       Bun workspaces: `packages/@wisp/*`, `apps/main-app`, `apps/hosting-service`

     

       187
       105
        
       

     

       188
       188
       -
       ### OAuth Implementation

     

       189
       189
       -
       - **State & Session Storage**: PostgreSQL (with expiration)

     

       190
       190
       -
       - **Key Rotation**: Periodic rotation + expiration cleanup (hourly)

     

       191
       191
       -
       - **OAuth Flow**: Redirects to PDS, returns to /api/auth/callback

     

       192
       192
       -
       - **Session Timeout**: 30 days

     

       193
       193
       -
       - **State Timeout**: 1 hour

     

       106
       106
       +
       There are two typescript apps

     

       107
       107
       +
       **`apps/main-app`** - Main backend (Bun + Elysia)

     

       194
       108
        
       

     

       195
       195
       -
       ### Security Headers

     

       196
       196
       -
       - X-Frame-Options: DENY

     

       197
       197
       -
       - X-Content-Type-Options: nosniff

     

       198
       198
       -
       - Strict-Transport-Security: max-age=31536000

     

       199
       199
       -
       - Content-Security-Policy (configured for Elysia + React)

     

       200
       200
       -
       - X-XSS-Protection: 1; mode=block

     

       201
       201
       -
       - Referrer-Policy: strict-origin-when-cross-origin

     

       202
       202
       -
       

     

       203
       203
       -
       ### Admin Authentication

     

       204
       204
       -
       - Simple username/password (hashed with bcrypt)

     

       205
       205
       -
       - Session-based cookie auth (24hr expiration)

     

       206
       206
       -
       - Separate `admin_session` cookie

     

       207
       207
       -
       - Initial setup prompted on startup

     

       208
       208
       -
       

     

       209
       209
       -
       ### Observability

     

       210
       210
       -
       - **Logging**: Structured logging with service tags + event types

     

       211
       211
       -
       - **Error Tracking**: Captures error context (message, stack, etc.)

     

       212
       212
       -
       - **Metrics**: Request counts, latencies, error rates

     

       213
       213
       -
       - **Log Levels**: debug, info, warn, error

     

       214
       214
       -
       - **Collection**: Centralized log collector with in-memory buffer

     

       215
       215
       -
       

     

       216
       216
       -
       ---

     

       217
       217
       -
       

     

       218
       218
       -
       ## 📝 Database Schema

     

       219
       219
       -
       

     

       220
       220
       -
       ### oauth_states

     

       221
       221
       -
       - key (primary key)

     

       222
       222
       -
       - data (JSON)

     

       223
       223
       -
       - created_at, expires_at (timestamps)

     

       224
       224
       -
       

     

       225
       225
       -
       ### oauth_sessions

     

       226
       226
       -
       - sub (primary key - subject/DID)

     

       227
       227
       -
       - data (JSON with OAuth session)

     

       228
       228
       -
       - updated_at, expires_at

     

       229
       229
       -
       

     

       230
       230
       -
       ### oauth_keys

     

       231
       231
       -
       - kid (primary key - key ID)

     

       232
       232
       -
       - jwk (JSON Web Key)

     

       233
       233
       -
       - created_at

     

       234
       234
       -
       

     

       235
       235
       -
       ### domains

     

       236
       236
       -
       - domain (primary key - e.g., alice.wisp.place)

     

       237
       237
       -
       - did (unique - user's DID)

     

       238
       238
       -
       - rkey (optional - record key)

     

       239
       239
       -
       - created_at

     

       240
       240
       -
       

     

       241
       241
       -
       ### custom_domains

     

       242
       242
       -
       - id (primary key - UUID)

     

       243
       243
       -
       - domain (unique - e.g., example.com)

     

       244
       244
       -
       - did (user's DID)

     

       245
       245
       -
       - rkey (optional)

     

       246
       246
       -
       - verified (boolean)

     

       247
       247
       -
       - last_verified_at (timestamp)

     

       248
       248
       -
       - created_at

     

       249
       249
       -
       

     

       250
       250
       -
       ### sites

     

       251
       251
       -
       - id, did, rkey, site_name

     

       252
       252
       -
       - created_at, updated_at

     

       253
       253
       -
       - Indexes on (did), (did, rkey), (rkey)

     

       254
       254
       -
       

     

       255
       255
       -
       ### admin_users

     

       256
       256
       -
       - username (primary key)

     

       257
       257
       -
       - password_hash (bcrypt)

     

       258
       258
       -
       - created_at

     

       259
       259
       -
       

     

       260
       260
       -
       ---

     

       261
       261
       -
       

     

       262
       262
       -
       ## 🚀 Key Workflows

     

       263
       263
       -
       

     

       264
       264
       -
       ### Sign In Flow

     

       265
       265
       -
       1. POST /api/auth/signin with handle

     

       266
       266
       -
       2. System generates state token

     

       267
       267
       -
       3. Redirects to PDS OAuth endpoint

     

       268
       268
       -
       4. PDS redirects back to /api/auth/callback?code=X&state=Y

     

       269
       269
       -
       5. Validate state (CSRF protection)

     

       270
       270
       -
       6. Exchange code for session

     

       271
       271
       -
       7. Store session in DB, set DID cookie

     

       272
       272
       -
       8. Sync sites from PDS

     

       273
       273
       -
       9. Redirect to /editor or /onboarding

     

       274
       274
       -
       

     

       275
       275
       -
       ### File Upload Flow

     

       276
       276
       -
       1. POST /wisp/upload-files with siteName + files

     

       277
       277
       -
       2. Validate site name (rkey format rules)

     

       278
       278
       -
       3. For each file:

     

       279
       279
       -
          - Check size limits

     

       280
       280
       -
          - Read as ArrayBuffer

     

       281
       281
       -
          - Gzip compress

     

       282
       282
       -
          - Base64 encode

     

       283
       283
       -
       4. Upload all blobs in parallel via agent.com.atproto.repo.uploadBlob()

     

       284
       284
       -
       5. Create manifest with all blob refs

     

       285
       285
       -
       6. putRecord() for place.wisp.fs with manifest

     

       286
       286
       -
       7. Upsert to sites table

     

       287
       287
       -
       8. Return URI + CID

     

       288
       288
       -
       

     

       289
       289
       -
       ### Domain Verification Flow

     

       290
       290
       -
       1. POST /api/custom-domains/claim

     

       291
       291
       -
       2. Generate hash = SHA256(domain + secret)

     

       292
       292
       -
       3. Store in custom_domains with verified=false

     

       293
       293
       -
       4. Return hash for user to configure DNS

     

       294
       294
       -
       5. Background worker periodically:

     

       295
       295
       -
          - Query custom_domains where verified=false

     

       296
       296
       -
          - Verify TXT record at _wisp.domain

     

       297
       297
       -
          - Verify CNAME points to hash.dns.wisp.place

     

       298
       298
       -
          - Update verified flag + last_verified_at

     

       299
       299
       -
       6. Hosting service routes when verified=true

     

       300
       300
       -
       

     

       301
       301
       -
       ---

     

       302
       302
       -
       

     

       303
       303
       -
       ## 🎨 Frontend Structure

     

       304
       304
       -
       

     

       305
       305
       -
       ### `/public`

     

       306
       306
       -
       - **index.tsx** - Landing page with sign-in form

     

       307
       307
       -
       - **editor/editor.tsx** - Site editor/management UI

     

       308
       308
       -
       - **admin/admin.tsx** - Admin dashboard

     

       309
       309
       -
       - **components/ui/** - Reusable components (Button, Card, Dialog, etc.)

     

       310
       310
       -
       - **styles/global.css** - Tailwind + custom styles

     

       311
       311
       -
       

     

       312
       312
       -
       ### Page Flow

     

       313
       313
       -
       1. `/` - Landing page (sign in / get started)

     

       314
       314
       -
       2. `/editor` - Main app (requires auth)

     

       315
       315
       -
       3. `/admin` - Admin console (requires admin auth)

     

       316
       316
       -
       4. `/onboarding` - First-time user setup

     

       317
       317
       -
       

     

       318
       318
       -
       ---

     

       109
       109
       +
       - OAuth authentication and session management

     

       110
       110
       +
       - Site CRUD operations via PDS

     

       111
       111
       +
       - Custom domain management

     

       112
       112
       +
       - Admin database view in /admin

     

       113
       113
       +
       - React frontend in public/

     

       319
       114
        
       

     

       320
       320
       -
       ## 🔍 Notable Implementation Patterns

     

       115
       115
       +
       **`apps/hosting-service`** - CDN static file server (Node + Hono)

     

       321
       116
        
       

     

       322
       322
       -
       ### File Handling

     

       323
       323
       -
       - Files stored as base64-encoded gzip in PDS blobs

     

       324
       324
       -
       - Metadata preserves original MIME type

     

       325
       325
       -
       - Hosting service decompresses on serve

     

       326
       326
       -
       - Workaround for PDS image pipeline issues with HTML

     

       117
       117
       +
       - Watches AT Protocol firehose for `place.wisp.fs` record changes

     

       118
       118
       +
       - Downloads and caches site files to disk

     

       119
       119
       +
       - Serves sites at `https://sites.wisp.place/{did}/{site-name}` and custom domains

     

       120
       120
       +
       - Handles redirects (`_redirects` file support) and routing logic

     

       121
       121
       +
       - Backfill mode for syncing existing sites

     

       327
       122
        
       

     

       328
       328
       -
       ### Error Handling

     

       329
       329
       -
       - Comprehensive logging with context

     

       330
       330
       -
       - Graceful degradation (e.g., site sync failure doesn't break auth)

     

       331
       331
       -
       - Structured error responses with details

     

       123
       123
       +
       ### Shared Packages (`packages/@wisp/*`)

     

       332
       124
        
       

     

       333
       333
       -
       ### Performance

     

       334
       334
       -
       - Site sync: Batch fetch up to 100 records per request

     

       335
       335
       -
       - Blob upload: Parallel promises for all files

     

       336
       336
       -
       - DNS verification: Batched background worker (10 min intervals)

     

       337
       337
       -
       - Caching: Two-tier (DB + disk in hosting service)

     

       125
       125
       +
       - **`lexicons`** - AT Protocol lexicons (`place.wisp.fs`, `place.wisp.subfs`, `place.wisp.settings`) with

     

       126
       126
       +
         generated TypeScript types

     

       127
       127
       +
       - **`fs-utils`** - Filesystem tree building, manifest creation, subfs splitting logic

     

       128
       128
       +
       - **`atproto-utils`** - AT Protocol helpers (blob upload, record operations, CID handling)

     

       129
       129
       +
       - **`database`** - PostgreSQL schema and queries

     

       130
       130
       +
       - **`constants`** - Shared constants (limits, file patterns, default settings)

     

       131
       131
       +
       - **`observability`** - OpenTelemetry instrumentation

     

       132
       132
       +
       - **`safe-fetch`** - Wrapped fetch with timeout/retry logic

     

       338
       133
        
       

     

       339
       339
       -
       ### Validation

     

       340
       340
       -
       - Lexicon validation on manifest creation

     

       341
       341
       -
       - Record type checking

     

       342
       342
       -
       - Domain format validation

     

       343
       343
       -
       - Site name format validation (AT Protocol rkey rules)

     

       344
       344
       -
       - File size limits enforced before upload

     

       345
       345
       -
       

     

       346
       346
       -
       ---

     

       347
       347
       -
       

     

       348
       348
       -
       ## 🐛 Known Quirks & Workarounds

     

       349
       349
       -
       

     

       350
       350
       -
       1. **PDS Content Sniffing**: Files must be uploaded as `application/octet-stream` (even HTML) and base64-encoded to prevent PDS from misinterpreting content

     

       351
       351
       -
       

     

       352
       352
       -
       2. **Max URL Query Size**: DNS verification worker queries in batch; may need pagination for users with many custom domains

     

       353
       353
       -
       

     

       354
       354
       -
       3. **File Count Limits**: Max 500 entries per directory (Lexicon constraint); large sites split across multiple directories

     

       355
       355
       -
       

     

       356
       356
       -
       4. **Blob Size Limits**: Individual blobs limited to 100MB by Lexicon; large files handled differently if needed

     

       357
       357
       -
       

     

       358
       358
       -
       5. **HTML Path Rewriting**: Only in hosting service for `/s.wisp.place/:identifier/:site/*` routes; custom domains handled differently

     

       134
       134
       +
       ### CLI

     

       359
       135
        
       

     

       360
       360
       -
       ---

     

       136
       136
       +
       **`cli/`** - Rust CLI using Jacquard (AT Protocol library)

     

       137
       137
       +
       - Direct PDS uploads without interacting with main-app

     

       138
       138
       +
       - Can also do the same firehose watching, caching, and serving hosting-service does, just without domain management

     

       361
       139
        
       

     

       362
       362
       -
       ## 📋 Environment Variables

     

       140
       140
       +
       ### Other Directories

     

       363
       141
        
       

     

       364
       364
       -
       - `DOMAIN` - Base domain with protocol (default: `https://wisp.place`)

     

       365
       365
       -
       - `CLIENT_NAME` - OAuth client name (default: `PDS-View`)

     

       366
       366
       -
       - `DATABASE_URL` - PostgreSQL connection (default: `postgres://postgres:postgres@localhost:5432/wisp`)

     

       367
       367
       -
       - `NODE_ENV` - production/development

     

       368
       368
       -
       - `HOSTING_PORT` - Hosting service port (default: 3001)

     

       369
       369
       -
       - `BASE_DOMAIN` - Domain for URLs (default: wisp.place)

     

       370
       370
       -
       

     

       371
       371
       -
       ---

     

       372
       372
       -
       

     

       373
       373
       -
       ## 🧑‍💻 Development Notes

     

       374
       374
       -
       

     

       375
       375
       -
       ### Adding New Features

     

       376
       376
       -
       1. **New routes**: Add to `/src/routes/*.ts`, import in index.ts

     

       377
       377
       -
       2. **DB changes**: Add migration in db.ts

     

       378
       378
       -
       3. **New lexicons**: Update `/lexicons/*.json`, regenerate types

     

       379
       379
       -
       4. **Admin features**: Add to /api/admin endpoints

     

       380
       380
       -
       

     

       381
       381
       -
       ### Testing

     

       382
       382
       -
       - Run with `bun test`

     

       383
       383
       -
       - CSRF tests in lib/csrf.test.ts

     

       384
       384
       -
       - Utility tests in lib/wisp-utils.test.ts

     

       385
       385
       -
       

     

       386
       386
       -
       ### Debugging

     

       387
       387
       -
       - Check logs via `/api/admin/logs` (requires admin auth)

     

       388
       388
       -
       - DNS verification manual trigger: POST /api/admin/verify-dns

     

       389
       389
       -
       - Health check: GET /api/health (includes DNS verifier status)

     

       390
       390
       -
       

     

       391
       391
       -
       ---

     

       392
       392
       -
       

     

       393
       393
       -
       ## 🚀 Deployment Considerations

     

       394
       394
       -
       

     

       395
       395
       -
       1. **Secrets**: Admin password, OAuth keys, database credentials

     

       396
       396
       -
       2. **HTTPS**: Required (HSTS header enforces it)

     

       397
       397
       -
       3. **CDN**: Custom domains require DNS configuration

     

       398
       398
       -
       4. **Scaling**: 

     

       399
       399
       -
          - Main server: Horizontal scaling with session DB

     

       400
       400
       -
          - Hosting service: Independent scaling, disk cache per instance

     

       401
       401
       -
       5. **Backups**: PostgreSQL database critical; firehose provides recovery

     

       402
       402
       -
       

     

       403
       403
       -
       ---

     

       404
       404
       -
       

     

       405
       405
       -
       ## 📚 Related Technologies

     

       406
       406
       -
       

     

       407
       407
       -
       - **AT Protocol**: Decentralized identity, OAuth 2.0

     

       408
       408
       -
       - **Jacquard**: Rust library for AT Protocol interactions

     

       409
       409
       -
       - **Elysia**: Bun web framework (similar to Express/Hono)

     

       410
       410
       -
       - **Lexicon**: AT Protocol's schema definition language

     

       411
       411
       -
       - **Firehose**: Real-time event stream of repo changes

     

       412
       412
       -
       - **PDS**: Personal Data Server (where users' data stored)

     

       413
       413
       -
       

     

       414
       414
       -
       ---

     

       415
       415
       -
       

     

       416
       416
       -
       ## 🎯 Project Goals

     

       417
       417
       -
       

     

       418
       418
       -
       ✅ Decentralized site hosting (data owned by users)

     

       419
       419
       -
       ✅ Custom domain support with DNS verification

     

       420
       420
       -
       ✅ Fast CDN distribution via hosting service

     

       421
       421
       -
       ✅ Developer tools (CLI + API)

     

       422
       422
       -
       ✅ Admin dashboard for monitoring

     

       423
       423
       -
       ✅ Zero user data retention (sites in PDS, sessions in DB only)

     

       424
       424
       -
       

     

       425
       425
       -
       ---

     

       426
       426
       -
       

     

       427
       427
       -
       **Last Updated**: November 2025

     

       428
       428
       -
       **Status**: Active development

     

       142
       142
       +
       - **`docs/`** - Astro documentation site

     

       143
       143
       +
       - **`binaries/`** - Compiled CLI binaries for distribution