Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
nekomimi.pet
1import { Elysia } from 'elysia'
2import { requireAuth, type AuthenticatedContext } from '../lib/wisp-auth'
3import { NodeOAuthClient } from '@atproto/oauth-client-node'
4import { Agent } from '@atproto/api'
5import {
6 type UploadedFile,
7 type FileUploadResult,
8 processUploadedFiles,
9 createManifest,
10 updateFileBlobs,
11 shouldCompressFile,
12 compressFile
13} from '../lib/wisp-utils'
14import { upsertSite } from '../lib/db'
15import { logger } from '../lib/logger'
16import { validateRecord } from '../lexicon/types/place/wisp/fs'
17
18function isValidSiteName(siteName: string): boolean {
19 if (!siteName || typeof siteName !== 'string') return false;
20
21 // Length check (AT Protocol rkey limit)
22 if (siteName.length < 1 || siteName.length > 512) return false;
23
24 // Check for path traversal
25 if (siteName === '.' || siteName === '..') return false;
26 if (siteName.includes('/') || siteName.includes('\\')) return false;
27 if (siteName.includes('\0')) return false;
28
29 // AT Protocol rkey format: alphanumeric, dots, dashes, underscores, tildes, colons
30 // Based on NSID format rules
31 const validRkeyPattern = /^[a-zA-Z0-9._~:-]+$/;
32 if (!validRkeyPattern.test(siteName)) return false;
33
34 return true;
35}
36
37export const wispRoutes = (client: NodeOAuthClient) =>
38 new Elysia({ prefix: '/wisp' })
39 .derive(async ({ cookie }) => {
40 const auth = await requireAuth(client, cookie)
41 return { auth }
42 })
43 .post(
44 '/upload-files',
45 async ({ body, auth }) => {
46 const { siteName, files } = body as {
47 siteName: string;
48 files: File | File[]
49 };
50
51 try {
52 if (!siteName) {
53 throw new Error('Site name is required')
54 }
55
56 if (!isValidSiteName(siteName)) {
57 throw new Error('Invalid site name: must be 1-512 characters and contain only alphanumeric, dots, dashes, underscores, tildes, and colons')
58 }
59
60 // Check if files were provided
61 const hasFiles = files && (Array.isArray(files) ? files.length > 0 : !!files);
62
63 if (!hasFiles) {
64 // Create agent with OAuth session
65 const agent = new Agent((url, init) => auth.session.fetchHandler(url, init))
66
67 // Create empty manifest
68 const emptyManifest = {
69 $type: 'place.wisp.fs',
70 site: siteName,
71 root: {
72 type: 'directory',
73 entries: []
74 },
75 fileCount: 0,
76 createdAt: new Date().toISOString()
77 };
78
79 // Validate the manifest
80 const validationResult = validateRecord(emptyManifest);
81 if (!validationResult.success) {
82 throw new Error(`Invalid manifest: ${validationResult.error?.message || 'Validation failed'}`);
83 }
84
85 // Use site name as rkey
86 const rkey = siteName;
87
88 const record = await agent.com.atproto.repo.putRecord({
89 repo: auth.did,
90 collection: 'place.wisp.fs',
91 rkey: rkey,
92 record: emptyManifest
93 });
94
95 await upsertSite(auth.did, rkey, siteName);
96
97 return {
98 success: true,
99 uri: record.data.uri,
100 cid: record.data.cid,
101 fileCount: 0,
102 siteName
103 };
104 }
105
106 // Create agent with OAuth session
107 const agent = new Agent((url, init) => auth.session.fetchHandler(url, init))
108
109 // Convert File objects to UploadedFile format
110 // Elysia gives us File objects directly, handle both single file and array
111 const fileArray = Array.isArray(files) ? files : [files];
112 const uploadedFiles: UploadedFile[] = [];
113 const skippedFiles: Array<{ name: string; reason: string }> = [];
114
115 // Define allowed file extensions for static site hosting
116 const allowedExtensions = new Set([
117 // HTML
118 '.html', '.htm',
119 // CSS
120 '.css',
121 // JavaScript
122 '.js', '.mjs', '.jsx', '.ts', '.tsx',
123 // Images
124 '.jpg', '.jpeg', '.png', '.gif', '.svg', '.webp', '.ico', '.avif',
125 // Fonts
126 '.woff', '.woff2', '.ttf', '.otf', '.eot',
127 // Documents
128 '.pdf', '.txt',
129 // JSON (for config files, but not .map files)
130 '.json',
131 // Audio/Video
132 '.mp3', '.mp4', '.webm', '.ogg', '.wav',
133 // Other web assets
134 '.xml', '.rss', '.atom'
135 ]);
136
137 // Files to explicitly exclude
138 const excludedFiles = new Set([
139 '.map', '.DS_Store', 'Thumbs.db'
140 ]);
141
142 for (let i = 0; i < fileArray.length; i++) {
143 const file = fileArray[i];
144 const fileExtension = '.' + file.name.split('.').pop()?.toLowerCase();
145
146 // Skip excluded files
147 if (excludedFiles.has(fileExtension)) {
148 skippedFiles.push({ name: file.name, reason: 'excluded file type' });
149 continue;
150 }
151
152 // Skip files that aren't in allowed extensions
153 if (!allowedExtensions.has(fileExtension)) {
154 skippedFiles.push({ name: file.name, reason: 'unsupported file type' });
155 continue;
156 }
157
158 // Skip files that are too large (limit to 100MB per file)
159 const maxSize = 100 * 1024 * 1024; // 100MB
160 if (file.size > maxSize) {
161 skippedFiles.push({
162 name: file.name,
163 reason: `file too large (${(file.size / 1024 / 1024).toFixed(2)}MB, max 100MB)`
164 });
165 continue;
166 }
167
168 const arrayBuffer = await file.arrayBuffer();
169 const originalContent = Buffer.from(arrayBuffer);
170 const originalMimeType = file.type || 'application/octet-stream';
171
172 // Determine if we should compress this file
173 const shouldCompress = shouldCompressFile(originalMimeType);
174
175 if (shouldCompress) {
176 const compressedContent = compressFile(originalContent);
177 // Base64 encode the gzipped content to prevent PDS content sniffing
178 const base64Content = Buffer.from(compressedContent.toString('base64'), 'utf-8');
179 const compressionRatio = (compressedContent.length / originalContent.length * 100).toFixed(1);
180 logger.info(`[Wisp] Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`);
181
182 uploadedFiles.push({
183 name: file.name,
184 content: base64Content,
185 mimeType: originalMimeType,
186 size: base64Content.length,
187 compressed: true,
188 originalMimeType
189 });
190 } else {
191 uploadedFiles.push({
192 name: file.name,
193 content: originalContent,
194 mimeType: originalMimeType,
195 size: file.size,
196 compressed: false
197 });
198 }
199 }
200
201 // Check total size limit (300MB)
202 const totalSize = uploadedFiles.reduce((sum, file) => sum + file.size, 0);
203 const maxTotalSize = 300 * 1024 * 1024; // 300MB
204
205 if (totalSize > maxTotalSize) {
206 throw new Error(`Total upload size ${(totalSize / 1024 / 1024).toFixed(2)}MB exceeds 300MB limit`);
207 }
208
209 if (uploadedFiles.length === 0) {
210
211 // Create empty manifest
212 const emptyManifest = {
213 $type: 'place.wisp.fs',
214 site: siteName,
215 root: {
216 type: 'directory',
217 entries: []
218 },
219 fileCount: 0,
220 createdAt: new Date().toISOString()
221 };
222
223 // Validate the manifest
224 const validationResult = validateRecord(emptyManifest);
225 if (!validationResult.success) {
226 throw new Error(`Invalid manifest: ${validationResult.error?.message || 'Validation failed'}`);
227 }
228
229 // Use site name as rkey
230 const rkey = siteName;
231
232 const record = await agent.com.atproto.repo.putRecord({
233 repo: auth.did,
234 collection: 'place.wisp.fs',
235 rkey: rkey,
236 record: emptyManifest
237 });
238
239 await upsertSite(auth.did, rkey, siteName);
240
241 return {
242 success: true,
243 uri: record.data.uri,
244 cid: record.data.cid,
245 fileCount: 0,
246 siteName,
247 skippedFiles,
248 message: 'Site created but no valid web files were found to upload'
249 };
250 }
251
252 // Process files into directory structure
253 const { directory, fileCount } = processUploadedFiles(uploadedFiles);
254
255 // Upload files as blobs in parallel
256 // For compressed files, we upload as octet-stream and store the original MIME type in metadata
257 // For text/html files, we also use octet-stream as a workaround for PDS image pipeline issues
258 const uploadPromises = uploadedFiles.map(async (file, i) => {
259 try {
260 // If compressed, always upload as octet-stream
261 // Otherwise, workaround: PDS incorrectly processes text/html through image pipeline
262 const uploadMimeType = file.compressed || file.mimeType.startsWith('text/html')
263 ? 'application/octet-stream'
264 : file.mimeType;
265
266 const compressionInfo = file.compressed ? ' (gzipped)' : '';
267 logger.info(`[Wisp] Uploading file: ${file.name} (original: ${file.mimeType}, sending as: ${uploadMimeType}, ${file.size} bytes${compressionInfo})`);
268
269 const uploadResult = await agent.com.atproto.repo.uploadBlob(
270 file.content,
271 {
272 encoding: uploadMimeType
273 }
274 );
275
276 const returnedBlobRef = uploadResult.data.blob;
277
278 // Use the blob ref exactly as returned from PDS
279 return {
280 result: {
281 hash: returnedBlobRef.ref.toString(),
282 blobRef: returnedBlobRef,
283 ...(file.compressed && {
284 encoding: 'gzip' as const,
285 mimeType: file.originalMimeType || file.mimeType,
286 base64: true
287 })
288 },
289 filePath: file.name,
290 sentMimeType: file.mimeType,
291 returnedMimeType: returnedBlobRef.mimeType
292 };
293 } catch (uploadError) {
294 logger.error('[Wisp] Upload failed for file', uploadError);
295 throw uploadError;
296 }
297 });
298
299 // Wait for all uploads to complete
300 const uploadedBlobs = await Promise.all(uploadPromises);
301
302 // Extract results and file paths in correct order
303 const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result);
304 const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath);
305
306 // Update directory with file blobs
307 const updatedDirectory = updateFileBlobs(directory, uploadResults, filePaths);
308
309 // Create manifest
310 const manifest = createManifest(siteName, updatedDirectory, fileCount);
311
312 // Use site name as rkey
313 const rkey = siteName;
314
315 let record;
316 try {
317 record = await agent.com.atproto.repo.putRecord({
318 repo: auth.did,
319 collection: 'place.wisp.fs',
320 rkey: rkey,
321 record: manifest
322 });
323 } catch (putRecordError: any) {
324 logger.error('[Wisp] Failed to create record on PDS');
325 logger.error('[Wisp] Record creation error', putRecordError);
326
327 throw putRecordError;
328 }
329
330 // Store site in database cache
331 await upsertSite(auth.did, rkey, siteName);
332
333 const result = {
334 success: true,
335 uri: record.data.uri,
336 cid: record.data.cid,
337 fileCount,
338 siteName,
339 skippedFiles,
340 uploadedCount: uploadedFiles.length
341 };
342
343 return result;
344 } catch (error) {
345 logger.error('[Wisp] Upload error', error);
346 logger.errorWithContext('[Wisp] Upload error details', {
347 message: error instanceof Error ? error.message : 'Unknown error',
348 name: error instanceof Error ? error.name : undefined
349 }, error);
350 throw new Error(`Failed to upload files: ${error instanceof Error ? error.message : 'Unknown error'}`);
351 }
352 }
353 )