Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
nekomimi.pet
1import { Hono } from 'hono';
2import { getWispDomain, getCustomDomain, getCustomDomainByHash } from './lib/db';
3import { resolveDid, getPdsForDid, fetchSiteRecord, downloadAndCacheSite, getCachedFilePath, isCached, sanitizePath, shouldCompressMimeType } from './lib/utils';
4import { rewriteHtmlPaths, isHtmlContent } from './lib/html-rewriter';
5import { existsSync } from 'fs';
6import { readFile, access } from 'fs/promises';
7import { lookup } from 'mime-types';
8import { logger, observabilityMiddleware, observabilityErrorHandler, logCollector, errorTracker, metricsCollector } from './lib/observability';
9import { fileCache, metadataCache, rewrittenHtmlCache, getCacheKey, type FileMetadata } from './lib/cache';
10import { loadRedirectRules, matchRedirectRule, parseCookies, parseQueryString, type RedirectRule } from './lib/redirects';
11
12const BASE_HOST = process.env.BASE_HOST || 'wisp.place';
13
14/**
15 * Validate site name (rkey) to prevent injection attacks
16 * Must match AT Protocol rkey format
17 */
18function isValidRkey(rkey: string): boolean {
19 if (!rkey || typeof rkey !== 'string') return false;
20 if (rkey.length < 1 || rkey.length > 512) return false;
21 if (rkey === '.' || rkey === '..') return false;
22 if (rkey.includes('/') || rkey.includes('\\') || rkey.includes('\0')) return false;
23 const validRkeyPattern = /^[a-zA-Z0-9._~:-]+$/;
24 return validRkeyPattern.test(rkey);
25}
26
27/**
28 * Async file existence check
29 */
30async function fileExists(path: string): Promise<boolean> {
31 try {
32 await access(path);
33 return true;
34 } catch {
35 return false;
36 }
37}
38
39// Cache for redirect rules (per site)
40const redirectRulesCache = new Map<string, RedirectRule[]>();
41
42/**
43 * Clear redirect rules cache for a specific site
44 * Should be called when a site is updated/recached
45 */
46export function clearRedirectRulesCache(did: string, rkey: string) {
47 const cacheKey = `${did}:${rkey}`;
48 redirectRulesCache.delete(cacheKey);
49}
50
51// Helper to serve files from cache
52async function serveFromCache(
53 did: string,
54 rkey: string,
55 filePath: string,
56 fullUrl?: string,
57 headers?: Record<string, string>
58) {
59 // Check for redirect rules first
60 const redirectCacheKey = `${did}:${rkey}`;
61 let redirectRules = redirectRulesCache.get(redirectCacheKey);
62
63 if (redirectRules === undefined) {
64 // Load rules for the first time
65 redirectRules = await loadRedirectRules(did, rkey);
66 redirectRulesCache.set(redirectCacheKey, redirectRules);
67 }
68
69 // Apply redirect rules if any exist
70 if (redirectRules.length > 0) {
71 const requestPath = '/' + (filePath || '');
72 const queryParams = fullUrl ? parseQueryString(fullUrl) : {};
73 const cookies = parseCookies(headers?.['cookie']);
74
75 const redirectMatch = matchRedirectRule(requestPath, redirectRules, {
76 queryParams,
77 headers,
78 cookies,
79 });
80
81 if (redirectMatch) {
82 const { targetPath, status } = redirectMatch;
83
84 // Handle different status codes
85 if (status === 200) {
86 // Rewrite: serve different content but keep URL the same
87 // Remove leading slash for internal path resolution
88 const rewritePath = targetPath.startsWith('/') ? targetPath.slice(1) : targetPath;
89 return serveFileInternal(did, rkey, rewritePath);
90 } else if (status === 301 || status === 302) {
91 // External redirect: change the URL
92 return new Response(null, {
93 status,
94 headers: {
95 'Location': targetPath,
96 'Cache-Control': status === 301 ? 'public, max-age=31536000' : 'public, max-age=0',
97 },
98 });
99 } else if (status === 404) {
100 // Custom 404 page
101 const custom404Path = targetPath.startsWith('/') ? targetPath.slice(1) : targetPath;
102 const response = await serveFileInternal(did, rkey, custom404Path);
103 // Override status to 404
104 return new Response(response.body, {
105 status: 404,
106 headers: response.headers,
107 });
108 }
109 }
110 }
111
112 // No redirect matched, serve normally
113 return serveFileInternal(did, rkey, filePath);
114}
115
116// Internal function to serve a file (used by both normal serving and rewrites)
117async function serveFileInternal(did: string, rkey: string, filePath: string) {
118 // Default to index.html if path is empty or ends with /
119 let requestPath = filePath || 'index.html';
120 if (requestPath.endsWith('/')) {
121 requestPath += 'index.html';
122 }
123
124 const cacheKey = getCacheKey(did, rkey, requestPath);
125 const cachedFile = getCachedFilePath(did, rkey, requestPath);
126
127 // Check in-memory cache first
128 let content = fileCache.get(cacheKey);
129 let meta = metadataCache.get(cacheKey);
130
131 if (!content && await fileExists(cachedFile)) {
132 // Read from disk and cache
133 content = await readFile(cachedFile);
134 fileCache.set(cacheKey, content, content.length);
135
136 const metaFile = `${cachedFile}.meta`;
137 if (await fileExists(metaFile)) {
138 const metaJson = await readFile(metaFile, 'utf-8');
139 meta = JSON.parse(metaJson);
140 metadataCache.set(cacheKey, meta!, JSON.stringify(meta).length);
141 }
142 }
143
144 if (content) {
145 // Build headers with caching
146 const headers: Record<string, string> = {};
147
148 if (meta && meta.encoding === 'gzip' && meta.mimeType) {
149 const shouldServeCompressed = shouldCompressMimeType(meta.mimeType);
150
151 if (!shouldServeCompressed) {
152 const { gunzipSync } = await import('zlib');
153 const decompressed = gunzipSync(content);
154 headers['Content-Type'] = meta.mimeType;
155 headers['Cache-Control'] = 'public, max-age=31536000, immutable';
156 return new Response(decompressed, { headers });
157 }
158
159 headers['Content-Type'] = meta.mimeType;
160 headers['Content-Encoding'] = 'gzip';
161 headers['Cache-Control'] = meta.mimeType.startsWith('text/html')
162 ? 'public, max-age=300'
163 : 'public, max-age=31536000, immutable';
164 return new Response(content, { headers });
165 }
166
167 // Non-compressed files
168 const mimeType = lookup(cachedFile) || 'application/octet-stream';
169 headers['Content-Type'] = mimeType;
170 headers['Cache-Control'] = mimeType.startsWith('text/html')
171 ? 'public, max-age=300'
172 : 'public, max-age=31536000, immutable';
173 return new Response(content, { headers });
174 }
175
176 // Try index.html for directory-like paths
177 if (!requestPath.includes('.')) {
178 const indexPath = `${requestPath}/index.html`;
179 const indexCacheKey = getCacheKey(did, rkey, indexPath);
180 const indexFile = getCachedFilePath(did, rkey, indexPath);
181
182 let indexContent = fileCache.get(indexCacheKey);
183 let indexMeta = metadataCache.get(indexCacheKey);
184
185 if (!indexContent && await fileExists(indexFile)) {
186 indexContent = await readFile(indexFile);
187 fileCache.set(indexCacheKey, indexContent, indexContent.length);
188
189 const indexMetaFile = `${indexFile}.meta`;
190 if (await fileExists(indexMetaFile)) {
191 const metaJson = await readFile(indexMetaFile, 'utf-8');
192 indexMeta = JSON.parse(metaJson);
193 metadataCache.set(indexCacheKey, indexMeta!, JSON.stringify(indexMeta).length);
194 }
195 }
196
197 if (indexContent) {
198 const headers: Record<string, string> = {
199 'Content-Type': 'text/html; charset=utf-8',
200 'Cache-Control': 'public, max-age=300',
201 };
202
203 if (indexMeta && indexMeta.encoding === 'gzip') {
204 headers['Content-Encoding'] = 'gzip';
205 }
206
207 return new Response(indexContent, { headers });
208 }
209 }
210
211 return new Response('Not Found', { status: 404 });
212}
213
214// Helper to serve files from cache with HTML path rewriting for sites.wisp.place routes
215async function serveFromCacheWithRewrite(
216 did: string,
217 rkey: string,
218 filePath: string,
219 basePath: string,
220 fullUrl?: string,
221 headers?: Record<string, string>
222) {
223 // Check for redirect rules first
224 const redirectCacheKey = `${did}:${rkey}`;
225 let redirectRules = redirectRulesCache.get(redirectCacheKey);
226
227 if (redirectRules === undefined) {
228 // Load rules for the first time
229 redirectRules = await loadRedirectRules(did, rkey);
230 redirectRulesCache.set(redirectCacheKey, redirectRules);
231 }
232
233 // Apply redirect rules if any exist
234 if (redirectRules.length > 0) {
235 const requestPath = '/' + (filePath || '');
236 const queryParams = fullUrl ? parseQueryString(fullUrl) : {};
237 const cookies = parseCookies(headers?.['cookie']);
238
239 const redirectMatch = matchRedirectRule(requestPath, redirectRules, {
240 queryParams,
241 headers,
242 cookies,
243 });
244
245 if (redirectMatch) {
246 const { targetPath, status } = redirectMatch;
247
248 // Handle different status codes
249 if (status === 200) {
250 // Rewrite: serve different content but keep URL the same
251 const rewritePath = targetPath.startsWith('/') ? targetPath.slice(1) : targetPath;
252 return serveFileInternalWithRewrite(did, rkey, rewritePath, basePath);
253 } else if (status === 301 || status === 302) {
254 // External redirect: change the URL
255 // For sites.wisp.place, we need to adjust the target path to include the base path
256 // unless it's an absolute URL
257 let redirectTarget = targetPath;
258 if (!targetPath.startsWith('http://') && !targetPath.startsWith('https://')) {
259 redirectTarget = basePath + (targetPath.startsWith('/') ? targetPath.slice(1) : targetPath);
260 }
261 return new Response(null, {
262 status,
263 headers: {
264 'Location': redirectTarget,
265 'Cache-Control': status === 301 ? 'public, max-age=31536000' : 'public, max-age=0',
266 },
267 });
268 } else if (status === 404) {
269 // Custom 404 page
270 const custom404Path = targetPath.startsWith('/') ? targetPath.slice(1) : targetPath;
271 const response = await serveFileInternalWithRewrite(did, rkey, custom404Path, basePath);
272 // Override status to 404
273 return new Response(response.body, {
274 status: 404,
275 headers: response.headers,
276 });
277 }
278 }
279 }
280
281 // No redirect matched, serve normally
282 return serveFileInternalWithRewrite(did, rkey, filePath, basePath);
283}
284
285// Internal function to serve a file with rewriting
286async function serveFileInternalWithRewrite(did: string, rkey: string, filePath: string, basePath: string) {
287 // Default to index.html if path is empty or ends with /
288 let requestPath = filePath || 'index.html';
289 if (requestPath.endsWith('/')) {
290 requestPath += 'index.html';
291 }
292
293 const cacheKey = getCacheKey(did, rkey, requestPath);
294 const cachedFile = getCachedFilePath(did, rkey, requestPath);
295
296 // Check for rewritten HTML in cache first (if it's HTML)
297 const mimeTypeGuess = lookup(requestPath) || 'application/octet-stream';
298 if (isHtmlContent(requestPath, mimeTypeGuess)) {
299 const rewrittenKey = getCacheKey(did, rkey, requestPath, `rewritten:${basePath}`);
300 const rewrittenContent = rewrittenHtmlCache.get(rewrittenKey);
301 if (rewrittenContent) {
302 return new Response(rewrittenContent, {
303 headers: {
304 'Content-Type': 'text/html; charset=utf-8',
305 'Content-Encoding': 'gzip',
306 'Cache-Control': 'public, max-age=300',
307 },
308 });
309 }
310 }
311
312 // Check in-memory file cache
313 let content = fileCache.get(cacheKey);
314 let meta = metadataCache.get(cacheKey);
315
316 if (!content && await fileExists(cachedFile)) {
317 // Read from disk and cache
318 content = await readFile(cachedFile);
319 fileCache.set(cacheKey, content, content.length);
320
321 const metaFile = `${cachedFile}.meta`;
322 if (await fileExists(metaFile)) {
323 const metaJson = await readFile(metaFile, 'utf-8');
324 meta = JSON.parse(metaJson);
325 metadataCache.set(cacheKey, meta!, JSON.stringify(meta).length);
326 }
327 }
328
329 if (content) {
330 const mimeType = meta?.mimeType || lookup(cachedFile) || 'application/octet-stream';
331 const isGzipped = meta?.encoding === 'gzip';
332
333 // Check if this is HTML content that needs rewriting
334 if (isHtmlContent(requestPath, mimeType)) {
335 let htmlContent: string;
336 if (isGzipped) {
337 const { gunzipSync } = await import('zlib');
338 htmlContent = gunzipSync(content).toString('utf-8');
339 } else {
340 htmlContent = content.toString('utf-8');
341 }
342 const rewritten = rewriteHtmlPaths(htmlContent, basePath, requestPath);
343
344 // Recompress and cache the rewritten HTML
345 const { gzipSync } = await import('zlib');
346 const recompressed = gzipSync(Buffer.from(rewritten, 'utf-8'));
347
348 const rewrittenKey = getCacheKey(did, rkey, requestPath, `rewritten:${basePath}`);
349 rewrittenHtmlCache.set(rewrittenKey, recompressed, recompressed.length);
350
351 return new Response(recompressed, {
352 headers: {
353 'Content-Type': 'text/html; charset=utf-8',
354 'Content-Encoding': 'gzip',
355 'Cache-Control': 'public, max-age=300',
356 },
357 });
358 }
359
360 // Non-HTML files: serve as-is
361 const headers: Record<string, string> = {
362 'Content-Type': mimeType,
363 'Cache-Control': 'public, max-age=31536000, immutable',
364 };
365
366 if (isGzipped) {
367 const shouldServeCompressed = shouldCompressMimeType(mimeType);
368 if (!shouldServeCompressed) {
369 const { gunzipSync } = await import('zlib');
370 const decompressed = gunzipSync(content);
371 return new Response(decompressed, { headers });
372 }
373 headers['Content-Encoding'] = 'gzip';
374 }
375
376 return new Response(content, { headers });
377 }
378
379 // Try index.html for directory-like paths
380 if (!requestPath.includes('.')) {
381 const indexPath = `${requestPath}/index.html`;
382 const indexCacheKey = getCacheKey(did, rkey, indexPath);
383 const indexFile = getCachedFilePath(did, rkey, indexPath);
384
385 // Check for rewritten index.html in cache
386 const rewrittenKey = getCacheKey(did, rkey, indexPath, `rewritten:${basePath}`);
387 const rewrittenContent = rewrittenHtmlCache.get(rewrittenKey);
388 if (rewrittenContent) {
389 return new Response(rewrittenContent, {
390 headers: {
391 'Content-Type': 'text/html; charset=utf-8',
392 'Content-Encoding': 'gzip',
393 'Cache-Control': 'public, max-age=300',
394 },
395 });
396 }
397
398 let indexContent = fileCache.get(indexCacheKey);
399 let indexMeta = metadataCache.get(indexCacheKey);
400
401 if (!indexContent && await fileExists(indexFile)) {
402 indexContent = await readFile(indexFile);
403 fileCache.set(indexCacheKey, indexContent, indexContent.length);
404
405 const indexMetaFile = `${indexFile}.meta`;
406 if (await fileExists(indexMetaFile)) {
407 const metaJson = await readFile(indexMetaFile, 'utf-8');
408 indexMeta = JSON.parse(metaJson);
409 metadataCache.set(indexCacheKey, indexMeta!, JSON.stringify(indexMeta).length);
410 }
411 }
412
413 if (indexContent) {
414 const isGzipped = indexMeta?.encoding === 'gzip';
415
416 let htmlContent: string;
417 if (isGzipped) {
418 const { gunzipSync } = await import('zlib');
419 htmlContent = gunzipSync(indexContent).toString('utf-8');
420 } else {
421 htmlContent = indexContent.toString('utf-8');
422 }
423 const rewritten = rewriteHtmlPaths(htmlContent, basePath, indexPath);
424
425 const { gzipSync } = await import('zlib');
426 const recompressed = gzipSync(Buffer.from(rewritten, 'utf-8'));
427
428 rewrittenHtmlCache.set(rewrittenKey, recompressed, recompressed.length);
429
430 return new Response(recompressed, {
431 headers: {
432 'Content-Type': 'text/html; charset=utf-8',
433 'Content-Encoding': 'gzip',
434 'Cache-Control': 'public, max-age=300',
435 },
436 });
437 }
438 }
439
440 return new Response('Not Found', { status: 404 });
441}
442
443// Helper to ensure site is cached
444async function ensureSiteCached(did: string, rkey: string): Promise<boolean> {
445 if (isCached(did, rkey)) {
446 return true;
447 }
448
449 // Fetch and cache the site
450 const siteData = await fetchSiteRecord(did, rkey);
451 if (!siteData) {
452 logger.error('Site record not found', null, { did, rkey });
453 return false;
454 }
455
456 const pdsEndpoint = await getPdsForDid(did);
457 if (!pdsEndpoint) {
458 logger.error('PDS not found for DID', null, { did });
459 return false;
460 }
461
462 try {
463 await downloadAndCacheSite(did, rkey, siteData.record, pdsEndpoint, siteData.cid);
464 // Clear redirect rules cache since the site was updated
465 clearRedirectRulesCache(did, rkey);
466 logger.info('Site cached successfully', { did, rkey });
467 return true;
468 } catch (err) {
469 logger.error('Failed to cache site', err, { did, rkey });
470 return false;
471 }
472}
473
474const app = new Hono();
475
476// Add observability middleware
477app.use('*', observabilityMiddleware('hosting-service'));
478
479// Error handler
480app.onError(observabilityErrorHandler('hosting-service'));
481
482// Main site serving route
483app.get('/*', async (c) => {
484 const url = new URL(c.req.url);
485 const hostname = c.req.header('host') || '';
486 const rawPath = url.pathname.replace(/^\//, '');
487 const path = sanitizePath(rawPath);
488
489 // Check if this is sites.wisp.place subdomain
490 if (hostname === `sites.${BASE_HOST}` || hostname === `sites.${BASE_HOST}:${process.env.PORT || 3000}`) {
491 // Sanitize the path FIRST to prevent path traversal
492 const sanitizedFullPath = sanitizePath(rawPath);
493
494 // Extract identifier and site from sanitized path: did:plc:123abc/sitename/file.html
495 const pathParts = sanitizedFullPath.split('/');
496 if (pathParts.length < 2) {
497 return c.text('Invalid path format. Expected: /identifier/sitename/path', 400);
498 }
499
500 const identifier = pathParts[0];
501 const site = pathParts[1];
502 const filePath = pathParts.slice(2).join('/');
503
504 // Additional validation: identifier must be a valid DID or handle format
505 if (!identifier || identifier.length < 3 || identifier.includes('..') || identifier.includes('\0')) {
506 return c.text('Invalid identifier', 400);
507 }
508
509 // Validate site parameter exists
510 if (!site) {
511 return c.text('Site name required', 400);
512 }
513
514 // Validate site name (rkey)
515 if (!isValidRkey(site)) {
516 return c.text('Invalid site name', 400);
517 }
518
519 // Resolve identifier to DID
520 const did = await resolveDid(identifier);
521 if (!did) {
522 return c.text('Invalid identifier', 400);
523 }
524
525 // Ensure site is cached
526 const cached = await ensureSiteCached(did, site);
527 if (!cached) {
528 return c.text('Site not found', 404);
529 }
530
531 // Serve with HTML path rewriting to handle absolute paths
532 const basePath = `/${identifier}/${site}/`;
533 const headers: Record<string, string> = {};
534 c.req.raw.headers.forEach((value, key) => {
535 headers[key.toLowerCase()] = value;
536 });
537 return serveFromCacheWithRewrite(did, site, filePath, basePath, c.req.url, headers);
538 }
539
540 // Check if this is a DNS hash subdomain
541 const dnsMatch = hostname.match(/^([a-f0-9]{16})\.dns\.(.+)$/);
542 if (dnsMatch) {
543 const hash = dnsMatch[1];
544 const baseDomain = dnsMatch[2];
545
546 if (!hash) {
547 return c.text('Invalid DNS hash', 400);
548 }
549
550 if (baseDomain !== BASE_HOST) {
551 return c.text('Invalid base domain', 400);
552 }
553
554 const customDomain = await getCustomDomainByHash(hash);
555 if (!customDomain) {
556 return c.text('Custom domain not found or not verified', 404);
557 }
558
559 if (!customDomain.rkey) {
560 return c.text('Domain not mapped to a site', 404);
561 }
562
563 const rkey = customDomain.rkey;
564 if (!isValidRkey(rkey)) {
565 return c.text('Invalid site configuration', 500);
566 }
567
568 const cached = await ensureSiteCached(customDomain.did, rkey);
569 if (!cached) {
570 return c.text('Site not found', 404);
571 }
572
573 const headers: Record<string, string> = {};
574 c.req.raw.headers.forEach((value, key) => {
575 headers[key.toLowerCase()] = value;
576 });
577 return serveFromCache(customDomain.did, rkey, path, c.req.url, headers);
578 }
579
580 // Route 2: Registered subdomains - /*.wisp.place/*
581 if (hostname.endsWith(`.${BASE_HOST}`)) {
582 const domainInfo = await getWispDomain(hostname);
583 if (!domainInfo) {
584 return c.text('Subdomain not registered', 404);
585 }
586
587 if (!domainInfo.rkey) {
588 return c.text('Domain not mapped to a site', 404);
589 }
590
591 const rkey = domainInfo.rkey;
592 if (!isValidRkey(rkey)) {
593 return c.text('Invalid site configuration', 500);
594 }
595
596 const cached = await ensureSiteCached(domainInfo.did, rkey);
597 if (!cached) {
598 return c.text('Site not found', 404);
599 }
600
601 const headers: Record<string, string> = {};
602 c.req.raw.headers.forEach((value, key) => {
603 headers[key.toLowerCase()] = value;
604 });
605 return serveFromCache(domainInfo.did, rkey, path, c.req.url, headers);
606 }
607
608 // Route 1: Custom domains - /*
609 const customDomain = await getCustomDomain(hostname);
610 if (!customDomain) {
611 return c.text('Custom domain not found or not verified', 404);
612 }
613
614 if (!customDomain.rkey) {
615 return c.text('Domain not mapped to a site', 404);
616 }
617
618 const rkey = customDomain.rkey;
619 if (!isValidRkey(rkey)) {
620 return c.text('Invalid site configuration', 500);
621 }
622
623 const cached = await ensureSiteCached(customDomain.did, rkey);
624 if (!cached) {
625 return c.text('Site not found', 404);
626 }
627
628 const headers: Record<string, string> = {};
629 c.req.raw.headers.forEach((value, key) => {
630 headers[key.toLowerCase()] = value;
631 });
632 return serveFromCache(customDomain.did, rkey, path, c.req.url, headers);
633});
634
635// Internal observability endpoints (for admin panel)
636app.get('/__internal__/observability/logs', (c) => {
637 const query = c.req.query();
638 const filter: any = {};
639 if (query.level) filter.level = query.level;
640 if (query.service) filter.service = query.service;
641 if (query.search) filter.search = query.search;
642 if (query.eventType) filter.eventType = query.eventType;
643 if (query.limit) filter.limit = parseInt(query.limit as string);
644 return c.json({ logs: logCollector.getLogs(filter) });
645});
646
647app.get('/__internal__/observability/errors', (c) => {
648 const query = c.req.query();
649 const filter: any = {};
650 if (query.service) filter.service = query.service;
651 if (query.limit) filter.limit = parseInt(query.limit as string);
652 return c.json({ errors: errorTracker.getErrors(filter) });
653});
654
655app.get('/__internal__/observability/metrics', (c) => {
656 const query = c.req.query();
657 const timeWindow = query.timeWindow ? parseInt(query.timeWindow as string) : 3600000;
658 const stats = metricsCollector.getStats('hosting-service', timeWindow);
659 return c.json({ stats, timeWindow });
660});
661
662app.get('/__internal__/observability/cache', async (c) => {
663 const { getCacheStats } = await import('./lib/cache');
664 const stats = getCacheStats();
665 return c.json({ cache: stats });
666});
667
668export default app;