/** * Example HTTP server serving static sites from tiered storage * * This demonstrates a real-world use case: serving static websites * with automatic caching across hot (memory), warm (disk), and cold (S3) tiers. * * Run with: bun run serve */ import { Hono } from 'hono'; import { TieredStorage, MemoryStorageTier, DiskStorageTier, S3StorageTier } from './src/index.js'; import { readFile, readdir } from 'node:fs/promises'; import { lookup } from 'mime-types'; const S3_BUCKET = process.env.S3_BUCKET || 'tiered-storage-example'; const S3_METADATA_BUCKET = process.env.S3_METADATA_BUCKET; const S3_REGION = process.env.S3_REGION || 'us-east-1'; const S3_ENDPOINT = process.env.S3_ENDPOINT; const S3_FORCE_PATH_STYLE = process.env.S3_FORCE_PATH_STYLE !== 'false'; const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID; const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY; const PORT = parseInt(process.env.PORT || '3000', 10); const storage = new TieredStorage({ tiers: { hot: new MemoryStorageTier({ maxSizeBytes: 50 * 1024 * 1024, maxItems: 500, }), warm: new DiskStorageTier({ directory: './cache/sites', maxSizeBytes: 1024 * 1024 * 1024, }), cold: new S3StorageTier({ bucket: S3_BUCKET, metadataBucket: S3_METADATA_BUCKET, region: S3_REGION, endpoint: S3_ENDPOINT, forcePathStyle: S3_FORCE_PATH_STYLE, credentials: AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY ? { accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY } : undefined, prefix: 'demo-sites/', }), }, placementRules: [ // index.html goes to all tiers for instant serving { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] }, // everything else: warm + cold only { pattern: '**', tiers: ['warm', 'cold'] }, ], compression: true, defaultTTL: 14 * 24 * 60 * 60 * 1000, promotionStrategy: 'lazy', }); const app = new Hono(); // Site metadata const siteId = 'did:plc:example123'; const siteName = 'tiered-cache-demo'; /** * Load the example site into storage */ async function loadExampleSite() { console.log('\n๐Ÿ“ฆ Loading example site into tiered storage...\n'); const files = [ { name: 'index.html', mimeType: 'text/html' }, { name: 'about.html', mimeType: 'text/html' }, { name: 'docs.html', mimeType: 'text/html' }, { name: 'style.css', mimeType: 'text/css' }, { name: 'script.js', mimeType: 'application/javascript' }, ]; for (const file of files) { const content = await readFile(`./example-site/${file.name}`, 'utf-8'); const key = `${siteId}/${siteName}/${file.name}`; await storage.set(key, content, { metadata: { mimeType: file.mimeType }, }); // Determine which tiers this file went to based on placement rules const isIndex = file.name === 'index.html'; const tierInfo = isIndex ? '๐Ÿ”ฅ hot + ๐Ÿ’พ warm + โ˜๏ธ cold' : '๐Ÿ’พ warm + โ˜๏ธ cold (skipped hot)'; const sizeKB = (content.length / 1024).toFixed(2); console.log(` โœ“ ${file.name.padEnd(15)} ${sizeKB.padStart(6)} KB โ†’ ${tierInfo}`); } console.log('\nโœ… Site loaded successfully!\n'); } /** * Serve a file from tiered storage */ app.get('/sites/:did/:siteName/:path{.*}', async (c) => { const { did, siteName, path } = c.req.param(); let filePath = path || 'index.html'; if (filePath === '' || filePath.endsWith('/')) { filePath += 'index.html'; } const key = `${did}/${siteName}/${filePath}`; try { const result = await storage.getWithMetadata(key); if (!result) { return c.text('404 Not Found', 404); } const mimeType = result.metadata.customMetadata?.mimeType || lookup(filePath) || 'application/octet-stream'; const headers: Record = { 'Content-Type': mimeType, 'X-Cache-Tier': result.source, // Which tier served this 'X-Cache-Size': result.metadata.size.toString(), 'X-Cache-Compressed': result.metadata.compressed.toString(), 'X-Cache-Access-Count': result.metadata.accessCount.toString(), }; // Add cache control based on tier if (result.source === 'hot') { headers['X-Cache-Status'] = 'HIT-MEMORY'; } else if (result.source === 'warm') { headers['X-Cache-Status'] = 'HIT-DISK'; } else { headers['X-Cache-Status'] = 'HIT-S3'; } const emoji = result.source === 'hot' ? '๐Ÿ”ฅ' : result.source === 'warm' ? '๐Ÿ’พ' : 'โ˜๏ธ'; console.log(`${emoji} ${filePath.padEnd(20)} served from ${result.source.padEnd(4)} (${(result.metadata.size / 1024).toFixed(2)} KB, access #${result.metadata.accessCount})`); return c.body(result.data as any, 200, headers); } catch (error: any) { console.error(`โŒ Error serving ${filePath}:`, error.message); return c.text('500 Internal Server Error', 500); } }); /** * Admin endpoint: Cache statistics */ app.get('/admin/stats', async (c) => { const stats = await storage.getStats(); const html = ` Tiered Storage Statistics

๐Ÿ“Š Tiered Storage Statistics

Real-time cache performance metrics โ€ข Auto-refresh every 5 seconds

๐Ÿ”ฅ Hot Tier (Memory)
Items
${stats.hot?.items || 0}
Size
${((stats.hot?.bytes || 0) / 1024).toFixed(2)} KB
Hits / Misses
${stats.hot?.hits || 0} / ${stats.hot?.misses || 0}
Evictions
${stats.hot?.evictions || 0}
๐Ÿ’พ Warm Tier (Disk)
Items
${stats.warm?.items || 0}
Size
${((stats.warm?.bytes || 0) / 1024).toFixed(2)} KB
Hits / Misses
${stats.warm?.hits || 0} / ${stats.warm?.misses || 0}
โ˜๏ธ Cold Tier (S3)
Items
${stats.cold.items}
Size
${(stats.cold.bytes / 1024).toFixed(2)} KB
๐Ÿ“ˆ Overall Performance
Total Hits
${stats.totalHits}
Total Misses
${stats.totalMisses}
Hit Rate
${(stats.hitRate * 100).toFixed(1)}%

Try it out:

Visit http://localhost:${PORT}/sites/${siteId}/${siteName}/ to see the site

Watch the stats update as you browse different pages!

`; return c.html(html); }); /** * Admin endpoint: Invalidate cache */ app.post('/admin/invalidate/:did/:siteName', async (c) => { const { did, siteName } = c.req.param(); const prefix = `${did}/${siteName}/`; const deleted = await storage.invalidate(prefix); console.log(`๐Ÿ—‘๏ธ Invalidated ${deleted} files for ${did}/${siteName}`); return c.json({ success: true, deleted, prefix }); }); /** * Admin endpoint: Bootstrap hot cache */ app.post('/admin/bootstrap/hot', async (c) => { const limit = parseInt(c.req.query('limit') || '100', 10); const loaded = await storage.bootstrapHot(limit); console.log(`๐Ÿ”ฅ Bootstrapped ${loaded} items into hot tier`); return c.json({ success: true, loaded, limit }); }); /** * Root redirect */ app.get('/', (c) => { return c.redirect(`/sites/${siteId}/${siteName}/`); }); /** * Health check */ app.get('/health', (c) => c.json({ status: 'ok' })); /** * Test S3 connection */ async function testS3Connection() { console.log('\n๐Ÿ” Testing S3 connection...\n'); try { // Try to get stats (which lists objects) const stats = await storage.getStats(); console.log(`โœ… S3 connection successful!`); console.log(` Found ${stats.cold.items} items (${(stats.cold.bytes / 1024).toFixed(2)} KB)\n`); return true; } catch (error: any) { console.error('โŒ S3 connection failed:', error.message); console.error('\nDebug Info:'); console.error(` Bucket: ${S3_BUCKET}`); console.error(` Region: ${S3_REGION}`); console.error(` Endpoint: ${S3_ENDPOINT || '(default AWS S3)'}`); console.error(` Access Key: ${AWS_ACCESS_KEY_ID?.substring(0, 8)}...`); console.error(` Force Path Style: ${S3_FORCE_PATH_STYLE}`); console.error('\nCommon issues:'); console.error(' โ€ข Check that bucket exists'); console.error(' โ€ข Verify credentials are correct'); console.error(' โ€ข Ensure endpoint URL is correct'); console.error(' โ€ข Check firewall/network access'); console.error(' โ€ข For S3-compatible services, verify region name\n'); return false; } } /** * Periodic cache clearing - demonstrates tier bootstrapping */ function startCacheClearInterval() { const CLEAR_INTERVAL_MS = 60 * 1000; // 1 minute setInterval(async () => { console.log('\n' + 'โ•'.repeat(60)); console.log('๐Ÿงน CACHE CLEAR - Clearing hot and warm tiers...'); console.log(' (Cold tier on S3 remains intact)'); console.log('โ•'.repeat(60) + '\n'); try { // Clear hot tier (memory) if (storage['config'].tiers.hot) { await storage['config'].tiers.hot.clear(); console.log('โœ“ Hot tier (memory) cleared'); } // Clear warm tier (disk) if (storage['config'].tiers.warm) { await storage['config'].tiers.warm.clear(); console.log('โœ“ Warm tier (disk) cleared'); } console.log('\n๐Ÿ’ก Next request will bootstrap from S3 (cold tier)\n'); console.log('โ”€'.repeat(60) + '\n'); } catch (error: any) { console.error('โŒ Error clearing cache:', error.message); } }, CLEAR_INTERVAL_MS); console.log(`โฐ Cache clear interval started (every ${CLEAR_INTERVAL_MS / 1000}s)\n`); } /** * Main startup */ async function main() { console.log('โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—'); console.log('โ•‘ Tiered Storage Demo Server โ•‘'); console.log('โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•\n'); console.log('Configuration:'); console.log(` S3 Bucket: ${S3_BUCKET}`); console.log(` S3 Region: ${S3_REGION}`); console.log(` S3 Endpoint: ${S3_ENDPOINT || '(default AWS S3)'}`); console.log(` Force Path Style: ${S3_FORCE_PATH_STYLE}`); console.log(` Port: ${PORT}`); try { // Test S3 connection first const s3Connected = await testS3Connection(); if (!s3Connected) { process.exit(1); } // Load the example site await loadExampleSite(); // Start periodic cache clearing startCacheClearInterval(); // Start the server console.log('๐Ÿš€ Starting server...\n'); const server = Bun.serve({ port: PORT, fetch: app.fetch, }); console.log('โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—'); console.log('โ•‘ Server Running! โ•‘'); console.log('โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•\n'); console.log(`๐Ÿ“ Demo Site: http://localhost:${PORT}/sites/${siteId}/${siteName}/`); console.log(`๐Ÿ“Š Statistics: http://localhost:${PORT}/admin/stats`); console.log(`๐Ÿ’š Health: http://localhost:${PORT}/health`); console.log('\n๐ŸŽฏ Try browsing the site and watch which tier serves each file!\n'); console.log('๐Ÿ’ก Caches clear every 60 seconds - watch files get re-fetched from S3!\n'); if (S3_METADATA_BUCKET) { console.log(`โœจ Metadata bucket: ${S3_METADATA_BUCKET} (fast updates enabled!)\n`); } else { console.log('โš ๏ธ No metadata bucket - using legacy mode (slower updates)\n'); } console.log('Press Ctrl+C to stop\n'); console.log('โ”€'.repeat(60)); console.log('Request Log:\n'); } catch (error: any) { console.error('\nโŒ Failed to start server:', error.message); if (error.message.includes('Forbidden')) { console.error('\nS3 connection issue. Check:'); console.error(' 1. Bucket exists on S3 service'); console.error(' 2. Credentials are correct'); console.error(' 3. Permissions allow read/write'); } process.exit(1); } } main().catch(console.error);