wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1/**
2 * Example HTTP server serving static sites from tiered storage
3 *
4 * This demonstrates a real-world use case: serving static websites
5 * with automatic caching across hot (memory), warm (disk), and cold (S3) tiers.
6 *
7 * Run with: bun run serve
8 */
9
10import { Hono } from 'hono';
11import { TieredStorage, MemoryStorageTier, DiskStorageTier, S3StorageTier } from './src/index.js';
12import { readFile, readdir } from 'node:fs/promises';
13import { lookup } from 'mime-types';
14
15const S3_BUCKET = process.env.S3_BUCKET || 'tiered-storage-example';
16const S3_METADATA_BUCKET = process.env.S3_METADATA_BUCKET;
17const S3_REGION = process.env.S3_REGION || 'us-east-1';
18const S3_ENDPOINT = process.env.S3_ENDPOINT;
19const S3_FORCE_PATH_STYLE = process.env.S3_FORCE_PATH_STYLE !== 'false';
20const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
21const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
22const PORT = parseInt(process.env.PORT || '3000', 10);
23
24const storage = new TieredStorage({
25 tiers: {
26 hot: new MemoryStorageTier({
27 maxSizeBytes: 50 * 1024 * 1024,
28 maxItems: 500,
29 }),
30 warm: new DiskStorageTier({
31 directory: './cache/sites',
32 maxSizeBytes: 1024 * 1024 * 1024,
33 }),
34 cold: new S3StorageTier({
35 bucket: S3_BUCKET,
36 metadataBucket: S3_METADATA_BUCKET,
37 region: S3_REGION,
38 endpoint: S3_ENDPOINT,
39 forcePathStyle: S3_FORCE_PATH_STYLE,
40 credentials:
41 AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY
42 ? { accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY }
43 : undefined,
44 prefix: 'demo-sites/',
45 }),
46 },
47 placementRules: [
48 // index.html goes to all tiers for instant serving
49 { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
50
51 // everything else: warm + cold only
52 { pattern: '**', tiers: ['warm', 'cold'] },
53 ],
54 compression: true,
55 defaultTTL: 14 * 24 * 60 * 60 * 1000,
56 promotionStrategy: 'lazy',
57});
58
59const app = new Hono();
60
61// Site metadata
62const siteId = 'did:plc:example123';
63const siteName = 'tiered-cache-demo';
64
65/**
66 * Load the example site into storage
67 */
68async function loadExampleSite() {
69 console.log('\n📦 Loading example site into tiered storage...\n');
70
71 const files = [
72 { name: 'index.html', mimeType: 'text/html' },
73 { name: 'about.html', mimeType: 'text/html' },
74 { name: 'docs.html', mimeType: 'text/html' },
75 { name: 'style.css', mimeType: 'text/css' },
76 { name: 'script.js', mimeType: 'application/javascript' },
77 ];
78
79 for (const file of files) {
80 const content = await readFile(`./example-site/${file.name}`, 'utf-8');
81 const key = `${siteId}/${siteName}/${file.name}`;
82
83 await storage.set(key, content, {
84 metadata: { mimeType: file.mimeType },
85 });
86
87 // Determine which tiers this file went to based on placement rules
88 const isIndex = file.name === 'index.html';
89 const tierInfo = isIndex
90 ? '🔥 hot + 💾 warm + ☁️ cold'
91 : '💾 warm + ☁️ cold (skipped hot)';
92 const sizeKB = (content.length / 1024).toFixed(2);
93 console.log(` ✓ ${file.name.padEnd(15)} ${sizeKB.padStart(6)} KB → ${tierInfo}`);
94 }
95
96 console.log('\n✅ Site loaded successfully!\n');
97}
98
99/**
100 * Serve a file from tiered storage
101 */
102app.get('/sites/:did/:siteName/:path{.*}', async (c) => {
103 const { did, siteName, path } = c.req.param();
104 let filePath = path || 'index.html';
105
106 if (filePath === '' || filePath.endsWith('/')) {
107 filePath += 'index.html';
108 }
109
110 const key = `${did}/${siteName}/${filePath}`;
111
112 try {
113 const result = await storage.getWithMetadata(key);
114
115 if (!result) {
116 return c.text('404 Not Found', 404);
117 }
118
119 const mimeType = result.metadata.customMetadata?.mimeType || lookup(filePath) || 'application/octet-stream';
120
121 const headers: Record<string, string> = {
122 'Content-Type': mimeType,
123 'X-Cache-Tier': result.source, // Which tier served this
124 'X-Cache-Size': result.metadata.size.toString(),
125 'X-Cache-Compressed': result.metadata.compressed.toString(),
126 'X-Cache-Access-Count': result.metadata.accessCount.toString(),
127 };
128
129 // Add cache control based on tier
130 if (result.source === 'hot') {
131 headers['X-Cache-Status'] = 'HIT-MEMORY';
132 } else if (result.source === 'warm') {
133 headers['X-Cache-Status'] = 'HIT-DISK';
134 } else {
135 headers['X-Cache-Status'] = 'HIT-S3';
136 }
137
138 const emoji = result.source === 'hot' ? '🔥' : result.source === 'warm' ? '💾' : '☁️';
139 console.log(`${emoji} ${filePath.padEnd(20)} served from ${result.source.padEnd(4)} (${(result.metadata.size / 1024).toFixed(2)} KB, access #${result.metadata.accessCount})`);
140
141 return c.body(result.data as any, 200, headers);
142 } catch (error: any) {
143 console.error(`❌ Error serving ${filePath}:`, error.message);
144 return c.text('500 Internal Server Error', 500);
145 }
146});
147
148/**
149 * Admin endpoint: Cache statistics
150 */
151app.get('/admin/stats', async (c) => {
152 const stats = await storage.getStats();
153
154 const html = `
155<!DOCTYPE html>
156<html>
157<head>
158 <title>Tiered Storage Statistics</title>
159 <meta name="viewport" content="width=device-width, initial-scale=1.0">
160 <style>
161 * { margin: 0; padding: 0; box-sizing: border-box; }
162 body {
163 font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
164 background: #0f172a;
165 color: #f1f5f9;
166 padding: 2rem;
167 line-height: 1.6;
168 }
169 .container { max-width: 1200px; margin: 0 auto; }
170 h1 {
171 font-size: 2rem;
172 margin-bottom: 0.5rem;
173 background: linear-gradient(135deg, #3b82f6, #8b5cf6);
174 -webkit-background-clip: text;
175 -webkit-text-fill-color: transparent;
176 }
177 .subtitle { color: #94a3b8; margin-bottom: 2rem; }
178 .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1.5rem; margin-bottom: 2rem; }
179 .card {
180 background: #1e293b;
181 border: 1px solid #334155;
182 border-radius: 0.5rem;
183 padding: 1.5rem;
184 }
185 .tier-hot { border-left: 4px solid #ef4444; }
186 .tier-warm { border-left: 4px solid #f59e0b; }
187 .tier-cold { border-left: 4px solid #3b82f6; }
188 .card-title {
189 font-size: 1.2rem;
190 font-weight: 600;
191 margin-bottom: 1rem;
192 display: flex;
193 align-items: center;
194 gap: 0.5rem;
195 }
196 .stat { margin-bottom: 0.75rem; }
197 .stat-label { color: #94a3b8; font-size: 0.9rem; }
198 .stat-value { color: #f1f5f9; font-size: 1.5rem; font-weight: 700; }
199 .overall { background: linear-gradient(135deg, rgba(59, 130, 246, 0.1), rgba(139, 92, 246, 0.1)); }
200 .refresh {
201 display: inline-block;
202 background: #3b82f6;
203 color: white;
204 padding: 0.75rem 1.5rem;
205 border-radius: 0.5rem;
206 text-decoration: none;
207 font-weight: 600;
208 margin-top: 1rem;
209 }
210 .refresh:hover { background: #2563eb; }
211 code {
212 background: #334155;
213 padding: 0.2rem 0.5rem;
214 border-radius: 0.25rem;
215 font-size: 0.9em;
216 }
217 </style>
218</head>
219<body>
220 <div class="container">
221 <h1>📊 Tiered Storage Statistics</h1>
222 <p class="subtitle">Real-time cache performance metrics • Auto-refresh every 5 seconds</p>
223
224 <div class="grid">
225 <div class="card tier-hot">
226 <div class="card-title">🔥 Hot Tier (Memory)</div>
227 <div class="stat">
228 <div class="stat-label">Items</div>
229 <div class="stat-value">${stats.hot?.items || 0}</div>
230 </div>
231 <div class="stat">
232 <div class="stat-label">Size</div>
233 <div class="stat-value">${((stats.hot?.bytes || 0) / 1024).toFixed(2)} KB</div>
234 </div>
235 <div class="stat">
236 <div class="stat-label">Hits / Misses</div>
237 <div class="stat-value">${stats.hot?.hits || 0} / ${stats.hot?.misses || 0}</div>
238 </div>
239 <div class="stat">
240 <div class="stat-label">Evictions</div>
241 <div class="stat-value">${stats.hot?.evictions || 0}</div>
242 </div>
243 </div>
244
245 <div class="card tier-warm">
246 <div class="card-title">💾 Warm Tier (Disk)</div>
247 <div class="stat">
248 <div class="stat-label">Items</div>
249 <div class="stat-value">${stats.warm?.items || 0}</div>
250 </div>
251 <div class="stat">
252 <div class="stat-label">Size</div>
253 <div class="stat-value">${((stats.warm?.bytes || 0) / 1024).toFixed(2)} KB</div>
254 </div>
255 <div class="stat">
256 <div class="stat-label">Hits / Misses</div>
257 <div class="stat-value">${stats.warm?.hits || 0} / ${stats.warm?.misses || 0}</div>
258 </div>
259 </div>
260
261 <div class="card tier-cold">
262 <div class="card-title">☁️ Cold Tier (S3)</div>
263 <div class="stat">
264 <div class="stat-label">Items</div>
265 <div class="stat-value">${stats.cold.items}</div>
266 </div>
267 <div class="stat">
268 <div class="stat-label">Size</div>
269 <div class="stat-value">${(stats.cold.bytes / 1024).toFixed(2)} KB</div>
270 </div>
271 </div>
272 </div>
273
274 <div class="card overall">
275 <div class="card-title">📈 Overall Performance</div>
276 <div class="grid" style="grid-template-columns: repeat(3, 1fr);">
277 <div class="stat">
278 <div class="stat-label">Total Hits</div>
279 <div class="stat-value">${stats.totalHits}</div>
280 </div>
281 <div class="stat">
282 <div class="stat-label">Total Misses</div>
283 <div class="stat-value">${stats.totalMisses}</div>
284 </div>
285 <div class="stat">
286 <div class="stat-label">Hit Rate</div>
287 <div class="stat-value">${(stats.hitRate * 100).toFixed(1)}%</div>
288 </div>
289 </div>
290 </div>
291
292 <div style="margin-top: 2rem; padding: 1rem; background: #1e293b; border-radius: 0.5rem; border: 1px solid #334155;">
293 <p style="margin-bottom: 0.5rem;"><strong>Try it out:</strong></p>
294 <p>Visit <code>http://localhost:${PORT}/sites/${siteId}/${siteName}/</code> to see the site</p>
295 <p>Watch the stats update as you browse different pages!</p>
296 </div>
297 </div>
298
299 <script>
300 // Auto-refresh stats every 5 seconds
301 setTimeout(() => window.location.reload(), 5000);
302 </script>
303</body>
304</html>
305 `;
306
307 return c.html(html);
308});
309
310/**
311 * Admin endpoint: Invalidate cache
312 */
313app.post('/admin/invalidate/:did/:siteName', async (c) => {
314 const { did, siteName } = c.req.param();
315 const prefix = `${did}/${siteName}/`;
316 const deleted = await storage.invalidate(prefix);
317
318 console.log(`🗑️ Invalidated ${deleted} files for ${did}/${siteName}`);
319
320 return c.json({ success: true, deleted, prefix });
321});
322
323/**
324 * Admin endpoint: Bootstrap hot cache
325 */
326app.post('/admin/bootstrap/hot', async (c) => {
327 const limit = parseInt(c.req.query('limit') || '100', 10);
328 const loaded = await storage.bootstrapHot(limit);
329
330 console.log(`🔥 Bootstrapped ${loaded} items into hot tier`);
331
332 return c.json({ success: true, loaded, limit });
333});
334
335/**
336 * Root redirect
337 */
338app.get('/', (c) => {
339 return c.redirect(`/sites/${siteId}/${siteName}/`);
340});
341
342/**
343 * Health check
344 */
345app.get('/health', (c) => c.json({ status: 'ok' }));
346
347/**
348 * Test S3 connection
349 */
350async function testS3Connection() {
351 console.log('\n🔍 Testing S3 connection...\n');
352
353 try {
354 // Try to get stats (which lists objects)
355 const stats = await storage.getStats();
356 console.log(`✅ S3 connection successful!`);
357 console.log(` Found ${stats.cold.items} items (${(stats.cold.bytes / 1024).toFixed(2)} KB)\n`);
358 return true;
359 } catch (error: any) {
360 console.error('❌ S3 connection failed:', error.message);
361 console.error('\nDebug Info:');
362 console.error(` Bucket: ${S3_BUCKET}`);
363 console.error(` Region: ${S3_REGION}`);
364 console.error(` Endpoint: ${S3_ENDPOINT || '(default AWS S3)'}`);
365 console.error(` Access Key: ${AWS_ACCESS_KEY_ID?.substring(0, 8)}...`);
366 console.error(` Force Path Style: ${S3_FORCE_PATH_STYLE}`);
367 console.error('\nCommon issues:');
368 console.error(' • Check that bucket exists');
369 console.error(' • Verify credentials are correct');
370 console.error(' • Ensure endpoint URL is correct');
371 console.error(' • Check firewall/network access');
372 console.error(' • For S3-compatible services, verify region name\n');
373 return false;
374 }
375}
376
377/**
378 * Periodic cache clearing - demonstrates tier bootstrapping
379 */
380function startCacheClearInterval() {
381 const CLEAR_INTERVAL_MS = 60 * 1000; // 1 minute
382
383 setInterval(async () => {
384 console.log('\n' + '═'.repeat(60));
385 console.log('🧹 CACHE CLEAR - Clearing hot and warm tiers...');
386 console.log(' (Cold tier on S3 remains intact)');
387 console.log('═'.repeat(60) + '\n');
388
389 try {
390 // Clear hot tier (memory)
391 if (storage['config'].tiers.hot) {
392 await storage['config'].tiers.hot.clear();
393 console.log('✓ Hot tier (memory) cleared');
394 }
395
396 // Clear warm tier (disk)
397 if (storage['config'].tiers.warm) {
398 await storage['config'].tiers.warm.clear();
399 console.log('✓ Warm tier (disk) cleared');
400 }
401
402 console.log('\n💡 Next request will bootstrap from S3 (cold tier)\n');
403 console.log('─'.repeat(60) + '\n');
404 } catch (error: any) {
405 console.error('❌ Error clearing cache:', error.message);
406 }
407 }, CLEAR_INTERVAL_MS);
408
409 console.log(`⏰ Cache clear interval started (every ${CLEAR_INTERVAL_MS / 1000}s)\n`);
410}
411
412/**
413 * Main startup
414 */
415async function main() {
416 console.log('╔════════════════════════════════════════════════╗');
417 console.log('║ Tiered Storage Demo Server ║');
418 console.log('╚════════════════════════════════════════════════╝\n');
419
420 console.log('Configuration:');
421 console.log(` S3 Bucket: ${S3_BUCKET}`);
422 console.log(` S3 Region: ${S3_REGION}`);
423 console.log(` S3 Endpoint: ${S3_ENDPOINT || '(default AWS S3)'}`);
424 console.log(` Force Path Style: ${S3_FORCE_PATH_STYLE}`);
425 console.log(` Port: ${PORT}`);
426
427 try {
428 // Test S3 connection first
429 const s3Connected = await testS3Connection();
430 if (!s3Connected) {
431 process.exit(1);
432 }
433
434 // Load the example site
435 await loadExampleSite();
436
437 // Start periodic cache clearing
438 startCacheClearInterval();
439
440 // Start the server
441 console.log('🚀 Starting server...\n');
442
443 const server = Bun.serve({
444 port: PORT,
445 fetch: app.fetch,
446 });
447
448 console.log('╔════════════════════════════════════════════════╗');
449 console.log('║ Server Running! ║');
450 console.log('╚════════════════════════════════════════════════╝\n');
451 console.log(`📍 Demo Site: http://localhost:${PORT}/sites/${siteId}/${siteName}/`);
452 console.log(`📊 Statistics: http://localhost:${PORT}/admin/stats`);
453 console.log(`💚 Health: http://localhost:${PORT}/health`);
454 console.log('\n🎯 Try browsing the site and watch which tier serves each file!\n');
455 console.log('💡 Caches clear every 60 seconds - watch files get re-fetched from S3!\n');
456 if (S3_METADATA_BUCKET) {
457 console.log(`✨ Metadata bucket: ${S3_METADATA_BUCKET} (fast updates enabled!)\n`);
458 } else {
459 console.log('⚠️ No metadata bucket - using legacy mode (slower updates)\n');
460 }
461 console.log('Press Ctrl+C to stop\n');
462 console.log('─'.repeat(60));
463 console.log('Request Log:\n');
464 } catch (error: any) {
465 console.error('\n❌ Failed to start server:', error.message);
466 if (error.message.includes('Forbidden')) {
467 console.error('\nS3 connection issue. Check:');
468 console.error(' 1. Bucket exists on S3 service');
469 console.error(' 2. Credentials are correct');
470 console.error(' 3. Permissions allow read/write');
471 }
472 process.exit(1);
473 }
474}
475
476main().catch(console.error);