wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1/**
2 * Example HTTP server serving static sites from tiered storage
3 *
4 * This demonstrates a real-world use case: serving static websites
5 * with automatic caching across hot (memory), warm (disk), and cold (S3) tiers.
6 *
7 * Run with: bun run serve
8 */
9
10import { Hono } from 'hono';
11import { TieredStorage, MemoryStorageTier, DiskStorageTier, S3StorageTier } from './src/index.js';
12import { readFile, readdir } from 'node:fs/promises';
13import { lookup } from 'mime-types';
14
15const S3_BUCKET = process.env.S3_BUCKET || 'tiered-storage-example';
16const S3_METADATA_BUCKET = process.env.S3_METADATA_BUCKET;
17const S3_REGION = process.env.S3_REGION || 'us-east-1';
18const S3_ENDPOINT = process.env.S3_ENDPOINT;
19const S3_FORCE_PATH_STYLE = process.env.S3_FORCE_PATH_STYLE !== 'false';
20const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
21const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
22const PORT = parseInt(process.env.PORT || '3000', 10);
23
24const storage = new TieredStorage({
25 tiers: {
26 hot: new MemoryStorageTier({
27 maxSizeBytes: 50 * 1024 * 1024,
28 maxItems: 500,
29 }),
30 warm: new DiskStorageTier({
31 directory: './cache/sites',
32 maxSizeBytes: 1024 * 1024 * 1024,
33 }),
34 cold: new S3StorageTier({
35 bucket: S3_BUCKET,
36 metadataBucket: S3_METADATA_BUCKET,
37 region: S3_REGION,
38 endpoint: S3_ENDPOINT,
39 forcePathStyle: S3_FORCE_PATH_STYLE,
40 credentials:
41 AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY
42 ? { accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY }
43 : undefined,
44 prefix: 'demo-sites/',
45 }),
46 },
47 compression: true,
48 defaultTTL: 14 * 24 * 60 * 60 * 1000,
49 promotionStrategy: 'lazy',
50});
51
52const app = new Hono();
53
54// Site metadata
55const siteId = 'did:plc:example123';
56const siteName = 'tiered-cache-demo';
57
58/**
59 * Load the example site into storage
60 */
61async function loadExampleSite() {
62 console.log('\n📦 Loading example site into tiered storage...\n');
63
64 const files = [
65 { name: 'index.html', skipTiers: [], mimeType: 'text/html' },
66 { name: 'about.html', skipTiers: ['hot'], mimeType: 'text/html' },
67 { name: 'docs.html', skipTiers: ['hot'], mimeType: 'text/html' },
68 { name: 'style.css', skipTiers: ['hot'], mimeType: 'text/css' },
69 { name: 'script.js', skipTiers: ['hot'], mimeType: 'application/javascript' },
70 ];
71
72 for (const file of files) {
73 const content = await readFile(`./example-site/${file.name}`, 'utf-8');
74 const key = `${siteId}/${siteName}/${file.name}`;
75
76 await storage.set(key, content, {
77 skipTiers: file.skipTiers as ('hot' | 'warm')[],
78 metadata: { mimeType: file.mimeType },
79 });
80
81 const tierInfo =
82 file.skipTiers.length === 0
83 ? '🔥 hot + 💾 warm + ☁️ cold'
84 : `💾 warm + ☁️ cold (skipped hot)`;
85 const sizeKB = (content.length / 1024).toFixed(2);
86 console.log(` ✓ ${file.name.padEnd(15)} ${sizeKB.padStart(6)} KB → ${tierInfo}`);
87 }
88
89 console.log('\n✅ Site loaded successfully!\n');
90}
91
92/**
93 * Serve a file from tiered storage
94 */
95app.get('/sites/:did/:siteName/:path{.*}', async (c) => {
96 const { did, siteName, path } = c.req.param();
97 let filePath = path || 'index.html';
98
99 if (filePath === '' || filePath.endsWith('/')) {
100 filePath += 'index.html';
101 }
102
103 const key = `${did}/${siteName}/${filePath}`;
104
105 try {
106 const result = await storage.getWithMetadata(key);
107
108 if (!result) {
109 return c.text('404 Not Found', 404);
110 }
111
112 const mimeType = result.metadata.customMetadata?.mimeType || lookup(filePath) || 'application/octet-stream';
113
114 const headers: Record<string, string> = {
115 'Content-Type': mimeType,
116 'X-Cache-Tier': result.source, // Which tier served this
117 'X-Cache-Size': result.metadata.size.toString(),
118 'X-Cache-Compressed': result.metadata.compressed.toString(),
119 'X-Cache-Access-Count': result.metadata.accessCount.toString(),
120 };
121
122 // Add cache control based on tier
123 if (result.source === 'hot') {
124 headers['X-Cache-Status'] = 'HIT-MEMORY';
125 } else if (result.source === 'warm') {
126 headers['X-Cache-Status'] = 'HIT-DISK';
127 } else {
128 headers['X-Cache-Status'] = 'HIT-S3';
129 }
130
131 const emoji = result.source === 'hot' ? '🔥' : result.source === 'warm' ? '💾' : '☁️';
132 console.log(`${emoji} ${filePath.padEnd(20)} served from ${result.source.padEnd(4)} (${(result.metadata.size / 1024).toFixed(2)} KB, access #${result.metadata.accessCount})`);
133
134 return c.body(result.data as any, 200, headers);
135 } catch (error: any) {
136 console.error(`❌ Error serving ${filePath}:`, error.message);
137 return c.text('500 Internal Server Error', 500);
138 }
139});
140
141/**
142 * Admin endpoint: Cache statistics
143 */
144app.get('/admin/stats', async (c) => {
145 const stats = await storage.getStats();
146
147 const html = `
148<!DOCTYPE html>
149<html>
150<head>
151 <title>Tiered Storage Statistics</title>
152 <meta name="viewport" content="width=device-width, initial-scale=1.0">
153 <style>
154 * { margin: 0; padding: 0; box-sizing: border-box; }
155 body {
156 font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
157 background: #0f172a;
158 color: #f1f5f9;
159 padding: 2rem;
160 line-height: 1.6;
161 }
162 .container { max-width: 1200px; margin: 0 auto; }
163 h1 {
164 font-size: 2rem;
165 margin-bottom: 0.5rem;
166 background: linear-gradient(135deg, #3b82f6, #8b5cf6);
167 -webkit-background-clip: text;
168 -webkit-text-fill-color: transparent;
169 }
170 .subtitle { color: #94a3b8; margin-bottom: 2rem; }
171 .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1.5rem; margin-bottom: 2rem; }
172 .card {
173 background: #1e293b;
174 border: 1px solid #334155;
175 border-radius: 0.5rem;
176 padding: 1.5rem;
177 }
178 .tier-hot { border-left: 4px solid #ef4444; }
179 .tier-warm { border-left: 4px solid #f59e0b; }
180 .tier-cold { border-left: 4px solid #3b82f6; }
181 .card-title {
182 font-size: 1.2rem;
183 font-weight: 600;
184 margin-bottom: 1rem;
185 display: flex;
186 align-items: center;
187 gap: 0.5rem;
188 }
189 .stat { margin-bottom: 0.75rem; }
190 .stat-label { color: #94a3b8; font-size: 0.9rem; }
191 .stat-value { color: #f1f5f9; font-size: 1.5rem; font-weight: 700; }
192 .overall { background: linear-gradient(135deg, rgba(59, 130, 246, 0.1), rgba(139, 92, 246, 0.1)); }
193 .refresh {
194 display: inline-block;
195 background: #3b82f6;
196 color: white;
197 padding: 0.75rem 1.5rem;
198 border-radius: 0.5rem;
199 text-decoration: none;
200 font-weight: 600;
201 margin-top: 1rem;
202 }
203 .refresh:hover { background: #2563eb; }
204 code {
205 background: #334155;
206 padding: 0.2rem 0.5rem;
207 border-radius: 0.25rem;
208 font-size: 0.9em;
209 }
210 </style>
211</head>
212<body>
213 <div class="container">
214 <h1>📊 Tiered Storage Statistics</h1>
215 <p class="subtitle">Real-time cache performance metrics • Auto-refresh every 5 seconds</p>
216
217 <div class="grid">
218 <div class="card tier-hot">
219 <div class="card-title">🔥 Hot Tier (Memory)</div>
220 <div class="stat">
221 <div class="stat-label">Items</div>
222 <div class="stat-value">${stats.hot?.items || 0}</div>
223 </div>
224 <div class="stat">
225 <div class="stat-label">Size</div>
226 <div class="stat-value">${((stats.hot?.bytes || 0) / 1024).toFixed(2)} KB</div>
227 </div>
228 <div class="stat">
229 <div class="stat-label">Hits / Misses</div>
230 <div class="stat-value">${stats.hot?.hits || 0} / ${stats.hot?.misses || 0}</div>
231 </div>
232 <div class="stat">
233 <div class="stat-label">Evictions</div>
234 <div class="stat-value">${stats.hot?.evictions || 0}</div>
235 </div>
236 </div>
237
238 <div class="card tier-warm">
239 <div class="card-title">💾 Warm Tier (Disk)</div>
240 <div class="stat">
241 <div class="stat-label">Items</div>
242 <div class="stat-value">${stats.warm?.items || 0}</div>
243 </div>
244 <div class="stat">
245 <div class="stat-label">Size</div>
246 <div class="stat-value">${((stats.warm?.bytes || 0) / 1024).toFixed(2)} KB</div>
247 </div>
248 <div class="stat">
249 <div class="stat-label">Hits / Misses</div>
250 <div class="stat-value">${stats.warm?.hits || 0} / ${stats.warm?.misses || 0}</div>
251 </div>
252 </div>
253
254 <div class="card tier-cold">
255 <div class="card-title">☁️ Cold Tier (S3)</div>
256 <div class="stat">
257 <div class="stat-label">Items</div>
258 <div class="stat-value">${stats.cold.items}</div>
259 </div>
260 <div class="stat">
261 <div class="stat-label">Size</div>
262 <div class="stat-value">${(stats.cold.bytes / 1024).toFixed(2)} KB</div>
263 </div>
264 </div>
265 </div>
266
267 <div class="card overall">
268 <div class="card-title">📈 Overall Performance</div>
269 <div class="grid" style="grid-template-columns: repeat(3, 1fr);">
270 <div class="stat">
271 <div class="stat-label">Total Hits</div>
272 <div class="stat-value">${stats.totalHits}</div>
273 </div>
274 <div class="stat">
275 <div class="stat-label">Total Misses</div>
276 <div class="stat-value">${stats.totalMisses}</div>
277 </div>
278 <div class="stat">
279 <div class="stat-label">Hit Rate</div>
280 <div class="stat-value">${(stats.hitRate * 100).toFixed(1)}%</div>
281 </div>
282 </div>
283 </div>
284
285 <div style="margin-top: 2rem; padding: 1rem; background: #1e293b; border-radius: 0.5rem; border: 1px solid #334155;">
286 <p style="margin-bottom: 0.5rem;"><strong>Try it out:</strong></p>
287 <p>Visit <code>http://localhost:${PORT}/sites/${siteId}/${siteName}/</code> to see the site</p>
288 <p>Watch the stats update as you browse different pages!</p>
289 </div>
290 </div>
291
292 <script>
293 // Auto-refresh stats every 5 seconds
294 setTimeout(() => window.location.reload(), 5000);
295 </script>
296</body>
297</html>
298 `;
299
300 return c.html(html);
301});
302
303/**
304 * Admin endpoint: Invalidate cache
305 */
306app.post('/admin/invalidate/:did/:siteName', async (c) => {
307 const { did, siteName } = c.req.param();
308 const prefix = `${did}/${siteName}/`;
309 const deleted = await storage.invalidate(prefix);
310
311 console.log(`🗑️ Invalidated ${deleted} files for ${did}/${siteName}`);
312
313 return c.json({ success: true, deleted, prefix });
314});
315
316/**
317 * Admin endpoint: Bootstrap hot cache
318 */
319app.post('/admin/bootstrap/hot', async (c) => {
320 const limit = parseInt(c.req.query('limit') || '100', 10);
321 const loaded = await storage.bootstrapHot(limit);
322
323 console.log(`🔥 Bootstrapped ${loaded} items into hot tier`);
324
325 return c.json({ success: true, loaded, limit });
326});
327
328/**
329 * Root redirect
330 */
331app.get('/', (c) => {
332 return c.redirect(`/sites/${siteId}/${siteName}/`);
333});
334
335/**
336 * Health check
337 */
338app.get('/health', (c) => c.json({ status: 'ok' }));
339
340/**
341 * Test S3 connection
342 */
343async function testS3Connection() {
344 console.log('\n🔍 Testing S3 connection...\n');
345
346 try {
347 // Try to get stats (which lists objects)
348 const stats = await storage.getStats();
349 console.log(`✅ S3 connection successful!`);
350 console.log(` Found ${stats.cold.items} items (${(stats.cold.bytes / 1024).toFixed(2)} KB)\n`);
351 return true;
352 } catch (error: any) {
353 console.error('❌ S3 connection failed:', error.message);
354 console.error('\nDebug Info:');
355 console.error(` Bucket: ${S3_BUCKET}`);
356 console.error(` Region: ${S3_REGION}`);
357 console.error(` Endpoint: ${S3_ENDPOINT || '(default AWS S3)'}`);
358 console.error(` Access Key: ${AWS_ACCESS_KEY_ID?.substring(0, 8)}...`);
359 console.error(` Force Path Style: ${S3_FORCE_PATH_STYLE}`);
360 console.error('\nCommon issues:');
361 console.error(' • Check that bucket exists');
362 console.error(' • Verify credentials are correct');
363 console.error(' • Ensure endpoint URL is correct');
364 console.error(' • Check firewall/network access');
365 console.error(' • For S3-compatible services, verify region name\n');
366 return false;
367 }
368}
369
370/**
371 * Periodic cache clearing - demonstrates tier bootstrapping
372 */
373function startCacheClearInterval() {
374 const CLEAR_INTERVAL_MS = 60 * 1000; // 1 minute
375
376 setInterval(async () => {
377 console.log('\n' + '═'.repeat(60));
378 console.log('🧹 CACHE CLEAR - Clearing hot and warm tiers...');
379 console.log(' (Cold tier on S3 remains intact)');
380 console.log('═'.repeat(60) + '\n');
381
382 try {
383 // Clear hot tier (memory)
384 if (storage['config'].tiers.hot) {
385 await storage['config'].tiers.hot.clear();
386 console.log('✓ Hot tier (memory) cleared');
387 }
388
389 // Clear warm tier (disk)
390 if (storage['config'].tiers.warm) {
391 await storage['config'].tiers.warm.clear();
392 console.log('✓ Warm tier (disk) cleared');
393 }
394
395 console.log('\n💡 Next request will bootstrap from S3 (cold tier)\n');
396 console.log('─'.repeat(60) + '\n');
397 } catch (error: any) {
398 console.error('❌ Error clearing cache:', error.message);
399 }
400 }, CLEAR_INTERVAL_MS);
401
402 console.log(`⏰ Cache clear interval started (every ${CLEAR_INTERVAL_MS / 1000}s)\n`);
403}
404
405/**
406 * Main startup
407 */
408async function main() {
409 console.log('╔════════════════════════════════════════════════╗');
410 console.log('║ Tiered Storage Demo Server ║');
411 console.log('╚════════════════════════════════════════════════╝\n');
412
413 console.log('Configuration:');
414 console.log(` S3 Bucket: ${S3_BUCKET}`);
415 console.log(` S3 Region: ${S3_REGION}`);
416 console.log(` S3 Endpoint: ${S3_ENDPOINT || '(default AWS S3)'}`);
417 console.log(` Force Path Style: ${S3_FORCE_PATH_STYLE}`);
418 console.log(` Port: ${PORT}`);
419
420 try {
421 // Test S3 connection first
422 const s3Connected = await testS3Connection();
423 if (!s3Connected) {
424 process.exit(1);
425 }
426
427 // Load the example site
428 await loadExampleSite();
429
430 // Start periodic cache clearing
431 startCacheClearInterval();
432
433 // Start the server
434 console.log('🚀 Starting server...\n');
435
436 const server = Bun.serve({
437 port: PORT,
438 fetch: app.fetch,
439 });
440
441 console.log('╔════════════════════════════════════════════════╗');
442 console.log('║ Server Running! ║');
443 console.log('╚════════════════════════════════════════════════╝\n');
444 console.log(`📍 Demo Site: http://localhost:${PORT}/sites/${siteId}/${siteName}/`);
445 console.log(`📊 Statistics: http://localhost:${PORT}/admin/stats`);
446 console.log(`💚 Health: http://localhost:${PORT}/health`);
447 console.log('\n🎯 Try browsing the site and watch which tier serves each file!\n');
448 console.log('💡 Caches clear every 60 seconds - watch files get re-fetched from S3!\n');
449 if (S3_METADATA_BUCKET) {
450 console.log(`✨ Metadata bucket: ${S3_METADATA_BUCKET} (fast updates enabled!)\n`);
451 } else {
452 console.log('⚠️ No metadata bucket - using legacy mode (slower updates)\n');
453 }
454 console.log('Press Ctrl+C to stop\n');
455 console.log('─'.repeat(60));
456 console.log('Request Log:\n');
457 } catch (error: any) {
458 console.error('\n❌ Failed to start server:', error.message);
459 if (error.message.includes('Forbidden')) {
460 console.error('\nS3 connection issue. Check:');
461 console.error(' 1. Bucket exists on S3 service');
462 console.error(' 2. Credentials are correct');
463 console.error(' 3. Permissions allow read/write');
464 }
465 process.exit(1);
466 }
467}
468
469main().catch(console.error);