/** * Example usage of the tiered-storage library * * Run with: bun run example * * Note: This example uses S3 for cold storage. You'll need to configure * AWS credentials and an S3 bucket in .env (see .env.example) */ import { TieredStorage, MemoryStorageTier, DiskStorageTier, S3StorageTier } from './src/index.js'; import { rm } from 'node:fs/promises'; // Configuration from environment variables const S3_BUCKET = process.env.S3_BUCKET || 'tiered-storage-example'; const S3_REGION = process.env.S3_REGION || 'us-east-1'; const S3_ENDPOINT = process.env.S3_ENDPOINT; const S3_FORCE_PATH_STYLE = process.env.S3_FORCE_PATH_STYLE !== 'false'; // Default true const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID; const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY; async function basicExample() { console.log('\n=== Basic Example ===\n'); const storage = new TieredStorage({ tiers: { hot: new MemoryStorageTier({ maxSizeBytes: 10 * 1024 * 1024 }), // 10MB warm: new DiskStorageTier({ directory: './example-cache/basic/warm' }), cold: new S3StorageTier({ bucket: S3_BUCKET, region: S3_REGION, endpoint: S3_ENDPOINT, forcePathStyle: S3_FORCE_PATH_STYLE, credentials: AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY ? { accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY, } : undefined, prefix: 'example/basic/', }), }, compression: true, defaultTTL: 60 * 60 * 1000, // 1 hour }); // Store some data console.log('Storing user data...'); await storage.set('user:alice', { name: 'Alice', email: 'alice@example.com', role: 'admin', }); await storage.set('user:bob', { name: 'Bob', email: 'bob@example.com', role: 'user', }); // Retrieve with metadata const result = await storage.getWithMetadata('user:alice'); if (result) { console.log(`Retrieved user:alice from ${result.source} tier:`); console.log(result.data); console.log('Metadata:', { size: result.metadata.size, compressed: result.metadata.compressed, accessCount: result.metadata.accessCount, }); } // Get statistics const stats = await storage.getStats(); console.log('\nStorage Statistics:'); console.log(`Hot tier: ${stats.hot?.items} items, ${stats.hot?.bytes} bytes`); console.log(`Warm tier: ${stats.warm?.items} items, ${stats.warm?.bytes} bytes`); console.log(`Cold tier (S3): ${stats.cold.items} items, ${stats.cold.bytes} bytes`); console.log(`Hit rate: ${(stats.hitRate * 100).toFixed(2)}%`); // List all keys with prefix console.log('\nAll user keys:'); for await (const key of storage.listKeys('user:')) { console.log(` - ${key}`); } // Invalidate by prefix console.log('\nInvalidating all user keys...'); const deleted = await storage.invalidate('user:'); console.log(`Deleted ${deleted} keys`); } async function staticSiteHostingExample() { console.log('\n=== Static Site Hosting Example (wisp.place pattern) ===\n'); const storage = new TieredStorage({ tiers: { hot: new MemoryStorageTier({ maxSizeBytes: 50 * 1024 * 1024, // 50MB maxItems: 500, }), warm: new DiskStorageTier({ directory: './example-cache/sites/warm', maxSizeBytes: 1024 * 1024 * 1024, // 1GB }), cold: new S3StorageTier({ bucket: S3_BUCKET, region: S3_REGION, endpoint: S3_ENDPOINT, forcePathStyle: S3_FORCE_PATH_STYLE, credentials: AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY ? { accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY, } : undefined, prefix: 'example/sites/', }), }, compression: true, defaultTTL: 14 * 24 * 60 * 60 * 1000, // 14 days promotionStrategy: 'lazy', // Don't auto-promote large files }); const siteId = 'did:plc:abc123'; const siteName = 'tiered-cache-demo'; console.log('Loading real static site from example-site/...\n'); // Load actual site files const { readFile } = await import('node:fs/promises'); const files = [ { name: 'index.html', skipTiers: [], mimeType: 'text/html' }, { name: 'about.html', skipTiers: ['hot'], mimeType: 'text/html' }, { name: 'docs.html', skipTiers: ['hot'], mimeType: 'text/html' }, { name: 'style.css', skipTiers: ['hot'], mimeType: 'text/css' }, { name: 'script.js', skipTiers: ['hot'], mimeType: 'application/javascript' }, ]; console.log('Storing site files with selective tier placement:\n'); for (const file of files) { const content = await readFile(`./example-site/${file.name}`, 'utf-8'); const key = `${siteId}/${siteName}/${file.name}`; await storage.set(key, content, { skipTiers: file.skipTiers as ('hot' | 'warm')[], metadata: { mimeType: file.mimeType }, }); const tierInfo = file.skipTiers.length === 0 ? 'hot + warm + cold (S3)' : `warm + cold (S3) - skipped ${file.skipTiers.join(', ')}`; const sizeKB = (content.length / 1024).toFixed(2); console.log(`✓ ${file.name} (${sizeKB} KB) → ${tierInfo}`); } // Check where each file is served from console.log('\nServing files (checking which tier):'); for (const file of files) { const result = await storage.getWithMetadata(`${siteId}/${siteName}/${file.name}`); if (result) { const sizeKB = (result.metadata.size / 1024).toFixed(2); console.log(` ${file.name}: served from ${result.source} (${sizeKB} KB)`); } } // Show hot tier only has index.html console.log('\nHot tier contents (should only contain index.html):'); const stats = await storage.getStats(); console.log(` Items: ${stats.hot?.items}`); console.log(` Size: ${((stats.hot?.bytes ?? 0) / 1024).toFixed(2)} KB`); console.log(` Files: index.html only`); console.log('\nWarm tier contents (all site files):'); console.log(` Items: ${stats.warm?.items}`); console.log(` Size: ${((stats.warm?.bytes ?? 0) / 1024).toFixed(2)} KB`); console.log(` Files: all ${files.length} files`); // Demonstrate accessing a page console.log('\nSimulating page request for about.html:'); const aboutPage = await storage.getWithMetadata(`${siteId}/${siteName}/about.html`); if (aboutPage) { console.log(` Source: ${aboutPage.source} tier`); console.log(` Access count: ${aboutPage.metadata.accessCount}`); console.log(` Preview: ${aboutPage.data.toString().slice(0, 100)}...`); } // Invalidate entire site console.log(`\nInvalidating entire site: ${siteId}/${siteName}/`); const deleted = await storage.invalidate(`${siteId}/${siteName}/`); console.log(`Deleted ${deleted} files from all tiers`); } async function bootstrapExample() { console.log('\n=== Bootstrap Example ===\n'); const hot = new MemoryStorageTier({ maxSizeBytes: 10 * 1024 * 1024 }); const warm = new DiskStorageTier({ directory: './example-cache/bootstrap/warm' }); const cold = new S3StorageTier({ bucket: S3_BUCKET, region: S3_REGION, endpoint: S3_ENDPOINT, forcePathStyle: S3_FORCE_PATH_STYLE, credentials: AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY ? { accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY, } : undefined, prefix: 'example/bootstrap/', }); const storage = new TieredStorage({ tiers: { hot, warm, cold }, }); // Populate with some data console.log('Populating storage with test data...'); for (let i = 0; i < 10; i++) { await storage.set(`item:${i}`, { id: i, name: `Item ${i}`, description: `This is item number ${i}`, }); } // Access some items to build up access counts console.log('Accessing some items to simulate usage patterns...'); await storage.get('item:0'); // Most accessed await storage.get('item:0'); await storage.get('item:0'); await storage.get('item:1'); // Second most accessed await storage.get('item:1'); await storage.get('item:2'); // Third most accessed // Clear hot tier to simulate server restart console.log('\nSimulating server restart (clearing hot tier)...'); await hot.clear(); let hotStats = await hot.getStats(); console.log(`Hot tier after clear: ${hotStats.items} items`); // Bootstrap hot from warm (loads most accessed items) console.log('\nBootstrapping hot tier from warm (loading top 3 items)...'); const loaded = await storage.bootstrapHot(3); console.log(`Loaded ${loaded} items into hot tier`); hotStats = await hot.getStats(); console.log(`Hot tier after bootstrap: ${hotStats.items} items`); // Verify the right items were loaded console.log('\nVerifying loaded items are served from hot:'); for (let i = 0; i < 3; i++) { const result = await storage.getWithMetadata(`item:${i}`); console.log(` item:${i}: ${result?.source}`); } // Cleanup this example's data console.log('\nCleaning up bootstrap example data...'); await storage.invalidate('item:'); } async function promotionStrategyExample() { console.log('\n=== Promotion Strategy Example ===\n'); // Lazy promotion (default) console.log('Testing LAZY promotion:'); const lazyStorage = new TieredStorage({ tiers: { hot: new MemoryStorageTier({ maxSizeBytes: 10 * 1024 * 1024 }), warm: new DiskStorageTier({ directory: './example-cache/promo-lazy/warm' }), cold: new S3StorageTier({ bucket: S3_BUCKET, region: S3_REGION, endpoint: S3_ENDPOINT, forcePathStyle: S3_FORCE_PATH_STYLE, credentials: AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY ? { accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY, } : undefined, prefix: 'example/promo-lazy/', }), }, promotionStrategy: 'lazy', }); // Write data and clear hot await lazyStorage.set('test:lazy', { value: 'lazy test' }); await lazyStorage.clearTier('hot'); // Read from cold (should NOT auto-promote to hot) const lazyResult = await lazyStorage.getWithMetadata('test:lazy'); console.log(` First read served from: ${lazyResult?.source}`); const lazyResult2 = await lazyStorage.getWithMetadata('test:lazy'); console.log(` Second read served from: ${lazyResult2?.source} (lazy = no auto-promotion)`); // Eager promotion console.log('\nTesting EAGER promotion:'); const eagerStorage = new TieredStorage({ tiers: { hot: new MemoryStorageTier({ maxSizeBytes: 10 * 1024 * 1024 }), warm: new DiskStorageTier({ directory: './example-cache/promo-eager/warm' }), cold: new S3StorageTier({ bucket: S3_BUCKET, region: S3_REGION, endpoint: S3_ENDPOINT, forcePathStyle: S3_FORCE_PATH_STYLE, credentials: AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY ? { accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY, } : undefined, prefix: 'example/promo-eager/', }), }, promotionStrategy: 'eager', }); // Write data and clear hot await eagerStorage.set('test:eager', { value: 'eager test' }); await eagerStorage.clearTier('hot'); // Read from cold (SHOULD auto-promote to hot) const eagerResult = await eagerStorage.getWithMetadata('test:eager'); console.log(` First read served from: ${eagerResult?.source}`); const eagerResult2 = await eagerStorage.getWithMetadata('test:eager'); console.log(` Second read served from: ${eagerResult2?.source} (eager = promoted to hot)`); // Cleanup await lazyStorage.invalidate('test:'); await eagerStorage.invalidate('test:'); } async function cleanup() { console.log('\n=== Cleanup ===\n'); console.log('Removing example cache directories...'); await rm('./example-cache', { recursive: true, force: true }); console.log('✓ Local cache directories removed'); console.log('\nNote: S3 objects with prefix "example/" remain in bucket'); console.log(' (remove manually if needed)'); } async function main() { console.log('╔════════════════════════════════════════════════╗'); console.log('║ Tiered Storage Library - Usage Examples ║'); console.log('║ Cold Tier: S3 (or S3-compatible storage) ║'); console.log('╚════════════════════════════════════════════════╝'); // Check for S3 configuration if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY) { console.log('\n⚠️ Warning: AWS credentials not configured'); console.log(' Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in .env'); console.log(' (See .env.example for configuration options)\n'); } console.log('\nConfiguration:'); console.log(` S3 Bucket: ${S3_BUCKET}`); console.log(` S3 Region: ${S3_REGION}`); console.log(` S3 Endpoint: ${S3_ENDPOINT || '(default AWS S3)'}`); console.log(` Force Path Style: ${S3_FORCE_PATH_STYLE}`); console.log(` Credentials: ${AWS_ACCESS_KEY_ID ? '✓ Configured' : '✗ Not configured (using IAM role)'}`); try { // Test S3 connection first console.log('\nTesting S3 connection...'); const testStorage = new S3StorageTier({ bucket: S3_BUCKET, region: S3_REGION, endpoint: S3_ENDPOINT, forcePathStyle: S3_FORCE_PATH_STYLE, credentials: AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY ? { accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY, } : undefined, prefix: 'test/', }); try { await testStorage.set('connection-test', new TextEncoder().encode('test'), { key: 'connection-test', size: 4, createdAt: new Date(), lastAccessed: new Date(), accessCount: 0, compressed: false, checksum: 'test', }); console.log('✓ S3 connection successful!\n'); await testStorage.delete('connection-test'); } catch (error: any) { console.error('✗ S3 connection failed:', error.message); console.error('\nPossible issues:'); console.error(' 1. Check that the bucket exists on your S3 service'); console.error(' 2. Verify credentials have read/write permissions'); console.error(' 3. Confirm the endpoint URL is correct'); console.error(' 4. Try setting S3_REGION to a different value (e.g., "us-east-1" or "auto")'); console.error('\nSkipping examples due to S3 connection error.\n'); return; } await basicExample(); await staticSiteHostingExample(); await bootstrapExample(); await promotionStrategyExample(); } catch (error: any) { console.error('\n❌ Error:', error.message); if (error.name === 'NoSuchBucket') { console.error(`\n The S3 bucket "${S3_BUCKET}" does not exist.`); console.error(' Create it first or set S3_BUCKET in .env to an existing bucket.\n'); } } finally { await cleanup(); } console.log('\n✅ All examples completed successfully!'); console.log('\nTry modifying this file to experiment with different patterns.'); } main().catch(console.error);