wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1/**
2 * Example usage of the tiered-storage library
3 *
4 * Run with: bun run example
5 *
6 * Note: This example uses S3 for cold storage. You'll need to configure
7 * AWS credentials and an S3 bucket in .env (see .env.example)
8 */
9
10import { TieredStorage, MemoryStorageTier, DiskStorageTier, S3StorageTier } from './src/index.js';
11import { rm } from 'node:fs/promises';
12
13// Configuration from environment variables
14const S3_BUCKET = process.env.S3_BUCKET || 'tiered-storage-example';
15const S3_REGION = process.env.S3_REGION || 'us-east-1';
16const S3_ENDPOINT = process.env.S3_ENDPOINT;
17const S3_FORCE_PATH_STYLE = process.env.S3_FORCE_PATH_STYLE !== 'false'; // Default true
18const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
19const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
20
21async function basicExample() {
22 console.log('\n=== Basic Example ===\n');
23
24 const storage = new TieredStorage({
25 tiers: {
26 hot: new MemoryStorageTier({ maxSizeBytes: 10 * 1024 * 1024 }), // 10MB
27 warm: new DiskStorageTier({ directory: './example-cache/basic/warm' }),
28 cold: new S3StorageTier({
29 bucket: S3_BUCKET,
30 region: S3_REGION,
31 endpoint: S3_ENDPOINT,
32 forcePathStyle: S3_FORCE_PATH_STYLE,
33 credentials:
34 AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY
35 ? {
36 accessKeyId: AWS_ACCESS_KEY_ID,
37 secretAccessKey: AWS_SECRET_ACCESS_KEY,
38 }
39 : undefined,
40 prefix: 'example/basic/',
41 }),
42 },
43 compression: true,
44 defaultTTL: 60 * 60 * 1000, // 1 hour
45 });
46
47 // Store some data
48 console.log('Storing user data...');
49 await storage.set('user:alice', {
50 name: 'Alice',
51 email: 'alice@example.com',
52 role: 'admin',
53 });
54
55 await storage.set('user:bob', {
56 name: 'Bob',
57 email: 'bob@example.com',
58 role: 'user',
59 });
60
61 // Retrieve with metadata
62 const result = await storage.getWithMetadata('user:alice');
63 if (result) {
64 console.log(`Retrieved user:alice from ${result.source} tier:`);
65 console.log(result.data);
66 console.log('Metadata:', {
67 size: result.metadata.size,
68 compressed: result.metadata.compressed,
69 accessCount: result.metadata.accessCount,
70 });
71 }
72
73 // Get statistics
74 const stats = await storage.getStats();
75 console.log('\nStorage Statistics:');
76 console.log(`Hot tier: ${stats.hot?.items} items, ${stats.hot?.bytes} bytes`);
77 console.log(`Warm tier: ${stats.warm?.items} items, ${stats.warm?.bytes} bytes`);
78 console.log(`Cold tier (S3): ${stats.cold.items} items, ${stats.cold.bytes} bytes`);
79 console.log(`Hit rate: ${(stats.hitRate * 100).toFixed(2)}%`);
80
81 // List all keys with prefix
82 console.log('\nAll user keys:');
83 for await (const key of storage.listKeys('user:')) {
84 console.log(` - ${key}`);
85 }
86
87 // Invalidate by prefix
88 console.log('\nInvalidating all user keys...');
89 const deleted = await storage.invalidate('user:');
90 console.log(`Deleted ${deleted} keys`);
91}
92
93async function staticSiteHostingExample() {
94 console.log('\n=== Static Site Hosting Example (wisp.place pattern) ===\n');
95
96 const storage = new TieredStorage({
97 tiers: {
98 hot: new MemoryStorageTier({
99 maxSizeBytes: 50 * 1024 * 1024, // 50MB
100 maxItems: 500,
101 }),
102 warm: new DiskStorageTier({
103 directory: './example-cache/sites/warm',
104 maxSizeBytes: 1024 * 1024 * 1024, // 1GB
105 }),
106 cold: new S3StorageTier({
107 bucket: S3_BUCKET,
108 region: S3_REGION,
109 endpoint: S3_ENDPOINT,
110 forcePathStyle: S3_FORCE_PATH_STYLE,
111 credentials:
112 AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY
113 ? {
114 accessKeyId: AWS_ACCESS_KEY_ID,
115 secretAccessKey: AWS_SECRET_ACCESS_KEY,
116 }
117 : undefined,
118 prefix: 'example/sites/',
119 }),
120 },
121 compression: true,
122 defaultTTL: 14 * 24 * 60 * 60 * 1000, // 14 days
123 promotionStrategy: 'lazy', // Don't auto-promote large files
124 });
125
126 const siteId = 'did:plc:abc123';
127 const siteName = 'tiered-cache-demo';
128
129 console.log('Loading real static site from example-site/...\n');
130
131 // Load actual site files
132 const { readFile } = await import('node:fs/promises');
133
134 const files = [
135 { name: 'index.html', skipTiers: [], mimeType: 'text/html' },
136 { name: 'about.html', skipTiers: ['hot'], mimeType: 'text/html' },
137 { name: 'docs.html', skipTiers: ['hot'], mimeType: 'text/html' },
138 { name: 'style.css', skipTiers: ['hot'], mimeType: 'text/css' },
139 { name: 'script.js', skipTiers: ['hot'], mimeType: 'application/javascript' },
140 ];
141
142 console.log('Storing site files with selective tier placement:\n');
143
144 for (const file of files) {
145 const content = await readFile(`./example-site/${file.name}`, 'utf-8');
146 const key = `${siteId}/${siteName}/${file.name}`;
147
148 await storage.set(key, content, {
149 skipTiers: file.skipTiers as ('hot' | 'warm')[],
150 metadata: { mimeType: file.mimeType },
151 });
152
153 const tierInfo =
154 file.skipTiers.length === 0
155 ? 'hot + warm + cold (S3)'
156 : `warm + cold (S3) - skipped ${file.skipTiers.join(', ')}`;
157 const sizeKB = (content.length / 1024).toFixed(2);
158 console.log(`✓ ${file.name} (${sizeKB} KB) → ${tierInfo}`);
159 }
160
161 // Check where each file is served from
162 console.log('\nServing files (checking which tier):');
163 for (const file of files) {
164 const result = await storage.getWithMetadata(`${siteId}/${siteName}/${file.name}`);
165 if (result) {
166 const sizeKB = (result.metadata.size / 1024).toFixed(2);
167 console.log(` ${file.name}: served from ${result.source} (${sizeKB} KB)`);
168 }
169 }
170
171 // Show hot tier only has index.html
172 console.log('\nHot tier contents (should only contain index.html):');
173 const stats = await storage.getStats();
174 console.log(` Items: ${stats.hot?.items}`);
175 console.log(` Size: ${((stats.hot?.bytes ?? 0) / 1024).toFixed(2)} KB`);
176 console.log(` Files: index.html only`);
177
178 console.log('\nWarm tier contents (all site files):');
179 console.log(` Items: ${stats.warm?.items}`);
180 console.log(` Size: ${((stats.warm?.bytes ?? 0) / 1024).toFixed(2)} KB`);
181 console.log(` Files: all ${files.length} files`);
182
183 // Demonstrate accessing a page
184 console.log('\nSimulating page request for about.html:');
185 const aboutPage = await storage.getWithMetadata(`${siteId}/${siteName}/about.html`);
186 if (aboutPage) {
187 console.log(` Source: ${aboutPage.source} tier`);
188 console.log(` Access count: ${aboutPage.metadata.accessCount}`);
189 console.log(` Preview: ${aboutPage.data.toString().slice(0, 100)}...`);
190 }
191
192 // Invalidate entire site
193 console.log(`\nInvalidating entire site: ${siteId}/${siteName}/`);
194 const deleted = await storage.invalidate(`${siteId}/${siteName}/`);
195 console.log(`Deleted ${deleted} files from all tiers`);
196}
197
198async function bootstrapExample() {
199 console.log('\n=== Bootstrap Example ===\n');
200
201 const hot = new MemoryStorageTier({ maxSizeBytes: 10 * 1024 * 1024 });
202 const warm = new DiskStorageTier({ directory: './example-cache/bootstrap/warm' });
203 const cold = new S3StorageTier({
204 bucket: S3_BUCKET,
205 region: S3_REGION,
206 endpoint: S3_ENDPOINT,
207 forcePathStyle: S3_FORCE_PATH_STYLE,
208 credentials:
209 AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY
210 ? {
211 accessKeyId: AWS_ACCESS_KEY_ID,
212 secretAccessKey: AWS_SECRET_ACCESS_KEY,
213 }
214 : undefined,
215 prefix: 'example/bootstrap/',
216 });
217
218 const storage = new TieredStorage({
219 tiers: { hot, warm, cold },
220 });
221
222 // Populate with some data
223 console.log('Populating storage with test data...');
224 for (let i = 0; i < 10; i++) {
225 await storage.set(`item:${i}`, {
226 id: i,
227 name: `Item ${i}`,
228 description: `This is item number ${i}`,
229 });
230 }
231
232 // Access some items to build up access counts
233 console.log('Accessing some items to simulate usage patterns...');
234 await storage.get('item:0'); // Most accessed
235 await storage.get('item:0');
236 await storage.get('item:0');
237 await storage.get('item:1'); // Second most accessed
238 await storage.get('item:1');
239 await storage.get('item:2'); // Third most accessed
240
241 // Clear hot tier to simulate server restart
242 console.log('\nSimulating server restart (clearing hot tier)...');
243 await hot.clear();
244
245 let hotStats = await hot.getStats();
246 console.log(`Hot tier after clear: ${hotStats.items} items`);
247
248 // Bootstrap hot from warm (loads most accessed items)
249 console.log('\nBootstrapping hot tier from warm (loading top 3 items)...');
250 const loaded = await storage.bootstrapHot(3);
251 console.log(`Loaded ${loaded} items into hot tier`);
252
253 hotStats = await hot.getStats();
254 console.log(`Hot tier after bootstrap: ${hotStats.items} items`);
255
256 // Verify the right items were loaded
257 console.log('\nVerifying loaded items are served from hot:');
258 for (let i = 0; i < 3; i++) {
259 const result = await storage.getWithMetadata(`item:${i}`);
260 console.log(` item:${i}: ${result?.source}`);
261 }
262
263 // Cleanup this example's data
264 console.log('\nCleaning up bootstrap example data...');
265 await storage.invalidate('item:');
266}
267
268async function promotionStrategyExample() {
269 console.log('\n=== Promotion Strategy Example ===\n');
270
271 // Lazy promotion (default)
272 console.log('Testing LAZY promotion:');
273 const lazyStorage = new TieredStorage({
274 tiers: {
275 hot: new MemoryStorageTier({ maxSizeBytes: 10 * 1024 * 1024 }),
276 warm: new DiskStorageTier({ directory: './example-cache/promo-lazy/warm' }),
277 cold: new S3StorageTier({
278 bucket: S3_BUCKET,
279 region: S3_REGION,
280 endpoint: S3_ENDPOINT,
281 forcePathStyle: S3_FORCE_PATH_STYLE,
282 credentials:
283 AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY
284 ? {
285 accessKeyId: AWS_ACCESS_KEY_ID,
286 secretAccessKey: AWS_SECRET_ACCESS_KEY,
287 }
288 : undefined,
289 prefix: 'example/promo-lazy/',
290 }),
291 },
292 promotionStrategy: 'lazy',
293 });
294
295 // Write data and clear hot
296 await lazyStorage.set('test:lazy', { value: 'lazy test' });
297 await lazyStorage.clearTier('hot');
298
299 // Read from cold (should NOT auto-promote to hot)
300 const lazyResult = await lazyStorage.getWithMetadata('test:lazy');
301 console.log(` First read served from: ${lazyResult?.source}`);
302
303 const lazyResult2 = await lazyStorage.getWithMetadata('test:lazy');
304 console.log(` Second read served from: ${lazyResult2?.source} (lazy = no auto-promotion)`);
305
306 // Eager promotion
307 console.log('\nTesting EAGER promotion:');
308 const eagerStorage = new TieredStorage({
309 tiers: {
310 hot: new MemoryStorageTier({ maxSizeBytes: 10 * 1024 * 1024 }),
311 warm: new DiskStorageTier({ directory: './example-cache/promo-eager/warm' }),
312 cold: new S3StorageTier({
313 bucket: S3_BUCKET,
314 region: S3_REGION,
315 endpoint: S3_ENDPOINT,
316 forcePathStyle: S3_FORCE_PATH_STYLE,
317 credentials:
318 AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY
319 ? {
320 accessKeyId: AWS_ACCESS_KEY_ID,
321 secretAccessKey: AWS_SECRET_ACCESS_KEY,
322 }
323 : undefined,
324 prefix: 'example/promo-eager/',
325 }),
326 },
327 promotionStrategy: 'eager',
328 });
329
330 // Write data and clear hot
331 await eagerStorage.set('test:eager', { value: 'eager test' });
332 await eagerStorage.clearTier('hot');
333
334 // Read from cold (SHOULD auto-promote to hot)
335 const eagerResult = await eagerStorage.getWithMetadata('test:eager');
336 console.log(` First read served from: ${eagerResult?.source}`);
337
338 const eagerResult2 = await eagerStorage.getWithMetadata('test:eager');
339 console.log(` Second read served from: ${eagerResult2?.source} (eager = promoted to hot)`);
340
341 // Cleanup
342 await lazyStorage.invalidate('test:');
343 await eagerStorage.invalidate('test:');
344}
345
346async function cleanup() {
347 console.log('\n=== Cleanup ===\n');
348 console.log('Removing example cache directories...');
349 await rm('./example-cache', { recursive: true, force: true });
350 console.log('✓ Local cache directories removed');
351 console.log('\nNote: S3 objects with prefix "example/" remain in bucket');
352 console.log(' (remove manually if needed)');
353}
354
355async function main() {
356 console.log('╔════════════════════════════════════════════════╗');
357 console.log('║ Tiered Storage Library - Usage Examples ║');
358 console.log('║ Cold Tier: S3 (or S3-compatible storage) ║');
359 console.log('╚════════════════════════════════════════════════╝');
360
361 // Check for S3 configuration
362 if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY) {
363 console.log('\n⚠️ Warning: AWS credentials not configured');
364 console.log(' Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in .env');
365 console.log(' (See .env.example for configuration options)\n');
366 }
367
368 console.log('\nConfiguration:');
369 console.log(` S3 Bucket: ${S3_BUCKET}`);
370 console.log(` S3 Region: ${S3_REGION}`);
371 console.log(` S3 Endpoint: ${S3_ENDPOINT || '(default AWS S3)'}`);
372 console.log(` Force Path Style: ${S3_FORCE_PATH_STYLE}`);
373 console.log(` Credentials: ${AWS_ACCESS_KEY_ID ? '✓ Configured' : '✗ Not configured (using IAM role)'}`);
374
375 try {
376 // Test S3 connection first
377 console.log('\nTesting S3 connection...');
378 const testStorage = new S3StorageTier({
379 bucket: S3_BUCKET,
380 region: S3_REGION,
381 endpoint: S3_ENDPOINT,
382 forcePathStyle: S3_FORCE_PATH_STYLE,
383 credentials:
384 AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY
385 ? {
386 accessKeyId: AWS_ACCESS_KEY_ID,
387 secretAccessKey: AWS_SECRET_ACCESS_KEY,
388 }
389 : undefined,
390 prefix: 'test/',
391 });
392
393 try {
394 await testStorage.set('connection-test', new TextEncoder().encode('test'), {
395 key: 'connection-test',
396 size: 4,
397 createdAt: new Date(),
398 lastAccessed: new Date(),
399 accessCount: 0,
400 compressed: false,
401 checksum: 'test',
402 });
403 console.log('✓ S3 connection successful!\n');
404 await testStorage.delete('connection-test');
405 } catch (error: any) {
406 console.error('✗ S3 connection failed:', error.message);
407 console.error('\nPossible issues:');
408 console.error(' 1. Check that the bucket exists on your S3 service');
409 console.error(' 2. Verify credentials have read/write permissions');
410 console.error(' 3. Confirm the endpoint URL is correct');
411 console.error(' 4. Try setting S3_REGION to a different value (e.g., "us-east-1" or "auto")');
412 console.error('\nSkipping examples due to S3 connection error.\n');
413 return;
414 }
415
416 await basicExample();
417 await staticSiteHostingExample();
418 await bootstrapExample();
419 await promotionStrategyExample();
420 } catch (error: any) {
421 console.error('\n❌ Error:', error.message);
422 if (error.name === 'NoSuchBucket') {
423 console.error(`\n The S3 bucket "${S3_BUCKET}" does not exist.`);
424 console.error(' Create it first or set S3_BUCKET in .env to an existing bucket.\n');
425 }
426 } finally {
427 await cleanup();
428 }
429
430 console.log('\n✅ All examples completed successfully!');
431 console.log('\nTry modifying this file to experiment with different patterns.');
432}
433
434main().catch(console.error);