wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1import type {
2 TieredStorageConfig,
3 SetOptions,
4 StorageResult,
5 SetResult,
6 StorageMetadata,
7 AllTierStats,
8 StorageSnapshot,
9} from './types/index.js';
10import { compress, decompress } from './utils/compression.js';
11import { defaultSerialize, defaultDeserialize } from './utils/serialization.js';
12import { calculateChecksum } from './utils/checksum.js';
13
14/**
15 * Main orchestrator for tiered storage system.
16 *
17 * @typeParam T - The type of data being stored
18 *
19 * @remarks
20 * Implements a cascading containment model:
21 * - **Write Strategy (Cascading Down):** Write to hot → also writes to warm and cold
22 * - **Read Strategy (Bubbling Up):** Check hot first → if miss, check warm → if miss, check cold
23 * - **Bootstrap Strategy:** Hot can bootstrap from warm, warm can bootstrap from cold
24 *
25 * The cold tier is the source of truth and is required.
26 * Hot and warm tiers are optional performance optimizations.
27 *
28 * @example
29 * ```typescript
30 * const storage = new TieredStorage({
31 * tiers: {
32 * hot: new MemoryStorageTier({ maxSizeBytes: 100 * 1024 * 1024 }), // 100MB
33 * warm: new DiskStorageTier({ directory: './cache' }),
34 * cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),
35 * },
36 * compression: true,
37 * defaultTTL: 14 * 24 * 60 * 60 * 1000, // 14 days
38 * promotionStrategy: 'lazy',
39 * });
40 *
41 * // Store data (cascades to all tiers)
42 * await storage.set('user:123', { name: 'Alice' });
43 *
44 * // Retrieve data (bubbles up from cold → warm → hot)
45 * const user = await storage.get('user:123');
46 *
47 * // Invalidate all keys with prefix
48 * await storage.invalidate('user:');
49 * ```
50 */
51export class TieredStorage<T = unknown> {
52 private serialize: (data: unknown) => Promise<Uint8Array>;
53 private deserialize: (data: Uint8Array) => Promise<unknown>;
54
55 constructor(private config: TieredStorageConfig) {
56 if (!config.tiers.cold) {
57 throw new Error('Cold tier is required');
58 }
59
60 this.serialize = config.serialization?.serialize ?? defaultSerialize;
61 this.deserialize = config.serialization?.deserialize ?? defaultDeserialize;
62 }
63
64 /**
65 * Retrieve data for a key.
66 *
67 * @param key - The key to retrieve
68 * @returns The data, or null if not found or expired
69 *
70 * @remarks
71 * Checks tiers in order: hot → warm → cold.
72 * On cache miss, promotes data to upper tiers based on promotionStrategy.
73 * Automatically handles decompression and deserialization.
74 * Returns null if key doesn't exist or has expired (TTL).
75 */
76 async get(key: string): Promise<T | null> {
77 const result = await this.getWithMetadata(key);
78 return result ? result.data : null;
79 }
80
81 /**
82 * Retrieve data with metadata and source tier information.
83 *
84 * @param key - The key to retrieve
85 * @returns The data, metadata, and source tier, or null if not found
86 *
87 * @remarks
88 * Use this when you need to know:
89 * - Which tier served the data (for observability)
90 * - Metadata like access count, TTL, checksum
91 * - When the data was created/last accessed
92 */
93 async getWithMetadata(key: string): Promise<StorageResult<T> | null> {
94 // 1. Check hot tier first
95 if (this.config.tiers.hot) {
96 const data = await this.config.tiers.hot.get(key);
97 if (data) {
98 const metadata = await this.config.tiers.hot.getMetadata(key);
99 if (!metadata) {
100 await this.delete(key);
101 } else if (this.isExpired(metadata)) {
102 await this.delete(key);
103 return null;
104 } else {
105 await this.updateAccessStats(key, 'hot');
106 return {
107 data: (await this.deserializeData(data)) as T,
108 metadata,
109 source: 'hot',
110 };
111 }
112 }
113 }
114
115 // 2. Check warm tier
116 if (this.config.tiers.warm) {
117 const data = await this.config.tiers.warm.get(key);
118 if (data) {
119 const metadata = await this.config.tiers.warm.getMetadata(key);
120 if (!metadata) {
121 await this.delete(key);
122 } else if (this.isExpired(metadata)) {
123 await this.delete(key);
124 return null;
125 } else {
126 if (this.config.tiers.hot && this.config.promotionStrategy === 'eager') {
127 await this.config.tiers.hot.set(key, data, metadata);
128 }
129
130 await this.updateAccessStats(key, 'warm');
131 return {
132 data: (await this.deserializeData(data)) as T,
133 metadata,
134 source: 'warm',
135 };
136 }
137 }
138 }
139
140 // 3. Check cold tier (source of truth)
141 const data = await this.config.tiers.cold.get(key);
142 if (data) {
143 const metadata = await this.config.tiers.cold.getMetadata(key);
144 if (!metadata) {
145 await this.config.tiers.cold.delete(key);
146 return null;
147 }
148
149 if (this.isExpired(metadata)) {
150 await this.delete(key);
151 return null;
152 }
153
154 // Promote to warm and hot (if configured)
155 if (this.config.promotionStrategy === 'eager') {
156 if (this.config.tiers.warm) {
157 await this.config.tiers.warm.set(key, data, metadata);
158 }
159 if (this.config.tiers.hot) {
160 await this.config.tiers.hot.set(key, data, metadata);
161 }
162 }
163
164 await this.updateAccessStats(key, 'cold');
165 return {
166 data: (await this.deserializeData(data)) as T,
167 metadata,
168 source: 'cold',
169 };
170 }
171
172 return null;
173 }
174
175 /**
176 * Store data with optional configuration.
177 *
178 * @param key - The key to store under
179 * @param data - The data to store
180 * @param options - Optional configuration (TTL, metadata, tier skipping)
181 * @returns Information about what was stored and where
182 *
183 * @remarks
184 * Data cascades down through tiers:
185 * - If written to hot, also written to warm and cold
186 * - If written to warm (hot skipped), also written to cold
187 * - Cold is always written (source of truth)
188 *
189 * Use `skipTiers` to control placement. For example:
190 * - Large files: `skipTiers: ['hot']` to avoid memory bloat
191 * - Critical small files: Write to all tiers for fastest access
192 *
193 * Automatically handles serialization and optional compression.
194 */
195 async set(key: string, data: T, options?: SetOptions): Promise<SetResult> {
196 // 1. Serialize data
197 const serialized = await this.serialize(data);
198
199 // 2. Optionally compress
200 const finalData = this.config.compression ? await compress(serialized) : serialized;
201
202 // 3. Create metadata
203 const metadata = this.createMetadata(key, finalData, options);
204
205 // 4. Write to all tiers (cascading down)
206 const tiersWritten: ('hot' | 'warm' | 'cold')[] = [];
207
208 // Write to hot (if configured and not skipped)
209 if (this.config.tiers.hot && !options?.skipTiers?.includes('hot')) {
210 await this.config.tiers.hot.set(key, finalData, metadata);
211 tiersWritten.push('hot');
212
213 // Hot writes cascade to warm
214 if (this.config.tiers.warm && !options?.skipTiers?.includes('warm')) {
215 await this.config.tiers.warm.set(key, finalData, metadata);
216 tiersWritten.push('warm');
217 }
218 } else if (this.config.tiers.warm && !options?.skipTiers?.includes('warm')) {
219 // Write to warm (if hot skipped)
220 await this.config.tiers.warm.set(key, finalData, metadata);
221 tiersWritten.push('warm');
222 }
223
224 // Always write to cold (source of truth)
225 await this.config.tiers.cold.set(key, finalData, metadata);
226 tiersWritten.push('cold');
227
228 return { key, metadata, tiersWritten };
229 }
230
231 /**
232 * Delete data from all tiers.
233 *
234 * @param key - The key to delete
235 *
236 * @remarks
237 * Deletes from all configured tiers in parallel.
238 * Does not throw if the key doesn't exist.
239 */
240 async delete(key: string): Promise<void> {
241 await Promise.all([
242 this.config.tiers.hot?.delete(key),
243 this.config.tiers.warm?.delete(key),
244 this.config.tiers.cold.delete(key),
245 ]);
246 }
247
248 /**
249 * Check if a key exists in any tier.
250 *
251 * @param key - The key to check
252 * @returns true if the key exists and hasn't expired
253 *
254 * @remarks
255 * Checks tiers in order: hot → warm → cold.
256 * Returns false if key exists but has expired.
257 */
258 async exists(key: string): Promise<boolean> {
259 // Check hot first (fastest)
260 if (this.config.tiers.hot && (await this.config.tiers.hot.exists(key))) {
261 const metadata = await this.config.tiers.hot.getMetadata(key);
262 if (metadata && !this.isExpired(metadata)) {
263 return true;
264 }
265 }
266
267 // Check warm
268 if (this.config.tiers.warm && (await this.config.tiers.warm.exists(key))) {
269 const metadata = await this.config.tiers.warm.getMetadata(key);
270 if (metadata && !this.isExpired(metadata)) {
271 return true;
272 }
273 }
274
275 // Check cold (source of truth)
276 if (await this.config.tiers.cold.exists(key)) {
277 const metadata = await this.config.tiers.cold.getMetadata(key);
278 if (metadata && !this.isExpired(metadata)) {
279 return true;
280 }
281 }
282
283 return false;
284 }
285
286 /**
287 * Renew TTL for a key.
288 *
289 * @param key - The key to touch
290 * @param ttlMs - Optional new TTL in milliseconds (uses default if not provided)
291 *
292 * @remarks
293 * Updates the TTL and lastAccessed timestamp in all tiers.
294 * Useful for implementing "keep alive" behavior for actively used keys.
295 * Does nothing if no TTL is configured.
296 */
297 async touch(key: string, ttlMs?: number): Promise<void> {
298 const ttl = ttlMs ?? this.config.defaultTTL;
299 if (!ttl) return;
300
301 const newTTL = new Date(Date.now() + ttl);
302
303 for (const tier of [this.config.tiers.hot, this.config.tiers.warm, this.config.tiers.cold]) {
304 if (!tier) continue;
305
306 const metadata = await tier.getMetadata(key);
307 if (metadata) {
308 metadata.ttl = newTTL;
309 metadata.lastAccessed = new Date();
310 await tier.setMetadata(key, metadata);
311 }
312 }
313 }
314
315 /**
316 * Invalidate all keys matching a prefix.
317 *
318 * @param prefix - The prefix to match (e.g., 'user:' matches 'user:123', 'user:456')
319 * @returns Number of keys deleted
320 *
321 * @remarks
322 * Useful for bulk invalidation:
323 * - Site invalidation: `invalidate('site:abc:')`
324 * - User invalidation: `invalidate('user:123:')`
325 * - Global invalidation: `invalidate('')` (deletes everything)
326 *
327 * Deletes from all tiers in parallel for efficiency.
328 */
329 async invalidate(prefix: string): Promise<number> {
330 const keysToDelete = new Set<string>();
331
332 // Collect all keys matching prefix from all tiers
333 if (this.config.tiers.hot) {
334 for await (const key of this.config.tiers.hot.listKeys(prefix)) {
335 keysToDelete.add(key);
336 }
337 }
338
339 if (this.config.tiers.warm) {
340 for await (const key of this.config.tiers.warm.listKeys(prefix)) {
341 keysToDelete.add(key);
342 }
343 }
344
345 for await (const key of this.config.tiers.cold.listKeys(prefix)) {
346 keysToDelete.add(key);
347 }
348
349 // Delete from all tiers in parallel
350 const keys = Array.from(keysToDelete);
351
352 await Promise.all([
353 this.config.tiers.hot?.deleteMany(keys),
354 this.config.tiers.warm?.deleteMany(keys),
355 this.config.tiers.cold.deleteMany(keys),
356 ]);
357
358 return keys.length;
359 }
360
361 /**
362 * List all keys, optionally filtered by prefix.
363 *
364 * @param prefix - Optional prefix to filter keys
365 * @returns Async iterator of keys
366 *
367 * @remarks
368 * Returns keys from the cold tier (source of truth).
369 * Memory-efficient - streams keys rather than loading all into memory.
370 *
371 * @example
372 * ```typescript
373 * for await (const key of storage.listKeys('user:')) {
374 * console.log(key);
375 * }
376 * ```
377 */
378 async *listKeys(prefix?: string): AsyncIterableIterator<string> {
379 // List from cold tier (source of truth)
380 for await (const key of this.config.tiers.cold.listKeys(prefix)) {
381 yield key;
382 }
383 }
384
385 /**
386 * Get aggregated statistics across all tiers.
387 *
388 * @returns Statistics including size, item count, hits, misses, hit rate
389 *
390 * @remarks
391 * Useful for monitoring and capacity planning.
392 * Hit rate is calculated as: hits / (hits + misses).
393 */
394 async getStats(): Promise<AllTierStats> {
395 const [hot, warm, cold] = await Promise.all([
396 this.config.tiers.hot?.getStats(),
397 this.config.tiers.warm?.getStats(),
398 this.config.tiers.cold.getStats(),
399 ]);
400
401 const totalHits = (hot?.hits ?? 0) + (warm?.hits ?? 0) + (cold?.hits ?? 0);
402 const totalMisses = (hot?.misses ?? 0) + (warm?.misses ?? 0) + (cold?.misses ?? 0);
403 const hitRate = totalHits + totalMisses > 0 ? totalHits / (totalHits + totalMisses) : 0;
404
405 return {
406 ...(hot && { hot }),
407 ...(warm && { warm }),
408 cold,
409 totalHits,
410 totalMisses,
411 hitRate,
412 };
413 }
414
415 /**
416 * Clear all data from all tiers.
417 *
418 * @remarks
419 * Use with extreme caution! This will delete all data in the entire storage system.
420 * Cannot be undone.
421 */
422 async clear(): Promise<void> {
423 await Promise.all([
424 this.config.tiers.hot?.clear(),
425 this.config.tiers.warm?.clear(),
426 this.config.tiers.cold.clear(),
427 ]);
428 }
429
430 /**
431 * Clear a specific tier.
432 *
433 * @param tier - Which tier to clear
434 *
435 * @remarks
436 * Useful for:
437 * - Clearing hot tier to test warm/cold performance
438 * - Clearing warm tier to force rebuilding from cold
439 * - Clearing cold tier to start fresh (⚠️ loses source of truth!)
440 */
441 async clearTier(tier: 'hot' | 'warm' | 'cold'): Promise<void> {
442 switch (tier) {
443 case 'hot':
444 await this.config.tiers.hot?.clear();
445 break;
446 case 'warm':
447 await this.config.tiers.warm?.clear();
448 break;
449 case 'cold':
450 await this.config.tiers.cold.clear();
451 break;
452 }
453 }
454
455 /**
456 * Export metadata snapshot for backup or migration.
457 *
458 * @returns Snapshot containing all keys, metadata, and statistics
459 *
460 * @remarks
461 * The snapshot includes metadata but not the actual data (data remains in tiers).
462 * Useful for:
463 * - Backup and restore
464 * - Migration between storage systems
465 * - Auditing and compliance
466 */
467 async export(): Promise<StorageSnapshot> {
468 const keys: string[] = [];
469 const metadata: Record<string, StorageMetadata> = {};
470
471 // Export from cold tier (source of truth)
472 for await (const key of this.config.tiers.cold.listKeys()) {
473 keys.push(key);
474 const meta = await this.config.tiers.cold.getMetadata(key);
475 if (meta) {
476 metadata[key] = meta;
477 }
478 }
479
480 const stats = await this.getStats();
481
482 return {
483 version: 1,
484 exportedAt: new Date(),
485 keys,
486 metadata,
487 stats,
488 };
489 }
490
491 /**
492 * Import metadata snapshot.
493 *
494 * @param snapshot - Snapshot to import
495 *
496 * @remarks
497 * Validates version compatibility before importing.
498 * Only imports metadata - assumes data already exists in cold tier.
499 */
500 async import(snapshot: StorageSnapshot): Promise<void> {
501 if (snapshot.version !== 1) {
502 throw new Error(`Unsupported snapshot version: ${snapshot.version}`);
503 }
504
505 // Import metadata into all configured tiers
506 for (const key of snapshot.keys) {
507 const metadata = snapshot.metadata[key];
508 if (!metadata) continue;
509
510 if (this.config.tiers.hot) {
511 await this.config.tiers.hot.setMetadata(key, metadata);
512 }
513
514 if (this.config.tiers.warm) {
515 await this.config.tiers.warm.setMetadata(key, metadata);
516 }
517
518 await this.config.tiers.cold.setMetadata(key, metadata);
519 }
520 }
521
522 /**
523 * Bootstrap hot tier from warm tier.
524 *
525 * @param limit - Optional limit on number of items to load
526 * @returns Number of items loaded
527 *
528 * @remarks
529 * Loads the most frequently accessed items from warm into hot.
530 * Useful for warming up the cache after a restart.
531 * Items are sorted by: accessCount * lastAccessed timestamp (higher is better).
532 */
533 async bootstrapHot(limit?: number): Promise<number> {
534 if (!this.config.tiers.hot || !this.config.tiers.warm) {
535 return 0;
536 }
537
538 let loaded = 0;
539 const keyMetadata: Array<[string, StorageMetadata]> = [];
540
541 // Load metadata for all keys
542 for await (const key of this.config.tiers.warm.listKeys()) {
543 const metadata = await this.config.tiers.warm.getMetadata(key);
544 if (metadata) {
545 keyMetadata.push([key, metadata]);
546 }
547 }
548
549 // Sort by access count * recency (simple scoring)
550 keyMetadata.sort((a, b) => {
551 const scoreA = a[1].accessCount * a[1].lastAccessed.getTime();
552 const scoreB = b[1].accessCount * b[1].lastAccessed.getTime();
553 return scoreB - scoreA;
554 });
555
556 // Load top N keys into hot tier
557 const keysToLoad = limit ? keyMetadata.slice(0, limit) : keyMetadata;
558
559 for (const [key, metadata] of keysToLoad) {
560 const data = await this.config.tiers.warm.get(key);
561 if (data) {
562 await this.config.tiers.hot.set(key, data, metadata);
563 loaded++;
564 }
565 }
566
567 return loaded;
568 }
569
570 /**
571 * Bootstrap warm tier from cold tier.
572 *
573 * @param options - Optional limit and date filter
574 * @returns Number of items loaded
575 *
576 * @remarks
577 * Loads recent items from cold into warm.
578 * Useful for:
579 * - Initial cache population
580 * - Recovering from warm tier failure
581 * - Migrating to a new warm tier implementation
582 */
583 async bootstrapWarm(options?: { limit?: number; sinceDate?: Date }): Promise<number> {
584 if (!this.config.tiers.warm) {
585 return 0;
586 }
587
588 let loaded = 0;
589
590 for await (const key of this.config.tiers.cold.listKeys()) {
591 const metadata = await this.config.tiers.cold.getMetadata(key);
592 if (!metadata) continue;
593
594 // Skip if too old
595 if (options?.sinceDate && metadata.lastAccessed < options.sinceDate) {
596 continue;
597 }
598
599 const data = await this.config.tiers.cold.get(key);
600 if (data) {
601 await this.config.tiers.warm.set(key, data, metadata);
602 loaded++;
603
604 if (options?.limit && loaded >= options.limit) {
605 break;
606 }
607 }
608 }
609
610 return loaded;
611 }
612
613 /**
614 * Check if data has expired based on TTL.
615 */
616 private isExpired(metadata: StorageMetadata): boolean {
617 if (!metadata.ttl) return false;
618 return Date.now() > metadata.ttl.getTime();
619 }
620
621 /**
622 * Update access statistics for a key.
623 */
624 private async updateAccessStats(key: string, tier: 'hot' | 'warm' | 'cold'): Promise<void> {
625 const tierObj =
626 tier === 'hot'
627 ? this.config.tiers.hot
628 : tier === 'warm'
629 ? this.config.tiers.warm
630 : this.config.tiers.cold;
631
632 if (!tierObj) return;
633
634 const metadata = await tierObj.getMetadata(key);
635 if (metadata) {
636 metadata.lastAccessed = new Date();
637 metadata.accessCount++;
638 await tierObj.setMetadata(key, metadata);
639 }
640 }
641
642 /**
643 * Create metadata for new data.
644 */
645 private createMetadata(key: string, data: Uint8Array, options?: SetOptions): StorageMetadata {
646 const now = new Date();
647 const ttl = options?.ttl ?? this.config.defaultTTL;
648
649 const metadata: StorageMetadata = {
650 key,
651 size: data.byteLength,
652 createdAt: now,
653 lastAccessed: now,
654 accessCount: 0,
655 compressed: this.config.compression ?? false,
656 checksum: calculateChecksum(data),
657 };
658
659 if (ttl) {
660 metadata.ttl = new Date(now.getTime() + ttl);
661 }
662
663 if (options?.metadata) {
664 metadata.customMetadata = options.metadata;
665 }
666
667 return metadata;
668 }
669
670 /**
671 * Deserialize data, handling compression automatically.
672 */
673 private async deserializeData(data: Uint8Array): Promise<unknown> {
674 // Decompress if needed (check for gzip magic bytes)
675 const finalData =
676 this.config.compression && data[0] === 0x1f && data[1] === 0x8b
677 ? await decompress(data)
678 : data;
679
680 return this.deserialize(finalData);
681 }
682}