wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1import { describe, it, expect, beforeEach, afterEach } from 'vitest';
2import { TieredStorage } from '../src/TieredStorage.js';
3import { MemoryStorageTier } from '../src/tiers/MemoryStorageTier.js';
4import { DiskStorageTier } from '../src/tiers/DiskStorageTier.js';
5import { rm } from 'node:fs/promises';
6
7describe('TieredStorage', () => {
8 const testDir = './test-cache';
9
10 afterEach(async () => {
11 await rm(testDir, { recursive: true, force: true });
12 });
13
14 describe('Basic Operations', () => {
15 it('should store and retrieve data', async () => {
16 const storage = new TieredStorage({
17 tiers: {
18 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
19 warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
20 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
21 },
22 });
23
24 await storage.set('test-key', { message: 'Hello, world!' });
25 const result = await storage.get('test-key');
26
27 expect(result).toEqual({ message: 'Hello, world!' });
28 });
29
30 it('should return null for non-existent key', async () => {
31 const storage = new TieredStorage({
32 tiers: {
33 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
34 },
35 });
36
37 const result = await storage.get('non-existent');
38 expect(result).toBeNull();
39 });
40
41 it('should delete data from all tiers', async () => {
42 const storage = new TieredStorage({
43 tiers: {
44 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
45 warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
46 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
47 },
48 });
49
50 await storage.set('test-key', { data: 'test' });
51 await storage.delete('test-key');
52 const result = await storage.get('test-key');
53
54 expect(result).toBeNull();
55 });
56
57 it('should check if key exists', async () => {
58 const storage = new TieredStorage({
59 tiers: {
60 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
61 },
62 });
63
64 await storage.set('test-key', { data: 'test' });
65
66 expect(await storage.exists('test-key')).toBe(true);
67 expect(await storage.exists('non-existent')).toBe(false);
68 });
69 });
70
71 describe('Cascading Write', () => {
72 it('should write to all configured tiers', async () => {
73 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
74 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
75 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
76
77 const storage = new TieredStorage({
78 tiers: { hot, warm, cold },
79 });
80
81 await storage.set('test-key', { data: 'test' });
82
83 // Verify data exists in all tiers
84 expect(await hot.exists('test-key')).toBe(true);
85 expect(await warm.exists('test-key')).toBe(true);
86 expect(await cold.exists('test-key')).toBe(true);
87 });
88
89 it('should skip tiers when specified', async () => {
90 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
91 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
92 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
93
94 const storage = new TieredStorage({
95 tiers: { hot, warm, cold },
96 });
97
98 // Skip hot tier
99 await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
100
101 expect(await hot.exists('test-key')).toBe(false);
102 expect(await warm.exists('test-key')).toBe(true);
103 expect(await cold.exists('test-key')).toBe(true);
104 });
105 });
106
107 describe('Bubbling Read', () => {
108 it('should read from hot tier first', async () => {
109 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
110 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
111 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
112
113 const storage = new TieredStorage({
114 tiers: { hot, warm, cold },
115 });
116
117 await storage.set('test-key', { data: 'test' });
118 const result = await storage.getWithMetadata('test-key');
119
120 expect(result?.source).toBe('hot');
121 expect(result?.data).toEqual({ data: 'test' });
122 });
123
124 it('should fall back to warm tier on hot miss', async () => {
125 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
126 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
127 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
128
129 const storage = new TieredStorage({
130 tiers: { hot, warm, cold },
131 });
132
133 // Write to warm and cold, skip hot
134 await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
135
136 const result = await storage.getWithMetadata('test-key');
137
138 expect(result?.source).toBe('warm');
139 expect(result?.data).toEqual({ data: 'test' });
140 });
141
142 it('should fall back to cold tier on hot and warm miss', async () => {
143 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
144 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
145
146 const storage = new TieredStorage({
147 tiers: { hot, cold },
148 });
149
150 // Write only to cold
151 await cold.set(
152 'test-key',
153 new TextEncoder().encode(JSON.stringify({ data: 'test' })),
154 {
155 key: 'test-key',
156 size: 100,
157 createdAt: new Date(),
158 lastAccessed: new Date(),
159 accessCount: 0,
160 compressed: false,
161 checksum: 'abc123',
162 }
163 );
164
165 const result = await storage.getWithMetadata('test-key');
166
167 expect(result?.source).toBe('cold');
168 expect(result?.data).toEqual({ data: 'test' });
169 });
170 });
171
172 describe('Promotion Strategy', () => {
173 it('should eagerly promote data to upper tiers', async () => {
174 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
175 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
176 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
177
178 const storage = new TieredStorage({
179 tiers: { hot, warm, cold },
180 promotionStrategy: 'eager',
181 });
182
183 // Write only to cold
184 await cold.set(
185 'test-key',
186 new TextEncoder().encode(JSON.stringify({ data: 'test' })),
187 {
188 key: 'test-key',
189 size: 100,
190 createdAt: new Date(),
191 lastAccessed: new Date(),
192 accessCount: 0,
193 compressed: false,
194 checksum: 'abc123',
195 }
196 );
197
198 // Read should promote to hot and warm
199 await storage.get('test-key');
200
201 expect(await hot.exists('test-key')).toBe(true);
202 expect(await warm.exists('test-key')).toBe(true);
203 });
204
205 it('should lazily promote data (not automatic)', async () => {
206 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
207 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
208 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
209
210 const storage = new TieredStorage({
211 tiers: { hot, warm, cold },
212 promotionStrategy: 'lazy',
213 });
214
215 // Write only to cold
216 await cold.set(
217 'test-key',
218 new TextEncoder().encode(JSON.stringify({ data: 'test' })),
219 {
220 key: 'test-key',
221 size: 100,
222 createdAt: new Date(),
223 lastAccessed: new Date(),
224 accessCount: 0,
225 compressed: false,
226 checksum: 'abc123',
227 }
228 );
229
230 // Read should NOT promote to hot and warm
231 await storage.get('test-key');
232
233 expect(await hot.exists('test-key')).toBe(false);
234 expect(await warm.exists('test-key')).toBe(false);
235 });
236 });
237
238 describe('TTL Management', () => {
239 it('should expire data after TTL', async () => {
240 const storage = new TieredStorage({
241 tiers: {
242 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
243 },
244 });
245
246 // Set with 100ms TTL
247 await storage.set('test-key', { data: 'test' }, { ttl: 100 });
248
249 // Should exist immediately
250 expect(await storage.get('test-key')).toEqual({ data: 'test' });
251
252 // Wait for expiration
253 await new Promise((resolve) => setTimeout(resolve, 150));
254
255 // Should be null after expiration
256 expect(await storage.get('test-key')).toBeNull();
257 });
258
259 it('should renew TTL with touch', async () => {
260 const storage = new TieredStorage({
261 tiers: {
262 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
263 },
264 defaultTTL: 100,
265 });
266
267 await storage.set('test-key', { data: 'test' });
268
269 // Wait 50ms
270 await new Promise((resolve) => setTimeout(resolve, 50));
271
272 // Renew TTL
273 await storage.touch('test-key', 200);
274
275 // Wait another 100ms (would have expired without touch)
276 await new Promise((resolve) => setTimeout(resolve, 100));
277
278 // Should still exist
279 expect(await storage.get('test-key')).toEqual({ data: 'test' });
280 });
281 });
282
283 describe('Prefix Invalidation', () => {
284 it('should invalidate all keys with prefix', async () => {
285 const storage = new TieredStorage({
286 tiers: {
287 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
288 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
289 },
290 });
291
292 await storage.set('user:123', { name: 'Alice' });
293 await storage.set('user:456', { name: 'Bob' });
294 await storage.set('post:789', { title: 'Test' });
295
296 const deleted = await storage.invalidate('user:');
297
298 expect(deleted).toBe(2);
299 expect(await storage.exists('user:123')).toBe(false);
300 expect(await storage.exists('user:456')).toBe(false);
301 expect(await storage.exists('post:789')).toBe(true);
302 });
303 });
304
305 describe('Compression', () => {
306 it('should compress data when enabled', async () => {
307 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
308
309 const storage = new TieredStorage({
310 tiers: { cold },
311 compression: true,
312 });
313
314 const largeData = { data: 'x'.repeat(10000) };
315 const result = await storage.set('test-key', largeData);
316
317 // Check that compressed flag is set
318 expect(result.metadata.compressed).toBe(true);
319
320 // Verify data can be retrieved correctly
321 const retrieved = await storage.get('test-key');
322 expect(retrieved).toEqual(largeData);
323 });
324 });
325
326 describe('Bootstrap', () => {
327 it('should bootstrap hot from warm', async () => {
328 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
329 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
330 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
331
332 const storage = new TieredStorage({
333 tiers: { hot, warm, cold },
334 });
335
336 // Write some data
337 await storage.set('key1', { data: '1' });
338 await storage.set('key2', { data: '2' });
339 await storage.set('key3', { data: '3' });
340
341 // Clear hot tier
342 await hot.clear();
343
344 // Bootstrap hot from warm
345 const loaded = await storage.bootstrapHot();
346
347 expect(loaded).toBe(3);
348 expect(await hot.exists('key1')).toBe(true);
349 expect(await hot.exists('key2')).toBe(true);
350 expect(await hot.exists('key3')).toBe(true);
351 });
352
353 it('should bootstrap warm from cold', async () => {
354 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
355 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
356
357 const storage = new TieredStorage({
358 tiers: { warm, cold },
359 });
360
361 // Write directly to cold
362 await cold.set(
363 'key1',
364 new TextEncoder().encode(JSON.stringify({ data: '1' })),
365 {
366 key: 'key1',
367 size: 100,
368 createdAt: new Date(),
369 lastAccessed: new Date(),
370 accessCount: 0,
371 compressed: false,
372 checksum: 'abc',
373 }
374 );
375
376 // Bootstrap warm from cold
377 const loaded = await storage.bootstrapWarm({ limit: 10 });
378
379 expect(loaded).toBe(1);
380 expect(await warm.exists('key1')).toBe(true);
381 });
382 });
383
384 describe('Statistics', () => {
385 it('should return statistics for all tiers', async () => {
386 const storage = new TieredStorage({
387 tiers: {
388 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
389 warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
390 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
391 },
392 });
393
394 await storage.set('key1', { data: 'test1' });
395 await storage.set('key2', { data: 'test2' });
396
397 const stats = await storage.getStats();
398
399 expect(stats.cold.items).toBe(2);
400 expect(stats.warm?.items).toBe(2);
401 expect(stats.hot?.items).toBe(2);
402 });
403 });
404
405 describe('Placement Rules', () => {
406 it('should place index.html in all tiers based on rule', async () => {
407 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
408 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
409 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
410
411 const storage = new TieredStorage({
412 tiers: { hot, warm, cold },
413 placementRules: [
414 { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
415 { pattern: '**', tiers: ['warm', 'cold'] },
416 ],
417 });
418
419 await storage.set('site:abc/index.html', { content: 'hello' });
420
421 expect(await hot.exists('site:abc/index.html')).toBe(true);
422 expect(await warm.exists('site:abc/index.html')).toBe(true);
423 expect(await cold.exists('site:abc/index.html')).toBe(true);
424 });
425
426 it('should skip hot tier for non-matching files', async () => {
427 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
428 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
429 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
430
431 const storage = new TieredStorage({
432 tiers: { hot, warm, cold },
433 placementRules: [
434 { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
435 { pattern: '**', tiers: ['warm', 'cold'] },
436 ],
437 });
438
439 await storage.set('site:abc/about.html', { content: 'about' });
440
441 expect(await hot.exists('site:abc/about.html')).toBe(false);
442 expect(await warm.exists('site:abc/about.html')).toBe(true);
443 expect(await cold.exists('site:abc/about.html')).toBe(true);
444 });
445
446 it('should match directory patterns', async () => {
447 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
448 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
449 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
450
451 const storage = new TieredStorage({
452 tiers: { hot, warm, cold },
453 placementRules: [
454 { pattern: 'assets/**', tiers: ['warm', 'cold'] },
455 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },
456 ],
457 });
458
459 await storage.set('assets/images/logo.png', { data: 'png' });
460 await storage.set('index.html', { data: 'html' });
461
462 // assets/** should skip hot
463 expect(await hot.exists('assets/images/logo.png')).toBe(false);
464 expect(await warm.exists('assets/images/logo.png')).toBe(true);
465
466 // everything else goes to all tiers
467 expect(await hot.exists('index.html')).toBe(true);
468 });
469
470 it('should match file extension patterns', async () => {
471 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
472 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
473 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
474
475 const storage = new TieredStorage({
476 tiers: { hot, warm, cold },
477 placementRules: [
478 { pattern: '**/*.{jpg,png,gif,mp4}', tiers: ['warm', 'cold'] },
479 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },
480 ],
481 });
482
483 await storage.set('site/hero.png', { data: 'image' });
484 await storage.set('site/video.mp4', { data: 'video' });
485 await storage.set('site/index.html', { data: 'html' });
486
487 // Images and video skip hot
488 expect(await hot.exists('site/hero.png')).toBe(false);
489 expect(await hot.exists('site/video.mp4')).toBe(false);
490
491 // HTML goes everywhere
492 expect(await hot.exists('site/index.html')).toBe(true);
493 });
494
495 it('should use first matching rule', async () => {
496 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
497 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
498 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
499
500 const storage = new TieredStorage({
501 tiers: { hot, warm, cold },
502 placementRules: [
503 // Specific rule first
504 { pattern: 'assets/critical.css', tiers: ['hot', 'warm', 'cold'] },
505 // General rule second
506 { pattern: 'assets/**', tiers: ['warm', 'cold'] },
507 { pattern: '**', tiers: ['warm', 'cold'] },
508 ],
509 });
510
511 await storage.set('assets/critical.css', { data: 'css' });
512 await storage.set('assets/style.css', { data: 'css' });
513
514 // critical.css matches first rule -> hot
515 expect(await hot.exists('assets/critical.css')).toBe(true);
516
517 // style.css matches second rule -> no hot
518 expect(await hot.exists('assets/style.css')).toBe(false);
519 });
520
521 it('should allow skipTiers to override placement rules', async () => {
522 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
523 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
524 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
525
526 const storage = new TieredStorage({
527 tiers: { hot, warm, cold },
528 placementRules: [
529 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },
530 ],
531 });
532
533 // Explicit skipTiers should override the rule
534 await storage.set('large-file.bin', { data: 'big' }, { skipTiers: ['hot'] });
535
536 expect(await hot.exists('large-file.bin')).toBe(false);
537 expect(await warm.exists('large-file.bin')).toBe(true);
538 expect(await cold.exists('large-file.bin')).toBe(true);
539 });
540
541 it('should always include cold tier even if not in rule', async () => {
542 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
543 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
544 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
545
546 const storage = new TieredStorage({
547 tiers: { hot, warm, cold },
548 placementRules: [
549 // Rule doesn't include cold (should be auto-added)
550 { pattern: '**', tiers: ['hot', 'warm'] },
551 ],
552 });
553
554 await storage.set('test-key', { data: 'test' });
555
556 expect(await cold.exists('test-key')).toBe(true);
557 });
558
559 it('should write to all tiers when no rules match', async () => {
560 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
561 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
562 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
563
564 const storage = new TieredStorage({
565 tiers: { hot, warm, cold },
566 placementRules: [
567 { pattern: 'specific-pattern-only', tiers: ['warm', 'cold'] },
568 ],
569 });
570
571 // This doesn't match any rule
572 await storage.set('other-key', { data: 'test' });
573
574 expect(await hot.exists('other-key')).toBe(true);
575 expect(await warm.exists('other-key')).toBe(true);
576 expect(await cold.exists('other-key')).toBe(true);
577 });
578 });
579});