wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1import { describe, it, expect, beforeEach, afterEach } from 'vitest';
2import { TieredStorage } from '../src/TieredStorage.js';
3import { MemoryStorageTier } from '../src/tiers/MemoryStorageTier.js';
4import { DiskStorageTier } from '../src/tiers/DiskStorageTier.js';
5import { rm } from 'node:fs/promises';
6
7describe('TieredStorage', () => {
8 const testDir = './test-cache';
9
10 afterEach(async () => {
11 await rm(testDir, { recursive: true, force: true });
12 });
13
14 describe('Basic Operations', () => {
15 it('should store and retrieve data', async () => {
16 const storage = new TieredStorage({
17 tiers: {
18 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
19 warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
20 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
21 },
22 });
23
24 await storage.set('test-key', { message: 'Hello, world!' });
25 const result = await storage.get('test-key');
26
27 expect(result).toEqual({ message: 'Hello, world!' });
28 });
29
30 it('should return null for non-existent key', async () => {
31 const storage = new TieredStorage({
32 tiers: {
33 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
34 },
35 });
36
37 const result = await storage.get('non-existent');
38 expect(result).toBeNull();
39 });
40
41 it('should delete data from all tiers', async () => {
42 const storage = new TieredStorage({
43 tiers: {
44 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
45 warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
46 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
47 },
48 });
49
50 await storage.set('test-key', { data: 'test' });
51 await storage.delete('test-key');
52 const result = await storage.get('test-key');
53
54 expect(result).toBeNull();
55 });
56
57 it('should check if key exists', async () => {
58 const storage = new TieredStorage({
59 tiers: {
60 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
61 },
62 });
63
64 await storage.set('test-key', { data: 'test' });
65
66 expect(await storage.exists('test-key')).toBe(true);
67 expect(await storage.exists('non-existent')).toBe(false);
68 });
69 });
70
71 describe('Cascading Write', () => {
72 it('should write to all configured tiers', async () => {
73 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
74 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
75 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
76
77 const storage = new TieredStorage({
78 tiers: { hot, warm, cold },
79 });
80
81 await storage.set('test-key', { data: 'test' });
82
83 // Verify data exists in all tiers
84 expect(await hot.exists('test-key')).toBe(true);
85 expect(await warm.exists('test-key')).toBe(true);
86 expect(await cold.exists('test-key')).toBe(true);
87 });
88
89 it('should skip tiers when specified', async () => {
90 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
91 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
92 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
93
94 const storage = new TieredStorage({
95 tiers: { hot, warm, cold },
96 });
97
98 // Skip hot tier
99 await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
100
101 expect(await hot.exists('test-key')).toBe(false);
102 expect(await warm.exists('test-key')).toBe(true);
103 expect(await cold.exists('test-key')).toBe(true);
104 });
105 });
106
107 describe('Bubbling Read', () => {
108 it('should read from hot tier first', async () => {
109 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
110 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
111 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
112
113 const storage = new TieredStorage({
114 tiers: { hot, warm, cold },
115 });
116
117 await storage.set('test-key', { data: 'test' });
118 const result = await storage.getWithMetadata('test-key');
119
120 expect(result?.source).toBe('hot');
121 expect(result?.data).toEqual({ data: 'test' });
122 });
123
124 it('should fall back to warm tier on hot miss', async () => {
125 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
126 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
127 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
128
129 const storage = new TieredStorage({
130 tiers: { hot, warm, cold },
131 });
132
133 // Write to warm and cold, skip hot
134 await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
135
136 const result = await storage.getWithMetadata('test-key');
137
138 expect(result?.source).toBe('warm');
139 expect(result?.data).toEqual({ data: 'test' });
140 });
141
142 it('should fall back to cold tier on hot and warm miss', async () => {
143 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
144 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
145
146 const storage = new TieredStorage({
147 tiers: { hot, cold },
148 });
149
150 // Write only to cold
151 await cold.set(
152 'test-key',
153 new TextEncoder().encode(JSON.stringify({ data: 'test' })),
154 {
155 key: 'test-key',
156 size: 100,
157 createdAt: new Date(),
158 lastAccessed: new Date(),
159 accessCount: 0,
160 compressed: false,
161 checksum: 'abc123',
162 }
163 );
164
165 const result = await storage.getWithMetadata('test-key');
166
167 expect(result?.source).toBe('cold');
168 expect(result?.data).toEqual({ data: 'test' });
169 });
170 });
171
172 describe('Promotion Strategy', () => {
173 it('should eagerly promote data to upper tiers', async () => {
174 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
175 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
176 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
177
178 const storage = new TieredStorage({
179 tiers: { hot, warm, cold },
180 promotionStrategy: 'eager',
181 });
182
183 // Write only to cold
184 await cold.set(
185 'test-key',
186 new TextEncoder().encode(JSON.stringify({ data: 'test' })),
187 {
188 key: 'test-key',
189 size: 100,
190 createdAt: new Date(),
191 lastAccessed: new Date(),
192 accessCount: 0,
193 compressed: false,
194 checksum: 'abc123',
195 }
196 );
197
198 // Read should promote to hot and warm
199 await storage.get('test-key');
200
201 expect(await hot.exists('test-key')).toBe(true);
202 expect(await warm.exists('test-key')).toBe(true);
203 });
204
205 it('should lazily promote data (not automatic)', async () => {
206 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
207 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
208 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
209
210 const storage = new TieredStorage({
211 tiers: { hot, warm, cold },
212 promotionStrategy: 'lazy',
213 });
214
215 // Write only to cold
216 await cold.set(
217 'test-key',
218 new TextEncoder().encode(JSON.stringify({ data: 'test' })),
219 {
220 key: 'test-key',
221 size: 100,
222 createdAt: new Date(),
223 lastAccessed: new Date(),
224 accessCount: 0,
225 compressed: false,
226 checksum: 'abc123',
227 }
228 );
229
230 // Read should NOT promote to hot and warm
231 await storage.get('test-key');
232
233 expect(await hot.exists('test-key')).toBe(false);
234 expect(await warm.exists('test-key')).toBe(false);
235 });
236 });
237
238 describe('TTL Management', () => {
239 it('should expire data after TTL', async () => {
240 const storage = new TieredStorage({
241 tiers: {
242 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
243 },
244 });
245
246 // Set with 100ms TTL
247 await storage.set('test-key', { data: 'test' }, { ttl: 100 });
248
249 // Should exist immediately
250 expect(await storage.get('test-key')).toEqual({ data: 'test' });
251
252 // Wait for expiration
253 await new Promise((resolve) => setTimeout(resolve, 150));
254
255 // Should be null after expiration
256 expect(await storage.get('test-key')).toBeNull();
257 });
258
259 it('should renew TTL with touch', async () => {
260 const storage = new TieredStorage({
261 tiers: {
262 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
263 },
264 defaultTTL: 100,
265 });
266
267 await storage.set('test-key', { data: 'test' });
268
269 // Wait 50ms
270 await new Promise((resolve) => setTimeout(resolve, 50));
271
272 // Renew TTL
273 await storage.touch('test-key', 200);
274
275 // Wait another 100ms (would have expired without touch)
276 await new Promise((resolve) => setTimeout(resolve, 100));
277
278 // Should still exist
279 expect(await storage.get('test-key')).toEqual({ data: 'test' });
280 });
281 });
282
283 describe('Prefix Invalidation', () => {
284 it('should invalidate all keys with prefix', async () => {
285 const storage = new TieredStorage({
286 tiers: {
287 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
288 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
289 },
290 });
291
292 await storage.set('user:123', { name: 'Alice' });
293 await storage.set('user:456', { name: 'Bob' });
294 await storage.set('post:789', { title: 'Test' });
295
296 const deleted = await storage.invalidate('user:');
297
298 expect(deleted).toBe(2);
299 expect(await storage.exists('user:123')).toBe(false);
300 expect(await storage.exists('user:456')).toBe(false);
301 expect(await storage.exists('post:789')).toBe(true);
302 });
303 });
304
305 describe('Compression', () => {
306 it('should compress data when enabled', async () => {
307 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
308
309 const storage = new TieredStorage({
310 tiers: { cold },
311 compression: true,
312 });
313
314 const largeData = { data: 'x'.repeat(10000) };
315 const result = await storage.set('test-key', largeData);
316
317 // Check that compressed flag is set
318 expect(result.metadata.compressed).toBe(true);
319
320 // Verify data can be retrieved correctly
321 const retrieved = await storage.get('test-key');
322 expect(retrieved).toEqual(largeData);
323 });
324 });
325
326 describe('Bootstrap', () => {
327 it('should bootstrap hot from warm', async () => {
328 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
329 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
330 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
331
332 const storage = new TieredStorage({
333 tiers: { hot, warm, cold },
334 });
335
336 // Write some data
337 await storage.set('key1', { data: '1' });
338 await storage.set('key2', { data: '2' });
339 await storage.set('key3', { data: '3' });
340
341 // Clear hot tier
342 await hot.clear();
343
344 // Bootstrap hot from warm
345 const loaded = await storage.bootstrapHot();
346
347 expect(loaded).toBe(3);
348 expect(await hot.exists('key1')).toBe(true);
349 expect(await hot.exists('key2')).toBe(true);
350 expect(await hot.exists('key3')).toBe(true);
351 });
352
353 it('should bootstrap warm from cold', async () => {
354 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
355 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
356
357 const storage = new TieredStorage({
358 tiers: { warm, cold },
359 });
360
361 // Write directly to cold
362 await cold.set(
363 'key1',
364 new TextEncoder().encode(JSON.stringify({ data: '1' })),
365 {
366 key: 'key1',
367 size: 100,
368 createdAt: new Date(),
369 lastAccessed: new Date(),
370 accessCount: 0,
371 compressed: false,
372 checksum: 'abc',
373 }
374 );
375
376 // Bootstrap warm from cold
377 const loaded = await storage.bootstrapWarm({ limit: 10 });
378
379 expect(loaded).toBe(1);
380 expect(await warm.exists('key1')).toBe(true);
381 });
382 });
383
384 describe('Statistics', () => {
385 it('should return statistics for all tiers', async () => {
386 const storage = new TieredStorage({
387 tiers: {
388 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
389 warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
390 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
391 },
392 });
393
394 await storage.set('key1', { data: 'test1' });
395 await storage.set('key2', { data: 'test2' });
396
397 const stats = await storage.getStats();
398
399 expect(stats.cold.items).toBe(2);
400 expect(stats.warm?.items).toBe(2);
401 expect(stats.hot?.items).toBe(2);
402 });
403 });
404});