wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1import { readFile, writeFile, unlink, readdir, stat, mkdir, rm, rename } from 'node:fs/promises';
2import { existsSync } from 'node:fs';
3import { join, dirname } from 'node:path';
4import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js';
5import { encodeKey } from '../utils/path-encoding.js';
6
7/**
8 * Eviction policy for disk tier when size limit is reached.
9 */
10export type EvictionPolicy = 'lru' | 'fifo' | 'size';
11
12/**
13 * Configuration for DiskStorageTier.
14 */
15export interface DiskStorageTierConfig {
16 /**
17 * Directory path where files will be stored.
18 *
19 * @remarks
20 * Created automatically if it doesn't exist.
21 * Files are stored as: `{directory}/{encoded-key}`
22 * Metadata is stored as: `{directory}/{encoded-key}.meta`
23 */
24 directory: string;
25
26 /**
27 * Optional maximum size in bytes.
28 *
29 * @remarks
30 * When this limit is reached, files are evicted according to the eviction policy.
31 * If not set, no size limit is enforced (grows unbounded).
32 */
33 maxSizeBytes?: number;
34
35 /**
36 * Eviction policy when maxSizeBytes is reached.
37 *
38 * @defaultValue 'lru'
39 *
40 * @remarks
41 * - 'lru': Evict least-recently-accessed files (based on metadata.lastAccessed)
42 * - 'fifo': Evict oldest files (based on metadata.createdAt)
43 * - 'size': Evict largest files first
44 */
45 evictionPolicy?: EvictionPolicy;
46}
47
48/**
49 * Filesystem-based storage tier.
50 *
51 * @remarks
52 * - Stores data files and `.meta` JSON files side-by-side
53 * - Keys are encoded to be filesystem-safe
54 * - Human-readable file structure for debugging
55 * - Optional size-based eviction with configurable policy
56 * - Zero external dependencies (uses Node.js fs APIs)
57 *
58 * File structure:
59 * ```
60 * cache/
61 * ├── user%3A123 # Data file (encoded key)
62 * ├── user%3A123.meta # Metadata JSON
63 * ├── site%3Aabc%2Findex.html
64 * └── site%3Aabc%2Findex.html.meta
65 * ```
66 *
67 * @example
68 * ```typescript
69 * const tier = new DiskStorageTier({
70 * directory: './cache',
71 * maxSizeBytes: 10 * 1024 * 1024 * 1024, // 10GB
72 * evictionPolicy: 'lru',
73 * });
74 *
75 * await tier.set('key', data, metadata);
76 * const retrieved = await tier.get('key');
77 * ```
78 */
79export class DiskStorageTier implements StorageTier {
80 private metadataIndex = new Map<
81 string,
82 { size: number; createdAt: Date; lastAccessed: Date }
83 >();
84 private currentSize = 0;
85
86 constructor(private config: DiskStorageTierConfig) {
87 if (!config.directory) {
88 throw new Error('directory is required');
89 }
90 if (config.maxSizeBytes !== undefined && config.maxSizeBytes <= 0) {
91 throw new Error('maxSizeBytes must be positive');
92 }
93
94 void this.ensureDirectory();
95 void this.rebuildIndex();
96 }
97
98 private async rebuildIndex(): Promise<void> {
99 if (!existsSync(this.config.directory)) {
100 return;
101 }
102
103 const files = await readdir(this.config.directory);
104
105 for (const file of files) {
106 if (file.endsWith('.meta')) {
107 continue;
108 }
109
110 try {
111 const metaPath = join(this.config.directory, `${file}.meta`);
112 const metaContent = await readFile(metaPath, 'utf-8');
113 const metadata = JSON.parse(metaContent) as StorageMetadata;
114 const filePath = join(this.config.directory, file);
115 const fileStats = await stat(filePath);
116
117 this.metadataIndex.set(metadata.key, {
118 size: fileStats.size,
119 createdAt: new Date(metadata.createdAt),
120 lastAccessed: new Date(metadata.lastAccessed),
121 });
122
123 this.currentSize += fileStats.size;
124 } catch {
125 continue;
126 }
127 }
128 }
129
130 async get(key: string): Promise<Uint8Array | null> {
131 const filePath = this.getFilePath(key);
132
133 try {
134 const data = await readFile(filePath);
135 return new Uint8Array(data);
136 } catch (error) {
137 if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
138 return null;
139 }
140 throw error;
141 }
142 }
143
144 /**
145 * Retrieve data and metadata together in a single operation.
146 *
147 * @param key - The key to retrieve
148 * @returns The data and metadata, or null if not found
149 *
150 * @remarks
151 * Reads data and metadata files in parallel for better performance.
152 */
153 async getWithMetadata(key: string): Promise<TierGetResult | null> {
154 const filePath = this.getFilePath(key);
155 const metaPath = this.getMetaPath(key);
156
157 try {
158 // Read data and metadata in parallel
159 const [dataBuffer, metaContent] = await Promise.all([
160 readFile(filePath),
161 readFile(metaPath, 'utf-8'),
162 ]);
163
164 const metadata = JSON.parse(metaContent) as StorageMetadata;
165
166 // Convert date strings back to Date objects
167 metadata.createdAt = new Date(metadata.createdAt);
168 metadata.lastAccessed = new Date(metadata.lastAccessed);
169 if (metadata.ttl) {
170 metadata.ttl = new Date(metadata.ttl);
171 }
172
173 return { data: new Uint8Array(dataBuffer), metadata };
174 } catch (error) {
175 if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
176 return null;
177 }
178 throw error;
179 }
180 }
181
182 async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {
183 const filePath = this.getFilePath(key);
184 const metaPath = this.getMetaPath(key);
185
186 const dir = dirname(filePath);
187 if (!existsSync(dir)) {
188 await mkdir(dir, { recursive: true });
189 }
190
191 const existingEntry = this.metadataIndex.get(key);
192 if (existingEntry) {
193 this.currentSize -= existingEntry.size;
194 }
195
196 if (this.config.maxSizeBytes) {
197 await this.evictIfNeeded(data.byteLength);
198 }
199
200 const tempMetaPath = `${metaPath}.tmp`;
201 await writeFile(tempMetaPath, JSON.stringify(metadata, null, 2));
202 await writeFile(filePath, data);
203 await rename(tempMetaPath, metaPath);
204
205 this.metadataIndex.set(key, {
206 size: data.byteLength,
207 createdAt: metadata.createdAt,
208 lastAccessed: metadata.lastAccessed,
209 });
210 this.currentSize += data.byteLength;
211 }
212
213 async delete(key: string): Promise<void> {
214 const filePath = this.getFilePath(key);
215 const metaPath = this.getMetaPath(key);
216
217 const entry = this.metadataIndex.get(key);
218 if (entry) {
219 this.currentSize -= entry.size;
220 this.metadataIndex.delete(key);
221 }
222
223 await Promise.all([
224 unlink(filePath).catch(() => {}),
225 unlink(metaPath).catch(() => {}),
226 ]);
227 }
228
229 async exists(key: string): Promise<boolean> {
230 const filePath = this.getFilePath(key);
231 return existsSync(filePath);
232 }
233
234 async *listKeys(prefix?: string): AsyncIterableIterator<string> {
235 if (!existsSync(this.config.directory)) {
236 return;
237 }
238
239 const files = await readdir(this.config.directory);
240
241 for (const file of files) {
242 // Skip metadata files
243 if (file.endsWith('.meta')) {
244 continue;
245 }
246
247 // The file name is the encoded key
248 // We need to read metadata to get the original key for prefix matching
249 const metaPath = join(this.config.directory, `${file}.meta`);
250 try {
251 const metaContent = await readFile(metaPath, 'utf-8');
252 const metadata = JSON.parse(metaContent) as StorageMetadata;
253 const originalKey = metadata.key;
254
255 if (!prefix || originalKey.startsWith(prefix)) {
256 yield originalKey;
257 }
258 } catch {
259 // If metadata is missing or invalid, skip this file
260 continue;
261 }
262 }
263 }
264
265 async deleteMany(keys: string[]): Promise<void> {
266 await Promise.all(keys.map((key) => this.delete(key)));
267 }
268
269 async getMetadata(key: string): Promise<StorageMetadata | null> {
270 const metaPath = this.getMetaPath(key);
271
272 try {
273 const content = await readFile(metaPath, 'utf-8');
274 const metadata = JSON.parse(content) as StorageMetadata;
275
276 // Convert date strings back to Date objects
277 metadata.createdAt = new Date(metadata.createdAt);
278 metadata.lastAccessed = new Date(metadata.lastAccessed);
279 if (metadata.ttl) {
280 metadata.ttl = new Date(metadata.ttl);
281 }
282
283 return metadata;
284 } catch (error) {
285 if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
286 return null;
287 }
288 throw error;
289 }
290 }
291
292 async setMetadata(key: string, metadata: StorageMetadata): Promise<void> {
293 const metaPath = this.getMetaPath(key);
294
295 // Ensure parent directory exists
296 const dir = dirname(metaPath);
297 if (!existsSync(dir)) {
298 await mkdir(dir, { recursive: true });
299 }
300
301 await writeFile(metaPath, JSON.stringify(metadata, null, 2));
302 }
303
304 async getStats(): Promise<TierStats> {
305 let bytes = 0;
306 let items = 0;
307
308 if (!existsSync(this.config.directory)) {
309 return { bytes: 0, items: 0 };
310 }
311
312 const files = await readdir(this.config.directory);
313
314 for (const file of files) {
315 if (file.endsWith('.meta')) {
316 continue;
317 }
318
319 const filePath = join(this.config.directory, file);
320 const stats = await stat(filePath);
321 bytes += stats.size;
322 items++;
323 }
324
325 return { bytes, items };
326 }
327
328 async clear(): Promise<void> {
329 if (existsSync(this.config.directory)) {
330 await rm(this.config.directory, { recursive: true, force: true });
331 await this.ensureDirectory();
332 this.metadataIndex.clear();
333 this.currentSize = 0;
334 }
335 }
336
337 /**
338 * Get the filesystem path for a key's data file.
339 */
340 private getFilePath(key: string): string {
341 const encoded = encodeKey(key);
342 return join(this.config.directory, encoded);
343 }
344
345 /**
346 * Get the filesystem path for a key's metadata file.
347 */
348 private getMetaPath(key: string): string {
349 return `${this.getFilePath(key)}.meta`;
350 }
351
352 private async ensureDirectory(): Promise<void> {
353 await mkdir(this.config.directory, { recursive: true }).catch(() => {});
354 }
355
356 private async evictIfNeeded(incomingSize: number): Promise<void> {
357 if (!this.config.maxSizeBytes) {
358 return;
359 }
360
361 if (this.currentSize + incomingSize <= this.config.maxSizeBytes) {
362 return;
363 }
364
365 const entries = Array.from(this.metadataIndex.entries()).map(([key, info]) => ({
366 key,
367 ...info,
368 }));
369
370 const policy = this.config.evictionPolicy ?? 'lru';
371 entries.sort((a, b) => {
372 switch (policy) {
373 case 'lru':
374 return a.lastAccessed.getTime() - b.lastAccessed.getTime();
375 case 'fifo':
376 return a.createdAt.getTime() - b.createdAt.getTime();
377 case 'size':
378 return b.size - a.size;
379 default:
380 return 0;
381 }
382 });
383
384 for (const entry of entries) {
385 if (this.currentSize + incomingSize <= this.config.maxSizeBytes) {
386 break;
387 }
388
389 await this.delete(entry.key);
390 }
391 }
392}