wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1import { readFile, writeFile, unlink, readdir, stat, mkdir, rm, rename } from 'node:fs/promises';
2import { existsSync } from 'node:fs';
3import { join, dirname } from 'node:path';
4import type { StorageTier, StorageMetadata, TierStats } from '../types/index.js';
5import { encodeKey } from '../utils/path-encoding.js';
6
7/**
8 * Eviction policy for disk tier when size limit is reached.
9 */
10export type EvictionPolicy = 'lru' | 'fifo' | 'size';
11
12/**
13 * Configuration for DiskStorageTier.
14 */
15export interface DiskStorageTierConfig {
16 /**
17 * Directory path where files will be stored.
18 *
19 * @remarks
20 * Created automatically if it doesn't exist.
21 * Files are stored as: `{directory}/{encoded-key}`
22 * Metadata is stored as: `{directory}/{encoded-key}.meta`
23 */
24 directory: string;
25
26 /**
27 * Optional maximum size in bytes.
28 *
29 * @remarks
30 * When this limit is reached, files are evicted according to the eviction policy.
31 * If not set, no size limit is enforced (grows unbounded).
32 */
33 maxSizeBytes?: number;
34
35 /**
36 * Eviction policy when maxSizeBytes is reached.
37 *
38 * @defaultValue 'lru'
39 *
40 * @remarks
41 * - 'lru': Evict least-recently-accessed files (based on metadata.lastAccessed)
42 * - 'fifo': Evict oldest files (based on metadata.createdAt)
43 * - 'size': Evict largest files first
44 */
45 evictionPolicy?: EvictionPolicy;
46}
47
48/**
49 * Filesystem-based storage tier.
50 *
51 * @remarks
52 * - Stores data files and `.meta` JSON files side-by-side
53 * - Keys are encoded to be filesystem-safe
54 * - Human-readable file structure for debugging
55 * - Optional size-based eviction with configurable policy
56 * - Zero external dependencies (uses Node.js fs APIs)
57 *
58 * File structure:
59 * ```
60 * cache/
61 * ├── user%3A123 # Data file (encoded key)
62 * ├── user%3A123.meta # Metadata JSON
63 * ├── site%3Aabc%2Findex.html
64 * └── site%3Aabc%2Findex.html.meta
65 * ```
66 *
67 * @example
68 * ```typescript
69 * const tier = new DiskStorageTier({
70 * directory: './cache',
71 * maxSizeBytes: 10 * 1024 * 1024 * 1024, // 10GB
72 * evictionPolicy: 'lru',
73 * });
74 *
75 * await tier.set('key', data, metadata);
76 * const retrieved = await tier.get('key');
77 * ```
78 */
79export class DiskStorageTier implements StorageTier {
80 private metadataIndex = new Map<
81 string,
82 { size: number; createdAt: Date; lastAccessed: Date }
83 >();
84 private currentSize = 0;
85
86 constructor(private config: DiskStorageTierConfig) {
87 if (!config.directory) {
88 throw new Error('directory is required');
89 }
90 if (config.maxSizeBytes !== undefined && config.maxSizeBytes <= 0) {
91 throw new Error('maxSizeBytes must be positive');
92 }
93
94 void this.ensureDirectory();
95 void this.rebuildIndex();
96 }
97
98 private async rebuildIndex(): Promise<void> {
99 if (!existsSync(this.config.directory)) {
100 return;
101 }
102
103 const files = await readdir(this.config.directory);
104
105 for (const file of files) {
106 if (file.endsWith('.meta')) {
107 continue;
108 }
109
110 try {
111 const metaPath = join(this.config.directory, `${file}.meta`);
112 const metaContent = await readFile(metaPath, 'utf-8');
113 const metadata = JSON.parse(metaContent) as StorageMetadata;
114 const filePath = join(this.config.directory, file);
115 const fileStats = await stat(filePath);
116
117 this.metadataIndex.set(metadata.key, {
118 size: fileStats.size,
119 createdAt: new Date(metadata.createdAt),
120 lastAccessed: new Date(metadata.lastAccessed),
121 });
122
123 this.currentSize += fileStats.size;
124 } catch {
125 continue;
126 }
127 }
128 }
129
130 async get(key: string): Promise<Uint8Array | null> {
131 const filePath = this.getFilePath(key);
132
133 try {
134 const data = await readFile(filePath);
135
136 const metadata = await this.getMetadata(key);
137 if (metadata) {
138 metadata.lastAccessed = new Date();
139 metadata.accessCount++;
140 await this.setMetadata(key, metadata);
141
142 const entry = this.metadataIndex.get(key);
143 if (entry) {
144 entry.lastAccessed = metadata.lastAccessed;
145 }
146 }
147
148 return new Uint8Array(data);
149 } catch (error) {
150 if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
151 return null;
152 }
153 throw error;
154 }
155 }
156
157 async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {
158 const filePath = this.getFilePath(key);
159 const metaPath = this.getMetaPath(key);
160
161 const dir = dirname(filePath);
162 if (!existsSync(dir)) {
163 await mkdir(dir, { recursive: true });
164 }
165
166 const existingEntry = this.metadataIndex.get(key);
167 if (existingEntry) {
168 this.currentSize -= existingEntry.size;
169 }
170
171 if (this.config.maxSizeBytes) {
172 await this.evictIfNeeded(data.byteLength);
173 }
174
175 const tempMetaPath = `${metaPath}.tmp`;
176 await writeFile(tempMetaPath, JSON.stringify(metadata, null, 2));
177 await writeFile(filePath, data);
178 await rename(tempMetaPath, metaPath);
179
180 this.metadataIndex.set(key, {
181 size: data.byteLength,
182 createdAt: metadata.createdAt,
183 lastAccessed: metadata.lastAccessed,
184 });
185 this.currentSize += data.byteLength;
186 }
187
188 async delete(key: string): Promise<void> {
189 const filePath = this.getFilePath(key);
190 const metaPath = this.getMetaPath(key);
191
192 const entry = this.metadataIndex.get(key);
193 if (entry) {
194 this.currentSize -= entry.size;
195 this.metadataIndex.delete(key);
196 }
197
198 await Promise.all([
199 unlink(filePath).catch(() => {}),
200 unlink(metaPath).catch(() => {}),
201 ]);
202 }
203
204 async exists(key: string): Promise<boolean> {
205 const filePath = this.getFilePath(key);
206 return existsSync(filePath);
207 }
208
209 async *listKeys(prefix?: string): AsyncIterableIterator<string> {
210 if (!existsSync(this.config.directory)) {
211 return;
212 }
213
214 const files = await readdir(this.config.directory);
215
216 for (const file of files) {
217 // Skip metadata files
218 if (file.endsWith('.meta')) {
219 continue;
220 }
221
222 // The file name is the encoded key
223 // We need to read metadata to get the original key for prefix matching
224 const metaPath = join(this.config.directory, `${file}.meta`);
225 try {
226 const metaContent = await readFile(metaPath, 'utf-8');
227 const metadata = JSON.parse(metaContent) as StorageMetadata;
228 const originalKey = metadata.key;
229
230 if (!prefix || originalKey.startsWith(prefix)) {
231 yield originalKey;
232 }
233 } catch {
234 // If metadata is missing or invalid, skip this file
235 continue;
236 }
237 }
238 }
239
240 async deleteMany(keys: string[]): Promise<void> {
241 await Promise.all(keys.map((key) => this.delete(key)));
242 }
243
244 async getMetadata(key: string): Promise<StorageMetadata | null> {
245 const metaPath = this.getMetaPath(key);
246
247 try {
248 const content = await readFile(metaPath, 'utf-8');
249 const metadata = JSON.parse(content) as StorageMetadata;
250
251 // Convert date strings back to Date objects
252 metadata.createdAt = new Date(metadata.createdAt);
253 metadata.lastAccessed = new Date(metadata.lastAccessed);
254 if (metadata.ttl) {
255 metadata.ttl = new Date(metadata.ttl);
256 }
257
258 return metadata;
259 } catch (error) {
260 if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
261 return null;
262 }
263 throw error;
264 }
265 }
266
267 async setMetadata(key: string, metadata: StorageMetadata): Promise<void> {
268 const metaPath = this.getMetaPath(key);
269
270 // Ensure parent directory exists
271 const dir = dirname(metaPath);
272 if (!existsSync(dir)) {
273 await mkdir(dir, { recursive: true });
274 }
275
276 await writeFile(metaPath, JSON.stringify(metadata, null, 2));
277 }
278
279 async getStats(): Promise<TierStats> {
280 let bytes = 0;
281 let items = 0;
282
283 if (!existsSync(this.config.directory)) {
284 return { bytes: 0, items: 0 };
285 }
286
287 const files = await readdir(this.config.directory);
288
289 for (const file of files) {
290 if (file.endsWith('.meta')) {
291 continue;
292 }
293
294 const filePath = join(this.config.directory, file);
295 const stats = await stat(filePath);
296 bytes += stats.size;
297 items++;
298 }
299
300 return { bytes, items };
301 }
302
303 async clear(): Promise<void> {
304 if (existsSync(this.config.directory)) {
305 await rm(this.config.directory, { recursive: true, force: true });
306 await this.ensureDirectory();
307 this.metadataIndex.clear();
308 this.currentSize = 0;
309 }
310 }
311
312 /**
313 * Get the filesystem path for a key's data file.
314 */
315 private getFilePath(key: string): string {
316 const encoded = encodeKey(key);
317 return join(this.config.directory, encoded);
318 }
319
320 /**
321 * Get the filesystem path for a key's metadata file.
322 */
323 private getMetaPath(key: string): string {
324 return `${this.getFilePath(key)}.meta`;
325 }
326
327 private async ensureDirectory(): Promise<void> {
328 await mkdir(this.config.directory, { recursive: true }).catch(() => {});
329 }
330
331 private async evictIfNeeded(incomingSize: number): Promise<void> {
332 if (!this.config.maxSizeBytes) {
333 return;
334 }
335
336 if (this.currentSize + incomingSize <= this.config.maxSizeBytes) {
337 return;
338 }
339
340 const entries = Array.from(this.metadataIndex.entries()).map(([key, info]) => ({
341 key,
342 ...info,
343 }));
344
345 const policy = this.config.evictionPolicy ?? 'lru';
346 entries.sort((a, b) => {
347 switch (policy) {
348 case 'lru':
349 return a.lastAccessed.getTime() - b.lastAccessed.getTime();
350 case 'fifo':
351 return a.createdAt.getTime() - b.createdAt.getTime();
352 case 'size':
353 return b.size - a.size;
354 default:
355 return 0;
356 }
357 });
358
359 for (const entry of entries) {
360 if (this.currentSize + incomingSize <= this.config.maxSizeBytes) {
361 break;
362 }
363
364 await this.delete(entry.key);
365 }
366 }
367}