import { readFile, writeFile, unlink, readdir, stat, mkdir, rm, rename } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { join, dirname } from 'node:path'; import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js'; import { encodeKey } from '../utils/path-encoding.js'; /** * Eviction policy for disk tier when size limit is reached. */ export type EvictionPolicy = 'lru' | 'fifo' | 'size'; /** * Configuration for DiskStorageTier. */ export interface DiskStorageTierConfig { /** * Directory path where files will be stored. * * @remarks * Created automatically if it doesn't exist. * Files are stored as: `{directory}/{encoded-key}` * Metadata is stored as: `{directory}/{encoded-key}.meta` */ directory: string; /** * Optional maximum size in bytes. * * @remarks * When this limit is reached, files are evicted according to the eviction policy. * If not set, no size limit is enforced (grows unbounded). */ maxSizeBytes?: number; /** * Eviction policy when maxSizeBytes is reached. * * @defaultValue 'lru' * * @remarks * - 'lru': Evict least-recently-accessed files (based on metadata.lastAccessed) * - 'fifo': Evict oldest files (based on metadata.createdAt) * - 'size': Evict largest files first */ evictionPolicy?: EvictionPolicy; } /** * Filesystem-based storage tier. * * @remarks * - Stores data files and `.meta` JSON files side-by-side * - Keys are encoded to be filesystem-safe * - Human-readable file structure for debugging * - Optional size-based eviction with configurable policy * - Zero external dependencies (uses Node.js fs APIs) * * File structure: * ``` * cache/ * ├── user%3A123 # Data file (encoded key) * ├── user%3A123.meta # Metadata JSON * ├── site%3Aabc%2Findex.html * └── site%3Aabc%2Findex.html.meta * ``` * * @example * ```typescript * const tier = new DiskStorageTier({ * directory: './cache', * maxSizeBytes: 10 * 1024 * 1024 * 1024, // 10GB * evictionPolicy: 'lru', * }); * * await tier.set('key', data, metadata); * const retrieved = await tier.get('key'); * ``` */ export class DiskStorageTier implements StorageTier { private metadataIndex = new Map< string, { size: number; createdAt: Date; lastAccessed: Date } >(); private currentSize = 0; constructor(private config: DiskStorageTierConfig) { if (!config.directory) { throw new Error('directory is required'); } if (config.maxSizeBytes !== undefined && config.maxSizeBytes <= 0) { throw new Error('maxSizeBytes must be positive'); } void this.ensureDirectory(); void this.rebuildIndex(); } private async rebuildIndex(): Promise { if (!existsSync(this.config.directory)) { return; } const files = await readdir(this.config.directory); for (const file of files) { if (file.endsWith('.meta')) { continue; } try { const metaPath = join(this.config.directory, `${file}.meta`); const metaContent = await readFile(metaPath, 'utf-8'); const metadata = JSON.parse(metaContent) as StorageMetadata; const filePath = join(this.config.directory, file); const fileStats = await stat(filePath); this.metadataIndex.set(metadata.key, { size: fileStats.size, createdAt: new Date(metadata.createdAt), lastAccessed: new Date(metadata.lastAccessed), }); this.currentSize += fileStats.size; } catch { continue; } } } async get(key: string): Promise { const filePath = this.getFilePath(key); try { const data = await readFile(filePath); return new Uint8Array(data); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return null; } throw error; } } /** * Retrieve data and metadata together in a single operation. * * @param key - The key to retrieve * @returns The data and metadata, or null if not found * * @remarks * Reads data and metadata files in parallel for better performance. */ async getWithMetadata(key: string): Promise { const filePath = this.getFilePath(key); const metaPath = this.getMetaPath(key); try { // Read data and metadata in parallel const [dataBuffer, metaContent] = await Promise.all([ readFile(filePath), readFile(metaPath, 'utf-8'), ]); const metadata = JSON.parse(metaContent) as StorageMetadata; // Convert date strings back to Date objects metadata.createdAt = new Date(metadata.createdAt); metadata.lastAccessed = new Date(metadata.lastAccessed); if (metadata.ttl) { metadata.ttl = new Date(metadata.ttl); } return { data: new Uint8Array(dataBuffer), metadata }; } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return null; } throw error; } } async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise { const filePath = this.getFilePath(key); const metaPath = this.getMetaPath(key); const dir = dirname(filePath); if (!existsSync(dir)) { await mkdir(dir, { recursive: true }); } const existingEntry = this.metadataIndex.get(key); if (existingEntry) { this.currentSize -= existingEntry.size; } if (this.config.maxSizeBytes) { await this.evictIfNeeded(data.byteLength); } const tempMetaPath = `${metaPath}.tmp`; await writeFile(tempMetaPath, JSON.stringify(metadata, null, 2)); await writeFile(filePath, data); await rename(tempMetaPath, metaPath); this.metadataIndex.set(key, { size: data.byteLength, createdAt: metadata.createdAt, lastAccessed: metadata.lastAccessed, }); this.currentSize += data.byteLength; } async delete(key: string): Promise { const filePath = this.getFilePath(key); const metaPath = this.getMetaPath(key); const entry = this.metadataIndex.get(key); if (entry) { this.currentSize -= entry.size; this.metadataIndex.delete(key); } await Promise.all([ unlink(filePath).catch(() => {}), unlink(metaPath).catch(() => {}), ]); } async exists(key: string): Promise { const filePath = this.getFilePath(key); return existsSync(filePath); } async *listKeys(prefix?: string): AsyncIterableIterator { if (!existsSync(this.config.directory)) { return; } const files = await readdir(this.config.directory); for (const file of files) { // Skip metadata files if (file.endsWith('.meta')) { continue; } // The file name is the encoded key // We need to read metadata to get the original key for prefix matching const metaPath = join(this.config.directory, `${file}.meta`); try { const metaContent = await readFile(metaPath, 'utf-8'); const metadata = JSON.parse(metaContent) as StorageMetadata; const originalKey = metadata.key; if (!prefix || originalKey.startsWith(prefix)) { yield originalKey; } } catch { // If metadata is missing or invalid, skip this file continue; } } } async deleteMany(keys: string[]): Promise { await Promise.all(keys.map((key) => this.delete(key))); } async getMetadata(key: string): Promise { const metaPath = this.getMetaPath(key); try { const content = await readFile(metaPath, 'utf-8'); const metadata = JSON.parse(content) as StorageMetadata; // Convert date strings back to Date objects metadata.createdAt = new Date(metadata.createdAt); metadata.lastAccessed = new Date(metadata.lastAccessed); if (metadata.ttl) { metadata.ttl = new Date(metadata.ttl); } return metadata; } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return null; } throw error; } } async setMetadata(key: string, metadata: StorageMetadata): Promise { const metaPath = this.getMetaPath(key); // Ensure parent directory exists const dir = dirname(metaPath); if (!existsSync(dir)) { await mkdir(dir, { recursive: true }); } await writeFile(metaPath, JSON.stringify(metadata, null, 2)); } async getStats(): Promise { let bytes = 0; let items = 0; if (!existsSync(this.config.directory)) { return { bytes: 0, items: 0 }; } const files = await readdir(this.config.directory); for (const file of files) { if (file.endsWith('.meta')) { continue; } const filePath = join(this.config.directory, file); const stats = await stat(filePath); bytes += stats.size; items++; } return { bytes, items }; } async clear(): Promise { if (existsSync(this.config.directory)) { await rm(this.config.directory, { recursive: true, force: true }); await this.ensureDirectory(); this.metadataIndex.clear(); this.currentSize = 0; } } /** * Get the filesystem path for a key's data file. */ private getFilePath(key: string): string { const encoded = encodeKey(key); return join(this.config.directory, encoded); } /** * Get the filesystem path for a key's metadata file. */ private getMetaPath(key: string): string { return `${this.getFilePath(key)}.meta`; } private async ensureDirectory(): Promise { await mkdir(this.config.directory, { recursive: true }).catch(() => {}); } private async evictIfNeeded(incomingSize: number): Promise { if (!this.config.maxSizeBytes) { return; } if (this.currentSize + incomingSize <= this.config.maxSizeBytes) { return; } const entries = Array.from(this.metadataIndex.entries()).map(([key, info]) => ({ key, ...info, })); const policy = this.config.evictionPolicy ?? 'lru'; entries.sort((a, b) => { switch (policy) { case 'lru': return a.lastAccessed.getTime() - b.lastAccessed.getTime(); case 'fifo': return a.createdAt.getTime() - b.createdAt.getTime(); case 'size': return b.size - a.size; default: return 0; } }); for (const entry of entries) { if (this.currentSize + incomingSize <= this.config.maxSizeBytes) { break; } await this.delete(entry.key); } } }