commit d9e314336e150b0cf559deed70f42a51e6f18cfd · nekomimi.pet/tiered-storage

README.md

···

       234
       234
        
         setMetadata(key: string, metadata: StorageMetadata): Promise<void>

     

       235
       235
        
         getStats(): Promise<TierStats>

     

       236
       236
        
         clear(): Promise<void>

     

       237
       237
       +
       

     

       238
       238
       +
         // Optional: combine get + getMetadata for better performance

     

       239
       239
       +
         getWithMetadata?(key: string): Promise<{ data: Uint8Array; metadata: StorageMetadata } | null>

     

       237
       240
        
       }

     

       238
       241
        
       ```

     

       242
       242
       +
       

     

       243
       243
       +
       The optional `getWithMetadata` method returns both data and metadata in a single call. Implement it if your backend can fetch both efficiently (e.g., parallel I/O, single query). Falls back to separate `get()` + `getMetadata()` calls if not implemented.

     

       239
       244
        
       

     

       240
       245
        
       ## Running the demo

     

       241
       246

+65 -44

src/TieredStorage.ts

···

       4
       4
        
         StorageResult,

     

       5
       5
        
         SetResult,

     

       6
       6
        
         StorageMetadata,

     

       7
       7
       +
         StorageTier,

     

       7
       8
        
         AllTierStats,

     

       8
       9
        
         StorageSnapshot,

     

       9
       10
        
         PlacementRule,

     
···

       95
       96
        
         async getWithMetadata(key: string): Promise<StorageResult<T> | null> {

     

       96
       97
        
           // 1. Check hot tier first

     

       97
       98
        
           if (this.config.tiers.hot) {

     

       98
       98
       -
             const data = await this.config.tiers.hot.get(key);

     

       99
       99
       -
             if (data) {

     

       100
       100
       -
               const metadata = await this.config.tiers.hot.getMetadata(key);

     

       101
       101
       -
               if (!metadata) {

     

       102
       102
       -
                 await this.delete(key);

     

       103
       103
       -
               } else if (this.isExpired(metadata)) {

     

       99
       99
       +
             const result = await this.getFromTier(this.config.tiers.hot, key);

     

       100
       100
       +
             if (result) {

     

       101
       101
       +
               if (this.isExpired(result.metadata)) {

     

       104
       102
        
                 await this.delete(key);

     

       105
       103
        
                 return null;

     

       106
       106
       -
               } else {

     

       107
       107
       -
                 await this.updateAccessStats(key, 'hot');

     

       108
       108
       -
                 return {

     

       109
       109
       -
                   data: (await this.deserializeData(data)) as T,

     

       110
       110
       -
                   metadata,

     

       111
       111
       -
                   source: 'hot',

     

       112
       112
       -
                 };

     

       113
       104
        
               }

     

       105
       105
       +
               // Fire-and-forget access stats update (non-critical)

     

       106
       106
       +
               void this.updateAccessStats(key, 'hot');

     

       107
       107
       +
               return {

     

       108
       108
       +
                 data: (await this.deserializeData(result.data)) as T,

     

       109
       109
       +
                 metadata: result.metadata,

     

       110
       110
       +
                 source: 'hot',

     

       111
       111
       +
               };

     

       114
       112
        
             }

     

       115
       113
        
           }

     

       116
       114
        
       

     

       117
       115
        
           // 2. Check warm tier

     

       118
       116
        
           if (this.config.tiers.warm) {

     

       119
       119
       -
             const data = await this.config.tiers.warm.get(key);

     

       120
       120
       -
             if (data) {

     

       121
       121
       -
               const metadata = await this.config.tiers.warm.getMetadata(key);

     

       122
       122
       -
               if (!metadata) {

     

       123
       123
       -
                 await this.delete(key);

     

       124
       124
       -
               } else if (this.isExpired(metadata)) {

     

       117
       117
       +
             const result = await this.getFromTier(this.config.tiers.warm, key);

     

       118
       118
       +
             if (result) {

     

       119
       119
       +
               if (this.isExpired(result.metadata)) {

     

       125
       120
        
                 await this.delete(key);

     

       126
       121
        
                 return null;

     

       127
       127
       -
               } else {

     

       128
       128
       -
                 if (this.config.tiers.hot && this.config.promotionStrategy === 'eager') {

     

       129
       129
       -
                   await this.config.tiers.hot.set(key, data, metadata);

     

       130
       130
       -
                 }

     

       131
       131
       -
       

     

       132
       132
       -
                 await this.updateAccessStats(key, 'warm');

     

       133
       133
       -
                 return {

     

       134
       134
       -
                   data: (await this.deserializeData(data)) as T,

     

       135
       135
       -
                   metadata,

     

       136
       136
       -
                   source: 'warm',

     

       137
       137
       -
                 };

     

       122
       122
       +
               }

     

       123
       123
       +
               // Eager promotion to hot tier (awaited - guaranteed to complete)

     

       124
       124
       +
               if (this.config.tiers.hot && this.config.promotionStrategy === 'eager') {

     

       125
       125
       +
                 await this.config.tiers.hot.set(key, result.data, result.metadata);

     

       138
       126
        
               }

     

       127
       127
       +
               // Fire-and-forget access stats update (non-critical)

     

       128
       128
       +
               void this.updateAccessStats(key, 'warm');

     

       129
       129
       +
               return {

     

       130
       130
       +
                 data: (await this.deserializeData(result.data)) as T,

     

       131
       131
       +
                 metadata: result.metadata,

     

       132
       132
       +
                 source: 'warm',

     

       133
       133
       +
               };

     

       139
       134
        
             }

     

       140
       135
        
           }

     

       141
       136
        
       

     

       142
       137
        
           // 3. Check cold tier (source of truth)

     

       143
       143
       -
           const data = await this.config.tiers.cold.get(key);

     

       144
       144
       -
           if (data) {

     

       145
       145
       -
             const metadata = await this.config.tiers.cold.getMetadata(key);

     

       146
       146
       -
             if (!metadata) {

     

       147
       147
       -
               await this.config.tiers.cold.delete(key);

     

       148
       148
       -
               return null;

     

       149
       149
       -
             }

     

       150
       150
       -
       

     

       151
       151
       -
             if (this.isExpired(metadata)) {

     

       138
       138
       +
           const result = await this.getFromTier(this.config.tiers.cold, key);

     

       139
       139
       +
           if (result) {

     

       140
       140
       +
             if (this.isExpired(result.metadata)) {

     

       152
       141
        
               await this.delete(key);

     

       153
       142
        
               return null;

     

       154
       143
        
             }

     

       155
       144
        
       

     

       156
       145
        
             // Promote to warm and hot (if configured)

     

       146
       146
       +
             // Eager promotion is awaited to guarantee completion

     

       157
       147
        
             if (this.config.promotionStrategy === 'eager') {

     

       148
       148
       +
               const promotions: Promise<void>[] = [];

     

       158
       149
        
               if (this.config.tiers.warm) {

     

       159
       159
       -
                 await this.config.tiers.warm.set(key, data, metadata);

     

       150
       150
       +
                 promotions.push(this.config.tiers.warm.set(key, result.data, result.metadata));

     

       160
       151
        
               }

     

       161
       152
        
               if (this.config.tiers.hot) {

     

       162
       162
       -
                 await this.config.tiers.hot.set(key, data, metadata);

     

       153
       153
       +
                 promotions.push(this.config.tiers.hot.set(key, result.data, result.metadata));

     

       163
       154
        
               }

     

       155
       155
       +
               await Promise.all(promotions);

     

       164
       156
        
             }

     

       165
       157
        
       

     

       166
       166
       -
             await this.updateAccessStats(key, 'cold');

     

       158
       158
       +
             // Fire-and-forget access stats update (non-critical)

     

       159
       159
       +
             void this.updateAccessStats(key, 'cold');

     

       167
       160
        
             return {

     

       168
       168
       -
               data: (await this.deserializeData(data)) as T,

     

       169
       169
       -
               metadata,

     

       161
       161
       +
               data: (await this.deserializeData(result.data)) as T,

     

       162
       162
       +
               metadata: result.metadata,

     

       170
       163
        
               source: 'cold',

     

       171
       164
        
             };

     

       172
       165
        
           }

     

       173
       166
        
       

     

       174
       167
        
           return null;

     

       168
       168
       +
         }

     

       169
       169
       +
       

     

       170
       170
       +
         /**

     

       171
       171
       +
          * Get data and metadata from a tier using the most efficient method.

     

       172
       172
       +
          *

     

       173
       173
       +
          * @remarks

     

       174
       174
       +
          * Uses the tier's getWithMetadata if available, otherwise falls back

     

       175
       175
       +
          * to separate get() and getMetadata() calls.

     

       176
       176
       +
          */

     

       177
       177
       +
         private async getFromTier(

     

       178
       178
       +
           tier: StorageTier,

     

       179
       179
       +
           key: string

     

       180
       180
       +
         ): Promise<{ data: Uint8Array; metadata: StorageMetadata } | null> {

     

       181
       181
       +
           // Use optimized combined method if available

     

       182
       182
       +
           if (tier.getWithMetadata) {

     

       183
       183
       +
             return tier.getWithMetadata(key);

     

       184
       184
       +
           }

     

       185
       185
       +
       

     

       186
       186
       +
           // Fallback: separate calls

     

       187
       187
       +
           const data = await tier.get(key);

     

       188
       188
       +
           if (!data) {

     

       189
       189
       +
             return null;

     

       190
       190
       +
           }

     

       191
       191
       +
           const metadata = await tier.getMetadata(key);

     

       192
       192
       +
           if (!metadata) {

     

       193
       193
       +
             return null;

     

       194
       194
       +
           }

     

       195
       195
       +
           return { data, metadata };

     

       175
       196
        
         }

     

       176
       197
        
       

     

       177
       198
        
         /**

src/index.ts

···

       20
       20
        
         StorageTier,

     

       21
       21
        
         StorageMetadata,

     

       22
       22
        
         TierStats,

     

       23
       23
       +
         TierGetResult,

     

       23
       24
        
         AllTierStats,

     

       24
       25
        
         TieredStorageConfig,

     

       25
       26
        
         PlacementRule,

+36 -11

src/tiers/DiskStorageTier.ts

···

       1
       1
        
       import { readFile, writeFile, unlink, readdir, stat, mkdir, rm, rename } from 'node:fs/promises';

     

       2
       2
        
       import { existsSync } from 'node:fs';

     

       3
       3
        
       import { join, dirname } from 'node:path';

     

       4
       4
       -
       import type { StorageTier, StorageMetadata, TierStats } from '../types/index.js';

     

       4
       4
       +
       import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js';

     

       5
       5
        
       import { encodeKey } from '../utils/path-encoding.js';

     

       6
       6
        
       

     

       7
       7
        
       /**

     
···

       132
       132
        
       

     

       133
       133
        
           try {

     

       134
       134
        
             const data = await readFile(filePath);

     

       135
       135
       +
             return new Uint8Array(data);

     

       136
       136
       +
           } catch (error) {

     

       137
       137
       +
             if ((error as NodeJS.ErrnoException).code === 'ENOENT') {

     

       138
       138
       +
               return null;

     

       139
       139
       +
             }

     

       140
       140
       +
             throw error;

     

       141
       141
       +
           }

     

       142
       142
       +
         }

     

       135
       143
        
       

     

       136
       136
       -
             const metadata = await this.getMetadata(key);

     

       137
       137
       -
             if (metadata) {

     

       138
       138
       -
               metadata.lastAccessed = new Date();

     

       139
       139
       -
               metadata.accessCount++;

     

       140
       140
       -
               await this.setMetadata(key, metadata);

     

       144
       144
       +
         /**

     

       145
       145
       +
          * Retrieve data and metadata together in a single operation.

     

       146
       146
       +
          *

     

       147
       147
       +
          * @param key - The key to retrieve

     

       148
       148
       +
          * @returns The data and metadata, or null if not found

     

       149
       149
       +
          *

     

       150
       150
       +
          * @remarks

     

       151
       151
       +
          * Reads data and metadata files in parallel for better performance.

     

       152
       152
       +
          */

     

       153
       153
       +
         async getWithMetadata(key: string): Promise<TierGetResult | null> {

     

       154
       154
       +
           const filePath = this.getFilePath(key);

     

       155
       155
       +
           const metaPath = this.getMetaPath(key);

     

       156
       156
       +
       

     

       157
       157
       +
           try {

     

       158
       158
       +
             // Read data and metadata in parallel

     

       159
       159
       +
             const [dataBuffer, metaContent] = await Promise.all([

     

       160
       160
       +
               readFile(filePath),

     

       161
       161
       +
               readFile(metaPath, 'utf-8'),

     

       162
       162
       +
             ]);

     

       141
       163
        
       

     

       142
       142
       -
               const entry = this.metadataIndex.get(key);

     

       143
       143
       -
               if (entry) {

     

       144
       144
       -
                 entry.lastAccessed = metadata.lastAccessed;

     

       145
       145
       -
               }

     

       164
       164
       +
             const metadata = JSON.parse(metaContent) as StorageMetadata;

     

       165
       165
       +
       

     

       166
       166
       +
             // Convert date strings back to Date objects

     

       167
       167
       +
             metadata.createdAt = new Date(metadata.createdAt);

     

       168
       168
       +
             metadata.lastAccessed = new Date(metadata.lastAccessed);

     

       169
       169
       +
             if (metadata.ttl) {

     

       170
       170
       +
               metadata.ttl = new Date(metadata.ttl);

     

       146
       171
        
             }

     

       147
       172
        
       

     

       148
       148
       -
             return new Uint8Array(data);

     

       173
       173
       +
             return { data: new Uint8Array(dataBuffer), metadata };

     

       149
       174
        
           } catch (error) {

     

       150
       175
        
             if ((error as NodeJS.ErrnoException).code === 'ENOENT') {

     

       151
       176
        
               return null;

+19 -1

src/tiers/MemoryStorageTier.ts

···

       1
       1
        
       import { lru } from 'tiny-lru';

     

       2
       2
       -
       import type { StorageTier, StorageMetadata, TierStats } from '../types/index.js';

     

       2
       2
       +
       import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js';

     

       3
       3
        
       

     

       4
       4
        
       interface CacheEntry {

     

       5
       5
        
         data: Uint8Array;

     
···

       82
       82
        
       

     

       83
       83
        
           this.stats.hits++;

     

       84
       84
        
           return entry.data;

     

       85
       85
       +
         }

     

       86
       86
       +
       

     

       87
       87
       +
         /**

     

       88
       88
       +
          * Retrieve data and metadata together in a single cache lookup.

     

       89
       89
       +
          *

     

       90
       90
       +
          * @param key - The key to retrieve

     

       91
       91
       +
          * @returns The data and metadata, or null if not found

     

       92
       92
       +
          */

     

       93
       93
       +
         async getWithMetadata(key: string): Promise<TierGetResult | null> {

     

       94
       94
       +
           const entry = this.cache.get(key);

     

       95
       95
       +
       

     

       96
       96
       +
           if (!entry) {

     

       97
       97
       +
             this.stats.misses++;

     

       98
       98
       +
             return null;

     

       99
       99
       +
           }

     

       100
       100
       +
       

     

       101
       101
       +
           this.stats.hits++;

     

       102
       102
       +
           return { data: entry.data, metadata: entry.metadata };

     

       85
       103
        
         }

     

       86
       104
        
       

     

       87
       105
        
         async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {

+70 -1

src/tiers/S3StorageTier.ts

···

       10
       10
        
         type S3ClientConfig,

     

       11
       11
        
       } from '@aws-sdk/client-s3';

     

       12
       12
        
       import type { Readable } from 'node:stream';

     

       13
       13
       -
       import type { StorageTier, StorageMetadata, TierStats } from '../types/index.js';

     

       13
       13
       +
       import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js';

     

       14
       14
        
       

     

       15
       15
        
       /**

     

       16
       16
        
        * Configuration for S3StorageTier.

     
···

       180
       180
        
             }

     

       181
       181
        
       

     

       182
       182
        
             return await this.streamToUint8Array(response.Body as Readable);

     

       183
       183
       +
           } catch (error) {

     

       184
       184
       +
             if (this.isNoSuchKeyError(error)) {

     

       185
       185
       +
               return null;

     

       186
       186
       +
             }

     

       187
       187
       +
             throw error;

     

       188
       188
       +
           }

     

       189
       189
       +
         }

     

       190
       190
       +
       

     

       191
       191
       +
         /**

     

       192
       192
       +
          * Retrieve data and metadata together in a single operation.

     

       193
       193
       +
          *

     

       194
       194
       +
          * @param key - The key to retrieve

     

       195
       195
       +
          * @returns The data and metadata, or null if not found

     

       196
       196
       +
          *

     

       197
       197
       +
          * @remarks

     

       198
       198
       +
          * When using a separate metadata bucket, fetches data and metadata in parallel.

     

       199
       199
       +
          * Otherwise, uses the data object's embedded metadata.

     

       200
       200
       +
          */

     

       201
       201
       +
         async getWithMetadata(key: string): Promise<TierGetResult | null> {

     

       202
       202
       +
           const s3Key = this.getS3Key(key);

     

       203
       203
       +
       

     

       204
       204
       +
           try {

     

       205
       205
       +
             if (this.metadataBucket) {

     

       206
       206
       +
               // Fetch data and metadata in parallel

     

       207
       207
       +
               const [dataResponse, metadataResponse] = await Promise.all([

     

       208
       208
       +
                 this.client.send(new GetObjectCommand({

     

       209
       209
       +
                   Bucket: this.config.bucket,

     

       210
       210
       +
                   Key: s3Key,

     

       211
       211
       +
                 })),

     

       212
       212
       +
                 this.client.send(new GetObjectCommand({

     

       213
       213
       +
                   Bucket: this.metadataBucket,

     

       214
       214
       +
                   Key: s3Key + '.meta',

     

       215
       215
       +
                 })),

     

       216
       216
       +
               ]);

     

       217
       217
       +
       

     

       218
       218
       +
               if (!dataResponse.Body || !metadataResponse.Body) {

     

       219
       219
       +
                 return null;

     

       220
       220
       +
               }

     

       221
       221
       +
       

     

       222
       222
       +
               const [data, metaBuffer] = await Promise.all([

     

       223
       223
       +
                 this.streamToUint8Array(dataResponse.Body as Readable),

     

       224
       224
       +
                 this.streamToUint8Array(metadataResponse.Body as Readable),

     

       225
       225
       +
               ]);

     

       226
       226
       +
       

     

       227
       227
       +
               const json = new TextDecoder().decode(metaBuffer);

     

       228
       228
       +
               const metadata = JSON.parse(json) as StorageMetadata;

     

       229
       229
       +
               metadata.createdAt = new Date(metadata.createdAt);

     

       230
       230
       +
               metadata.lastAccessed = new Date(metadata.lastAccessed);

     

       231
       231
       +
               if (metadata.ttl) {

     

       232
       232
       +
                 metadata.ttl = new Date(metadata.ttl);

     

       233
       233
       +
               }

     

       234
       234
       +
       

     

       235
       235
       +
               return { data, metadata };

     

       236
       236
       +
             } else {

     

       237
       237
       +
               // Get data with embedded metadata from response headers

     

       238
       238
       +
               const response = await this.client.send(new GetObjectCommand({

     

       239
       239
       +
                 Bucket: this.config.bucket,

     

       240
       240
       +
                 Key: s3Key,

     

       241
       241
       +
               }));

     

       242
       242
       +
       

     

       243
       243
       +
               if (!response.Body || !response.Metadata) {

     

       244
       244
       +
                 return null;

     

       245
       245
       +
               }

     

       246
       246
       +
       

     

       247
       247
       +
               const data = await this.streamToUint8Array(response.Body as Readable);

     

       248
       248
       +
               const metadata = this.s3ToMetadata(response.Metadata);

     

       249
       249
       +
       

     

       250
       250
       +
               return { data, metadata };

     

       251
       251
       +
             }

     

       183
       252
        
           } catch (error) {

     

       184
       253
        
             if (this.isNoSuchKeyError(error)) {

     

       185
       254
        
               return null;

+22

src/types/index.ts

···

       114
       114
        
        * }

     

       115
       115
        
        * ```

     

       116
       116
        
        */

     

       117
       117
       +
       /**

     

       118
       118
       +
        * Result from a combined get+metadata operation on a tier.

     

       119
       119
       +
        */

     

       120
       120
       +
       export interface TierGetResult {

     

       121
       121
       +
         /** The retrieved data */

     

       122
       122
       +
         data: Uint8Array;

     

       123
       123
       +
         /** Metadata associated with the data */

     

       124
       124
       +
         metadata: StorageMetadata;

     

       125
       125
       +
       }

     

       126
       126
       +
       

     

       117
       127
        
       export interface StorageTier {

     

       118
       128
        
         /**

     

       119
       129
        
          * Retrieve data for a key.

     
···

       122
       132
        
          * @returns The data as a Uint8Array, or null if not found

     

       123
       133
        
          */

     

       124
       134
        
         get(key: string): Promise<Uint8Array | null>;

     

       135
       135
       +
       

     

       136
       136
       +
         /**

     

       137
       137
       +
          * Retrieve data and metadata together in a single operation.

     

       138
       138
       +
          *

     

       139
       139
       +
          * @param key - The key to retrieve

     

       140
       140
       +
          * @returns The data and metadata, or null if not found

     

       141
       141
       +
          *

     

       142
       142
       +
          * @remarks

     

       143
       143
       +
          * This is more efficient than calling get() and getMetadata() separately,

     

       144
       144
       +
          * especially for disk and network-based tiers.

     

       145
       145
       +
          */

     

       146
       146
       +
         getWithMetadata?(key: string): Promise<TierGetResult | null>;

     

       125
       147
        
       

     

       126
       148
        
         /**

     

       127
       149
        
          * Store data with associated metadata.