comparing 5bc23a4aa6abfd49543e645d8510d452e20db2b4 and main on nekomimi.pet/tiered-storage

+82 -11

README.md

···

       28
       28
        
           warm: new DiskStorageTier({ directory: './cache' }),

     

       29
       29
        
           cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),

     

       30
       30
        
         },

     

       31
       31
       -
         compression: true,

     

       31
       31
       +
         placementRules: [

     

       32
       32
       +
           { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       33
       33
       +
           { pattern: '**/*.{jpg,png,gif,mp4}', tiers: ['warm', 'cold'] },

     

       34
       34
       +
           { pattern: '**', tiers: ['warm', 'cold'] },

     

       35
       35
       +
         ],

     

       32
       36
        
       })

     

       33
       37
        
       

     

       34
       34
       -
       // critical file: keep in memory for instant serving

     

       35
       35
       -
       await storage.set('site:abc/index.html', indexHtml)

     

       36
       36
       -
       

     

       37
       37
       -
       // big files: skip hot, let them live in warm + cold

     

       38
       38
       -
       await storage.set('site:abc/video.mp4', videoData, { skipTiers: ['hot'] })

     

       39
       39
       -
       await storage.set('site:abc/hero.png', imageData, { skipTiers: ['hot'] })

     

       38
       38
       +
       // just set - rules decide where it goes

     

       39
       39
       +
       await storage.set('site:abc/index.html', indexHtml)  // → hot + warm + cold

     

       40
       40
       +
       await storage.set('site:abc/hero.png', imageData)    // → warm + cold

     

       41
       41
       +
       await storage.set('site:abc/video.mp4', videoData)   // → warm + cold

     

       40
       42
        
       

     

       41
       41
       -
       // on read, bubbles up from wherever it lives

     

       42
       42
       -
       const result = await storage.getWithMetadata('site:abc/index.html')

     

       43
       43
       -
       console.log(result.source) // 'hot' - served from memory

     

       43
       43
       +
       // reads bubble up from wherever it lives

     

       44
       44
       +
       const page = await storage.getWithMetadata('site:abc/index.html')

     

       45
       45
       +
       console.log(page.source) // 'hot'

     

       44
       46
        
       

     

       45
       47
        
       const video = await storage.getWithMetadata('site:abc/video.mp4')

     

       46
       46
       -
       console.log(video.source) // 'warm' - served from disk, never touches memory

     

       48
       48
       +
       console.log(video.source) // 'warm'

     

       47
       49
        
       

     

       48
       50
        
       // nuke entire site

     

       49
       51
        
       await storage.invalidate('site:abc/')

     
···

       65
       67
        
       

     

       66
       68
        
       Writes cascade **down**. Reads bubble **up**.

     

       67
       69
        
       

     

       70
       70
       +
       ## Eviction

     

       71
       71
       +
       

     

       72
       72
       +
       Items leave upper tiers through eviction or TTL expiration:

     

       73
       73
       +
       

     

       74
       74
       +
       ```typescript

     

       75
       75
       +
       const storage = new TieredStorage({

     

       76
       76
       +
         tiers: {

     

       77
       77
       +
           // hot: LRU eviction when size/count limits hit

     

       78
       78
       +
           hot: new MemoryStorageTier({

     

       79
       79
       +
             maxSizeBytes: 100 * 1024 * 1024,

     

       80
       80
       +
             maxItems: 500,

     

       81
       81
       +
           }),

     

       82
       82
       +
       

     

       83
       83
       +
           // warm: evicts when maxSizeBytes hit, policy controls which items go

     

       84
       84
       +
           warm: new DiskStorageTier({

     

       85
       85
       +
             directory: './cache',

     

       86
       86
       +
             maxSizeBytes: 10 * 1024 * 1024 * 1024,

     

       87
       87
       +
             evictionPolicy: 'lru',  // 'lru' | 'fifo' | 'size'

     

       88
       88
       +
           }),

     

       89
       89
       +
       

     

       90
       90
       +
           // cold: never evicts, keeps everything

     

       91
       91
       +
           cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),

     

       92
       92
       +
         },

     

       93
       93
       +
         defaultTTL: 14 * 24 * 60 * 60 * 1000, // TTL checked on read

     

       94
       94
       +
       })

     

       95
       95
       +
       ```

     

       96
       96
       +
       

     

       97
       97
       +
       A file that hasn't been accessed eventually gets evicted from hot (LRU), then warm (size limit + policy). Next request fetches from cold and promotes it back up.

     

       98
       98
       +
       

     

       99
       99
       +
       ## Placement rules

     

       100
       100
       +
       

     

       101
       101
       +
       Define once which keys go where, instead of passing `skipTiers` on every `set()`:

     

       102
       102
       +
       

     

       103
       103
       +
       ```typescript

     

       104
       104
       +
       const storage = new TieredStorage({

     

       105
       105
       +
         tiers: {

     

       106
       106
       +
           hot: new MemoryStorageTier({ maxSizeBytes: 50 * 1024 * 1024 }),

     

       107
       107
       +
           warm: new DiskStorageTier({ directory: './cache' }),

     

       108
       108
       +
           cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),

     

       109
       109
       +
         },

     

       110
       110
       +
         placementRules: [

     

       111
       111
       +
           // index.html goes everywhere for instant serving

     

       112
       112
       +
           { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       113
       113
       +
       

     

       114
       114
       +
           // images and video skip hot

     

       115
       115
       +
           { pattern: '**/*.{jpg,png,gif,webp,mp4}', tiers: ['warm', 'cold'] },

     

       116
       116
       +
       

     

       117
       117
       +
           // assets directory skips hot

     

       118
       118
       +
           { pattern: 'assets/**', tiers: ['warm', 'cold'] },

     

       119
       119
       +
       

     

       120
       120
       +
           // everything else: warm + cold only

     

       121
       121
       +
           { pattern: '**', tiers: ['warm', 'cold'] },

     

       122
       122
       +
         ],

     

       123
       123
       +
       })

     

       124
       124
       +
       

     

       125
       125
       +
       // just call set() - rules handle placement

     

       126
       126
       +
       await storage.set('site:abc/index.html', html)       // → hot + warm + cold

     

       127
       127
       +
       await storage.set('site:abc/hero.png', image)        // → warm + cold

     

       128
       128
       +
       await storage.set('site:abc/assets/font.woff', font) // → warm + cold

     

       129
       129
       +
       await storage.set('site:abc/about.html', html)       // → warm + cold

     

       130
       130
       +
       ```

     

       131
       131
       +
       

     

       132
       132
       +
       Rules are evaluated in order. First match wins. Cold is always included.

     

       133
       133
       +
       

     

       68
       134
        
       ## API

     

       69
       135
        
       

     

       70
       136
        
       ### `storage.get(key)`

     
···

       168
       234
        
         setMetadata(key: string, metadata: StorageMetadata): Promise<void>

     

       169
       235
        
         getStats(): Promise<TierStats>

     

       170
       236
        
         clear(): Promise<void>

     

       237
       237
       +
       

     

       238
       238
       +
         // Optional: combine get + getMetadata for better performance

     

       239
       239
       +
         getWithMetadata?(key: string): Promise<{ data: Uint8Array; metadata: StorageMetadata } | null>

     

       171
       240
        
       }

     

       172
       241
        
       ```

     

       242
       242
       +
       

     

       243
       243
       +
       The optional `getWithMetadata` method returns both data and metadata in a single call. Implement it if your backend can fetch both efficiently (e.g., parallel I/O, single query). Falls back to separate `get()` + `getMetadata()` calls if not implemented.

     

       173
       244
        
       

     

       174
       245
        
       ## Running the demo

     

       175
       246

+17 -10

serve-example.ts

···

       44
       44
        
             prefix: 'demo-sites/',

     

       45
       45
        
           }),

     

       46
       46
        
         },

     

       47
       47
       +
         placementRules: [

     

       48
       48
       +
           // index.html goes to all tiers for instant serving

     

       49
       49
       +
           { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       50
       50
       +
           

     

       51
       51
       +
           // everything else: warm + cold only

     

       52
       52
       +
           { pattern: '**', tiers: ['warm', 'cold'] },

     

       53
       53
       +
         ],

     

       47
       54
        
         compression: true,

     

       48
       55
        
         defaultTTL: 14 * 24 * 60 * 60 * 1000,

     

       49
       56
        
         promotionStrategy: 'lazy',

     
···

       62
       69
        
         console.log('\n📦 Loading example site into tiered storage...\n');

     

       63
       70
        
       

     

       64
       71
        
         const files = [

     

       65
       65
       -
           { name: 'index.html', skipTiers: [], mimeType: 'text/html' },

     

       66
       66
       -
           { name: 'about.html', skipTiers: ['hot'], mimeType: 'text/html' },

     

       67
       67
       -
           { name: 'docs.html', skipTiers: ['hot'], mimeType: 'text/html' },

     

       68
       68
       -
           { name: 'style.css', skipTiers: ['hot'], mimeType: 'text/css' },

     

       69
       69
       -
           { name: 'script.js', skipTiers: ['hot'], mimeType: 'application/javascript' },

     

       72
       72
       +
           { name: 'index.html', mimeType: 'text/html' },

     

       73
       73
       +
           { name: 'about.html', mimeType: 'text/html' },

     

       74
       74
       +
           { name: 'docs.html', mimeType: 'text/html' },

     

       75
       75
       +
           { name: 'style.css', mimeType: 'text/css' },

     

       76
       76
       +
           { name: 'script.js', mimeType: 'application/javascript' },

     

       70
       77
        
         ];

     

       71
       78
        
       

     

       72
       79
        
         for (const file of files) {

     
···

       74
       81
        
           const key = `${siteId}/${siteName}/${file.name}`;

     

       75
       82
        
       

     

       76
       83
        
           await storage.set(key, content, {

     

       77
       77
       -
             skipTiers: file.skipTiers as ('hot' | 'warm')[],

     

       78
       84
        
             metadata: { mimeType: file.mimeType },

     

       79
       85
        
           });

     

       80
       86
        
       

     

       81
       81
       -
           const tierInfo =

     

       82
       82
       -
             file.skipTiers.length === 0

     

       83
       83
       -
               ? '🔥 hot + 💾 warm + ☁️  cold'

     

       84
       84
       -
               : `💾 warm + ☁️  cold (skipped hot)`;

     

       87
       87
       +
           // Determine which tiers this file went to based on placement rules

     

       88
       88
       +
           const isIndex = file.name === 'index.html';

     

       89
       89
       +
           const tierInfo = isIndex

     

       90
       90
       +
             ? '🔥 hot + 💾 warm + ☁️  cold'

     

       91
       91
       +
             : '💾 warm + ☁️  cold (skipped hot)';

     

       85
       92
        
           const sizeKB = (content.length / 1024).toFixed(2);

     

       86
       93
        
           console.log(`   ✓ ${file.name.padEnd(15)} ${sizeKB.padStart(6)} KB → ${tierInfo}`);

     

       87
       94
        
         }

+112 -55

src/TieredStorage.ts

···

       4
       4
        
         StorageResult,

     

       5
       5
        
         SetResult,

     

       6
       6
        
         StorageMetadata,

     

       7
       7
       +
         StorageTier,

     

       7
       8
        
         AllTierStats,

     

       8
       9
        
         StorageSnapshot,

     

       9
       9
       -
       } from './types/index.js';

     

       10
       10
       +
         PlacementRule,

     

       11
       11
       +
       } from './types/index';

     

       10
       12
        
       import { compress, decompress } from './utils/compression.js';

     

       11
       13
        
       import { defaultSerialize, defaultDeserialize } from './utils/serialization.js';

     

       12
       14
        
       import { calculateChecksum } from './utils/checksum.js';

     

       15
       15
       +
       import { matchGlob } from './utils/glob.js';

     

       13
       16
        
       

     

       14
       17
        
       /**

     

       15
       18
        
        * Main orchestrator for tiered storage system.

     
···

       93
       96
        
         async getWithMetadata(key: string): Promise<StorageResult<T> | null> {

     

       94
       97
        
           // 1. Check hot tier first

     

       95
       98
        
           if (this.config.tiers.hot) {

     

       96
       96
       -
             const data = await this.config.tiers.hot.get(key);

     

       97
       97
       -
             if (data) {

     

       98
       98
       -
               const metadata = await this.config.tiers.hot.getMetadata(key);

     

       99
       99
       -
               if (!metadata) {

     

       100
       100
       -
                 await this.delete(key);

     

       101
       101
       -
               } else if (this.isExpired(metadata)) {

     

       99
       99
       +
             const result = await this.getFromTier(this.config.tiers.hot, key);

     

       100
       100
       +
             if (result) {

     

       101
       101
       +
               if (this.isExpired(result.metadata)) {

     

       102
       102
        
                 await this.delete(key);

     

       103
       103
        
                 return null;

     

       104
       104
       -
               } else {

     

       105
       105
       -
                 await this.updateAccessStats(key, 'hot');

     

       106
       106
       -
                 return {

     

       107
       107
       -
                   data: (await this.deserializeData(data)) as T,

     

       108
       108
       -
                   metadata,

     

       109
       109
       -
                   source: 'hot',

     

       110
       110
       -
                 };

     

       111
       104
        
               }

     

       105
       105
       +
               // Fire-and-forget access stats update (non-critical)

     

       106
       106
       +
               void this.updateAccessStats(key, 'hot');

     

       107
       107
       +
               return {

     

       108
       108
       +
                 data: (await this.deserializeData(result.data)) as T,

     

       109
       109
       +
                 metadata: result.metadata,

     

       110
       110
       +
                 source: 'hot',

     

       111
       111
       +
               };

     

       112
       112
        
             }

     

       113
       113
        
           }

     

       114
       114
        
       

     

       115
       115
        
           // 2. Check warm tier

     

       116
       116
        
           if (this.config.tiers.warm) {

     

       117
       117
       -
             const data = await this.config.tiers.warm.get(key);

     

       118
       118
       -
             if (data) {

     

       119
       119
       -
               const metadata = await this.config.tiers.warm.getMetadata(key);

     

       120
       120
       -
               if (!metadata) {

     

       121
       121
       -
                 await this.delete(key);

     

       122
       122
       -
               } else if (this.isExpired(metadata)) {

     

       117
       117
       +
             const result = await this.getFromTier(this.config.tiers.warm, key);

     

       118
       118
       +
             if (result) {

     

       119
       119
       +
               if (this.isExpired(result.metadata)) {

     

       123
       120
        
                 await this.delete(key);

     

       124
       121
        
                 return null;

     

       125
       125
       -
               } else {

     

       126
       126
       -
                 if (this.config.tiers.hot && this.config.promotionStrategy === 'eager') {

     

       127
       127
       -
                   await this.config.tiers.hot.set(key, data, metadata);

     

       128
       128
       -
                 }

     

       129
       129
       -
       

     

       130
       130
       -
                 await this.updateAccessStats(key, 'warm');

     

       131
       131
       -
                 return {

     

       132
       132
       -
                   data: (await this.deserializeData(data)) as T,

     

       133
       133
       -
                   metadata,

     

       134
       134
       -
                   source: 'warm',

     

       135
       135
       -
                 };

     

       122
       122
       +
               }

     

       123
       123
       +
               // Eager promotion to hot tier (awaited - guaranteed to complete)

     

       124
       124
       +
               if (this.config.tiers.hot && this.config.promotionStrategy === 'eager') {

     

       125
       125
       +
                 await this.config.tiers.hot.set(key, result.data, result.metadata);

     

       136
       126
        
               }

     

       127
       127
       +
               // Fire-and-forget access stats update (non-critical)

     

       128
       128
       +
               void this.updateAccessStats(key, 'warm');

     

       129
       129
       +
               return {

     

       130
       130
       +
                 data: (await this.deserializeData(result.data)) as T,

     

       131
       131
       +
                 metadata: result.metadata,

     

       132
       132
       +
                 source: 'warm',

     

       133
       133
       +
               };

     

       137
       134
        
             }

     

       138
       135
        
           }

     

       139
       136
        
       

     

       140
       137
        
           // 3. Check cold tier (source of truth)

     

       141
       141
       -
           const data = await this.config.tiers.cold.get(key);

     

       142
       142
       -
           if (data) {

     

       143
       143
       -
             const metadata = await this.config.tiers.cold.getMetadata(key);

     

       144
       144
       -
             if (!metadata) {

     

       145
       145
       -
               await this.config.tiers.cold.delete(key);

     

       146
       146
       -
               return null;

     

       147
       147
       -
             }

     

       148
       148
       -
       

     

       149
       149
       -
             if (this.isExpired(metadata)) {

     

       138
       138
       +
           const result = await this.getFromTier(this.config.tiers.cold, key);

     

       139
       139
       +
           if (result) {

     

       140
       140
       +
             if (this.isExpired(result.metadata)) {

     

       150
       141
        
               await this.delete(key);

     

       151
       142
        
               return null;

     

       152
       143
        
             }

     

       153
       144
        
       

     

       154
       145
        
             // Promote to warm and hot (if configured)

     

       146
       146
       +
             // Eager promotion is awaited to guarantee completion

     

       155
       147
        
             if (this.config.promotionStrategy === 'eager') {

     

       148
       148
       +
               const promotions: Promise<void>[] = [];

     

       156
       149
        
               if (this.config.tiers.warm) {

     

       157
       157
       -
                 await this.config.tiers.warm.set(key, data, metadata);

     

       150
       150
       +
                 promotions.push(this.config.tiers.warm.set(key, result.data, result.metadata));

     

       158
       151
        
               }

     

       159
       152
        
               if (this.config.tiers.hot) {

     

       160
       160
       -
                 await this.config.tiers.hot.set(key, data, metadata);

     

       153
       153
       +
                 promotions.push(this.config.tiers.hot.set(key, result.data, result.metadata));

     

       161
       154
        
               }

     

       155
       155
       +
               await Promise.all(promotions);

     

       162
       156
        
             }

     

       163
       157
        
       

     

       164
       164
       -
             await this.updateAccessStats(key, 'cold');

     

       158
       158
       +
             // Fire-and-forget access stats update (non-critical)

     

       159
       159
       +
             void this.updateAccessStats(key, 'cold');

     

       165
       160
        
             return {

     

       166
       166
       -
               data: (await this.deserializeData(data)) as T,

     

       167
       167
       -
               metadata,

     

       161
       161
       +
               data: (await this.deserializeData(result.data)) as T,

     

       162
       162
       +
               metadata: result.metadata,

     

       168
       163
        
               source: 'cold',

     

       169
       164
        
             };

     

       170
       165
        
           }

     
···

       173
       168
        
         }

     

       174
       169
        
       

     

       175
       170
        
         /**

     

       171
       171
       +
          * Get data and metadata from a tier using the most efficient method.

     

       172
       172
       +
          *

     

       173
       173
       +
          * @remarks

     

       174
       174
       +
          * Uses the tier's getWithMetadata if available, otherwise falls back

     

       175
       175
       +
          * to separate get() and getMetadata() calls.

     

       176
       176
       +
          */

     

       177
       177
       +
         private async getFromTier(

     

       178
       178
       +
           tier: StorageTier,

     

       179
       179
       +
           key: string

     

       180
       180
       +
         ): Promise<{ data: Uint8Array; metadata: StorageMetadata } | null> {

     

       181
       181
       +
           // Use optimized combined method if available

     

       182
       182
       +
           if (tier.getWithMetadata) {

     

       183
       183
       +
             return tier.getWithMetadata(key);

     

       184
       184
       +
           }

     

       185
       185
       +
       

     

       186
       186
       +
           // Fallback: separate calls

     

       187
       187
       +
           const data = await tier.get(key);

     

       188
       188
       +
           if (!data) {

     

       189
       189
       +
             return null;

     

       190
       190
       +
           }

     

       191
       191
       +
           const metadata = await tier.getMetadata(key);

     

       192
       192
       +
           if (!metadata) {

     

       193
       193
       +
             return null;

     

       194
       194
       +
           }

     

       195
       195
       +
           return { data, metadata };

     

       196
       196
       +
         }

     

       197
       197
       +
       

     

       198
       198
       +
         /**

     

       176
       199
        
          * Store data with optional configuration.

     

       177
       200
        
          *

     

       178
       201
        
          * @param key - The key to store under

     
···

       202
       225
        
           // 3. Create metadata

     

       203
       226
        
           const metadata = this.createMetadata(key, finalData, options);

     

       204
       227
        
       

     

       205
       205
       -
           // 4. Write to all tiers (cascading down)

     

       228
       228
       +
           // 4. Determine which tiers to write to

     

       229
       229
       +
           const allowedTiers = this.getTiersForKey(key, options?.skipTiers);

     

       230
       230
       +
       

     

       231
       231
       +
           // 5. Write to tiers

     

       206
       232
        
           const tiersWritten: ('hot' | 'warm' | 'cold')[] = [];

     

       207
       233
        
       

     

       208
       208
       -
           // Write to hot (if configured and not skipped)

     

       209
       209
       -
           if (this.config.tiers.hot && !options?.skipTiers?.includes('hot')) {

     

       234
       234
       +
           if (this.config.tiers.hot && allowedTiers.includes('hot')) {

     

       210
       235
        
             await this.config.tiers.hot.set(key, finalData, metadata);

     

       211
       236
        
             tiersWritten.push('hot');

     

       237
       237
       +
           }

     

       212
       238
        
       

     

       213
       213
       -
             // Hot writes cascade to warm

     

       214
       214
       -
             if (this.config.tiers.warm && !options?.skipTiers?.includes('warm')) {

     

       215
       215
       -
               await this.config.tiers.warm.set(key, finalData, metadata);

     

       216
       216
       -
               tiersWritten.push('warm');

     

       217
       217
       -
             }

     

       218
       218
       -
           } else if (this.config.tiers.warm && !options?.skipTiers?.includes('warm')) {

     

       219
       219
       -
             // Write to warm (if hot skipped)

     

       239
       239
       +
           if (this.config.tiers.warm && allowedTiers.includes('warm')) {

     

       220
       240
        
             await this.config.tiers.warm.set(key, finalData, metadata);

     

       221
       241
        
             tiersWritten.push('warm');

     

       222
       242
        
           }

     
···

       226
       246
        
           tiersWritten.push('cold');

     

       227
       247
        
       

     

       228
       248
        
           return { key, metadata, tiersWritten };

     

       249
       249
       +
         }

     

       250
       250
       +
       

     

       251
       251
       +
         /**

     

       252
       252
       +
          * Determine which tiers a key should be written to.

     

       253
       253
       +
          *

     

       254
       254
       +
          * @param key - The key being stored

     

       255
       255
       +
          * @param skipTiers - Explicit tiers to skip (overrides placement rules)

     

       256
       256
       +
          * @returns Array of tiers to write to

     

       257
       257
       +
          *

     

       258
       258
       +
          * @remarks

     

       259
       259
       +
          * Priority: skipTiers option > placementRules > all configured tiers

     

       260
       260
       +
          */

     

       261
       261
       +
         private getTiersForKey(

     

       262
       262
       +
           key: string,

     

       263
       263
       +
           skipTiers?: ('hot' | 'warm')[]

     

       264
       264
       +
         ): ('hot' | 'warm' | 'cold')[] {

     

       265
       265
       +
           // If explicit skipTiers provided, use that

     

       266
       266
       +
           if (skipTiers && skipTiers.length > 0) {

     

       267
       267
       +
             const allTiers: ('hot' | 'warm' | 'cold')[] = ['hot', 'warm', 'cold'];

     

       268
       268
       +
             return allTiers.filter((t) => !skipTiers.includes(t as 'hot' | 'warm'));

     

       269
       269
       +
           }

     

       270
       270
       +
       

     

       271
       271
       +
           // Check placement rules

     

       272
       272
       +
           if (this.config.placementRules) {

     

       273
       273
       +
             for (const rule of this.config.placementRules) {

     

       274
       274
       +
               if (matchGlob(rule.pattern, key)) {

     

       275
       275
       +
                 // Ensure cold is always included

     

       276
       276
       +
                 if (!rule.tiers.includes('cold')) {

     

       277
       277
       +
                   return [...rule.tiers, 'cold'];

     

       278
       278
       +
                 }

     

       279
       279
       +
                 return rule.tiers;

     

       280
       280
       +
               }

     

       281
       281
       +
             }

     

       282
       282
       +
           }

     

       283
       283
       +
       

     

       284
       284
       +
           // Default: write to all configured tiers

     

       285
       285
       +
           return ['hot', 'warm', 'cold'];

     

       229
       286
        
         }

     

       230
       287
        
       

     

       231
       288
        
         /**

src/index.ts

···

       20
       20
        
         StorageTier,

     

       21
       21
        
         StorageMetadata,

     

       22
       22
        
         TierStats,

     

       23
       23
       +
         TierGetResult,

     

       23
       24
        
         AllTierStats,

     

       24
       25
        
         TieredStorageConfig,

     

       26
       26
       +
         PlacementRule,

     

       25
       27
        
         SetOptions,

     

       26
       28
        
         StorageResult,

     

       27
       29
        
         SetResult,

+36 -11

src/tiers/DiskStorageTier.ts

···

       1
       1
        
       import { readFile, writeFile, unlink, readdir, stat, mkdir, rm, rename } from 'node:fs/promises';

     

       2
       2
        
       import { existsSync } from 'node:fs';

     

       3
       3
        
       import { join, dirname } from 'node:path';

     

       4
       4
       -
       import type { StorageTier, StorageMetadata, TierStats } from '../types/index.js';

     

       4
       4
       +
       import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js';

     

       5
       5
        
       import { encodeKey } from '../utils/path-encoding.js';

     

       6
       6
        
       

     

       7
       7
        
       /**

     
···

       132
       132
        
       

     

       133
       133
        
           try {

     

       134
       134
        
             const data = await readFile(filePath);

     

       135
       135
       +
             return new Uint8Array(data);

     

       136
       136
       +
           } catch (error) {

     

       137
       137
       +
             if ((error as NodeJS.ErrnoException).code === 'ENOENT') {

     

       138
       138
       +
               return null;

     

       139
       139
       +
             }

     

       140
       140
       +
             throw error;

     

       141
       141
       +
           }

     

       142
       142
       +
         }

     

       135
       143
        
       

     

       136
       136
       -
             const metadata = await this.getMetadata(key);

     

       137
       137
       -
             if (metadata) {

     

       138
       138
       -
               metadata.lastAccessed = new Date();

     

       139
       139
       -
               metadata.accessCount++;

     

       140
       140
       -
               await this.setMetadata(key, metadata);

     

       144
       144
       +
         /**

     

       145
       145
       +
          * Retrieve data and metadata together in a single operation.

     

       146
       146
       +
          *

     

       147
       147
       +
          * @param key - The key to retrieve

     

       148
       148
       +
          * @returns The data and metadata, or null if not found

     

       149
       149
       +
          *

     

       150
       150
       +
          * @remarks

     

       151
       151
       +
          * Reads data and metadata files in parallel for better performance.

     

       152
       152
       +
          */

     

       153
       153
       +
         async getWithMetadata(key: string): Promise<TierGetResult | null> {

     

       154
       154
       +
           const filePath = this.getFilePath(key);

     

       155
       155
       +
           const metaPath = this.getMetaPath(key);

     

       156
       156
       +
       

     

       157
       157
       +
           try {

     

       158
       158
       +
             // Read data and metadata in parallel

     

       159
       159
       +
             const [dataBuffer, metaContent] = await Promise.all([

     

       160
       160
       +
               readFile(filePath),

     

       161
       161
       +
               readFile(metaPath, 'utf-8'),

     

       162
       162
       +
             ]);

     

       141
       163
        
       

     

       142
       142
       -
               const entry = this.metadataIndex.get(key);

     

       143
       143
       -
               if (entry) {

     

       144
       144
       -
                 entry.lastAccessed = metadata.lastAccessed;

     

       145
       145
       -
               }

     

       164
       164
       +
             const metadata = JSON.parse(metaContent) as StorageMetadata;

     

       165
       165
       +
       

     

       166
       166
       +
             // Convert date strings back to Date objects

     

       167
       167
       +
             metadata.createdAt = new Date(metadata.createdAt);

     

       168
       168
       +
             metadata.lastAccessed = new Date(metadata.lastAccessed);

     

       169
       169
       +
             if (metadata.ttl) {

     

       170
       170
       +
               metadata.ttl = new Date(metadata.ttl);

     

       146
       171
        
             }

     

       147
       172
        
       

     

       148
       148
       -
             return new Uint8Array(data);

     

       173
       173
       +
             return { data: new Uint8Array(dataBuffer), metadata };

     

       149
       174
        
           } catch (error) {

     

       150
       175
        
             if ((error as NodeJS.ErrnoException).code === 'ENOENT') {

     

       151
       176
        
               return null;

+19 -1

src/tiers/MemoryStorageTier.ts

···

       1
       1
        
       import { lru } from 'tiny-lru';

     

       2
       2
       -
       import type { StorageTier, StorageMetadata, TierStats } from '../types/index.js';

     

       2
       2
       +
       import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js';

     

       3
       3
        
       

     

       4
       4
        
       interface CacheEntry {

     

       5
       5
        
         data: Uint8Array;

     
···

       82
       82
        
       

     

       83
       83
        
           this.stats.hits++;

     

       84
       84
        
           return entry.data;

     

       85
       85
       +
         }

     

       86
       86
       +
       

     

       87
       87
       +
         /**

     

       88
       88
       +
          * Retrieve data and metadata together in a single cache lookup.

     

       89
       89
       +
          *

     

       90
       90
       +
          * @param key - The key to retrieve

     

       91
       91
       +
          * @returns The data and metadata, or null if not found

     

       92
       92
       +
          */

     

       93
       93
       +
         async getWithMetadata(key: string): Promise<TierGetResult | null> {

     

       94
       94
       +
           const entry = this.cache.get(key);

     

       95
       95
       +
       

     

       96
       96
       +
           if (!entry) {

     

       97
       97
       +
             this.stats.misses++;

     

       98
       98
       +
             return null;

     

       99
       99
       +
           }

     

       100
       100
       +
       

     

       101
       101
       +
           this.stats.hits++;

     

       102
       102
       +
           return { data: entry.data, metadata: entry.metadata };

     

       85
       103
        
         }

     

       86
       104
        
       

     

       87
       105
        
         async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {

+70 -1

src/tiers/S3StorageTier.ts

···

       10
       10
        
         type S3ClientConfig,

     

       11
       11
        
       } from '@aws-sdk/client-s3';

     

       12
       12
        
       import type { Readable } from 'node:stream';

     

       13
       13
       -
       import type { StorageTier, StorageMetadata, TierStats } from '../types/index.js';

     

       13
       13
       +
       import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js';

     

       14
       14
        
       

     

       15
       15
        
       /**

     

       16
       16
        
        * Configuration for S3StorageTier.

     
···

       180
       180
        
             }

     

       181
       181
        
       

     

       182
       182
        
             return await this.streamToUint8Array(response.Body as Readable);

     

       183
       183
       +
           } catch (error) {

     

       184
       184
       +
             if (this.isNoSuchKeyError(error)) {

     

       185
       185
       +
               return null;

     

       186
       186
       +
             }

     

       187
       187
       +
             throw error;

     

       188
       188
       +
           }

     

       189
       189
       +
         }

     

       190
       190
       +
       

     

       191
       191
       +
         /**

     

       192
       192
       +
          * Retrieve data and metadata together in a single operation.

     

       193
       193
       +
          *

     

       194
       194
       +
          * @param key - The key to retrieve

     

       195
       195
       +
          * @returns The data and metadata, or null if not found

     

       196
       196
       +
          *

     

       197
       197
       +
          * @remarks

     

       198
       198
       +
          * When using a separate metadata bucket, fetches data and metadata in parallel.

     

       199
       199
       +
          * Otherwise, uses the data object's embedded metadata.

     

       200
       200
       +
          */

     

       201
       201
       +
         async getWithMetadata(key: string): Promise<TierGetResult | null> {

     

       202
       202
       +
           const s3Key = this.getS3Key(key);

     

       203
       203
       +
       

     

       204
       204
       +
           try {

     

       205
       205
       +
             if (this.metadataBucket) {

     

       206
       206
       +
               // Fetch data and metadata in parallel

     

       207
       207
       +
               const [dataResponse, metadataResponse] = await Promise.all([

     

       208
       208
       +
                 this.client.send(new GetObjectCommand({

     

       209
       209
       +
                   Bucket: this.config.bucket,

     

       210
       210
       +
                   Key: s3Key,

     

       211
       211
       +
                 })),

     

       212
       212
       +
                 this.client.send(new GetObjectCommand({

     

       213
       213
       +
                   Bucket: this.metadataBucket,

     

       214
       214
       +
                   Key: s3Key + '.meta',

     

       215
       215
       +
                 })),

     

       216
       216
       +
               ]);

     

       217
       217
       +
       

     

       218
       218
       +
               if (!dataResponse.Body || !metadataResponse.Body) {

     

       219
       219
       +
                 return null;

     

       220
       220
       +
               }

     

       221
       221
       +
       

     

       222
       222
       +
               const [data, metaBuffer] = await Promise.all([

     

       223
       223
       +
                 this.streamToUint8Array(dataResponse.Body as Readable),

     

       224
       224
       +
                 this.streamToUint8Array(metadataResponse.Body as Readable),

     

       225
       225
       +
               ]);

     

       226
       226
       +
       

     

       227
       227
       +
               const json = new TextDecoder().decode(metaBuffer);

     

       228
       228
       +
               const metadata = JSON.parse(json) as StorageMetadata;

     

       229
       229
       +
               metadata.createdAt = new Date(metadata.createdAt);

     

       230
       230
       +
               metadata.lastAccessed = new Date(metadata.lastAccessed);

     

       231
       231
       +
               if (metadata.ttl) {

     

       232
       232
       +
                 metadata.ttl = new Date(metadata.ttl);

     

       233
       233
       +
               }

     

       234
       234
       +
       

     

       235
       235
       +
               return { data, metadata };

     

       236
       236
       +
             } else {

     

       237
       237
       +
               // Get data with embedded metadata from response headers

     

       238
       238
       +
               const response = await this.client.send(new GetObjectCommand({

     

       239
       239
       +
                 Bucket: this.config.bucket,

     

       240
       240
       +
                 Key: s3Key,

     

       241
       241
       +
               }));

     

       242
       242
       +
       

     

       243
       243
       +
               if (!response.Body || !response.Metadata) {

     

       244
       244
       +
                 return null;

     

       245
       245
       +
               }

     

       246
       246
       +
       

     

       247
       247
       +
               const data = await this.streamToUint8Array(response.Body as Readable);

     

       248
       248
       +
               const metadata = this.s3ToMetadata(response.Metadata);

     

       249
       249
       +
       

     

       250
       250
       +
               return { data, metadata };

     

       251
       251
       +
             }

     

       183
       252
        
           } catch (error) {

     

       184
       253
        
             if (this.isNoSuchKeyError(error)) {

     

       185
       254
        
               return null;

+67

src/types/index.ts

···

       114
       114
        
        * }

     

       115
       115
        
        * ```

     

       116
       116
        
        */

     

       117
       117
       +
       /**

     

       118
       118
       +
        * Result from a combined get+metadata operation on a tier.

     

       119
       119
       +
        */

     

       120
       120
       +
       export interface TierGetResult {

     

       121
       121
       +
         /** The retrieved data */

     

       122
       122
       +
         data: Uint8Array;

     

       123
       123
       +
         /** Metadata associated with the data */

     

       124
       124
       +
         metadata: StorageMetadata;

     

       125
       125
       +
       }

     

       126
       126
       +
       

     

       117
       127
        
       export interface StorageTier {

     

       118
       128
        
         /**

     

       119
       129
        
          * Retrieve data for a key.

     
···

       122
       132
        
          * @returns The data as a Uint8Array, or null if not found

     

       123
       133
        
          */

     

       124
       134
        
         get(key: string): Promise<Uint8Array | null>;

     

       135
       135
       +
       

     

       136
       136
       +
         /**

     

       137
       137
       +
          * Retrieve data and metadata together in a single operation.

     

       138
       138
       +
          *

     

       139
       139
       +
          * @param key - The key to retrieve

     

       140
       140
       +
          * @returns The data and metadata, or null if not found

     

       141
       141
       +
          *

     

       142
       142
       +
          * @remarks

     

       143
       143
       +
          * This is more efficient than calling get() and getMetadata() separately,

     

       144
       144
       +
          * especially for disk and network-based tiers.

     

       145
       145
       +
          */

     

       146
       146
       +
         getWithMetadata?(key: string): Promise<TierGetResult | null>;

     

       125
       147
        
       

     

       126
       148
        
         /**

     

       127
       149
        
          * Store data with associated metadata.

     
···

       222
       244
        
       }

     

       223
       245
        
       

     

       224
       246
        
       /**

     

       247
       247
       +
        * Rule for automatic tier placement based on key patterns.

     

       248
       248
       +
        *

     

       249
       249
       +
        * @remarks

     

       250
       250
       +
        * Rules are evaluated in order. First matching rule wins.

     

       251
       251
       +
        * Use this to define which keys go to which tiers without

     

       252
       252
       +
        * specifying skipTiers on every set() call.

     

       253
       253
       +
        *

     

       254
       254
       +
        * @example

     

       255
       255
       +
        * ```typescript

     

       256
       256
       +
        * placementRules: [

     

       257
       257
       +
        *   { pattern: 'index.html', tiers: ['hot', 'warm', 'cold'] },

     

       258
       258
       +
        *   { pattern: '*.html', tiers: ['warm', 'cold'] },

     

       259
       259
       +
        *   { pattern: 'assets/**', tiers: ['warm', 'cold'] },

     

       260
       260
       +
        *   { pattern: '**', tiers: ['warm', 'cold'] },  // default

     

       261
       261
       +
        * ]

     

       262
       262
       +
        * ```

     

       263
       263
       +
        */

     

       264
       264
       +
       export interface PlacementRule {

     

       265
       265
       +
         /**

     

       266
       266
       +
          * Glob pattern to match against keys.

     

       267
       267
       +
          *

     

       268
       268
       +
          * @remarks

     

       269
       269
       +
          * Supports basic globs:

     

       270
       270
       +
          * - `*` matches any characters except `/`

     

       271
       271
       +
          * - `**` matches any characters including `/`

     

       272
       272
       +
          * - Exact matches work too: `index.html`

     

       273
       273
       +
          */

     

       274
       274
       +
         pattern: string;

     

       275
       275
       +
       

     

       276
       276
       +
         /**

     

       277
       277
       +
          * Which tiers to write to for matching keys.

     

       278
       278
       +
          *

     

       279
       279
       +
          * @remarks

     

       280
       280
       +
          * Cold is always included (source of truth).

     

       281
       281
       +
          * Use `['hot', 'warm', 'cold']` for critical files.

     

       282
       282
       +
          * Use `['warm', 'cold']` for large files.

     

       283
       283
       +
          * Use `['cold']` for archival only.

     

       284
       284
       +
          */

     

       285
       285
       +
         tiers: ('hot' | 'warm' | 'cold')[];

     

       286
       286
       +
       }

     

       287
       287
       +
       

     

       288
       288
       +
       /**

     

       225
       289
        
        * Configuration for the TieredStorage system.

     

       226
       290
        
        *

     

       227
       291
        
        * @typeParam T - The type of data being stored (for serialization)

     
···

       246
       310
        
           /** Required cold tier - slowest, largest capacity (e.g., S3, R2, object storage) */

     

       247
       311
        
           cold: StorageTier;

     

       248
       312
        
         };

     

       313
       313
       +
       

     

       314
       314
       +
         /** Rules for automatic tier placement based on key patterns. First match wins. */

     

       315
       315
       +
         placementRules?: PlacementRule[];

     

       249
       316
        
       

     

       250
       317
        
         /**

     

       251
       318
        
          * Whether to automatically compress data before storing.

+40

src/utils/glob.ts

···

       1
       1
       +
       /**

     

       2
       2
       +
        * Simple glob pattern matching for key placement rules.

     

       3
       3
       +
        *

     

       4
       4
       +
        * Supports:

     

       5
       5
       +
        * - `*` matches any characters except `/`

     

       6
       6
       +
        * - `**` matches any characters including `/` (including empty string)

     

       7
       7
       +
        * - `{a,b,c}` matches any of the alternatives

     

       8
       8
       +
        * - Exact strings match exactly

     

       9
       9
       +
        */

     

       10
       10
       +
       export function matchGlob(pattern: string, key: string): boolean {

     

       11
       11
       +
         // Handle exact match

     

       12
       12
       +
         if (!pattern.includes('*') && !pattern.includes('{')) {

     

       13
       13
       +
           return pattern === key;

     

       14
       14
       +
         }

     

       15
       15
       +
       

     

       16
       16
       +
         // Escape regex special chars (except * and {})

     

       17
       17
       +
         let regex = pattern.replace(/[.+^$|\\()[\]]/g, '\\$&');

     

       18
       18
       +
       

     

       19
       19
       +
         // Handle {a,b,c} alternation

     

       20
       20
       +
         regex = regex.replace(/\{([^}]+)\}/g, (_, alts) => `(${alts.split(',').join('|')})`);

     

       21
       21
       +
       

     

       22
       22
       +
         // Use placeholder to avoid double-processing

     

       23
       23
       +
         const DOUBLE = '\x00DOUBLE\x00';

     

       24
       24
       +
         const SINGLE = '\x00SINGLE\x00';

     

       25
       25
       +
       

     

       26
       26
       +
         // Mark ** and * with placeholders

     

       27
       27
       +
         regex = regex.replace(/\*\*/g, DOUBLE);

     

       28
       28
       +
         regex = regex.replace(/\*/g, SINGLE);

     

       29
       29
       +
       

     

       30
       30
       +
         // Replace placeholders with regex patterns

     

       31
       31
       +
         // ** matches anything (including /)

     

       32
       32
       +
         // When followed by /, it's optional (matches zero or more path segments)

     

       33
       33
       +
         regex = regex

     

       34
       34
       +
           .replace(new RegExp(`${DOUBLE}/`, 'g'), '(?:.*/)?')  // **/ -> optional path prefix

     

       35
       35
       +
           .replace(new RegExp(`/${DOUBLE}`, 'g'), '(?:/.*)?')  // /** -> optional path suffix

     

       36
       36
       +
           .replace(new RegExp(DOUBLE, 'g'), '.*')              // ** alone -> match anything

     

       37
       37
       +
           .replace(new RegExp(SINGLE, 'g'), '[^/]*');          // * -> match non-slash

     

       38
       38
       +
       

     

       39
       39
       +
         return new RegExp(`^${regex}$`).test(key);

     

       40
       40
       +
       }

+175

test/TieredStorage.test.ts

···

       401
       401
        
             expect(stats.hot?.items).toBe(2);

     

       402
       402
        
           });

     

       403
       403
        
         });

     

       404
       404
       +
       

     

       405
       405
       +
         describe('Placement Rules', () => {

     

       406
       406
       +
           it('should place index.html in all tiers based on rule', async () => {

     

       407
       407
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       408
       408
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       409
       409
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       410
       410
       +
       

     

       411
       411
       +
             const storage = new TieredStorage({

     

       412
       412
       +
               tiers: { hot, warm, cold },

     

       413
       413
       +
               placementRules: [

     

       414
       414
       +
                 { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       415
       415
       +
                 { pattern: '**', tiers: ['warm', 'cold'] },

     

       416
       416
       +
               ],

     

       417
       417
       +
             });

     

       418
       418
       +
       

     

       419
       419
       +
             await storage.set('site:abc/index.html', { content: 'hello' });

     

       420
       420
       +
       

     

       421
       421
       +
             expect(await hot.exists('site:abc/index.html')).toBe(true);

     

       422
       422
       +
             expect(await warm.exists('site:abc/index.html')).toBe(true);

     

       423
       423
       +
             expect(await cold.exists('site:abc/index.html')).toBe(true);

     

       424
       424
       +
           });

     

       425
       425
       +
       

     

       426
       426
       +
           it('should skip hot tier for non-matching files', async () => {

     

       427
       427
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       428
       428
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       429
       429
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       430
       430
       +
       

     

       431
       431
       +
             const storage = new TieredStorage({

     

       432
       432
       +
               tiers: { hot, warm, cold },

     

       433
       433
       +
               placementRules: [

     

       434
       434
       +
                 { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       435
       435
       +
                 { pattern: '**', tiers: ['warm', 'cold'] },

     

       436
       436
       +
               ],

     

       437
       437
       +
             });

     

       438
       438
       +
       

     

       439
       439
       +
             await storage.set('site:abc/about.html', { content: 'about' });

     

       440
       440
       +
       

     

       441
       441
       +
             expect(await hot.exists('site:abc/about.html')).toBe(false);

     

       442
       442
       +
             expect(await warm.exists('site:abc/about.html')).toBe(true);

     

       443
       443
       +
             expect(await cold.exists('site:abc/about.html')).toBe(true);

     

       444
       444
       +
           });

     

       445
       445
       +
       

     

       446
       446
       +
           it('should match directory patterns', async () => {

     

       447
       447
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       448
       448
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       449
       449
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       450
       450
       +
       

     

       451
       451
       +
             const storage = new TieredStorage({

     

       452
       452
       +
               tiers: { hot, warm, cold },

     

       453
       453
       +
               placementRules: [

     

       454
       454
       +
                 { pattern: 'assets/**', tiers: ['warm', 'cold'] },

     

       455
       455
       +
                 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },

     

       456
       456
       +
               ],

     

       457
       457
       +
             });

     

       458
       458
       +
       

     

       459
       459
       +
             await storage.set('assets/images/logo.png', { data: 'png' });

     

       460
       460
       +
             await storage.set('index.html', { data: 'html' });

     

       461
       461
       +
       

     

       462
       462
       +
             // assets/** should skip hot

     

       463
       463
       +
             expect(await hot.exists('assets/images/logo.png')).toBe(false);

     

       464
       464
       +
             expect(await warm.exists('assets/images/logo.png')).toBe(true);

     

       465
       465
       +
       

     

       466
       466
       +
             // everything else goes to all tiers

     

       467
       467
       +
             expect(await hot.exists('index.html')).toBe(true);

     

       468
       468
       +
           });

     

       469
       469
       +
       

     

       470
       470
       +
           it('should match file extension patterns', async () => {

     

       471
       471
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       472
       472
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       473
       473
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       474
       474
       +
       

     

       475
       475
       +
             const storage = new TieredStorage({

     

       476
       476
       +
               tiers: { hot, warm, cold },

     

       477
       477
       +
               placementRules: [

     

       478
       478
       +
                 { pattern: '**/*.{jpg,png,gif,mp4}', tiers: ['warm', 'cold'] },

     

       479
       479
       +
                 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },

     

       480
       480
       +
               ],

     

       481
       481
       +
             });

     

       482
       482
       +
       

     

       483
       483
       +
             await storage.set('site/hero.png', { data: 'image' });

     

       484
       484
       +
             await storage.set('site/video.mp4', { data: 'video' });

     

       485
       485
       +
             await storage.set('site/index.html', { data: 'html' });

     

       486
       486
       +
       

     

       487
       487
       +
             // Images and video skip hot

     

       488
       488
       +
             expect(await hot.exists('site/hero.png')).toBe(false);

     

       489
       489
       +
             expect(await hot.exists('site/video.mp4')).toBe(false);

     

       490
       490
       +
       

     

       491
       491
       +
             // HTML goes everywhere

     

       492
       492
       +
             expect(await hot.exists('site/index.html')).toBe(true);

     

       493
       493
       +
           });

     

       494
       494
       +
       

     

       495
       495
       +
           it('should use first matching rule', async () => {

     

       496
       496
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       497
       497
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       498
       498
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       499
       499
       +
       

     

       500
       500
       +
             const storage = new TieredStorage({

     

       501
       501
       +
               tiers: { hot, warm, cold },

     

       502
       502
       +
               placementRules: [

     

       503
       503
       +
                 // Specific rule first

     

       504
       504
       +
                 { pattern: 'assets/critical.css', tiers: ['hot', 'warm', 'cold'] },

     

       505
       505
       +
                 // General rule second

     

       506
       506
       +
                 { pattern: 'assets/**', tiers: ['warm', 'cold'] },

     

       507
       507
       +
                 { pattern: '**', tiers: ['warm', 'cold'] },

     

       508
       508
       +
               ],

     

       509
       509
       +
             });

     

       510
       510
       +
       

     

       511
       511
       +
             await storage.set('assets/critical.css', { data: 'css' });

     

       512
       512
       +
             await storage.set('assets/style.css', { data: 'css' });

     

       513
       513
       +
       

     

       514
       514
       +
             // critical.css matches first rule -> hot

     

       515
       515
       +
             expect(await hot.exists('assets/critical.css')).toBe(true);

     

       516
       516
       +
       

     

       517
       517
       +
             // style.css matches second rule -> no hot

     

       518
       518
       +
             expect(await hot.exists('assets/style.css')).toBe(false);

     

       519
       519
       +
           });

     

       520
       520
       +
       

     

       521
       521
       +
           it('should allow skipTiers to override placement rules', async () => {

     

       522
       522
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       523
       523
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       524
       524
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       525
       525
       +
       

     

       526
       526
       +
             const storage = new TieredStorage({

     

       527
       527
       +
               tiers: { hot, warm, cold },

     

       528
       528
       +
               placementRules: [

     

       529
       529
       +
                 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },

     

       530
       530
       +
               ],

     

       531
       531
       +
             });

     

       532
       532
       +
       

     

       533
       533
       +
             // Explicit skipTiers should override the rule

     

       534
       534
       +
             await storage.set('large-file.bin', { data: 'big' }, { skipTiers: ['hot'] });

     

       535
       535
       +
       

     

       536
       536
       +
             expect(await hot.exists('large-file.bin')).toBe(false);

     

       537
       537
       +
             expect(await warm.exists('large-file.bin')).toBe(true);

     

       538
       538
       +
             expect(await cold.exists('large-file.bin')).toBe(true);

     

       539
       539
       +
           });

     

       540
       540
       +
       

     

       541
       541
       +
           it('should always include cold tier even if not in rule', async () => {

     

       542
       542
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       543
       543
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       544
       544
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       545
       545
       +
       

     

       546
       546
       +
             const storage = new TieredStorage({

     

       547
       547
       +
               tiers: { hot, warm, cold },

     

       548
       548
       +
               placementRules: [

     

       549
       549
       +
                 // Rule doesn't include cold (should be auto-added)

     

       550
       550
       +
                 { pattern: '**', tiers: ['hot', 'warm'] },

     

       551
       551
       +
               ],

     

       552
       552
       +
             });

     

       553
       553
       +
       

     

       554
       554
       +
             await storage.set('test-key', { data: 'test' });

     

       555
       555
       +
       

     

       556
       556
       +
             expect(await cold.exists('test-key')).toBe(true);

     

       557
       557
       +
           });

     

       558
       558
       +
       

     

       559
       559
       +
           it('should write to all tiers when no rules match', async () => {

     

       560
       560
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       561
       561
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       562
       562
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       563
       563
       +
       

     

       564
       564
       +
             const storage = new TieredStorage({

     

       565
       565
       +
               tiers: { hot, warm, cold },

     

       566
       566
       +
               placementRules: [

     

       567
       567
       +
                 { pattern: 'specific-pattern-only', tiers: ['warm', 'cold'] },

     

       568
       568
       +
               ],

     

       569
       569
       +
             });

     

       570
       570
       +
       

     

       571
       571
       +
             // This doesn't match any rule

     

       572
       572
       +
             await storage.set('other-key', { data: 'test' });

     

       573
       573
       +
       

     

       574
       574
       +
             expect(await hot.exists('other-key')).toBe(true);

     

       575
       575
       +
             expect(await warm.exists('other-key')).toBe(true);

     

       576
       576
       +
             expect(await cold.exists('other-key')).toBe(true);

     

       577
       577
       +
           });

     

       578
       578
       +
         });

     

       404
       579
        
       });

+95

test/glob.test.ts

···

       1
       1
       +
       import { describe, it, expect } from 'vitest';

     

       2
       2
       +
       import { matchGlob } from '../src/utils/glob.js';

     

       3
       3
       +
       

     

       4
       4
       +
       describe('matchGlob', () => {

     

       5
       5
       +
         describe('exact matches', () => {

     

       6
       6
       +
           it('should match exact strings', () => {

     

       7
       7
       +
             expect(matchGlob('index.html', 'index.html')).toBe(true);

     

       8
       8
       +
             expect(matchGlob('index.html', 'about.html')).toBe(false);

     

       9
       9
       +
           });

     

       10
       10
       +
       

     

       11
       11
       +
           it('should match paths exactly', () => {

     

       12
       12
       +
             expect(matchGlob('site/index.html', 'site/index.html')).toBe(true);

     

       13
       13
       +
             expect(matchGlob('site/index.html', 'other/index.html')).toBe(false);

     

       14
       14
       +
           });

     

       15
       15
       +
         });

     

       16
       16
       +
       

     

       17
       17
       +
         describe('* wildcard', () => {

     

       18
       18
       +
           it('should match any characters except /', () => {

     

       19
       19
       +
             expect(matchGlob('*.html', 'index.html')).toBe(true);

     

       20
       20
       +
             expect(matchGlob('*.html', 'about.html')).toBe(true);

     

       21
       21
       +
             expect(matchGlob('*.html', 'style.css')).toBe(false);

     

       22
       22
       +
           });

     

       23
       23
       +
       

     

       24
       24
       +
           it('should not match across path separators', () => {

     

       25
       25
       +
             expect(matchGlob('*.html', 'dir/index.html')).toBe(false);

     

       26
       26
       +
           });

     

       27
       27
       +
       

     

       28
       28
       +
           it('should work with prefix and suffix', () => {

     

       29
       29
       +
             expect(matchGlob('index.*', 'index.html')).toBe(true);

     

       30
       30
       +
             expect(matchGlob('index.*', 'index.css')).toBe(true);

     

       31
       31
       +
             expect(matchGlob('index.*', 'about.html')).toBe(false);

     

       32
       32
       +
           });

     

       33
       33
       +
         });

     

       34
       34
       +
       

     

       35
       35
       +
         describe('** wildcard', () => {

     

       36
       36
       +
           it('should match any characters including /', () => {

     

       37
       37
       +
             expect(matchGlob('**', 'anything')).toBe(true);

     

       38
       38
       +
             expect(matchGlob('**', 'path/to/file.txt')).toBe(true);

     

       39
       39
       +
           });

     

       40
       40
       +
       

     

       41
       41
       +
           it('should match deeply nested paths', () => {

     

       42
       42
       +
             expect(matchGlob('**/index.html', 'index.html')).toBe(true);

     

       43
       43
       +
             expect(matchGlob('**/index.html', 'site/index.html')).toBe(true);

     

       44
       44
       +
             expect(matchGlob('**/index.html', 'a/b/c/index.html')).toBe(true);

     

       45
       45
       +
             expect(matchGlob('**/index.html', 'a/b/c/about.html')).toBe(false);

     

       46
       46
       +
           });

     

       47
       47
       +
       

     

       48
       48
       +
           it('should match directory prefixes', () => {

     

       49
       49
       +
             expect(matchGlob('assets/**', 'assets/style.css')).toBe(true);

     

       50
       50
       +
             expect(matchGlob('assets/**', 'assets/images/logo.png')).toBe(true);

     

       51
       51
       +
             expect(matchGlob('assets/**', 'other/style.css')).toBe(false);

     

       52
       52
       +
           });

     

       53
       53
       +
       

     

       54
       54
       +
           it('should match in the middle of a path', () => {

     

       55
       55
       +
             expect(matchGlob('site/**/index.html', 'site/index.html')).toBe(true);

     

       56
       56
       +
             expect(matchGlob('site/**/index.html', 'site/pages/index.html')).toBe(true);

     

       57
       57
       +
             expect(matchGlob('site/**/index.html', 'site/a/b/c/index.html')).toBe(true);

     

       58
       58
       +
           });

     

       59
       59
       +
         });

     

       60
       60
       +
       

     

       61
       61
       +
         describe('{a,b,c} alternation', () => {

     

       62
       62
       +
           it('should match any of the alternatives', () => {

     

       63
       63
       +
             expect(matchGlob('*.{html,css,js}', 'index.html')).toBe(true);

     

       64
       64
       +
             expect(matchGlob('*.{html,css,js}', 'style.css')).toBe(true);

     

       65
       65
       +
             expect(matchGlob('*.{html,css,js}', 'app.js')).toBe(true);

     

       66
       66
       +
             expect(matchGlob('*.{html,css,js}', 'image.png')).toBe(false);

     

       67
       67
       +
           });

     

       68
       68
       +
       

     

       69
       69
       +
           it('should work with ** and alternation', () => {

     

       70
       70
       +
             expect(matchGlob('**/*.{jpg,png,gif}', 'logo.png')).toBe(true);

     

       71
       71
       +
             expect(matchGlob('**/*.{jpg,png,gif}', 'images/logo.png')).toBe(true);

     

       72
       72
       +
             expect(matchGlob('**/*.{jpg,png,gif}', 'a/b/photo.jpg')).toBe(true);

     

       73
       73
       +
             expect(matchGlob('**/*.{jpg,png,gif}', 'style.css')).toBe(false);

     

       74
       74
       +
           });

     

       75
       75
       +
         });

     

       76
       76
       +
       

     

       77
       77
       +
         describe('edge cases', () => {

     

       78
       78
       +
           it('should handle empty strings', () => {

     

       79
       79
       +
             expect(matchGlob('', '')).toBe(true);

     

       80
       80
       +
             expect(matchGlob('', 'something')).toBe(false);

     

       81
       81
       +
             expect(matchGlob('**', '')).toBe(true);

     

       82
       82
       +
           });

     

       83
       83
       +
       

     

       84
       84
       +
           it('should escape regex special characters', () => {

     

       85
       85
       +
             expect(matchGlob('file.txt', 'file.txt')).toBe(true);

     

       86
       86
       +
             expect(matchGlob('file.txt', 'filextxt')).toBe(false);

     

       87
       87
       +
             expect(matchGlob('file[1].txt', 'file[1].txt')).toBe(true);

     

       88
       88
       +
           });

     

       89
       89
       +
       

     

       90
       90
       +
           it('should handle keys with colons (common in storage)', () => {

     

       91
       91
       +
             expect(matchGlob('site:*/index.html', 'site:abc/index.html')).toBe(true);

     

       92
       92
       +
             expect(matchGlob('site:**/index.html', 'site:abc/pages/index.html')).toBe(true);

     

       93
       93
       +
           });

     

       94
       94
       +
         });

     

       95
       95
       +
       });

Compare changes