commit 2f40f17ba7733703835878c363e4ea6617c402eb · nekomimi.pet/tiered-storage

+48 -11

README.md

···

       28
       28
        
           warm: new DiskStorageTier({ directory: './cache' }),

     

       29
       29
        
           cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),

     

       30
       30
        
         },

     

       31
       31
       -
         compression: true,

     

       31
       31
       +
         placementRules: [

     

       32
       32
       +
           { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       33
       33
       +
           { pattern: '**/*.{jpg,png,gif,mp4}', tiers: ['warm', 'cold'] },

     

       34
       34
       +
           { pattern: '**', tiers: ['warm', 'cold'] },

     

       35
       35
       +
         ],

     

       32
       36
        
       })

     

       33
       37
        
       

     

       34
       34
       -
       // critical file: keep in memory for instant serving

     

       35
       35
       -
       await storage.set('site:abc/index.html', indexHtml)

     

       36
       36
       -
       

     

       37
       37
       -
       // big files: skip hot, let them live in warm + cold

     

       38
       38
       -
       await storage.set('site:abc/video.mp4', videoData, { skipTiers: ['hot'] })

     

       39
       39
       -
       await storage.set('site:abc/hero.png', imageData, { skipTiers: ['hot'] })

     

       38
       38
       +
       // just set - rules decide where it goes

     

       39
       39
       +
       await storage.set('site:abc/index.html', indexHtml)  // → hot + warm + cold

     

       40
       40
       +
       await storage.set('site:abc/hero.png', imageData)    // → warm + cold

     

       41
       41
       +
       await storage.set('site:abc/video.mp4', videoData)   // → warm + cold

     

       40
       42
        
       

     

       41
       41
       -
       // on read, bubbles up from wherever it lives

     

       42
       42
       -
       const result = await storage.getWithMetadata('site:abc/index.html')

     

       43
       43
       -
       console.log(result.source) // 'hot' - served from memory

     

       43
       43
       +
       // reads bubble up from wherever it lives

     

       44
       44
       +
       const page = await storage.getWithMetadata('site:abc/index.html')

     

       45
       45
       +
       console.log(page.source) // 'hot'

     

       44
       46
        
       

     

       45
       47
        
       const video = await storage.getWithMetadata('site:abc/video.mp4')

     

       46
       46
       -
       console.log(video.source) // 'warm' - served from disk, never touches memory

     

       48
       48
       +
       console.log(video.source) // 'warm'

     

       47
       49
        
       

     

       48
       50
        
       // nuke entire site

     

       49
       51
        
       await storage.invalidate('site:abc/')

     
···

       93
       95
        
       ```

     

       94
       96
        
       

     

       95
       97
        
       A file that hasn't been accessed eventually gets evicted from hot (LRU), then warm (size limit + policy). Next request fetches from cold and promotes it back up.

     

       98
       98
       +
       

     

       99
       99
       +
       ## Placement rules

     

       100
       100
       +
       

     

       101
       101
       +
       Define once which keys go where, instead of passing `skipTiers` on every `set()`:

     

       102
       102
       +
       

     

       103
       103
       +
       ```typescript

     

       104
       104
       +
       const storage = new TieredStorage({

     

       105
       105
       +
         tiers: {

     

       106
       106
       +
           hot: new MemoryStorageTier({ maxSizeBytes: 50 * 1024 * 1024 }),

     

       107
       107
       +
           warm: new DiskStorageTier({ directory: './cache' }),

     

       108
       108
       +
           cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),

     

       109
       109
       +
         },

     

       110
       110
       +
         placementRules: [

     

       111
       111
       +
           // index.html goes everywhere for instant serving

     

       112
       112
       +
           { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       113
       113
       +
       

     

       114
       114
       +
           // images and video skip hot

     

       115
       115
       +
           { pattern: '**/*.{jpg,png,gif,webp,mp4}', tiers: ['warm', 'cold'] },

     

       116
       116
       +
       

     

       117
       117
       +
           // assets directory skips hot

     

       118
       118
       +
           { pattern: 'assets/**', tiers: ['warm', 'cold'] },

     

       119
       119
       +
       

     

       120
       120
       +
           // everything else: warm + cold only

     

       121
       121
       +
           { pattern: '**', tiers: ['warm', 'cold'] },

     

       122
       122
       +
         ],

     

       123
       123
       +
       })

     

       124
       124
       +
       

     

       125
       125
       +
       // just call set() - rules handle placement

     

       126
       126
       +
       await storage.set('site:abc/index.html', html)       // → hot + warm + cold

     

       127
       127
       +
       await storage.set('site:abc/hero.png', image)        // → warm + cold

     

       128
       128
       +
       await storage.set('site:abc/assets/font.woff', font) // → warm + cold

     

       129
       129
       +
       await storage.set('site:abc/about.html', html)       // → warm + cold

     

       130
       130
       +
       ```

     

       131
       131
       +
       

     

       132
       132
       +
       Rules are evaluated in order. First match wins. Cold is always included.

     

       96
       133
        
       

     

       97
       134
        
       ## API

     

       98
       135

+17 -10

serve-example.ts

···

       44
       44
        
             prefix: 'demo-sites/',

     

       45
       45
        
           }),

     

       46
       46
        
         },

     

       47
       47
       +
         placementRules: [

     

       48
       48
       +
           // index.html goes to all tiers for instant serving

     

       49
       49
       +
           { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       50
       50
       +
           

     

       51
       51
       +
           // everything else: warm + cold only

     

       52
       52
       +
           { pattern: '**', tiers: ['warm', 'cold'] },

     

       53
       53
       +
         ],

     

       47
       54
        
         compression: true,

     

       48
       55
        
         defaultTTL: 14 * 24 * 60 * 60 * 1000,

     

       49
       56
        
         promotionStrategy: 'lazy',

     
···

       62
       69
        
         console.log('\n📦 Loading example site into tiered storage...\n');

     

       63
       70
        
       

     

       64
       71
        
         const files = [

     

       65
       65
       -
           { name: 'index.html', skipTiers: [], mimeType: 'text/html' },

     

       66
       66
       -
           { name: 'about.html', skipTiers: ['hot'], mimeType: 'text/html' },

     

       67
       67
       -
           { name: 'docs.html', skipTiers: ['hot'], mimeType: 'text/html' },

     

       68
       68
       -
           { name: 'style.css', skipTiers: ['hot'], mimeType: 'text/css' },

     

       69
       69
       -
           { name: 'script.js', skipTiers: ['hot'], mimeType: 'application/javascript' },

     

       72
       72
       +
           { name: 'index.html', mimeType: 'text/html' },

     

       73
       73
       +
           { name: 'about.html', mimeType: 'text/html' },

     

       74
       74
       +
           { name: 'docs.html', mimeType: 'text/html' },

     

       75
       75
       +
           { name: 'style.css', mimeType: 'text/css' },

     

       76
       76
       +
           { name: 'script.js', mimeType: 'application/javascript' },

     

       70
       77
        
         ];

     

       71
       78
        
       

     

       72
       79
        
         for (const file of files) {

     
···

       74
       81
        
           const key = `${siteId}/${siteName}/${file.name}`;

     

       75
       82
        
       

     

       76
       83
        
           await storage.set(key, content, {

     

       77
       77
       -
             skipTiers: file.skipTiers as ('hot' | 'warm')[],

     

       78
       84
        
             metadata: { mimeType: file.mimeType },

     

       79
       85
        
           });

     

       80
       86
        
       

     

       81
       81
       -
           const tierInfo =

     

       82
       82
       -
             file.skipTiers.length === 0

     

       83
       83
       -
               ? '🔥 hot + 💾 warm + ☁️  cold'

     

       84
       84
       -
               : `💾 warm + ☁️  cold (skipped hot)`;

     

       87
       87
       +
           // Determine which tiers this file went to based on placement rules

     

       88
       88
       +
           const isIndex = file.name === 'index.html';

     

       89
       89
       +
           const tierInfo = isIndex

     

       90
       90
       +
             ? '🔥 hot + 💾 warm + ☁️  cold'

     

       91
       91
       +
             : '💾 warm + ☁️  cold (skipped hot)';

     

       85
       92
        
           const sizeKB = (content.length / 1024).toFixed(2);

     

       86
       93
        
           console.log(`   ✓ ${file.name.padEnd(15)} ${sizeKB.padStart(6)} KB → ${tierInfo}`);

     

       87
       94
        
         }

+47 -11

src/TieredStorage.ts

···

       6
       6
        
         StorageMetadata,

     

       7
       7
        
         AllTierStats,

     

       8
       8
        
         StorageSnapshot,

     

       9
       9
       -
       } from './types/index.js';

     

       9
       9
       +
         PlacementRule,

     

       10
       10
       +
       } from './types/index';

     

       10
       11
        
       import { compress, decompress } from './utils/compression.js';

     

       11
       12
        
       import { defaultSerialize, defaultDeserialize } from './utils/serialization.js';

     

       12
       13
        
       import { calculateChecksum } from './utils/checksum.js';

     

       14
       14
       +
       import { matchGlob } from './utils/glob.js';

     

       13
       15
        
       

     

       14
       16
        
       /**

     

       15
       17
        
        * Main orchestrator for tiered storage system.

     
···

       202
       204
        
           // 3. Create metadata

     

       203
       205
        
           const metadata = this.createMetadata(key, finalData, options);

     

       204
       206
        
       

     

       205
       205
       -
           // 4. Write to all tiers (cascading down)

     

       207
       207
       +
           // 4. Determine which tiers to write to

     

       208
       208
       +
           const allowedTiers = this.getTiersForKey(key, options?.skipTiers);

     

       209
       209
       +
       

     

       210
       210
       +
           // 5. Write to tiers

     

       206
       211
        
           const tiersWritten: ('hot' | 'warm' | 'cold')[] = [];

     

       207
       212
        
       

     

       208
       208
       -
           // Write to hot (if configured and not skipped)

     

       209
       209
       -
           if (this.config.tiers.hot && !options?.skipTiers?.includes('hot')) {

     

       213
       213
       +
           if (this.config.tiers.hot && allowedTiers.includes('hot')) {

     

       210
       214
        
             await this.config.tiers.hot.set(key, finalData, metadata);

     

       211
       215
        
             tiersWritten.push('hot');

     

       216
       216
       +
           }

     

       212
       217
        
       

     

       213
       213
       -
             // Hot writes cascade to warm

     

       214
       214
       -
             if (this.config.tiers.warm && !options?.skipTiers?.includes('warm')) {

     

       215
       215
       -
               await this.config.tiers.warm.set(key, finalData, metadata);

     

       216
       216
       -
               tiersWritten.push('warm');

     

       217
       217
       -
             }

     

       218
       218
       -
           } else if (this.config.tiers.warm && !options?.skipTiers?.includes('warm')) {

     

       219
       219
       -
             // Write to warm (if hot skipped)

     

       218
       218
       +
           if (this.config.tiers.warm && allowedTiers.includes('warm')) {

     

       220
       219
        
             await this.config.tiers.warm.set(key, finalData, metadata);

     

       221
       220
        
             tiersWritten.push('warm');

     

       222
       221
        
           }

     
···

       226
       225
        
           tiersWritten.push('cold');

     

       227
       226
        
       

     

       228
       227
        
           return { key, metadata, tiersWritten };

     

       228
       228
       +
         }

     

       229
       229
       +
       

     

       230
       230
       +
         /**

     

       231
       231
       +
          * Determine which tiers a key should be written to.

     

       232
       232
       +
          *

     

       233
       233
       +
          * @param key - The key being stored

     

       234
       234
       +
          * @param skipTiers - Explicit tiers to skip (overrides placement rules)

     

       235
       235
       +
          * @returns Array of tiers to write to

     

       236
       236
       +
          *

     

       237
       237
       +
          * @remarks

     

       238
       238
       +
          * Priority: skipTiers option > placementRules > all configured tiers

     

       239
       239
       +
          */

     

       240
       240
       +
         private getTiersForKey(

     

       241
       241
       +
           key: string,

     

       242
       242
       +
           skipTiers?: ('hot' | 'warm')[]

     

       243
       243
       +
         ): ('hot' | 'warm' | 'cold')[] {

     

       244
       244
       +
           // If explicit skipTiers provided, use that

     

       245
       245
       +
           if (skipTiers && skipTiers.length > 0) {

     

       246
       246
       +
             const allTiers: ('hot' | 'warm' | 'cold')[] = ['hot', 'warm', 'cold'];

     

       247
       247
       +
             return allTiers.filter((t) => !skipTiers.includes(t as 'hot' | 'warm'));

     

       248
       248
       +
           }

     

       249
       249
       +
       

     

       250
       250
       +
           // Check placement rules

     

       251
       251
       +
           if (this.config.placementRules) {

     

       252
       252
       +
             for (const rule of this.config.placementRules) {

     

       253
       253
       +
               if (matchGlob(rule.pattern, key)) {

     

       254
       254
       +
                 // Ensure cold is always included

     

       255
       255
       +
                 if (!rule.tiers.includes('cold')) {

     

       256
       256
       +
                   return [...rule.tiers, 'cold'];

     

       257
       257
       +
                 }

     

       258
       258
       +
                 return rule.tiers;

     

       259
       259
       +
               }

     

       260
       260
       +
             }

     

       261
       261
       +
           }

     

       262
       262
       +
       

     

       263
       263
       +
           // Default: write to all configured tiers

     

       264
       264
       +
           return ['hot', 'warm', 'cold'];

     

       229
       265
        
         }

     

       230
       266
        
       

     

       231
       267
        
         /**

src/index.ts

···

       22
       22
        
         TierStats,

     

       23
       23
        
         AllTierStats,

     

       24
       24
        
         TieredStorageConfig,

     

       25
       25
       +
         PlacementRule,

     

       25
       26
        
         SetOptions,

     

       26
       27
        
         StorageResult,

     

       27
       28
        
         SetResult,

+45

src/types/index.ts

···

       222
       222
        
       }

     

       223
       223
        
       

     

       224
       224
        
       /**

     

       225
       225
       +
        * Rule for automatic tier placement based on key patterns.

     

       226
       226
       +
        *

     

       227
       227
       +
        * @remarks

     

       228
       228
       +
        * Rules are evaluated in order. First matching rule wins.

     

       229
       229
       +
        * Use this to define which keys go to which tiers without

     

       230
       230
       +
        * specifying skipTiers on every set() call.

     

       231
       231
       +
        *

     

       232
       232
       +
        * @example

     

       233
       233
       +
        * ```typescript

     

       234
       234
       +
        * placementRules: [

     

       235
       235
       +
        *   { pattern: 'index.html', tiers: ['hot', 'warm', 'cold'] },

     

       236
       236
       +
        *   { pattern: '*.html', tiers: ['warm', 'cold'] },

     

       237
       237
       +
        *   { pattern: 'assets/**', tiers: ['warm', 'cold'] },

     

       238
       238
       +
        *   { pattern: '**', tiers: ['warm', 'cold'] },  // default

     

       239
       239
       +
        * ]

     

       240
       240
       +
        * ```

     

       241
       241
       +
        */

     

       242
       242
       +
       export interface PlacementRule {

     

       243
       243
       +
         /**

     

       244
       244
       +
          * Glob pattern to match against keys.

     

       245
       245
       +
          *

     

       246
       246
       +
          * @remarks

     

       247
       247
       +
          * Supports basic globs:

     

       248
       248
       +
          * - `*` matches any characters except `/`

     

       249
       249
       +
          * - `**` matches any characters including `/`

     

       250
       250
       +
          * - Exact matches work too: `index.html`

     

       251
       251
       +
          */

     

       252
       252
       +
         pattern: string;

     

       253
       253
       +
       

     

       254
       254
       +
         /**

     

       255
       255
       +
          * Which tiers to write to for matching keys.

     

       256
       256
       +
          *

     

       257
       257
       +
          * @remarks

     

       258
       258
       +
          * Cold is always included (source of truth).

     

       259
       259
       +
          * Use `['hot', 'warm', 'cold']` for critical files.

     

       260
       260
       +
          * Use `['warm', 'cold']` for large files.

     

       261
       261
       +
          * Use `['cold']` for archival only.

     

       262
       262
       +
          */

     

       263
       263
       +
         tiers: ('hot' | 'warm' | 'cold')[];

     

       264
       264
       +
       }

     

       265
       265
       +
       

     

       266
       266
       +
       /**

     

       225
       267
        
        * Configuration for the TieredStorage system.

     

       226
       268
        
        *

     

       227
       269
        
        * @typeParam T - The type of data being stored (for serialization)

     
···

       246
       288
        
           /** Required cold tier - slowest, largest capacity (e.g., S3, R2, object storage) */

     

       247
       289
        
           cold: StorageTier;

     

       248
       290
        
         };

     

       291
       291
       +
       

     

       292
       292
       +
         /** Rules for automatic tier placement based on key patterns. First match wins. */

     

       293
       293
       +
         placementRules?: PlacementRule[];

     

       249
       294
        
       

     

       250
       295
        
         /**

     

       251
       296
        
          * Whether to automatically compress data before storing.

+40

src/utils/glob.ts

···

       1
       1
       +
       /**

     

       2
       2
       +
        * Simple glob pattern matching for key placement rules.

     

       3
       3
       +
        *

     

       4
       4
       +
        * Supports:

     

       5
       5
       +
        * - `*` matches any characters except `/`

     

       6
       6
       +
        * - `**` matches any characters including `/` (including empty string)

     

       7
       7
       +
        * - `{a,b,c}` matches any of the alternatives

     

       8
       8
       +
        * - Exact strings match exactly

     

       9
       9
       +
        */

     

       10
       10
       +
       export function matchGlob(pattern: string, key: string): boolean {

     

       11
       11
       +
         // Handle exact match

     

       12
       12
       +
         if (!pattern.includes('*') && !pattern.includes('{')) {

     

       13
       13
       +
           return pattern === key;

     

       14
       14
       +
         }

     

       15
       15
       +
       

     

       16
       16
       +
         // Escape regex special chars (except * and {})

     

       17
       17
       +
         let regex = pattern.replace(/[.+^$|\\()[\]]/g, '\\$&');

     

       18
       18
       +
       

     

       19
       19
       +
         // Handle {a,b,c} alternation

     

       20
       20
       +
         regex = regex.replace(/\{([^}]+)\}/g, (_, alts) => `(${alts.split(',').join('|')})`);

     

       21
       21
       +
       

     

       22
       22
       +
         // Use placeholder to avoid double-processing

     

       23
       23
       +
         const DOUBLE = '\x00DOUBLE\x00';

     

       24
       24
       +
         const SINGLE = '\x00SINGLE\x00';

     

       25
       25
       +
       

     

       26
       26
       +
         // Mark ** and * with placeholders

     

       27
       27
       +
         regex = regex.replace(/\*\*/g, DOUBLE);

     

       28
       28
       +
         regex = regex.replace(/\*/g, SINGLE);

     

       29
       29
       +
       

     

       30
       30
       +
         // Replace placeholders with regex patterns

     

       31
       31
       +
         // ** matches anything (including /)

     

       32
       32
       +
         // When followed by /, it's optional (matches zero or more path segments)

     

       33
       33
       +
         regex = regex

     

       34
       34
       +
           .replace(new RegExp(`${DOUBLE}/`, 'g'), '(?:.*/)?')  // **/ -> optional path prefix

     

       35
       35
       +
           .replace(new RegExp(`/${DOUBLE}`, 'g'), '(?:/.*)?')  // /** -> optional path suffix

     

       36
       36
       +
           .replace(new RegExp(DOUBLE, 'g'), '.*')              // ** alone -> match anything

     

       37
       37
       +
           .replace(new RegExp(SINGLE, 'g'), '[^/]*');          // * -> match non-slash

     

       38
       38
       +
       

     

       39
       39
       +
         return new RegExp(`^${regex}$`).test(key);

     

       40
       40
       +
       }

+175

test/TieredStorage.test.ts

···

       401
       401
        
             expect(stats.hot?.items).toBe(2);

     

       402
       402
        
           });

     

       403
       403
        
         });

     

       404
       404
       +
       

     

       405
       405
       +
         describe('Placement Rules', () => {

     

       406
       406
       +
           it('should place index.html in all tiers based on rule', async () => {

     

       407
       407
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       408
       408
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       409
       409
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       410
       410
       +
       

     

       411
       411
       +
             const storage = new TieredStorage({

     

       412
       412
       +
               tiers: { hot, warm, cold },

     

       413
       413
       +
               placementRules: [

     

       414
       414
       +
                 { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       415
       415
       +
                 { pattern: '**', tiers: ['warm', 'cold'] },

     

       416
       416
       +
               ],

     

       417
       417
       +
             });

     

       418
       418
       +
       

     

       419
       419
       +
             await storage.set('site:abc/index.html', { content: 'hello' });

     

       420
       420
       +
       

     

       421
       421
       +
             expect(await hot.exists('site:abc/index.html')).toBe(true);

     

       422
       422
       +
             expect(await warm.exists('site:abc/index.html')).toBe(true);

     

       423
       423
       +
             expect(await cold.exists('site:abc/index.html')).toBe(true);

     

       424
       424
       +
           });

     

       425
       425
       +
       

     

       426
       426
       +
           it('should skip hot tier for non-matching files', async () => {

     

       427
       427
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       428
       428
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       429
       429
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       430
       430
       +
       

     

       431
       431
       +
             const storage = new TieredStorage({

     

       432
       432
       +
               tiers: { hot, warm, cold },

     

       433
       433
       +
               placementRules: [

     

       434
       434
       +
                 { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },

     

       435
       435
       +
                 { pattern: '**', tiers: ['warm', 'cold'] },

     

       436
       436
       +
               ],

     

       437
       437
       +
             });

     

       438
       438
       +
       

     

       439
       439
       +
             await storage.set('site:abc/about.html', { content: 'about' });

     

       440
       440
       +
       

     

       441
       441
       +
             expect(await hot.exists('site:abc/about.html')).toBe(false);

     

       442
       442
       +
             expect(await warm.exists('site:abc/about.html')).toBe(true);

     

       443
       443
       +
             expect(await cold.exists('site:abc/about.html')).toBe(true);

     

       444
       444
       +
           });

     

       445
       445
       +
       

     

       446
       446
       +
           it('should match directory patterns', async () => {

     

       447
       447
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       448
       448
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       449
       449
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       450
       450
       +
       

     

       451
       451
       +
             const storage = new TieredStorage({

     

       452
       452
       +
               tiers: { hot, warm, cold },

     

       453
       453
       +
               placementRules: [

     

       454
       454
       +
                 { pattern: 'assets/**', tiers: ['warm', 'cold'] },

     

       455
       455
       +
                 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },

     

       456
       456
       +
               ],

     

       457
       457
       +
             });

     

       458
       458
       +
       

     

       459
       459
       +
             await storage.set('assets/images/logo.png', { data: 'png' });

     

       460
       460
       +
             await storage.set('index.html', { data: 'html' });

     

       461
       461
       +
       

     

       462
       462
       +
             // assets/** should skip hot

     

       463
       463
       +
             expect(await hot.exists('assets/images/logo.png')).toBe(false);

     

       464
       464
       +
             expect(await warm.exists('assets/images/logo.png')).toBe(true);

     

       465
       465
       +
       

     

       466
       466
       +
             // everything else goes to all tiers

     

       467
       467
       +
             expect(await hot.exists('index.html')).toBe(true);

     

       468
       468
       +
           });

     

       469
       469
       +
       

     

       470
       470
       +
           it('should match file extension patterns', async () => {

     

       471
       471
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       472
       472
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       473
       473
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       474
       474
       +
       

     

       475
       475
       +
             const storage = new TieredStorage({

     

       476
       476
       +
               tiers: { hot, warm, cold },

     

       477
       477
       +
               placementRules: [

     

       478
       478
       +
                 { pattern: '**/*.{jpg,png,gif,mp4}', tiers: ['warm', 'cold'] },

     

       479
       479
       +
                 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },

     

       480
       480
       +
               ],

     

       481
       481
       +
             });

     

       482
       482
       +
       

     

       483
       483
       +
             await storage.set('site/hero.png', { data: 'image' });

     

       484
       484
       +
             await storage.set('site/video.mp4', { data: 'video' });

     

       485
       485
       +
             await storage.set('site/index.html', { data: 'html' });

     

       486
       486
       +
       

     

       487
       487
       +
             // Images and video skip hot

     

       488
       488
       +
             expect(await hot.exists('site/hero.png')).toBe(false);

     

       489
       489
       +
             expect(await hot.exists('site/video.mp4')).toBe(false);

     

       490
       490
       +
       

     

       491
       491
       +
             // HTML goes everywhere

     

       492
       492
       +
             expect(await hot.exists('site/index.html')).toBe(true);

     

       493
       493
       +
           });

     

       494
       494
       +
       

     

       495
       495
       +
           it('should use first matching rule', async () => {

     

       496
       496
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       497
       497
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       498
       498
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       499
       499
       +
       

     

       500
       500
       +
             const storage = new TieredStorage({

     

       501
       501
       +
               tiers: { hot, warm, cold },

     

       502
       502
       +
               placementRules: [

     

       503
       503
       +
                 // Specific rule first

     

       504
       504
       +
                 { pattern: 'assets/critical.css', tiers: ['hot', 'warm', 'cold'] },

     

       505
       505
       +
                 // General rule second

     

       506
       506
       +
                 { pattern: 'assets/**', tiers: ['warm', 'cold'] },

     

       507
       507
       +
                 { pattern: '**', tiers: ['warm', 'cold'] },

     

       508
       508
       +
               ],

     

       509
       509
       +
             });

     

       510
       510
       +
       

     

       511
       511
       +
             await storage.set('assets/critical.css', { data: 'css' });

     

       512
       512
       +
             await storage.set('assets/style.css', { data: 'css' });

     

       513
       513
       +
       

     

       514
       514
       +
             // critical.css matches first rule -> hot

     

       515
       515
       +
             expect(await hot.exists('assets/critical.css')).toBe(true);

     

       516
       516
       +
       

     

       517
       517
       +
             // style.css matches second rule -> no hot

     

       518
       518
       +
             expect(await hot.exists('assets/style.css')).toBe(false);

     

       519
       519
       +
           });

     

       520
       520
       +
       

     

       521
       521
       +
           it('should allow skipTiers to override placement rules', async () => {

     

       522
       522
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       523
       523
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       524
       524
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       525
       525
       +
       

     

       526
       526
       +
             const storage = new TieredStorage({

     

       527
       527
       +
               tiers: { hot, warm, cold },

     

       528
       528
       +
               placementRules: [

     

       529
       529
       +
                 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },

     

       530
       530
       +
               ],

     

       531
       531
       +
             });

     

       532
       532
       +
       

     

       533
       533
       +
             // Explicit skipTiers should override the rule

     

       534
       534
       +
             await storage.set('large-file.bin', { data: 'big' }, { skipTiers: ['hot'] });

     

       535
       535
       +
       

     

       536
       536
       +
             expect(await hot.exists('large-file.bin')).toBe(false);

     

       537
       537
       +
             expect(await warm.exists('large-file.bin')).toBe(true);

     

       538
       538
       +
             expect(await cold.exists('large-file.bin')).toBe(true);

     

       539
       539
       +
           });

     

       540
       540
       +
       

     

       541
       541
       +
           it('should always include cold tier even if not in rule', async () => {

     

       542
       542
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       543
       543
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       544
       544
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       545
       545
       +
       

     

       546
       546
       +
             const storage = new TieredStorage({

     

       547
       547
       +
               tiers: { hot, warm, cold },

     

       548
       548
       +
               placementRules: [

     

       549
       549
       +
                 // Rule doesn't include cold (should be auto-added)

     

       550
       550
       +
                 { pattern: '**', tiers: ['hot', 'warm'] },

     

       551
       551
       +
               ],

     

       552
       552
       +
             });

     

       553
       553
       +
       

     

       554
       554
       +
             await storage.set('test-key', { data: 'test' });

     

       555
       555
       +
       

     

       556
       556
       +
             expect(await cold.exists('test-key')).toBe(true);

     

       557
       557
       +
           });

     

       558
       558
       +
       

     

       559
       559
       +
           it('should write to all tiers when no rules match', async () => {

     

       560
       560
       +
             const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });

     

       561
       561
       +
             const warm = new DiskStorageTier({ directory: `${testDir}/warm` });

     

       562
       562
       +
             const cold = new DiskStorageTier({ directory: `${testDir}/cold` });

     

       563
       563
       +
       

     

       564
       564
       +
             const storage = new TieredStorage({

     

       565
       565
       +
               tiers: { hot, warm, cold },

     

       566
       566
       +
               placementRules: [

     

       567
       567
       +
                 { pattern: 'specific-pattern-only', tiers: ['warm', 'cold'] },

     

       568
       568
       +
               ],

     

       569
       569
       +
             });

     

       570
       570
       +
       

     

       571
       571
       +
             // This doesn't match any rule

     

       572
       572
       +
             await storage.set('other-key', { data: 'test' });

     

       573
       573
       +
       

     

       574
       574
       +
             expect(await hot.exists('other-key')).toBe(true);

     

       575
       575
       +
             expect(await warm.exists('other-key')).toBe(true);

     

       576
       576
       +
             expect(await cold.exists('other-key')).toBe(true);

     

       577
       577
       +
           });

     

       578
       578
       +
         });

     

       404
       579
        
       });

+95

test/glob.test.ts

···

       1
       1
       +
       import { describe, it, expect } from 'vitest';

     

       2
       2
       +
       import { matchGlob } from '../src/utils/glob.js';

     

       3
       3
       +
       

     

       4
       4
       +
       describe('matchGlob', () => {

     

       5
       5
       +
         describe('exact matches', () => {

     

       6
       6
       +
           it('should match exact strings', () => {

     

       7
       7
       +
             expect(matchGlob('index.html', 'index.html')).toBe(true);

     

       8
       8
       +
             expect(matchGlob('index.html', 'about.html')).toBe(false);

     

       9
       9
       +
           });

     

       10
       10
       +
       

     

       11
       11
       +
           it('should match paths exactly', () => {

     

       12
       12
       +
             expect(matchGlob('site/index.html', 'site/index.html')).toBe(true);

     

       13
       13
       +
             expect(matchGlob('site/index.html', 'other/index.html')).toBe(false);

     

       14
       14
       +
           });

     

       15
       15
       +
         });

     

       16
       16
       +
       

     

       17
       17
       +
         describe('* wildcard', () => {

     

       18
       18
       +
           it('should match any characters except /', () => {

     

       19
       19
       +
             expect(matchGlob('*.html', 'index.html')).toBe(true);

     

       20
       20
       +
             expect(matchGlob('*.html', 'about.html')).toBe(true);

     

       21
       21
       +
             expect(matchGlob('*.html', 'style.css')).toBe(false);

     

       22
       22
       +
           });

     

       23
       23
       +
       

     

       24
       24
       +
           it('should not match across path separators', () => {

     

       25
       25
       +
             expect(matchGlob('*.html', 'dir/index.html')).toBe(false);

     

       26
       26
       +
           });

     

       27
       27
       +
       

     

       28
       28
       +
           it('should work with prefix and suffix', () => {

     

       29
       29
       +
             expect(matchGlob('index.*', 'index.html')).toBe(true);

     

       30
       30
       +
             expect(matchGlob('index.*', 'index.css')).toBe(true);

     

       31
       31
       +
             expect(matchGlob('index.*', 'about.html')).toBe(false);

     

       32
       32
       +
           });

     

       33
       33
       +
         });

     

       34
       34
       +
       

     

       35
       35
       +
         describe('** wildcard', () => {

     

       36
       36
       +
           it('should match any characters including /', () => {

     

       37
       37
       +
             expect(matchGlob('**', 'anything')).toBe(true);

     

       38
       38
       +
             expect(matchGlob('**', 'path/to/file.txt')).toBe(true);

     

       39
       39
       +
           });

     

       40
       40
       +
       

     

       41
       41
       +
           it('should match deeply nested paths', () => {

     

       42
       42
       +
             expect(matchGlob('**/index.html', 'index.html')).toBe(true);

     

       43
       43
       +
             expect(matchGlob('**/index.html', 'site/index.html')).toBe(true);

     

       44
       44
       +
             expect(matchGlob('**/index.html', 'a/b/c/index.html')).toBe(true);

     

       45
       45
       +
             expect(matchGlob('**/index.html', 'a/b/c/about.html')).toBe(false);

     

       46
       46
       +
           });

     

       47
       47
       +
       

     

       48
       48
       +
           it('should match directory prefixes', () => {

     

       49
       49
       +
             expect(matchGlob('assets/**', 'assets/style.css')).toBe(true);

     

       50
       50
       +
             expect(matchGlob('assets/**', 'assets/images/logo.png')).toBe(true);

     

       51
       51
       +
             expect(matchGlob('assets/**', 'other/style.css')).toBe(false);

     

       52
       52
       +
           });

     

       53
       53
       +
       

     

       54
       54
       +
           it('should match in the middle of a path', () => {

     

       55
       55
       +
             expect(matchGlob('site/**/index.html', 'site/index.html')).toBe(true);

     

       56
       56
       +
             expect(matchGlob('site/**/index.html', 'site/pages/index.html')).toBe(true);

     

       57
       57
       +
             expect(matchGlob('site/**/index.html', 'site/a/b/c/index.html')).toBe(true);

     

       58
       58
       +
           });

     

       59
       59
       +
         });

     

       60
       60
       +
       

     

       61
       61
       +
         describe('{a,b,c} alternation', () => {

     

       62
       62
       +
           it('should match any of the alternatives', () => {

     

       63
       63
       +
             expect(matchGlob('*.{html,css,js}', 'index.html')).toBe(true);

     

       64
       64
       +
             expect(matchGlob('*.{html,css,js}', 'style.css')).toBe(true);

     

       65
       65
       +
             expect(matchGlob('*.{html,css,js}', 'app.js')).toBe(true);

     

       66
       66
       +
             expect(matchGlob('*.{html,css,js}', 'image.png')).toBe(false);

     

       67
       67
       +
           });

     

       68
       68
       +
       

     

       69
       69
       +
           it('should work with ** and alternation', () => {

     

       70
       70
       +
             expect(matchGlob('**/*.{jpg,png,gif}', 'logo.png')).toBe(true);

     

       71
       71
       +
             expect(matchGlob('**/*.{jpg,png,gif}', 'images/logo.png')).toBe(true);

     

       72
       72
       +
             expect(matchGlob('**/*.{jpg,png,gif}', 'a/b/photo.jpg')).toBe(true);

     

       73
       73
       +
             expect(matchGlob('**/*.{jpg,png,gif}', 'style.css')).toBe(false);

     

       74
       74
       +
           });

     

       75
       75
       +
         });

     

       76
       76
       +
       

     

       77
       77
       +
         describe('edge cases', () => {

     

       78
       78
       +
           it('should handle empty strings', () => {

     

       79
       79
       +
             expect(matchGlob('', '')).toBe(true);

     

       80
       80
       +
             expect(matchGlob('', 'something')).toBe(false);

     

       81
       81
       +
             expect(matchGlob('**', '')).toBe(true);

     

       82
       82
       +
           });

     

       83
       83
       +
       

     

       84
       84
       +
           it('should escape regex special characters', () => {

     

       85
       85
       +
             expect(matchGlob('file.txt', 'file.txt')).toBe(true);

     

       86
       86
       +
             expect(matchGlob('file.txt', 'filextxt')).toBe(false);

     

       87
       87
       +
             expect(matchGlob('file[1].txt', 'file[1].txt')).toBe(true);

     

       88
       88
       +
           });

     

       89
       89
       +
       

     

       90
       90
       +
           it('should handle keys with colons (common in storage)', () => {

     

       91
       91
       +
             expect(matchGlob('site:*/index.html', 'site:abc/index.html')).toBe(true);

     

       92
       92
       +
             expect(matchGlob('site:**/index.html', 'site:abc/pages/index.html')).toBe(true);

     

       93
       93
       +
           });

     

       94
       94
       +
         });

     

       95
       95
       +
       });