commit 1d01a11e924cad9ecd687b369d3fb5b226f44344 · dunkirk.sh/cachet

+37

README.md

···

       136
       136
        
       // { message: "User cache purged", userId: "U062UG485EE", success: true }

     

       137
       137
        
       ```

     

       138
       138
        
       

     

       139
       139
       +
       ## Development

     

       140
       140
       +
       

     

       141
       141
       +
       ### Migrations

     

       142
       142
       +
       

     

       143
       143
       +
       The app includes a migration system to handle database schema and data changes between versions. Migrations are automatically run when the app starts.

     

       144
       144
       +
       

     

       145
       145
       +
       Previous versions are tracked in a `migrations` table in the database, which records each applied migration with its version number and timestamp.

     

       146
       146
       +
       

     

       147
       147
       +
       To create a new migration:

     

       148
       148
       +
       

     

       149
       149
       +
       ```typescript

     

       150
       150
       +
       // src/migrations/myNewMigration.ts

     

       151
       151
       +
       import { Database } from "bun:sqlite";

     

       152
       152
       +
       import { Migration } from "./types";

     

       153
       153
       +
       

     

       154
       154
       +
       export const myNewMigration: Migration = {

     

       155
       155
       +
         version: "0.3.2", // Should match package.json version

     

       156
       156
       +
         description: "What this migration does",

     

       157
       157
       +
         

     

       158
       158
       +
         async up(db: Database): Promise<void> {

     

       159
       159
       +
           // Migration code here

     

       160
       160
       +
           db.run(`ALTER TABLE my_table ADD COLUMN new_column TEXT`);

     

       161
       161
       +
         }

     

       162
       162
       +
       };

     

       163
       163
       +
       

     

       164
       164
       +
       // Then add to src/migrations/index.ts

     

       165
       165
       +
       import { myNewMigration } from "./myNewMigration";

     

       166
       166
       +
       

     

       167
       167
       +
       export const migrations: Migration[] = [

     

       168
       168
       +
         endpointGroupingMigration,

     

       169
       169
       +
         myNewMigration,

     

       170
       170
       +
         // Add new migrations here

     

       171
       171
       +
       ];

     

       172
       172
       +
       ```

     

       173
       173
       +
       

     

       174
       174
       +
       Remember to update the version in `package.json` when adding new migrations.

     

       175
       175
       +
       

     

       139
       176
        
       <p align="center">

     

       140
       177
        
       	<img src="https://raw.githubusercontent.com/taciturnaxolotl/carriage/master/.github/images/line-break.svg" />

     

       141
       178
        
       </p>

+1 -1

package.json

···

       1
       1
        
       {

     

       2
       2
        
         "name": "cachet",

     

       3
       3
       -
         "version": "0.3.0",

     

       3
       3
       +
         "version": "0.3.1",

     

       4
       4
        
         "scripts": {

     

       5
       5
        
           "test": "echo \"Error: no test specified\" && exit 1",

     

       6
       6
        
           "dev": "bun run --watch src/index.ts",

+25

src/cache.ts

···

       1
       1
        
       import { Database } from "bun:sqlite";

     

       2
       2
        
       import { schedule } from "node-cron";

     

       3
       3
       +
       import { MigrationManager } from "./migrations/migrationManager";

     

       4
       4
       +
       import { endpointGroupingMigration } from "./migrations/endpointGroupingMigration";

     

       3
       5
        
       

     

       4
       6
        
       /**

     

       5
       7
        
        * @fileoverview This file contains the Cache class for storing user and emoji data with automatic expiration. To use the module in your project, import the default export and create a new instance of the Cache class. The class provides methods for inserting and retrieving user and emoji data from the cache. The cache automatically purges expired items every hour.

     
···

       61
       63
        
       

     

       62
       64
        
           this.initDatabase();

     

       63
       65
        
           this.setupPurgeSchedule();

     

       66
       66
       +
           

     

       67
       67
       +
           // Run migrations

     

       68
       68
       +
           this.runMigrations();

     

       64
       69
        
         }

     

       65
       70
        
       

     

       66
       71
        
         /**

     
···

       136
       141
        
           schedule("45 * * * *", async () => {

     

       137
       142
        
             await this.purgeExpiredItems();

     

       138
       143
        
           });

     

       144
       144
       +
         }

     

       145
       145
       +
       

     

       146
       146
       +
         /**

     

       147
       147
       +
          * Run database migrations

     

       148
       148
       +
          * @private

     

       149
       149
       +
          */

     

       150
       150
       +
         private async runMigrations() {

     

       151
       151
       +
           try {

     

       152
       152
       +
             const migrations = [endpointGroupingMigration];

     

       153
       153
       +
             const migrationManager = new MigrationManager(this.db, migrations);

     

       154
       154
       +
             const result = await migrationManager.runMigrations();

     

       155
       155
       +
             

     

       156
       156
       +
             if (result.migrationsApplied > 0) {

     

       157
       157
       +
               console.log(`Applied ${result.migrationsApplied} migrations. Latest version: ${result.lastAppliedVersion}`);

     

       158
       158
       +
             } else {

     

       159
       159
       +
               console.log("No new migrations to apply");

     

       160
       160
       +
             }

     

       161
       161
       +
           } catch (error) {

     

       162
       162
       +
             console.error("Error running migrations:", error);

     

       163
       163
       +
           }

     

       139
       164
        
         }

     

       140
       165
        
       

     

       141
       166
        
         /**

+66

src/migrations/endpointGroupingMigration.ts

···

       1
       1
       +
       import { Database } from "bun:sqlite";

     

       2
       2
       +
       import { Migration } from "./types";

     

       3
       3
       +
       

     

       4
       4
       +
       /**

     

       5
       5
       +
        * Migration to fix endpoint grouping in analytics

     

       6
       6
       +
        * This migration updates existing analytics data to use consistent endpoint grouping

     

       7
       7
       +
        */

     

       8
       8
       +
       export const endpointGroupingMigration: Migration = {

     

       9
       9
       +
         version: "0.3.1",

     

       10
       10
       +
         description: "Fix endpoint grouping in analytics data",

     

       11
       11
       +
         

     

       12
       12
       +
         async up(db: Database): Promise<void> {

     

       13
       13
       +
           console.log("Running endpoint grouping migration...");

     

       14
       14
       +
           

     

       15
       15
       +
           // Get all request_analytics entries with specific URLs

     

       16
       16
       +
           const results = db.query(`

     

       17
       17
       +
             SELECT id, endpoint FROM request_analytics 

     

       18
       18
       +
             WHERE endpoint LIKE '/users/%' OR endpoint LIKE '/emojis/%'

     

       19
       19
       +
           `).all() as Array<{ id: string; endpoint: string }>;

     

       20
       20
       +
           

     

       21
       21
       +
           console.log(`Found ${results.length} entries to update`);

     

       22
       22
       +
           

     

       23
       23
       +
           // Process each entry and update with the correct grouping

     

       24
       24
       +
           for (const entry of results) {

     

       25
       25
       +
             let newEndpoint = entry.endpoint;

     

       26
       26
       +
             

     

       27
       27
       +
             // Apply the same grouping logic we use in the analytics

     

       28
       28
       +
             if (entry.endpoint.match(/^\/users\/[^\/]+$/)) {

     

       29
       29
       +
               // Keep as is - these are already correctly grouped

     

       30
       30
       +
               continue;

     

       31
       31
       +
             } else if (entry.endpoint.match(/^\/users\/[^\/]+\/r$/)) {

     

       32
       32
       +
               // Keep as is - these are already correctly grouped

     

       33
       33
       +
               continue;

     

       34
       34
       +
             } else if (entry.endpoint.match(/^\/emojis\/[^\/]+$/)) {

     

       35
       35
       +
               // Keep as is - these are already correctly grouped

     

       36
       36
       +
               continue;

     

       37
       37
       +
             } else if (entry.endpoint.match(/^\/emojis\/[^\/]+\/r$/)) {

     

       38
       38
       +
               // Keep as is - these are already correctly grouped

     

       39
       39
       +
               continue;

     

       40
       40
       +
             } else if (entry.endpoint.includes("/users/") && entry.endpoint.includes("/r")) {

     

       41
       41
       +
               // This is a user redirect with a non-standard format

     

       42
       42
       +
               newEndpoint = "/users/USER_ID/r";

     

       43
       43
       +
             } else if (entry.endpoint.includes("/users/")) {

     

       44
       44
       +
               // This is a user data endpoint with a non-standard format

     

       45
       45
       +
               newEndpoint = "/users/USER_ID";

     

       46
       46
       +
             } else if (entry.endpoint.includes("/emojis/") && entry.endpoint.includes("/r")) {

     

       47
       47
       +
               // This is an emoji redirect with a non-standard format

     

       48
       48
       +
               newEndpoint = "/emojis/EMOJI_NAME/r";

     

       49
       49
       +
             } else if (entry.endpoint.includes("/emojis/")) {

     

       50
       50
       +
               // This is an emoji data endpoint with a non-standard format

     

       51
       51
       +
               newEndpoint = "/emojis/EMOJI_NAME";

     

       52
       52
       +
             }

     

       53
       53
       +
             

     

       54
       54
       +
             // Only update if the endpoint has changed

     

       55
       55
       +
             if (newEndpoint !== entry.endpoint) {

     

       56
       56
       +
               db.run(`

     

       57
       57
       +
                 UPDATE request_analytics 

     

       58
       58
       +
                 SET endpoint = ? 

     

       59
       59
       +
                 WHERE id = ?

     

       60
       60
       +
               `, [newEndpoint, entry.id]);

     

       61
       61
       +
             }

     

       62
       62
       +
           }

     

       63
       63
       +
           

     

       64
       64
       +
           console.log("Endpoint grouping migration completed");

     

       65
       65
       +
         }

     

       66
       66
       +
       };

+12

src/migrations/index.ts

···

       1
       1
       +
       import { endpointGroupingMigration } from "./endpointGroupingMigration";

     

       2
       2
       +
       import { Migration } from "./types";

     

       3
       3
       +
       import { MigrationManager } from "./migrationManager";

     

       4
       4
       +
       

     

       5
       5
       +
       // Export all migrations

     

       6
       6
       +
       export const migrations: Migration[] = [

     

       7
       7
       +
         endpointGroupingMigration,

     

       8
       8
       +
         // Add new migrations here

     

       9
       9
       +
       ];

     

       10
       10
       +
       

     

       11
       11
       +
       // Export the migration manager and types

     

       12
       12
       +
       export { MigrationManager, Migration };

+225

src/migrations/migrationManager.ts

···

       1
       1
       +
       import { Database } from "bun:sqlite";

     

       2
       2
       +
       import { version } from "../../package.json";

     

       3
       3
       +
       

     

       4
       4
       +
       /**

     

       5
       5
       +
        * Migration interface

     

       6
       6
       +
        */

     

       7
       7
       +
       export interface Migration {

     

       8
       8
       +
         version: string;

     

       9
       9
       +
         description: string;

     

       10
       10
       +
         up: (db: Database) => Promise<void>;

     

       11
       11
       +
         down?: (db: Database) => Promise<void>; // Optional downgrade function

     

       12
       12
       +
       }

     

       13
       13
       +
       

     

       14
       14
       +
       /**

     

       15
       15
       +
        * Migration Manager for handling database schema and data migrations

     

       16
       16
       +
        */

     

       17
       17
       +
       export class MigrationManager {

     

       18
       18
       +
         private db: Database;

     

       19
       19
       +
         private currentVersion: string;

     

       20
       20
       +
         private migrations: Migration[];

     

       21
       21
       +
       

     

       22
       22
       +
         /**

     

       23
       23
       +
          * Creates a new MigrationManager

     

       24
       24
       +
          * @param db SQLite database instance

     

       25
       25
       +
          * @param migrations Array of migrations to apply

     

       26
       26
       +
          */

     

       27
       27
       +
         constructor(db: Database, migrations: Migration[]) {

     

       28
       28
       +
           this.db = db;

     

       29
       29
       +
           this.currentVersion = version;

     

       30
       30
       +
           this.migrations = migrations;

     

       31
       31
       +
           this.initMigrationTable();

     

       32
       32
       +
         }

     

       33
       33
       +
       

     

       34
       34
       +
         /**

     

       35
       35
       +
          * Initialize the migrations table if it doesn't exist

     

       36
       36
       +
          */

     

       37
       37
       +
         private initMigrationTable() {

     

       38
       38
       +
           this.db.run(`

     

       39
       39
       +
             CREATE TABLE IF NOT EXISTS migrations (

     

       40
       40
       +
               id INTEGER PRIMARY KEY AUTOINCREMENT,

     

       41
       41
       +
               version TEXT NOT NULL,

     

       42
       42
       +
               applied_at INTEGER NOT NULL,

     

       43
       43
       +
               description TEXT

     

       44
       44
       +
             )

     

       45
       45
       +
           `);

     

       46
       46
       +
         }

     

       47
       47
       +
       

     

       48
       48
       +
         /**

     

       49
       49
       +
          * Get the last applied migration version

     

       50
       50
       +
          * @returns The last applied migration version or null if no migrations have been applied

     

       51
       51
       +
          */

     

       52
       52
       +
         private getLastAppliedMigration(): { version: string; applied_at: number } | null {

     

       53
       53
       +
           const result = this.db.query(`

     

       54
       54
       +
             SELECT version, applied_at FROM migrations 

     

       55
       55
       +
             ORDER BY applied_at DESC LIMIT 1

     

       56
       56
       +
           `).get() as { version: string; applied_at: number } | null;

     

       57
       57
       +
       

     

       58
       58
       +
           return result;

     

       59
       59
       +
         }

     

       60
       60
       +
       

     

       61
       61
       +
         /**

     

       62
       62
       +
          * Check if a migration has been applied

     

       63
       63
       +
          * @param version Migration version to check

     

       64
       64
       +
          * @returns True if the migration has been applied, false otherwise

     

       65
       65
       +
          */

     

       66
       66
       +
         private isMigrationApplied(version: string): boolean {

     

       67
       67
       +
           const result = this.db.query(`

     

       68
       68
       +
             SELECT COUNT(*) as count FROM migrations 

     

       69
       69
       +
             WHERE version = ?

     

       70
       70
       +
           `).get(version) as { count: number };

     

       71
       71
       +
       

     

       72
       72
       +
           return result.count > 0;

     

       73
       73
       +
         }

     

       74
       74
       +
       

     

       75
       75
       +
         /**

     

       76
       76
       +
          * Record a migration as applied

     

       77
       77
       +
          * @param version Migration version

     

       78
       78
       +
          * @param description Migration description

     

       79
       79
       +
          */

     

       80
       80
       +
         private recordMigration(version: string, description: string) {

     

       81
       81
       +
           this.db.run(`

     

       82
       82
       +
             INSERT INTO migrations (version, applied_at, description)

     

       83
       83
       +
             VALUES (?, ?, ?)

     

       84
       84
       +
           `, [version, Date.now(), description]);

     

       85
       85
       +
         }

     

       86
       86
       +
       

     

       87
       87
       +
         /**

     

       88
       88
       +
          * Run migrations up to the current version

     

       89
       89
       +
          * @returns Object containing migration results

     

       90
       90
       +
          */

     

       91
       91
       +
         async runMigrations(): Promise<{

     

       92
       92
       +
           success: boolean;

     

       93
       93
       +
           migrationsApplied: number;

     

       94
       94
       +
           lastAppliedVersion: string | null;

     

       95
       95
       +
           error?: string;

     

       96
       96
       +
         }> {

     

       97
       97
       +
           try {

     

       98
       98
       +
             // Sort migrations by version (semver)

     

       99
       99
       +
             const sortedMigrations = [...this.migrations].sort((a, b) => {

     

       100
       100
       +
               return this.compareVersions(a.version, b.version);

     

       101
       101
       +
             });

     

       102
       102
       +
       

     

       103
       103
       +
             const lastApplied = this.getLastAppliedMigration();

     

       104
       104
       +
             let migrationsApplied = 0;

     

       105
       105
       +
             let lastAppliedVersion = lastApplied?.version || null;

     

       106
       106
       +
       

     

       107
       107
       +
             console.log(`Current app version: ${this.currentVersion}`);

     

       108
       108
       +
             console.log(`Last applied migration: ${lastAppliedVersion || 'None'}`);

     

       109
       109
       +
       

     

       110
       110
       +
             // Special case for first run: if no migrations table exists yet,

     

       111
       111
       +
             // assume we're upgrading from the previous version without migrations

     

       112
       112
       +
             if (!lastAppliedVersion) {

     

       113
       113
       +
               // Record a "virtual" migration for the previous version

     

       114
       114
       +
               // This prevents all migrations from running on existing installations

     

       115
       115
       +
               const previousVersion = this.getPreviousVersion(this.currentVersion);

     

       116
       116
       +
               if (previousVersion) {

     

       117
       117
       +
                 console.log(`No migrations table found. Assuming upgrade from ${previousVersion}`);

     

       118
       118
       +
                 this.recordMigration(

     

       119
       119
       +
                   previousVersion, 

     

       120
       120
       +
                   "Virtual migration for existing installation"

     

       121
       121
       +
                 );

     

       122
       122
       +
                 lastAppliedVersion = previousVersion;

     

       123
       123
       +
               }

     

       124
       124
       +
             }

     

       125
       125
       +
       

     

       126
       126
       +
             // Apply migrations that haven't been applied yet

     

       127
       127
       +
             for (const migration of sortedMigrations) {

     

       128
       128
       +
               // Skip if this migration has already been applied

     

       129
       129
       +
               if (this.isMigrationApplied(migration.version)) {

     

       130
       130
       +
                 console.log(`Migration ${migration.version} already applied, skipping`);

     

       131
       131
       +
                 continue;

     

       132
       132
       +
               }

     

       133
       133
       +
       

     

       134
       134
       +
               // Skip if this migration is for a future version

     

       135
       135
       +
               if (this.compareVersions(migration.version, this.currentVersion) > 0) {

     

       136
       136
       +
                 console.log(`Migration ${migration.version} is for a future version, skipping`);

     

       137
       137
       +
                 continue;

     

       138
       138
       +
               }

     

       139
       139
       +
       

     

       140
       140
       +
               // If we have a last applied migration, only apply migrations that are newer

     

       141
       141
       +
               if (lastAppliedVersion && this.compareVersions(migration.version, lastAppliedVersion) <= 0) {

     

       142
       142
       +
                 console.log(`Migration ${migration.version} is older than last applied (${lastAppliedVersion}), skipping`);

     

       143
       143
       +
                 continue;

     

       144
       144
       +
               }

     

       145
       145
       +
       

     

       146
       146
       +
               console.log(`Applying migration ${migration.version}: ${migration.description}`);

     

       147
       147
       +
               

     

       148
       148
       +
               // Run the migration inside a transaction

     

       149
       149
       +
               this.db.transaction(() => {

     

       150
       150
       +
                 // Apply the migration

     

       151
       151
       +
                 migration.up(this.db);

     

       152
       152
       +
                 

     

       153
       153
       +
                 // Record the migration

     

       154
       154
       +
                 this.recordMigration(migration.version, migration.description);

     

       155
       155
       +
               })();

     

       156
       156
       +
       

     

       157
       157
       +
               migrationsApplied++;

     

       158
       158
       +
               lastAppliedVersion = migration.version;

     

       159
       159
       +
               console.log(`Migration ${migration.version} applied successfully`);

     

       160
       160
       +
             }

     

       161
       161
       +
       

     

       162
       162
       +
             return {

     

       163
       163
       +
               success: true,

     

       164
       164
       +
               migrationsApplied,

     

       165
       165
       +
               lastAppliedVersion

     

       166
       166
       +
             };

     

       167
       167
       +
           } catch (error) {

     

       168
       168
       +
             console.error('Error running migrations:', error);

     

       169
       169
       +
             return {

     

       170
       170
       +
               success: false,

     

       171
       171
       +
               migrationsApplied: 0,

     

       172
       172
       +
               lastAppliedVersion: null,

     

       173
       173
       +
               error: error instanceof Error ? error.message : String(error)

     

       174
       174
       +
             };

     

       175
       175
       +
           }

     

       176
       176
       +
         }

     

       177
       177
       +
       

     

       178
       178
       +
         /**

     

       179
       179
       +
          * Get the previous version from the current version

     

       180
       180
       +
          * @param version Current version

     

       181
       181
       +
          * @returns Previous version or null if can't determine

     

       182
       182
       +
          */

     

       183
       183
       +
         private getPreviousVersion(version: string): string | null {

     

       184
       184
       +
           const parts = version.split('.');

     

       185
       185
       +
           if (parts.length !== 3) return null;

     

       186
       186
       +
           

     

       187
       187
       +
           const [major, minor, patch] = parts.map(Number);

     

       188
       188
       +
           

     

       189
       189
       +
           // If patch > 0, decrement patch

     

       190
       190
       +
           if (patch > 0) {

     

       191
       191
       +
             return `${major}.${minor}.${patch - 1}`;

     

       192
       192
       +
           }

     

       193
       193
       +
           // If minor > 0, decrement minor and set patch to 0

     

       194
       194
       +
           else if (minor > 0) {

     

       195
       195
       +
             return `${major}.${minor - 1}.0`;

     

       196
       196
       +
           }

     

       197
       197
       +
           // If major > 0, decrement major and set minor and patch to 0

     

       198
       198
       +
           else if (major > 0) {

     

       199
       199
       +
             return `${major - 1}.0.0`;

     

       200
       200
       +
           }

     

       201
       201
       +
           

     

       202
       202
       +
           return null;

     

       203
       203
       +
         }

     

       204
       204
       +
       

     

       205
       205
       +
         /**

     

       206
       206
       +
          * Compare two version strings (semver)

     

       207
       207
       +
          * @param a First version

     

       208
       208
       +
          * @param b Second version

     

       209
       209
       +
          * @returns -1 if a < b, 0 if a = b, 1 if a > b

     

       210
       210
       +
          */

     

       211
       211
       +
         private compareVersions(a: string, b: string): number {

     

       212
       212
       +
           const partsA = a.split('.').map(Number);

     

       213
       213
       +
           const partsB = b.split('.').map(Number);

     

       214
       214
       +
           

     

       215
       215
       +
           for (let i = 0; i < Math.max(partsA.length, partsB.length); i++) {

     

       216
       216
       +
             const partA = i < partsA.length ? partsA[i] : 0;

     

       217
       217
       +
             const partB = i < partsB.length ? partsB[i] : 0;

     

       218
       218
       +
             

     

       219
       219
       +
             if (partA < partB) return -1;

     

       220
       220
       +
             if (partA > partB) return 1;

     

       221
       221
       +
           }

     

       222
       222
       +
           

     

       223
       223
       +
           return 0;

     

       224
       224
       +
         }

     

       225
       225
       +
       }

+11

src/migrations/types.ts

···

       1
       1
       +
       import { Database } from "bun:sqlite";

     

       2
       2
       +
       

     

       3
       3
       +
       /**

     

       4
       4
       +
        * Migration interface

     

       5
       5
       +
        */

     

       6
       6
       +
       export interface Migration {

     

       7
       7
       +
         version: string;

     

       8
       8
       +
         description: string;

     

       9
       9
       +
         up: (db: Database) => Promise<void>;

     

       10
       10
       +
         down?: (db: Database) => Promise<void>; // Optional downgrade function

     

       11
       11
       +
       }