commit d4a5fd0bbb0b90c609caf16515fcd82deb464d4e · dunkirk.sh/cachet

+1 -1

package.json

···

       1
        
       {

     

       2
        
         "name": "cachet",

     

       3
       -
         "version": "0.3.1",

     

       4
        
         "scripts": {

     

       5
        
           "test": "echo \"Error: no test specified\" && exit 1",

     

       6
        
           "dev": "bun run --watch src/index.ts",

···

       1
        
       {

     

       2
        
         "name": "cachet",

     

       3
       +
         "version": "0.3.2",

     

       4
        
         "scripts": {

     

       5
        
           "test": "echo \"Error: no test specified\" && exit 1",

     

       6
        
           "dev": "bun run --watch src/index.ts",

+5 -1

src/cache.ts

···

       2
        
       import { schedule } from "node-cron";

     

       3
        
       import { MigrationManager } from "./migrations/migrationManager";

     

       4
        
       import { endpointGroupingMigration } from "./migrations/endpointGroupingMigration";

     

       0
        
       
     

       5
        
       

     

       6
        
       /**

     

       7
        
        * @fileoverview This file contains the Cache class for storing user and emoji data with automatic expiration. To use the module in your project, import the default export and create a new instance of the Cache class. The class provides methods for inserting and retrieving user and emoji data from the cache. The cache automatically purges expired items every hour.

     
···

       149
        
          */

     

       150
        
         private async runMigrations() {

     

       151
        
           try {

     

       152
       -
             const migrations = [endpointGroupingMigration];

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       153
        
             const migrationManager = new MigrationManager(this.db, migrations);

     

       154
        
             const result = await migrationManager.runMigrations();

     

       155

···

       2
        
       import { schedule } from "node-cron";

     

       3
        
       import { MigrationManager } from "./migrations/migrationManager";

     

       4
        
       import { endpointGroupingMigration } from "./migrations/endpointGroupingMigration";

     

       5
       +
       import { logGroupingMigration } from "./migrations/logGroupingMigration";

     

       6
        
       

     

       7
        
       /**

     

       8
        
        * @fileoverview This file contains the Cache class for storing user and emoji data with automatic expiration. To use the module in your project, import the default export and create a new instance of the Cache class. The class provides methods for inserting and retrieving user and emoji data from the cache. The cache automatically purges expired items every hour.

     
···

       150
        
          */

     

       151
        
         private async runMigrations() {

     

       152
        
           try {

     

       153
       +
             const migrations = [

     

       154
       +
               endpointGroupingMigration,

     

       155
       +
               logGroupingMigration

     

       156
       +
             ];

     

       157
        
             const migrationManager = new MigrationManager(this.db, migrations);

     

       158
        
             const result = await migrationManager.runMigrations();

     

       159

src/migrations/index.ts

···

       1
        
       import { endpointGroupingMigration } from "./endpointGroupingMigration";

     

       0
        
       
     

       2
        
       import { Migration } from "./types";

     

       3
        
       import { MigrationManager } from "./migrationManager";

     

       4
        
       

     

       5
        
       // Export all migrations

     

       6
        
       export const migrations: Migration[] = [

     

       7
        
         endpointGroupingMigration,

     

       0
        
       
     

       8
        
         // Add new migrations here

     

       9
        
       ];

     

       10

···

       1
        
       import { endpointGroupingMigration } from "./endpointGroupingMigration";

     

       2
       +
       import { logGroupingMigration } from "./logGroupingMigration";

     

       3
        
       import { Migration } from "./types";

     

       4
        
       import { MigrationManager } from "./migrationManager";

     

       5
        
       

     

       6
        
       // Export all migrations

     

       7
        
       export const migrations: Migration[] = [

     

       8
        
         endpointGroupingMigration,

     

       9
       +
         logGroupingMigration,

     

       10
        
         // Add new migrations here

     

       11
        
       ];

     

       12

+97

src/migrations/logGroupingMigration.ts

···

       1
       +
       import { Database } from "bun:sqlite";

     

       2
       +
       import { Migration } from "./types";

     

       3
       +
       

     

       4
       +
       /**

     

       5
       +
        * Migration to group request logs that aren't already grouped

     

       6
       +
        * This migration normalizes request_analytics data to use consistent endpoint grouping

     

       7
       +
        */

     

       8
       +
       export const logGroupingMigration: Migration = {

     

       9
       +
         version: "0.3.2",

     

       10
       +
         description: "Group request logs that aren't already grouped",

     

       11
       +
         

     

       12
       +
         async up(db: Database): Promise<void> {

     

       13
       +
           console.log("Running log grouping migration...");

     

       14
       +
           

     

       15
       +
           // Get all request_analytics entries with specific URLs that need grouping

     

       16
       +
           const results = db.query(`

     

       17
       +
             SELECT id, endpoint FROM request_analytics 

     

       18
       +
             WHERE 

     

       19
       +
               endpoint NOT LIKE '/users/%/r' AND 

     

       20
       +
               endpoint NOT LIKE '/users/%' AND

     

       21
       +
               endpoint NOT LIKE '/emojis/%/r' AND

     

       22
       +
               endpoint NOT LIKE '/emojis/%' AND

     

       23
       +
               endpoint NOT LIKE '/health' AND

     

       24
       +
               endpoint NOT LIKE '/dashboard' AND

     

       25
       +
               endpoint NOT LIKE '/swagger%' AND

     

       26
       +
               endpoint NOT LIKE '/reset' AND

     

       27
       +
               endpoint NOT LIKE '/stats' AND

     

       28
       +
               endpoint NOT LIKE '/'

     

       29
       +
           `).all() as Array<{ id: string; endpoint: string }>;

     

       30
       +
           

     

       31
       +
           console.log(`Found ${results.length} entries to update`);

     

       32
       +
           

     

       33
       +
           // Process each entry and update with the correct grouping

     

       34
       +
           for (const entry of results) {

     

       35
       +
             let newEndpoint = entry.endpoint;

     

       36
       +
             

     

       37
       +
             // Apply grouping logic

     

       38
       +
             if (entry.endpoint.includes("localhost") || entry.endpoint.includes("http")) {

     

       39
       +
               // Extract the path from URLs

     

       40
       +
               try {

     

       41
       +
                 const url = new URL(entry.endpoint);

     

       42
       +
                 newEndpoint = url.pathname;

     

       43
       +
               } catch (e) {

     

       44
       +
                 // If URL parsing fails, try to extract the path manually

     

       45
       +
                 const pathMatch = entry.endpoint.match(/https?:\/\/[^\/]+(\/.*)/);

     

       46
       +
                 if (pathMatch && pathMatch[1]) {

     

       47
       +
                   newEndpoint = pathMatch[1];

     

       48
       +
                 }

     

       49
       +
               }

     

       50
       +
             }

     

       51
       +
             

     

       52
       +
             // Now apply the same grouping logic to the extracted path

     

       53
       +
             if (newEndpoint.match(/^\/users\/[^\/]+$/)) {

     

       54
       +
               newEndpoint = "/users/USER_ID";

     

       55
       +
             } else if (newEndpoint.match(/^\/users\/[^\/]+\/r$/)) {

     

       56
       +
               newEndpoint = "/users/USER_ID/r";

     

       57
       +
             } else if (newEndpoint.match(/^\/emojis\/[^\/]+$/)) {

     

       58
       +
               newEndpoint = "/emojis/EMOJI_NAME";

     

       59
       +
             } else if (newEndpoint.match(/^\/emojis\/[^\/]+\/r$/)) {

     

       60
       +
               newEndpoint = "/emojis/EMOJI_NAME/r";

     

       61
       +
             } else if (newEndpoint.includes("/users/") && newEndpoint.includes("/r")) {

     

       62
       +
               newEndpoint = "/users/USER_ID/r";

     

       63
       +
             } else if (newEndpoint.includes("/users/")) {

     

       64
       +
               newEndpoint = "/users/USER_ID";

     

       65
       +
             } else if (newEndpoint.includes("/emojis/") && newEndpoint.includes("/r")) {

     

       66
       +
               newEndpoint = "/emojis/EMOJI_NAME/r";

     

       67
       +
             } else if (newEndpoint.includes("/emojis/")) {

     

       68
       +
               newEndpoint = "/emojis/EMOJI_NAME";

     

       69
       +
             } else if (newEndpoint === "/") {

     

       70
       +
               newEndpoint = "/";

     

       71
       +
             } else if (newEndpoint === "/health") {

     

       72
       +
               newEndpoint = "/health";

     

       73
       +
             } else if (newEndpoint === "/dashboard") {

     

       74
       +
               newEndpoint = "/dashboard";

     

       75
       +
             } else if (newEndpoint.startsWith("/swagger")) {

     

       76
       +
               newEndpoint = "/swagger";

     

       77
       +
             } else if (newEndpoint === "/reset") {

     

       78
       +
               newEndpoint = "/reset";

     

       79
       +
             } else if (newEndpoint === "/stats") {

     

       80
       +
               newEndpoint = "/stats";

     

       81
       +
             } else {

     

       82
       +
               newEndpoint = "/other";

     

       83
       +
             }

     

       84
       +
             

     

       85
       +
             // Only update if the endpoint has changed

     

       86
       +
             if (newEndpoint !== entry.endpoint) {

     

       87
       +
               db.run(`

     

       88
       +
                 UPDATE request_analytics 

     

       89
       +
                 SET endpoint = ? 

     

       90
       +
                 WHERE id = ?

     

       91
       +
               `, [newEndpoint, entry.id]);

     

       92
       +
             }

     

       93
       +
           }

     

       94
       +
           

     

       95
       +
           console.log("Log grouping migration completed");

     

       96
       +
         }

     

       97
       +
       };