commit 909aa14c4f01f7eaba16396ab4be02f254b4c4d4 · ptr.pet/wisp.place-monorepo

+63 -25

hosting-service/src/lib/utils.ts

···

       15
       15
        
         rkey: string;

     

       16
       16
        
       }

     

       17
       17
        
       

     

       18
       18
       +
       /**

     

       19
       19
       +
        * Determines if a MIME type should benefit from gzip compression.

     

       20
       20
       +
        * Returns true for text-based web assets (HTML, CSS, JS, JSON, XML, SVG).

     

       21
       21
       +
        * Returns false for already-compressed formats (images, video, audio, PDFs).

     

       22
       22
       +
        * 

     

       23
       23
       +
        */

     

       24
       24
       +
       export function shouldCompressMimeType(mimeType: string | undefined): boolean {

     

       25
       25
       +
         if (!mimeType) return false;

     

       26
       26
       +
         

     

       27
       27
       +
         const mime = mimeType.toLowerCase();

     

       28
       28
       +
         

     

       29
       29
       +
         // Text-based web assets that benefit from compression

     

       30
       30
       +
         const compressibleTypes = [

     

       31
       31
       +
           'text/html',

     

       32
       32
       +
           'text/css',

     

       33
       33
       +
           'text/javascript',

     

       34
       34
       +
           'application/javascript',

     

       35
       35
       +
           'application/x-javascript',

     

       36
       36
       +
           'text/xml',

     

       37
       37
       +
           'application/xml',

     

       38
       38
       +
           'application/json',

     

       39
       39
       +
           'text/plain',

     

       40
       40
       +
           'image/svg+xml',

     

       41
       41
       +
         ];

     

       42
       42
       +
         

     

       43
       43
       +
         if (compressibleTypes.some(type => mime === type || mime.startsWith(type))) {

     

       44
       44
       +
           return true;

     

       45
       45
       +
         }

     

       46
       46
       +
         

     

       47
       47
       +
         // Already-compressed formats that should NOT be double-compressed

     

       48
       48
       +
         const alreadyCompressedPrefixes = [

     

       49
       49
       +
           'video/',

     

       50
       50
       +
           'audio/',

     

       51
       51
       +
           'image/',

     

       52
       52
       +
           'application/pdf',

     

       53
       53
       +
           'application/zip',

     

       54
       54
       +
           'application/gzip',

     

       55
       55
       +
         ];

     

       56
       56
       +
         

     

       57
       57
       +
         if (alreadyCompressedPrefixes.some(prefix => mime.startsWith(prefix))) {

     

       58
       58
       +
           return false;

     

       59
       59
       +
         }

     

       60
       60
       +
         

     

       61
       61
       +
         // Default to not compressing for unknown types

     

       62
       62
       +
         return false;

     

       63
       63
       +
       }

     

       64
       64
       +
       

     

       18
       65
        
       interface IpldLink {

     

       19
       66
        
         $link: string;

     

       20
       67
        
       }

     
···

       270
       317
        
       

     

       271
       318
        
         console.log(`[DEBUG] ${filePath}: fetched ${content.length} bytes, base64=${base64}, encoding=${encoding}, mimeType=${mimeType}`);

     

       272
       319
        
       

     

       273
       273
       -
         // If content is base64-encoded, decode it back to binary (gzipped or not)

     

       320
       320
       +
         // If content is base64-encoded, decode it back to raw binary (gzipped or not)

     

       274
       321
        
         if (base64) {

     

       275
       322
        
           const originalSize = content.length;

     

       276
       276
       -
           // The content from the blob is base64 text, decode it directly to binary

     

       277
       277
       -
           const buffer = Buffer.from(content);

     

       278
       278
       -
           const base64String = buffer.toString('ascii');  // Use ascii for base64 text, not utf-8

     

       279
       279
       -
           console.log(`[DEBUG] ${filePath}: base64 string first 100 chars: ${base64String.substring(0, 100)}`);

     

       323
       323
       +
           // Decode base64 directly from raw bytes - no string conversion

     

       324
       324
       +
           // The blob contains base64-encoded text as raw bytes, decode it in-place

     

       325
       325
       +
           const textDecoder = new TextDecoder();

     

       326
       326
       +
           const base64String = textDecoder.decode(content);

     

       280
       327
        
           content = Buffer.from(base64String, 'base64');

     

       281
       281
       -
           console.log(`[DEBUG] ${filePath}: decoded from ${originalSize} bytes to ${content.length} bytes`);

     

       328
       328
       +
           console.log(`[DEBUG] ${filePath}: decoded base64 from ${originalSize} bytes to ${content.length} bytes`);

     

       282
       329
        
           

     

       283
       330
        
           // Check if it's actually gzipped by looking at magic bytes

     

       284
       331
        
           if (content.length >= 2) {

     

       285
       285
       -
             const magic = content[0] === 0x1f && content[1] === 0x8b;

     

       286
       286
       -
             const byte0 = content[0];

     

       287
       287
       -
             const byte1 = content[1];

     

       288
       288
       -
             console.log(`[DEBUG] ${filePath}: has gzip magic bytes: ${magic} (0x${byte0?.toString(16)}, 0x${byte1?.toString(16)})`);

     

       332
       332
       +
             const hasGzipMagic = content[0] === 0x1f && content[1] === 0x8b;

     

       333
       333
       +
             console.log(`[DEBUG] ${filePath}: has gzip magic bytes: ${hasGzipMagic}`);

     

       289
       334
        
           }

     

       290
       335
        
         }

     

       291
       336
        
       

     
···

       296
       341
        
           mkdirSync(fileDir, { recursive: true });

     

       297
       342
        
         }

     

       298
       343
        
       

     

       299
       299
       -
         // Determine if this is a web asset that should remain compressed

     

       300
       300
       -
         const webAssetTypes = [

     

       301
       301
       -
           'text/html', 'text/css', 'application/javascript', 'text/javascript',

     

       302
       302
       -
           'application/json', 'text/xml', 'application/xml'

     

       303
       303
       -
         ];

     

       304
       304
       -
         

     

       305
       305
       -
         const isWebAsset = mimeType && webAssetTypes.some(type => 

     

       306
       306
       -
           mimeType.toLowerCase().startsWith(type) || mimeType.toLowerCase() === type

     

       307
       307
       -
         );

     

       344
       344
       +
         // Use the shared function to determine if this should remain compressed

     

       345
       345
       +
         const shouldStayCompressed = shouldCompressMimeType(mimeType);

     

       308
       346
        
       

     

       309
       309
       -
         // Decompress non-web assets that are gzipped

     

       310
       310
       -
         if (encoding === 'gzip' && !isWebAsset && content.length >= 2 && 

     

       347
       347
       +
         // Decompress files that shouldn't be stored compressed

     

       348
       348
       +
         if (encoding === 'gzip' && !shouldStayCompressed && content.length >= 2 && 

     

       311
       349
        
             content[0] === 0x1f && content[1] === 0x8b) {

     

       312
       312
       -
           console.log(`[DEBUG] ${filePath}: decompressing non-web asset (${mimeType}) before caching`);

     

       350
       350
       +
           console.log(`[DEBUG] ${filePath}: decompressing non-compressible type (${mimeType}) before caching`);

     

       313
       351
        
           try {

     

       314
       352
        
             const { gunzipSync } = await import('zlib');

     

       315
       353
        
             const decompressed = gunzipSync(content);

     
···

       318
       356
        
             // Clear the encoding flag since we're storing decompressed

     

       319
       357
        
             encoding = undefined;

     

       320
       358
        
           } catch (error) {

     

       321
       321
       -
             console.log(`[DEBUG] ${filePath}: failed to decompress, storing original gzipped content`);

     

       359
       359
       +
             console.log(`[DEBUG] ${filePath}: failed to decompress, storing original gzipped content. Error:`, error);

     

       322
       360
        
           }

     

       323
       361
        
         }

     

       324
       362
        
       

     

       325
       363
        
         await writeFile(cacheFile, content);

     

       326
       364
        
       

     

       327
       327
       -
         // Store metadata only if file is still compressed (web assets)

     

       365
       365
       +
         // Store metadata only if file is still compressed

     

       328
       366
        
         if (encoding === 'gzip' && mimeType) {

     

       329
       367
        
           const metaFile = `${cacheFile}.meta`;

     

       330
       368
        
           await writeFile(metaFile, JSON.stringify({ encoding, mimeType }));

     

       331
       369
        
           console.log('Cached file', filePath, content.length, 'bytes (gzipped,', mimeType + ')');

     

       332
       370
        
         } else {

     

       333
       333
       -
           console.log('Cached file', filePath, content.length, 'bytes (decompressed)');

     

       371
       371
       +
           console.log('Cached file', filePath, content.length, 'bytes');

     

       334
       372
        
         }

     

       335
       373
        
       }

     

       336
       374

+27 -34

hosting-service/src/server.ts

···

       1
       1
        
       import { Hono } from 'hono';

     

       2
       2
        
       import { getWispDomain, getCustomDomain, getCustomDomainByHash } from './lib/db';

     

       3
       3
       -
       import { resolveDid, getPdsForDid, fetchSiteRecord, downloadAndCacheSite, getCachedFilePath, isCached, sanitizePath } from './lib/utils';

     

       3
       3
       +
       import { resolveDid, getPdsForDid, fetchSiteRecord, downloadAndCacheSite, getCachedFilePath, isCached, sanitizePath, shouldCompressMimeType } from './lib/utils';

     

       4
       4
        
       import { rewriteHtmlPaths, isHtmlContent } from './lib/html-rewriter';

     

       5
       5
        
       import { existsSync, readFileSync } from 'fs';

     

       6
       6
        
       import { lookup } from 'mime-types';

     
···

       45
       45
        
             // Check actual content for gzip magic bytes

     

       46
       46
        
             if (content.length >= 2) {

     

       47
       47
        
               const hasGzipMagic = content[0] === 0x1f && content[1] === 0x8b;

     

       48
       48
       -
               const byte0 = content[0];

     

       49
       49
       -
               const byte1 = content[1];

     

       50
       50
       -
               console.log(`[DEBUG SERVE] ${requestPath}: has gzip magic bytes=${hasGzipMagic} (0x${byte0?.toString(16)}, 0x${byte1?.toString(16)})`);

     

       48
       48
       +
               console.log(`[DEBUG SERVE] ${requestPath}: has gzip magic bytes=${hasGzipMagic}`);

     

       51
       49
        
             }

     

       52
       50
        
             

     

       53
       51
        
             if (meta.encoding === 'gzip' && meta.mimeType) {

     

       54
       54
       -
               // Don't serve already-compressed media formats with Content-Encoding: gzip

     

       55
       55
       -
               // These formats (video, audio, images) are already compressed and the browser

     

       56
       56
       -
               // can't decode them if we add another layer of compression

     

       57
       57
       -
               const alreadyCompressedTypes = [

     

       58
       58
       -
                 'video/', 'audio/', 'image/jpeg', 'image/jpg', 'image/png', 

     

       59
       59
       -
                 'image/gif', 'image/webp', 'application/pdf'

     

       60
       60
       -
               ];

     

       52
       52
       +
               // Use shared function to determine if this should be served compressed

     

       53
       53
       +
               const shouldServeCompressed = shouldCompressMimeType(meta.mimeType);

     

       61
       54
        
               

     

       62
       62
       -
               const isAlreadyCompressed = alreadyCompressedTypes.some(type => 

     

       63
       63
       -
                 meta.mimeType.toLowerCase().startsWith(type)

     

       64
       64
       -
               );

     

       65
       65
       -
               

     

       66
       66
       -
               if (isAlreadyCompressed) {

     

       67
       67
       -
                 // Decompress the file before serving

     

       68
       68
       -
                 console.log(`[DEBUG SERVE] ${requestPath}: decompressing already-compressed media type`);

     

       55
       55
       +
               if (!shouldServeCompressed) {

     

       56
       56
       +
                 // This shouldn't happen if caching is working correctly, but handle it gracefully

     

       57
       57
       +
                 console.log(`[DEBUG SERVE] ${requestPath}: decompressing file that shouldn't be compressed (${meta.mimeType})`);

     

       69
       58
        
                 const { gunzipSync } = await import('zlib');

     

       70
       59
        
                 const decompressed = gunzipSync(content);

     

       71
       60
        
                 console.log(`[DEBUG SERVE] ${requestPath}: decompressed from ${content.length} to ${decompressed.length} bytes`);

     
···

       157
       146
        
           }

     

       158
       147
        
       

     

       159
       148
        
           // Check if this is HTML content that needs rewriting

     

       160
       160
       -
           // Note: For gzipped HTML with path rewriting, we need to decompress, rewrite, and serve uncompressed

     

       161
       161
       -
           // This is a trade-off for the sites.wisp.place domain which needs path rewriting

     

       149
       149
       +
           // We decompress, rewrite paths, then recompress for efficient delivery

     

       162
       150
        
           if (isHtmlContent(requestPath, mimeType)) {

     

       163
       151
        
             let content: string;

     

       164
       152
        
             if (isGzipped) {

     
···

       169
       157
        
               content = readFileSync(cachedFile, 'utf-8');

     

       170
       158
        
             }

     

       171
       159
        
             const rewritten = rewriteHtmlPaths(content, basePath);

     

       172
       172
       -
             return new Response(rewritten, {

     

       160
       160
       +
             

     

       161
       161
       +
             // Recompress the HTML for efficient delivery

     

       162
       162
       +
             const { gzipSync } = await import('zlib');

     

       163
       163
       +
             const recompressed = gzipSync(Buffer.from(rewritten, 'utf-8'));

     

       164
       164
       +
             

     

       165
       165
       +
             return new Response(recompressed, {

     

       173
       166
        
               headers: {

     

       174
       167
        
                 'Content-Type': 'text/html; charset=utf-8',

     

       168
       168
       +
                 'Content-Encoding': 'gzip',

     

       175
       169
        
               },

     

       176
       170
        
             });

     

       177
       171
        
           }

     
···

       179
       173
        
           // Non-HTML files: serve gzipped content as-is with proper headers

     

       180
       174
        
           const content = readFileSync(cachedFile);

     

       181
       175
        
           if (isGzipped) {

     

       182
       182
       -
             // Don't serve already-compressed media formats with Content-Encoding: gzip

     

       183
       183
       -
             const alreadyCompressedTypes = [

     

       184
       184
       -
               'video/', 'audio/', 'image/jpeg', 'image/jpg', 'image/png', 

     

       185
       185
       -
               'image/gif', 'image/webp', 'application/pdf'

     

       186
       186
       -
             ];

     

       187
       187
       -
             

     

       188
       188
       -
             const isAlreadyCompressed = alreadyCompressedTypes.some(type => 

     

       189
       189
       -
               mimeType.toLowerCase().startsWith(type)

     

       190
       190
       -
             );

     

       176
       176
       +
             // Use shared function to determine if this should be served compressed

     

       177
       177
       +
             const shouldServeCompressed = shouldCompressMimeType(mimeType);

     

       191
       178
        
             

     

       192
       192
       -
             if (isAlreadyCompressed) {

     

       193
       193
       -
               // Decompress the file before serving

     

       179
       179
       +
             if (!shouldServeCompressed) {

     

       180
       180
       +
               // This shouldn't happen if caching is working correctly, but handle it gracefully

     

       194
       181
        
               const { gunzipSync } = await import('zlib');

     

       195
       182
        
               const decompressed = gunzipSync(content);

     

       196
       183
        
               return new Response(decompressed, {

     
···

       228
       215
        
               }

     

       229
       216
        
             }

     

       230
       217
        
       

     

       231
       231
       -
             // HTML needs path rewriting, so decompress if needed

     

       218
       218
       +
             // HTML needs path rewriting, decompress, rewrite, then recompress

     

       232
       219
        
             let content: string;

     

       233
       220
        
             if (isGzipped) {

     

       234
       221
        
               const { gunzipSync } = await import('zlib');

     
···

       238
       225
        
               content = readFileSync(indexFile, 'utf-8');

     

       239
       226
        
             }

     

       240
       227
        
             const rewritten = rewriteHtmlPaths(content, basePath);

     

       241
       241
       -
             return new Response(rewritten, {

     

       228
       228
       +
             

     

       229
       229
       +
             // Recompress the HTML for efficient delivery

     

       230
       230
       +
             const { gzipSync } = await import('zlib');

     

       231
       231
       +
             const recompressed = gzipSync(Buffer.from(rewritten, 'utf-8'));

     

       232
       232
       +
             

     

       233
       233
       +
             return new Response(recompressed, {

     

       242
       234
        
               headers: {

     

       243
       235
        
                 'Content-Type': 'text/html; charset=utf-8',

     

       236
       236
       +
                 'Content-Encoding': 'gzip',

     

       244
       237
        
               },

     

       245
       238
        
             });

     

       246
       239
        
           }