commit b856f234ab33be0d21e9baaa1d3e69cb6d372663 · nekomimi.pet/wisp.place-monorepo

+8 -3

hosting-service/src/lexicon/lexicons.ts

···

       118
       118
        
                   type: 'string',

     

       119
       119
        
                   format: 'at-uri',

     

       120
       120
        
                   description:

     

       121
       121
       -
                     'AT-URI pointing to a place.wisp.subfs record containing this subtree',

     

       121
       121
       +
                     'AT-URI pointing to a place.wisp.subfs record containing this subtree.',

     

       122
       122
       +
                 },

     

       123
       123
       +
                 flat: {

     

       124
       124
       +
                   type: 'boolean',

     

       125
       125
       +
                   description:

     

       126
       126
       +
                     "If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure.",

     

       122
       127
        
                 },

     

       123
       128
        
               },

     

       124
       129
        
             },

     
···

       131
       136
        
             main: {

     

       132
       137
        
               type: 'record',

     

       133
       138
        
               description:

     

       134
       134
       -
                 'Virtual filesystem manifest within a place.wisp.fs record',

     

       139
       139
       +
                 'Virtual filesystem subtree referenced by place.wisp.fs records. When a subfs entry is expanded, its root entries are merged (flattened) into the parent directory, allowing large directories to be split across multiple records while maintaining a flat structure.',

     

       135
       140
        
               record: {

     

       136
       141
        
                 type: 'object',

     

       137
       142
        
                 required: ['root', 'createdAt'],

     
···

       230
       235
        
                   type: 'string',

     

       231
       236
        
                   format: 'at-uri',

     

       232
       237
        
                   description:

     

       233
       233
       -
                     'AT-URI pointing to another place.wisp.subfs record for nested subtrees',

     

       238
       238
       +
                     "AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures.",

     

       234
       239
        
                 },

     

       235
       240
        
               },

     

       236
       241
        
             },

+3 -1

hosting-service/src/lexicon/types/place/wisp/fs.ts

···

       93
       93
        
       export interface Subfs {

     

       94
       94
        
         $type?: 'place.wisp.fs#subfs'

     

       95
       95
        
         type: 'subfs'

     

       96
       96
       -
         /** AT-URI pointing to a place.wisp.subfs record containing this subtree */

     

       96
       96
       +
         /** AT-URI pointing to a place.wisp.subfs record containing this subtree. */

     

       97
       97
        
         subject: string

     

       98
       98
       +
         /** If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure. */

     

       99
       99
       +
         flat?: boolean

     

       98
       100
        
       }

     

       99
       101
        
       

     

       100
       102
        
       const hashSubfs = 'subfs'

+1 -1

hosting-service/src/lexicon/types/place/wisp/subfs.ts

···

       92
       92
        
       export interface Subfs {

     

       93
       93
        
         $type?: 'place.wisp.subfs#subfs'

     

       94
       94
        
         type: 'subfs'

     

       95
       95
       -
         /** AT-URI pointing to another place.wisp.subfs record for nested subtrees */

     

       95
       95
       +
         /** AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures. */

     

       96
       96
        
         subject: string

     

       97
       97
        
       }

     

       98
       98

+1

hosting-service/src/lib/firehose.ts

···

       82
       82
        
       								evt.cid?.toString()

     

       83
       83
        
       							)

     

       84
       84
        
       						} catch (err) {

     

       85
       85
       +
       							console.error('Full error details:', err);

     

       85
       86
        
       							this.log('Error handling event', {

     

       86
       87
        
       								did: evt.did,

     

       87
       88
        
       								event: evt.event,

+43 -7

hosting-service/src/lib/utils.ts

···

       295
       295
        
             const node = entry.node;

     

       296
       296
        
       

     

       297
       297
        
             if ('type' in node && node.type === 'subfs') {

     

       298
       298
       -
               // Merge subfs entries into parent directory

     

       298
       298
       +
               // Check if this is a flat merge or subdirectory merge (default to flat if not specified)

     

       299
       299
       +
               const subfsNode = node as any;

     

       300
       300
       +
               const isFlat = subfsNode.flat !== false; // Default to true

     

       299
       301
        
               const subfsEntries = subfsMap.get(fullPath);

     

       302
       302
       +
       

     

       300
       303
        
               if (subfsEntries) {

     

       301
       301
       -
                 console.log(`Merging subfs node at ${fullPath} (${subfsEntries.length} entries)`);

     

       302
       302
       -
                 // Recursively process the merged entries in case they contain nested subfs

     

       303
       303
       -
                 const processedEntries = replaceSubfsInEntries(subfsEntries, currentPath);

     

       304
       304
       -
                 result.push(...processedEntries);

     

       304
       304
       +
                 console.log(`Merging subfs node at ${fullPath} (${subfsEntries.length} entries, flat: ${isFlat})`);

     

       305
       305
       +
       

     

       306
       306
       +
                 if (isFlat) {

     

       307
       307
       +
                   // Flat merge: hoist entries directly into parent directory

     

       308
       308
       +
                   const processedEntries = replaceSubfsInEntries(subfsEntries, currentPath);

     

       309
       309
       +
                   result.push(...processedEntries);

     

       310
       310
       +
                 } else {

     

       311
       311
       +
                   // Subdirectory merge: create a directory with the subfs node's name

     

       312
       312
       +
                   const processedEntries = replaceSubfsInEntries(subfsEntries, fullPath);

     

       313
       313
       +
                   result.push({

     

       314
       314
       +
                     name: entry.name,

     

       315
       315
       +
                     node: {

     

       316
       316
       +
                       type: 'directory',

     

       317
       317
       +
                       entries: processedEntries

     

       318
       318
       +
                     }

     

       319
       319
       +
                   });

     

       320
       320
       +
                 }

     

       305
       321
        
               } else {

     

       306
       322
        
                 // If fetch failed, skip this entry

     

       307
       323
        
                 console.warn(`Failed to fetch subfs at ${fullPath}, skipping`);

     
···

       491
       507
        
       

     

       492
       508
        
         // Download new/changed files concurrently - increased from 3 to 20 for much better performance

     

       493
       509
        
         const downloadLimit = 20;

     

       510
       510
       +
         let successCount = 0;

     

       511
       511
       +
         let failureCount = 0;

     

       512
       512
       +
       

     

       494
       513
        
         for (let i = 0; i < downloadTasks.length; i += downloadLimit) {

     

       495
       514
        
           const batch = downloadTasks.slice(i, i + downloadLimit);

     

       496
       496
       -
           await Promise.all(batch.map(task => task()));

     

       515
       515
       +
           const results = await Promise.allSettled(batch.map(task => task()));

     

       516
       516
       +
       

     

       517
       517
       +
           // Count successes and failures

     

       518
       518
       +
           results.forEach((result, index) => {

     

       519
       519
       +
             if (result.status === 'fulfilled') {

     

       520
       520
       +
               successCount++;

     

       521
       521
       +
             } else {

     

       522
       522
       +
               failureCount++;

     

       523
       523
       +
               console.error(`[Cache] Failed to download file (continuing with others):`, result.reason);

     

       524
       524
       +
             }

     

       525
       525
       +
           });

     

       526
       526
       +
       

     

       497
       527
        
           if (downloadTasks.length > downloadLimit) {

     

       498
       498
       -
             console.log(`[Cache Progress] Downloaded ${Math.min(i + downloadLimit, downloadTasks.length)}/${downloadTasks.length} files`);

     

       528
       528
       +
             console.log(`[Cache Progress] Downloaded ${Math.min(i + downloadLimit, downloadTasks.length)}/${downloadTasks.length} files (${failureCount} failed)`);

     

       499
       529
        
           }

     

       530
       530
       +
         }

     

       531
       531
       +
       

     

       532
       532
       +
         if (failureCount > 0) {

     

       533
       533
       +
           console.warn(`[Cache] Completed with ${successCount} successful and ${failureCount} failed file downloads`);

     

       500
       534
        
         }

     

       501
       535
        
       }

     

       502
       536
        
       

     
···

       555
       589
        
         }

     

       556
       590
        
       

     

       557
       591
        
         const blobUrl = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`;

     

       592
       592
       +
       

     

       593
       593
       +
         console.log(`[Cache] Fetching blob for file: ${filePath}, CID: ${cid}`);

     

       558
       594
        
       

     

       559
       595
        
         // Allow up to 500MB per file blob, with 5 minute timeout

     

       560
       596
        
         let content = await safeFetchBlob(blobUrl, { maxSize: 500 * 1024 * 1024, timeout: 300000 });

+2 -1

lexicons/fs.json

···

       51
       51
        
             "required": ["type", "subject"],

     

       52
       52
        
             "properties": {

     

       53
       53
        
               "type": { "type": "string", "const": "subfs" },

     

       54
       54
       -
               "subject": { "type": "string", "format": "at-uri", "description": "AT-URI pointing to a place.wisp.subfs record containing this subtree. When expanded, the subfs record's root entries are merged (flattened) into the parent directory - the subfs entry itself is removed and replaced by all entries from the referenced record's root. This allows splitting large directories across multiple records while maintaining a flat structure." }

     

       54
       54
       +
               "subject": { "type": "string", "format": "at-uri", "description": "AT-URI pointing to a place.wisp.subfs record containing this subtree." },

     

       55
       55
       +
               "flat": { "type": "boolean", "description": "If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure." }

     

       55
       56
        
             }

     

       56
       57
        
           }

     

       57
       58
        
         }

+8 -3

src/lexicons/lexicons.ts

···

       118
       118
        
                   type: 'string',

     

       119
       119
        
                   format: 'at-uri',

     

       120
       120
        
                   description:

     

       121
       121
       -
                     'AT-URI pointing to a place.wisp.subfs record containing this subtree',

     

       121
       121
       +
                     'AT-URI pointing to a place.wisp.subfs record containing this subtree.',

     

       122
       122
       +
                 },

     

       123
       123
       +
                 flat: {

     

       124
       124
       +
                   type: 'boolean',

     

       125
       125
       +
                   description:

     

       126
       126
       +
                     "If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure.",

     

       122
       127
        
                 },

     

       123
       128
        
               },

     

       124
       129
        
             },

     
···

       131
       136
        
             main: {

     

       132
       137
        
               type: 'record',

     

       133
       138
        
               description:

     

       134
       134
       -
                 'Virtual filesystem manifest within a place.wisp.fs record',

     

       139
       139
       +
                 'Virtual filesystem subtree referenced by place.wisp.fs records. When a subfs entry is expanded, its root entries are merged (flattened) into the parent directory, allowing large directories to be split across multiple records while maintaining a flat structure.',

     

       135
       140
        
               record: {

     

       136
       141
        
                 type: 'object',

     

       137
       142
        
                 required: ['root', 'createdAt'],

     
···

       230
       235
        
                   type: 'string',

     

       231
       236
        
                   format: 'at-uri',

     

       232
       237
        
                   description:

     

       233
       233
       -
                     'AT-URI pointing to another place.wisp.subfs record for nested subtrees',

     

       238
       238
       +
                     "AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures.",

     

       234
       239
        
                 },

     

       235
       240
        
               },

     

       236
       241
        
             },

+3 -1

src/lexicons/types/place/wisp/fs.ts

···

       93
       93
        
       export interface Subfs {

     

       94
       94
        
         $type?: 'place.wisp.fs#subfs'

     

       95
       95
        
         type: 'subfs'

     

       96
       96
       -
         /** AT-URI pointing to a place.wisp.subfs record containing this subtree */

     

       96
       96
       +
         /** AT-URI pointing to a place.wisp.subfs record containing this subtree. */

     

       97
       97
        
         subject: string

     

       98
       98
       +
         /** If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure. */

     

       99
       99
       +
         flat?: boolean

     

       98
       100
        
       }

     

       99
       101
        
       

     

       100
       102
        
       const hashSubfs = 'subfs'

+1 -1

src/lexicons/types/place/wisp/subfs.ts

···

       92
       92
        
       export interface Subfs {

     

       93
       93
        
         $type?: 'place.wisp.subfs#subfs'

     

       94
       94
        
         type: 'subfs'

     

       95
       95
       -
         /** AT-URI pointing to another place.wisp.subfs record for nested subtrees */

     

       95
       95
       +
         /** AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures. */

     

       96
       96
        
         subject: string

     

       97
       97
        
       }

     

       98
       98

+2 -1

src/lib/wisp-utils.ts

···

       424
       424
        
       					node: {

     

       425
       425
        
       						$type: 'place.wisp.fs#subfs' as const,

     

       426
       426
        
       						type: 'subfs' as const,

     

       427
       427
       -
       						subject: subfsUri

     

       427
       427
       +
       						subject: subfsUri,

     

       428
       428
       +
       						flat: false  // Preserve directory structure

     

       428
       429
        
       					}

     

       429
       430
        
       				};

     

       430
       431
        
       			}

+2 -1

src/routes/wisp.ts

···

       719
       719
        
                               node: {

     

       720
       720
        
                                   $type: 'place.wisp.fs#subfs' as const,

     

       721
       721
        
                                   type: 'subfs' as const,

     

       722
       722
       -
                                   subject: subfsUri

     

       722
       722
       +
                                   subject: subfsUri,

     

       723
       723
       +
                                   flat: true  // Merge entries directly into parent (default, but explicit for clarity)

     

       723
       724
        
                               }

     

       724
       725
        
                           });

     

       725
       726