···
import { AtpAgent } from '@atproto/api';
import type { Record as WispFsRecord, Directory, Entry, File } from '../lexicon/types/place/wisp/fs';
3
+
import type { Record as SubfsRecord } from '../lexicon/types/place/wisp/subfs';
import { existsSync, mkdirSync, readFileSync, rmSync } from 'fs';
import { writeFile, readFile, rename } from 'fs/promises';
import { safeFetchJson, safeFetchBlob } from './safe-fetch';
···
194
+
* Extract all subfs URIs from a directory tree with their mount paths
196
+
function extractSubfsUris(directory: Directory, currentPath: string = ''): Array<{ uri: string; path: string }> {
197
+
const uris: Array<{ uri: string; path: string }> = [];
199
+
for (const entry of directory.entries) {
200
+
const fullPath = currentPath ? `${currentPath}/${entry.name}` : entry.name;
202
+
if ('type' in entry.node) {
203
+
if (entry.node.type === 'subfs') {
204
+
// Subfs node with subject URI
205
+
const subfsNode = entry.node as any;
206
+
if (subfsNode.subject) {
207
+
uris.push({ uri: subfsNode.subject, path: fullPath });
209
+
} else if (entry.node.type === 'directory') {
210
+
// Recursively search subdirectories
211
+
const subUris = extractSubfsUris(entry.node as Directory, fullPath);
212
+
uris.push(...subUris);
221
+
* Fetch a subfs record from the PDS
223
+
async function fetchSubfsRecord(uri: string, pdsEndpoint: string): Promise<SubfsRecord | null> {
225
+
// Parse URI: at://did/collection/rkey
226
+
const parts = uri.replace('at://', '').split('/');
227
+
if (parts.length < 3) {
228
+
console.error('Invalid subfs URI:', uri);
232
+
const did = parts[0];
233
+
const collection = parts[1];
234
+
const rkey = parts[2];
236
+
// Fetch the record from PDS
237
+
const url = `${pdsEndpoint}/xrpc/com.atproto.repo.getRecord?repo=${encodeURIComponent(did)}&collection=${encodeURIComponent(collection)}&rkey=${encodeURIComponent(rkey)}`;
238
+
const response = await safeFetchJson(url);
240
+
if (!response || !response.value) {
241
+
console.error('Subfs record not found:', uri);
245
+
return response.value as SubfsRecord;
247
+
console.error('Failed to fetch subfs record:', uri, err);
253
+
* Replace subfs nodes in a directory tree with their actual content
255
+
async function expandSubfsNodes(directory: Directory, pdsEndpoint: string): Promise<Directory> {
256
+
// Extract all subfs URIs
257
+
const subfsUris = extractSubfsUris(directory);
259
+
if (subfsUris.length === 0) {
260
+
// No subfs nodes, return as-is
264
+
console.log(`Found ${subfsUris.length} subfs records, fetching...`);
266
+
// Fetch all subfs records in parallel
267
+
const subfsRecords = await Promise.all(
268
+
subfsUris.map(async ({ uri, path }) => {
269
+
const record = await fetchSubfsRecord(uri, pdsEndpoint);
270
+
return { record, path };
274
+
// Build a map of path -> directory content
275
+
const subfsMap = new Map<string, Directory>();
276
+
for (const { record, path } of subfsRecords) {
277
+
if (record && record.root) {
278
+
subfsMap.set(path, record.root);
282
+
// Replace subfs nodes with their actual content
283
+
function replaceSubfsInEntries(entries: Entry[], currentPath: string = ''): Entry[] {
284
+
return entries.map(entry => {
285
+
const fullPath = currentPath ? `${currentPath}/${entry.name}` : entry.name;
286
+
const node = entry.node;
288
+
if ('type' in node && node.type === 'subfs') {
289
+
// Replace with actual directory content
290
+
const subfsDir = subfsMap.get(fullPath);
292
+
console.log(`Expanding subfs node at ${fullPath}`);
298
+
// If fetch failed, keep the subfs node (will be skipped later)
300
+
} else if ('type' in node && node.type === 'directory' && 'entries' in node) {
301
+
// Recursively process subdirectories
306
+
entries: replaceSubfsInEntries(node.entries, fullPath)
317
+
entries: replaceSubfsInEntries(directory.entries)
export async function downloadAndCacheSite(did: string, rkey: string, record: WispFsRecord, pdsEndpoint: string, recordCid: string): Promise<void> {
console.log('Caching site', did, rkey);
···
console.error('Record root missing entries array:', JSON.stringify(record.root, null, 2));
throw new Error('Invalid record structure: root missing entries array');
334
+
// Expand subfs nodes before caching
335
+
const expandedRoot = await expandSubfsNodes(record.root, pdsEndpoint);
// Get existing cache metadata to check for incremental updates
const existingMetadata = await getCacheMetadata(did, rkey);
···
const finalDir = `${CACHE_DIR}/${did}/${rkey}`;
215
-
// Collect file CIDs from the new record
347
+
// Collect file CIDs from the new record (using expanded root)
const newFileCids: Record<string, string> = {};
217
-
collectFileCidsFromEntries(record.root.entries, '', newFileCids);
349
+
collectFileCidsFromEntries(expandedRoot.entries, '', newFileCids);
219
-
// Download/copy files to temporary directory (with incremental logic)
220
-
await cacheFiles(did, rkey, record.root.entries, pdsEndpoint, '', tempSuffix, existingFileCids, finalDir);
351
+
// Download/copy files to temporary directory (with incremental logic, using expanded root)
352
+
await cacheFiles(did, rkey, expandedRoot.entries, pdsEndpoint, '', tempSuffix, existingFileCids, finalDir);
await saveCacheMetadata(did, rkey, recordCid, tempSuffix, newFileCids);
// Atomically replace old cache with new cache
nekomimi.pet