···
16
+
// Map of file path to blob CID for incremental updates
17
+
fileCids?: Record<string, string>;
···
throw new Error('Invalid record structure: root missing entries array');
205
+
// Get existing cache metadata to check for incremental updates
206
+
const existingMetadata = await getCacheMetadata(did, rkey);
207
+
const existingFileCids = existingMetadata?.fileCids || {};
// Use a temporary directory with timestamp to avoid collisions
const tempSuffix = `.tmp-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
const tempDir = `${CACHE_DIR}/${did}/${rkey}${tempSuffix}`;
const finalDir = `${CACHE_DIR}/${did}/${rkey}`;
209
-
// Download to temporary directory
210
-
await cacheFiles(did, rkey, record.root.entries, pdsEndpoint, '', tempSuffix);
211
-
await saveCacheMetadata(did, rkey, recordCid, tempSuffix);
215
+
// Collect file CIDs from the new record
216
+
const newFileCids: Record<string, string> = {};
217
+
collectFileCidsFromEntries(record.root.entries, '', newFileCids);
219
+
// Download/copy files to temporary directory (with incremental logic)
220
+
await cacheFiles(did, rkey, record.root.entries, pdsEndpoint, '', tempSuffix, existingFileCids, finalDir);
221
+
await saveCacheMetadata(did, rkey, recordCid, tempSuffix, newFileCids);
// Atomically replace old cache with new cache
// On POSIX systems (Linux/macOS), rename is atomic
···
259
+
* Recursively collect file CIDs from entries for incremental update tracking
261
+
function collectFileCidsFromEntries(entries: Entry[], pathPrefix: string, fileCids: Record<string, string>): void {
262
+
for (const entry of entries) {
263
+
const currentPath = pathPrefix ? `${pathPrefix}/${entry.name}` : entry.name;
264
+
const node = entry.node;
266
+
if ('type' in node && node.type === 'directory' && 'entries' in node) {
267
+
collectFileCidsFromEntries(node.entries, currentPath, fileCids);
268
+
} else if ('type' in node && node.type === 'file' && 'blob' in node) {
269
+
const fileNode = node as File;
270
+
const cid = extractBlobCid(fileNode.blob);
272
+
fileCids[currentPath] = cid;
async function cacheFiles(
254
-
dirSuffix: string = ''
284
+
dirSuffix: string = '',
285
+
existingFileCids: Record<string, string> = {},
286
+
existingCacheDir?: string
256
-
// Collect all file blob download tasks first
288
+
// Collect file tasks, separating unchanged files from new/changed files
const downloadTasks: Array<() => Promise<void>> = [];
290
+
const copyTasks: Array<() => Promise<void>> = [];
function collectFileTasks(
currentPathPrefix: string
···
collectFileTasks(node.entries, currentPath);
} else if ('type' in node && node.type === 'file' && 'blob' in node) {
const fileNode = node as File;
271
-
downloadTasks.push(() => cacheFileBlob(
304
+
const cid = extractBlobCid(fileNode.blob);
306
+
// Check if file is unchanged (same CID as existing cache)
307
+
if (cid && existingFileCids[currentPath] === cid && existingCacheDir) {
308
+
// File unchanged - copy from existing cache instead of downloading
309
+
copyTasks.push(() => copyExistingFile(
317
+
// File new or changed - download it
318
+
downloadTasks.push(() => cacheFileBlob(
collectFileTasks(entries, pathPrefix);
288
-
// Execute downloads concurrently with a limit of 3 at a time
289
-
const concurrencyLimit = 3;
290
-
for (let i = 0; i < downloadTasks.length; i += concurrencyLimit) {
291
-
const batch = downloadTasks.slice(i, i + concurrencyLimit);
336
+
console.log(`[Incremental Update] Files to copy: ${copyTasks.length}, Files to download: ${downloadTasks.length}`);
338
+
// Copy unchanged files in parallel (fast local operations)
339
+
const copyLimit = 10;
340
+
for (let i = 0; i < copyTasks.length; i += copyLimit) {
341
+
const batch = copyTasks.slice(i, i + copyLimit);
342
+
await Promise.all(batch.map(task => task()));
345
+
// Download new/changed files concurrently with a limit of 3 at a time
346
+
const downloadLimit = 3;
347
+
for (let i = 0; i < downloadTasks.length; i += downloadLimit) {
348
+
const batch = downloadTasks.slice(i, i + downloadLimit);
await Promise.all(batch.map(task => task()));
354
+
* Copy an unchanged file from existing cache to new cache location
356
+
async function copyExistingFile(
361
+
existingCacheDir: string
363
+
const { copyFile } = await import('fs/promises');
365
+
const sourceFile = `${existingCacheDir}/${filePath}`;
366
+
const destFile = `${CACHE_DIR}/${did}/${site}${dirSuffix}/${filePath}`;
367
+
const destDir = destFile.substring(0, destFile.lastIndexOf('/'));
369
+
// Create destination directory if needed
370
+
if (destDir && !existsSync(destDir)) {
371
+
mkdirSync(destDir, { recursive: true });
376
+
await copyFile(sourceFile, destFile);
378
+
// Copy metadata file if it exists
379
+
const sourceMetaFile = `${sourceFile}.meta`;
380
+
const destMetaFile = `${destFile}.meta`;
381
+
if (existsSync(sourceMetaFile)) {
382
+
await copyFile(sourceMetaFile, destMetaFile);
385
+
console.log(`[Incremental] Copied unchanged file: ${filePath}`);
387
+
console.error(`[Incremental] Failed to copy file ${filePath}, will attempt download:`, err);
···
return existsSync(`${CACHE_DIR}/${did}/${site}`);
407
-
async function saveCacheMetadata(did: string, rkey: string, recordCid: string, dirSuffix: string = ''): Promise<void> {
503
+
async function saveCacheMetadata(did: string, rkey: string, recordCid: string, dirSuffix: string = '', fileCids?: Record<string, string>): Promise<void> {
const metadata: CacheMetadata = {
const metadataPath = `${CACHE_DIR}/${did}/${rkey}${dirSuffix}/.metadata.json`;
nekomimi.pet