From fccf65cf0cbd4fc94f8a418ebeeacb74945020e4 Mon Sep 17 00:00:00 2001 From: "@nekomimi.pet" Date: Sat, 8 Nov 2025 23:44:57 -0500 Subject: [PATCH] save CIDs to local metadata file, incrementally download new files, copy old --- hosting-service/src/lib/utils.ts | 143 ++++++++++++++++++++++++++----- hosting-service/tsconfig.json | 4 +- 2 files changed, 123 insertions(+), 24 deletions(-) diff --git a/hosting-service/src/lib/utils.ts b/hosting-service/src/lib/utils.ts index f4fec2f..856b51c 100644 --- a/hosting-service/src/lib/utils.ts +++ b/hosting-service/src/lib/utils.ts @@ -13,6 +13,8 @@ interface CacheMetadata { cachedAt: number; did: string; rkey: string; + // Map of file path to blob CID for incremental updates + fileCids?: Record; } /** @@ -200,15 +202,23 @@ export async function downloadAndCacheSite(did: string, rkey: string, record: Wi throw new Error('Invalid record structure: root missing entries array'); } + // Get existing cache metadata to check for incremental updates + const existingMetadata = await getCacheMetadata(did, rkey); + const existingFileCids = existingMetadata?.fileCids || {}; + // Use a temporary directory with timestamp to avoid collisions const tempSuffix = `.tmp-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`; const tempDir = `${CACHE_DIR}/${did}/${rkey}${tempSuffix}`; const finalDir = `${CACHE_DIR}/${did}/${rkey}`; try { - // Download to temporary directory - await cacheFiles(did, rkey, record.root.entries, pdsEndpoint, '', tempSuffix); - await saveCacheMetadata(did, rkey, recordCid, tempSuffix); + // Collect file CIDs from the new record + const newFileCids: Record = {}; + collectFileCidsFromEntries(record.root.entries, '', newFileCids); + + // Download/copy files to temporary directory (with incremental logic) + await cacheFiles(did, rkey, record.root.entries, pdsEndpoint, '', tempSuffix, existingFileCids, finalDir); + await saveCacheMetadata(did, rkey, recordCid, tempSuffix, newFileCids); // Atomically replace old cache with new cache // On POSIX systems (Linux/macOS), rename is atomic @@ -245,17 +255,40 @@ export async function downloadAndCacheSite(did: string, rkey: string, record: Wi } } +/** + * Recursively collect file CIDs from entries for incremental update tracking + */ +function collectFileCidsFromEntries(entries: Entry[], pathPrefix: string, fileCids: Record): void { + for (const entry of entries) { + const currentPath = pathPrefix ? `${pathPrefix}/${entry.name}` : entry.name; + const node = entry.node; + + if ('type' in node && node.type === 'directory' && 'entries' in node) { + collectFileCidsFromEntries(node.entries, currentPath, fileCids); + } else if ('type' in node && node.type === 'file' && 'blob' in node) { + const fileNode = node as File; + const cid = extractBlobCid(fileNode.blob); + if (cid) { + fileCids[currentPath] = cid; + } + } + } +} + async function cacheFiles( did: string, site: string, entries: Entry[], pdsEndpoint: string, pathPrefix: string, - dirSuffix: string = '' + dirSuffix: string = '', + existingFileCids: Record = {}, + existingCacheDir?: string ): Promise { - // Collect all file blob download tasks first + // Collect file tasks, separating unchanged files from new/changed files const downloadTasks: Array<() => Promise> = []; - + const copyTasks: Array<() => Promise> = []; + function collectFileTasks( entries: Entry[], currentPathPrefix: string @@ -268,29 +301,92 @@ async function cacheFiles( collectFileTasks(node.entries, currentPath); } else if ('type' in node && node.type === 'file' && 'blob' in node) { const fileNode = node as File; - downloadTasks.push(() => cacheFileBlob( - did, - site, - currentPath, - fileNode.blob, - pdsEndpoint, - fileNode.encoding, - fileNode.mimeType, - fileNode.base64, - dirSuffix - )); + const cid = extractBlobCid(fileNode.blob); + + // Check if file is unchanged (same CID as existing cache) + if (cid && existingFileCids[currentPath] === cid && existingCacheDir) { + // File unchanged - copy from existing cache instead of downloading + copyTasks.push(() => copyExistingFile( + did, + site, + currentPath, + dirSuffix, + existingCacheDir + )); + } else { + // File new or changed - download it + downloadTasks.push(() => cacheFileBlob( + did, + site, + currentPath, + fileNode.blob, + pdsEndpoint, + fileNode.encoding, + fileNode.mimeType, + fileNode.base64, + dirSuffix + )); + } } } } collectFileTasks(entries, pathPrefix); - // Execute downloads concurrently with a limit of 3 at a time - const concurrencyLimit = 3; - for (let i = 0; i < downloadTasks.length; i += concurrencyLimit) { - const batch = downloadTasks.slice(i, i + concurrencyLimit); + console.log(`[Incremental Update] Files to copy: ${copyTasks.length}, Files to download: ${downloadTasks.length}`); + + // Copy unchanged files in parallel (fast local operations) + const copyLimit = 10; + for (let i = 0; i < copyTasks.length; i += copyLimit) { + const batch = copyTasks.slice(i, i + copyLimit); await Promise.all(batch.map(task => task())); } + + // Download new/changed files concurrently with a limit of 3 at a time + const downloadLimit = 3; + for (let i = 0; i < downloadTasks.length; i += downloadLimit) { + const batch = downloadTasks.slice(i, i + downloadLimit); + await Promise.all(batch.map(task => task())); + } +} + +/** + * Copy an unchanged file from existing cache to new cache location + */ +async function copyExistingFile( + did: string, + site: string, + filePath: string, + dirSuffix: string, + existingCacheDir: string +): Promise { + const { copyFile } = await import('fs/promises'); + + const sourceFile = `${existingCacheDir}/${filePath}`; + const destFile = `${CACHE_DIR}/${did}/${site}${dirSuffix}/${filePath}`; + const destDir = destFile.substring(0, destFile.lastIndexOf('/')); + + // Create destination directory if needed + if (destDir && !existsSync(destDir)) { + mkdirSync(destDir, { recursive: true }); + } + + try { + // Copy the file + await copyFile(sourceFile, destFile); + + // Copy metadata file if it exists + const sourceMetaFile = `${sourceFile}.meta`; + const destMetaFile = `${destFile}.meta`; + if (existsSync(sourceMetaFile)) { + await copyFile(sourceMetaFile, destMetaFile); + } + + console.log(`[Incremental] Copied unchanged file: ${filePath}`); + } catch (err) { + console.error(`[Incremental] Failed to copy file ${filePath}, will attempt download:`, err); + throw err; + } } async function cacheFileBlob( @@ -404,12 +500,13 @@ export function isCached(did: string, site: string): boolean { return existsSync(`${CACHE_DIR}/${did}/${site}`); } -async function saveCacheMetadata(did: string, rkey: string, recordCid: string, dirSuffix: string = ''): Promise { +async function saveCacheMetadata(did: string, rkey: string, recordCid: string, dirSuffix: string = '', fileCids?: Record): Promise { const metadata: CacheMetadata = { recordCid, cachedAt: Date.now(), did, - rkey + rkey, + fileCids }; const metadataPath = `${CACHE_DIR}/${did}/${rkey}${dirSuffix}/.metadata.json`; diff --git a/hosting-service/tsconfig.json b/hosting-service/tsconfig.json index aa23e3b..6de29f3 100644 --- a/hosting-service/tsconfig.json +++ b/hosting-service/tsconfig.json @@ -24,5 +24,7 @@ /* Code doesn't run in DOM */ "lib": ["es2022"], - } + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "cache", "dist"] } -- 2.51.0 From 69953ea1f4ee3c1008770a35846d789c931db13b Mon Sep 17 00:00:00 2001 From: "@nekomimi.pet" Date: Sat, 8 Nov 2025 23:46:03 -0500 Subject: [PATCH] update package.json to trust postinstalls --- bun.lock | 80 ++++++++++++++++++++++++++++++---------------------- package.json | 2 ++ 2 files changed, 48 insertions(+), 34 deletions(-) diff --git a/bun.lock b/bun.lock index 87c990b..7b3f2c7 100644 --- a/bun.lock +++ b/bun.lock @@ -44,6 +44,8 @@ }, "trustedDependencies": [ "core-js", + "cbor-extract", + "bun", "protobufjs", ], "packages": { @@ -51,11 +53,11 @@ "@atproto-labs/fetch": ["@atproto-labs/fetch@0.2.3", "", { "dependencies": { "@atproto-labs/pipe": "0.1.1" } }, "sha512-NZtbJOCbxKUFRFKMpamT38PUQMY0hX0p7TG5AEYOPhZKZEP7dHZ1K2s1aB8MdVH0qxmqX7nQleNrrvLf09Zfdw=="], - "@atproto-labs/fetch-node": ["@atproto-labs/fetch-node@0.1.10", "", { "dependencies": { "@atproto-labs/fetch": "0.2.3", "@atproto-labs/pipe": "0.1.1", "ipaddr.js": "^2.1.0", "undici": "^6.14.1" } }, "sha512-o7hGaonA71A6p7O107VhM6UBUN/g9tTyYohMp1q0Kf6xQ4npnuZYRSHSf2g6reSfGQJ1GoFNjBObETTT1ge/jQ=="], + "@atproto-labs/fetch-node": ["@atproto-labs/fetch-node@0.2.0", "", { "dependencies": { "@atproto-labs/fetch": "0.2.3", "@atproto-labs/pipe": "0.1.1", "ipaddr.js": "^2.1.0", "undici": "^6.14.1" } }, "sha512-Krq09nH/aeoiU2s9xdHA0FjTEFWG9B5FFenipv1iRixCcPc7V3DhTNDawxG9gI8Ny0k4dBVS9WTRN/IDzBx86Q=="], "@atproto-labs/handle-resolver": ["@atproto-labs/handle-resolver@0.3.2", "", { "dependencies": { "@atproto-labs/simple-store": "0.3.0", "@atproto-labs/simple-store-memory": "0.1.4", "@atproto/did": "0.2.1", "zod": "^3.23.8" } }, "sha512-KIerCzh3qb+zZoqWbIvTlvBY0XPq0r56kwViaJY/LTe/3oPO2JaqlYKS/F4dByWBhHK6YoUOJ0sWrh6PMJl40A=="], - "@atproto-labs/handle-resolver-node": ["@atproto-labs/handle-resolver-node@0.1.20", "", { "dependencies": { "@atproto-labs/fetch-node": "0.1.10", "@atproto-labs/handle-resolver": "0.3.2", "@atproto/did": "0.2.1" } }, "sha512-094EL61XN9M7vm22cloSOxk/gcTRaCK52vN7BYgXgdoEI8uJJMTFXenQqu+LRGwiCcjvyclcBqbaz0DzJep50Q=="], + "@atproto-labs/handle-resolver-node": ["@atproto-labs/handle-resolver-node@0.1.21", "", { "dependencies": { "@atproto-labs/fetch-node": "0.2.0", "@atproto-labs/handle-resolver": "0.3.2", "@atproto/did": "0.2.1" } }, "sha512-fuJy5Px5pGF3lJX/ATdurbT8tbmaFWtf+PPxAQDFy7ot2no3t+iaAgymhyxYymrssOuWs6BwOP8tyF3VrfdwtQ=="], "@atproto-labs/identity-resolver": ["@atproto-labs/identity-resolver@0.3.2", "", { "dependencies": { "@atproto-labs/did-resolver": "0.2.2", "@atproto-labs/handle-resolver": "0.3.2" } }, "sha512-MYxO9pe0WsFyi5HFdKAwqIqHfiF2kBPoVhAIuH/4PYHzGr799ED47xLhNMxR3ZUYrJm5+TQzWXypGZ0Btw1Ffw=="], @@ -65,7 +67,7 @@ "@atproto-labs/simple-store-memory": ["@atproto-labs/simple-store-memory@0.1.4", "", { "dependencies": { "@atproto-labs/simple-store": "0.3.0", "lru-cache": "^10.2.0" } }, "sha512-3mKY4dP8I7yKPFj9VKpYyCRzGJOi5CEpOLPlRhoJyLmgs3J4RzDrjn323Oakjz2Aj2JzRU/AIvWRAZVhpYNJHw=="], - "@atproto/api": ["@atproto/api@0.17.3", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/lexicon": "^0.5.1", "@atproto/syntax": "^0.4.1", "@atproto/xrpc": "^0.7.5", "await-lock": "^2.2.2", "multiformats": "^9.9.0", "tlds": "^1.234.0", "zod": "^3.23.8" } }, "sha512-pdQXhUAapNPdmN00W0vX5ta/aMkHqfgBHATt20X02XwxQpY2AnrPm2Iog4FyjsZqoHooAtCNV/NWJ4xfddJzsg=="], + "@atproto/api": ["@atproto/api@0.17.7", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/lexicon": "^0.5.1", "@atproto/syntax": "^0.4.1", "@atproto/xrpc": "^0.7.5", "await-lock": "^2.2.2", "multiformats": "^9.9.0", "tlds": "^1.234.0", "zod": "^3.23.8" } }, "sha512-V+OJBZq9chcrD21xk1bUa6oc5DSKfQj5DmUPf5rmZncqL1w9ZEbS38H5cMyqqdhfgo2LWeDRdZHD0rvNyJsIaw=="], "@atproto/common": ["@atproto/common@0.4.12", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@ipld/dag-cbor": "^7.0.3", "cbor-x": "^1.5.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "pino": "^8.21.0" } }, "sha512-NC+TULLQiqs6MvNymhQS5WDms3SlbIKGLf4n33tpftRJcalh507rI+snbcUb7TLIkKw7VO17qMqxEXtIdd5auQ=="], @@ -81,15 +83,15 @@ "@atproto/jwk-webcrypto": ["@atproto/jwk-webcrypto@0.2.0", "", { "dependencies": { "@atproto/jwk": "0.6.0", "@atproto/jwk-jose": "0.1.11", "zod": "^3.23.8" } }, "sha512-UmgRrrEAkWvxwhlwe30UmDOdTEFidlIzBC7C3cCbeJMcBN1x8B3KH+crXrsTqfWQBG58mXgt8wgSK3Kxs2LhFg=="], - "@atproto/lex-cli": ["@atproto/lex-cli@0.9.5", "", { "dependencies": { "@atproto/lexicon": "^0.5.1", "@atproto/syntax": "^0.4.1", "chalk": "^4.1.2", "commander": "^9.4.0", "prettier": "^3.2.5", "ts-morph": "^24.0.0", "yesno": "^0.4.0", "zod": "^3.23.8" }, "bin": { "lex": "dist/index.js" } }, "sha512-zun4jhD1dbjD7IHtLIjh/1UsMx+6E8+OyOT2GXYAKIj9N6wmLKM/v2OeQBKxcyqUmtRi57lxWnGikWjjU7pplQ=="], + "@atproto/lex-cli": ["@atproto/lex-cli@0.9.6", "", { "dependencies": { "@atproto/lexicon": "^0.5.1", "@atproto/syntax": "^0.4.1", "chalk": "^4.1.2", "commander": "^9.4.0", "prettier": "^3.2.5", "ts-morph": "^24.0.0", "yesno": "^0.4.0", "zod": "^3.23.8" }, "bin": { "lex": "dist/index.js" } }, "sha512-EedEKmURoSP735YwSDHsFrLOhZ4P2it8goCHv5ApWi/R9DFpOKOpmYfIXJ9MAprK8cw+yBnjDJbzpLJy7UXlTg=="], "@atproto/lexicon": ["@atproto/lexicon@0.5.1", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/syntax": "^0.4.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, "sha512-y8AEtYmfgVl4fqFxqXAeGvhesiGkxiy3CWoJIfsFDDdTlZUC8DFnZrYhcqkIop3OlCkkljvpSJi1hbeC1tbi8A=="], - "@atproto/oauth-client": ["@atproto/oauth-client@0.5.7", "", { "dependencies": { "@atproto-labs/did-resolver": "0.2.2", "@atproto-labs/fetch": "0.2.3", "@atproto-labs/handle-resolver": "0.3.2", "@atproto-labs/identity-resolver": "0.3.2", "@atproto-labs/simple-store": "0.3.0", "@atproto-labs/simple-store-memory": "0.1.4", "@atproto/did": "0.2.1", "@atproto/jwk": "0.6.0", "@atproto/oauth-types": "0.4.2", "@atproto/xrpc": "0.7.5", "core-js": "^3", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, "sha512-pDvbvy9DCxrAJv7bAbBUzWrHZKhFy091HvEMZhr+EyZA6gSCGYmmQJG/coDj0oICSVQeafAZd+IxR0YUCWwmEg=="], + "@atproto/oauth-client": ["@atproto/oauth-client@0.5.8", "", { "dependencies": { "@atproto-labs/did-resolver": "0.2.2", "@atproto-labs/fetch": "0.2.3", "@atproto-labs/handle-resolver": "0.3.2", "@atproto-labs/identity-resolver": "0.3.2", "@atproto-labs/simple-store": "0.3.0", "@atproto-labs/simple-store-memory": "0.1.4", "@atproto/did": "0.2.1", "@atproto/jwk": "0.6.0", "@atproto/oauth-types": "0.5.0", "@atproto/xrpc": "0.7.5", "core-js": "^3", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, "sha512-7YEym6d97+Dd73qGdkQTXi5La8xvCQxwRUDzzlR/NVAARa9a4YP7MCmqBJVeP2anT0By+DSAPyPDLTsxcjIcCg=="], - "@atproto/oauth-client-node": ["@atproto/oauth-client-node@0.3.9", "", { "dependencies": { "@atproto-labs/did-resolver": "0.2.2", "@atproto-labs/handle-resolver-node": "0.1.20", "@atproto-labs/simple-store": "0.3.0", "@atproto/did": "0.2.1", "@atproto/jwk": "0.6.0", "@atproto/jwk-jose": "0.1.11", "@atproto/jwk-webcrypto": "0.2.0", "@atproto/oauth-client": "0.5.7", "@atproto/oauth-types": "0.4.2" } }, "sha512-JdzwDQ8Gczl0lgfJNm7lG7omkJ4yu99IuGkkRWixpEvKY/jY/mDZaho+3pfd29SrUvwQOOx4Bm4l7DGeYwxxyA=="], + "@atproto/oauth-client-node": ["@atproto/oauth-client-node@0.3.10", "", { "dependencies": { "@atproto-labs/did-resolver": "0.2.2", "@atproto-labs/handle-resolver-node": "0.1.21", "@atproto-labs/simple-store": "0.3.0", "@atproto/did": "0.2.1", "@atproto/jwk": "0.6.0", "@atproto/jwk-jose": "0.1.11", "@atproto/jwk-webcrypto": "0.2.0", "@atproto/oauth-client": "0.5.8", "@atproto/oauth-types": "0.5.0" } }, "sha512-6khKlJqu1Ed5rt3rzcTD5hymB6JUjKdOHWYXwiphw4inkAIo6GxLCighI4eGOqZorYk2j8ueeTNB6KsgH0kcRw=="], - "@atproto/oauth-types": ["@atproto/oauth-types@0.4.2", "", { "dependencies": { "@atproto/did": "0.2.1", "@atproto/jwk": "0.6.0", "zod": "^3.23.8" } }, "sha512-gcfNTyFsPJcYDf79M0iKHykWqzxloscioKoerdIN3MTS3htiNOSgZjm2p8ho7pdrElLzea3qktuhTQI39j1XFQ=="], + "@atproto/oauth-types": ["@atproto/oauth-types@0.5.0", "", { "dependencies": { "@atproto/did": "0.2.1", "@atproto/jwk": "0.6.0", "zod": "^3.23.8" } }, "sha512-33xz7HcXhbl+XRqbIMVu3GE02iK1nKe2oMWENASsfZEYbCz2b9ZOarOFuwi7g4LKqpGowGp0iRKsQHFcq4SDaQ=="], "@atproto/syntax": ["@atproto/syntax@0.4.1", "", {}, "sha512-CJdImtLAiFO+0z3BWTtxwk6aY5w4t8orHTMVJgkf++QRJWTxPbIFko/0hrkADB7n2EruDxDSeAgfUGehpH6ngw=="], @@ -113,15 +115,15 @@ "@elysiajs/cors": ["@elysiajs/cors@1.4.0", "", { "peerDependencies": { "elysia": ">= 1.4.0" } }, "sha512-pb0SCzBfFbFSYA/U40HHO7R+YrcXBJXOWgL20eSViK33ol1e20ru2/KUaZYo5IMUn63yaTJI/bQERuQ+77ND8g=="], - "@elysiajs/eden": ["@elysiajs/eden@1.4.3", "", { "peerDependencies": { "elysia": ">= 1.4.0-exp.0" } }, "sha512-mX0v5cTvTJiDsDWNEEyuoqudOvW5J+tXsvp/ZOJXJF3iCIEJI0Brvm78ymPrvwiOG4nUr3lS8BxUfbNf32DSXA=="], + "@elysiajs/eden": ["@elysiajs/eden@1.4.4", "", { "peerDependencies": { "elysia": ">= 1.4.0-exp.0" } }, "sha512-/LVqflmgUcCiXb8rz1iRq9Rx3SWfIV/EkoNqDFGMx+TvOyo8QHAygFXAVQz7RHs+jk6n6mEgpI6KlKBANoErsQ=="], "@elysiajs/openapi": ["@elysiajs/openapi@1.4.11", "", { "peerDependencies": { "elysia": ">= 1.4.0" } }, "sha512-d75bMxYJpN6qSDi/z9L1S7SLk1S/8Px+cTb3W2lrYzU8uQ5E0kXdy1oOMJEfTyVsz3OA19NP9KNxE7ztSbLBLg=="], "@elysiajs/opentelemetry": ["@elysiajs/opentelemetry@1.4.6", "", { "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/instrumentation": "^0.200.0", "@opentelemetry/sdk-node": "^0.200.0" }, "peerDependencies": { "elysia": ">= 1.4.0" } }, "sha512-jR7t4M6ZvMnBqzzHsNTL6y3sNq9jbGi2vKxbkizi/OO5tlvlKl/rnBGyFjZUjQ1Hte7rCz+2kfmgOQMhkjk+Og=="], - "@elysiajs/static": ["@elysiajs/static@1.4.2", "", { "peerDependencies": { "elysia": ">= 1.4.0" } }, "sha512-lAEvdxeBhU/jX/hTzfoP+1AtqhsKNCwW4Q+tfNwAShWU6s4ZPQxR1hLoHBveeApofJt4HWEq/tBGvfFz3ODuKg=="], + "@elysiajs/static": ["@elysiajs/static@1.4.6", "", { "peerDependencies": { "elysia": ">= 1.4.0" } }, "sha512-cd61aY/DHOVhlnBjzTBX8E1XANIrsCH8MwEGHeLMaZzNrz0gD4Q8Qsde2dFMzu81I7ZDaaZ2Rim9blSLtUrYBg=="], - "@grpc/grpc-js": ["@grpc/grpc-js@1.14.0", "", { "dependencies": { "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-N8Jx6PaYzcTRNzirReJCtADVoq4z7+1KQ4E70jTg/koQiMoUSN1kbNjPOqpPbhMFhfU1/l7ixspPl8dNY+FoUg=="], + "@grpc/grpc-js": ["@grpc/grpc-js@1.14.1", "", { "dependencies": { "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-sPxgEWtPUR3EnRJCEtbGZG2iX8LQDUls2wUS3o27jg07KqJFMq6YDeWvMo1wfpmy3rqRdS0rivpLwhqQtEyCuQ=="], "@grpc/proto-loader": ["@grpc/proto-loader@0.8.0", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.5.3", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ=="], @@ -187,29 +189,29 @@ "@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.0.0", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.0.0", "@opentelemetry/core": "2.0.0", "@opentelemetry/sdk-trace-base": "2.0.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-omdilCZozUjQwY3uZRBwbaRMJ3p09l4t187Lsdf0dGMye9WKD4NGcpgZRvqhI1dwcH6og+YXQEtoO9Wx3ykilg=="], - "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.37.0", "", {}, "sha512-JD6DerIKdJGmRp4jQyX5FlrQjA4tjOw1cvfsPAZXfOOEErMUHjPcPSICS+6WnM0nB0efSFARh0KAZss+bvExOA=="], + "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.38.0", "", {}, "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg=="], - "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.3.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-WeXSaL29ylJEZMYHHW28QZ6rgAbxQ1KuNSZD9gvd3fPlo0s6s2PglvPArjjP07nmvIK9m4OffN0k4M98O7WmAg=="], + "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.3.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-licBDIbbLP5L5/S0+bwtJynso94XD3KyqSP48K59Sq7Mude6C7dR5ZujZm4Ut4BwZqUFfNOfYNMWBU5nlL7t1A=="], - "@oven/bun-darwin-x64": ["@oven/bun-darwin-x64@1.3.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-CFKjoUWQH0Oz3UHYfKbdKLq0wGryrFsTJEYq839qAwHQSECvVZYAnxVVDYUDa0yQFonhO2qSHY41f6HK+b7xtw=="], + "@oven/bun-darwin-x64": ["@oven/bun-darwin-x64@1.3.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-hn8lLzsYyyh6ULo2E8v2SqtrWOkdQKJwapeVy1rDw7juTTeHY3KDudGWf4mVYteC9riZU6HD88Fn3nGwyX0eIg=="], - "@oven/bun-darwin-x64-baseline": ["@oven/bun-darwin-x64-baseline@1.3.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-+FSr/ub5vA/EkD3fMhHJUzYioSf/sXd50OGxNDAntVxcDu4tXL/81Ka3R/gkZmjznpLFIzovU/1Ts+b7dlkrfw=="], + "@oven/bun-darwin-x64-baseline": ["@oven/bun-darwin-x64-baseline@1.3.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-UHxdtbyxdtNJUNcXtIrjx3Lmq8ji3KywlXtIHV/0vn9A8W5mulqOcryqUWMFVH9JTIIzmNn6Q/qVmXHTME63Ww=="], - "@oven/bun-linux-aarch64": ["@oven/bun-linux-aarch64@1.3.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-WHthS/eLkCNcp9pk4W8aubRl9fIUgt2XhHyLrP0GClB1FVvmodu/zIOtG0NXNpzlzB8+gglOkGo4dPjfVf4Z+g=="], + "@oven/bun-linux-aarch64": ["@oven/bun-linux-aarch64@1.3.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-5uZzxzvHU/z+3cZwN/A0H8G+enQ+9FkeJVZkE2fwK2XhiJZFUGAuWajCpy7GepvOWlqV7VjPaKi2+Qmr4IX7nQ=="], - "@oven/bun-linux-aarch64-musl": ["@oven/bun-linux-aarch64-musl@1.3.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-HT5sr7N8NDYbQRjAnT7ISpx64y+ewZZRQozOJb0+KQObKvg4UUNXGm4Pn1xA4/WPMZDDazjO8E2vtOQw1nJlAQ=="], + "@oven/bun-linux-aarch64-musl": ["@oven/bun-linux-aarch64-musl@1.3.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-OD9DYkjes7WXieBn4zQZGXWhRVZhIEWMDGCetZ3H4vxIuweZ++iul/CNX5jdpNXaJ17myb1ROMvmRbrqW44j3w=="], - "@oven/bun-linux-x64": ["@oven/bun-linux-x64@1.3.0", "", { "os": "linux", "cpu": "x64" }, "sha512-sGEWoJQXO4GDr0x4t/yJQ/Bq1yNkOdX9tHbZZ+DBGJt3z3r7jeb4Digv8xQUk6gdTFC9vnGHuin+KW3/yD1Aww=="], + "@oven/bun-linux-x64": ["@oven/bun-linux-x64@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-EoEuRP9bxAxVKuvi6tZ0ZENjueP4lvjz0mKsMzdG0kwg/2apGKiirH1l0RIcdmvfDGGuDmNiv/XBpkoXq1x8ug=="], - "@oven/bun-linux-x64-baseline": ["@oven/bun-linux-x64-baseline@1.3.0", "", { "os": "linux", "cpu": "x64" }, "sha512-OmlEH3nlxQyv7HOvTH21vyNAZGv9DIPnrTznzvKiOQxkOphhCyKvPTlF13ydw4s/i18iwaUrhHy+YG9HSSxa4Q=="], + "@oven/bun-linux-x64-baseline": ["@oven/bun-linux-x64-baseline@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-m9Ov9YH8KjRLui87eNtQQFKVnjGsNk3xgbrR9c8d2FS3NfZSxmVjSeBvEsDjzNf1TXLDriHb/NYOlpiMf/QzDg=="], - "@oven/bun-linux-x64-musl": ["@oven/bun-linux-x64-musl@1.3.0", "", { "os": "linux", "cpu": "x64" }, "sha512-rtzUEzCynl3Rhgn/iR9DQezSFiZMcAXAbU+xfROqsweMGKwvwIA2ckyyckO08psEP8XcUZTs3LT9CH7PnaMiEA=="], + "@oven/bun-linux-x64-musl": ["@oven/bun-linux-x64-musl@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-3TuOsRVoG8K+soQWRo+Cp5ACpRs6rTFSu5tAqc/6WrqwbNWmqjov/eWJPTgz3gPXnC7uNKVG7RxxAmV8r2EYTQ=="], - "@oven/bun-linux-x64-musl-baseline": ["@oven/bun-linux-x64-musl-baseline@1.3.0", "", { "os": "linux", "cpu": "x64" }, "sha512-hrr7mDvUjMX1tuJaXz448tMsgKIqGJBY8+rJqztKOw1U5+a/v2w5HuIIW1ce7ut0ZwEn+KIDvAujlPvpH33vpQ=="], + "@oven/bun-linux-x64-musl-baseline": ["@oven/bun-linux-x64-musl-baseline@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-q8Hto8hcpofPJjvuvjuwyYvhOaAzPw1F5vRUUeOJDmDwZ4lZhANFM0rUwchMzfWUJCD6jg8/EVQ8MiixnZWU0A=="], - "@oven/bun-windows-x64": ["@oven/bun-windows-x64@1.3.0", "", { "os": "win32", "cpu": "x64" }, "sha512-xXwtpZVVP7T+vkxcF/TUVVOGRjEfkByO4mKveKYb4xnHWV4u4NnV0oNmzyMKkvmj10to5j2h0oZxA4ZVVv4gfA=="], + "@oven/bun-windows-x64": ["@oven/bun-windows-x64@1.3.2", "", { "os": "win32", "cpu": "x64" }, "sha512-nZJUa5NprPYQ4Ii4cMwtP9PzlJJTp1XhxJ+A9eSn1Jfr6YygVWyN2KLjenyI93IcuBouBAaepDAVZZjH2lFBhg=="], - "@oven/bun-windows-x64-baseline": ["@oven/bun-windows-x64-baseline@1.3.0", "", { "os": "win32", "cpu": "x64" }, "sha512-/jVZ8eYjpYHLDFNoT86cP+AjuWvpkzFY+0R0a1bdeu0sQ6ILuy1FV6hz1hUAP390E09VCo5oP76fnx29giHTtA=="], + "@oven/bun-windows-x64-baseline": ["@oven/bun-windows-x64-baseline@1.3.2", "", { "os": "win32", "cpu": "x64" }, "sha512-s00T99MjB+xLOWq+t+wVaVBrry+oBOZNiTJijt+bmkp/MJptYS3FGvs7a+nkjLNzoNDoWQcXgKew6AaHES37Bg=="], "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], @@ -253,7 +255,7 @@ "@radix-ui/react-id": ["@radix-ui/react-id@1.1.1", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg=="], - "@radix-ui/react-label": ["@radix-ui/react-label@2.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-YT1GqPSL8kJn20djelMX7/cTRp/Y9w5IZHvfxQTVHrOqa2yMl7i/UfMqKRU5V7mEyKTrUVgJXhNQPVCG8PBLoQ=="], + "@radix-ui/react-label": ["@radix-ui/react-label@2.1.8", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.4" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-FmXs37I6hSBVDlO4y764TNz1rLgKwjJMQ0EGte6F3Cb3f4bIuHB/iLa/8I9VKkmOy+gNHq8rql3j686ACVV21A=="], "@radix-ui/react-portal": ["@radix-ui/react-portal@1.1.9", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-bpIxvq03if6UNwXZ+HTK71JLh4APvnXntDc6XOX8UVq4XQOVl7lwok0AvIl+b8zgCw3fSaVTZMpAPPagXbKmHQ=="], @@ -265,7 +267,7 @@ "@radix-ui/react-roving-focus": ["@radix-ui/react-roving-focus@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA=="], - "@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + "@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.4", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Jl+bCv8HxKnlTLVrcDE8zTMJ09R9/ukw4qBs/oZClOfoQk/cOTbDn+NceXfV7j09YPVQUryJPHurafcSg6EVKA=="], "@radix-ui/react-tabs": ["@radix-ui/react-tabs@1.1.13", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-7xdcatg7/U+7+Udyoj2zodtI9H/IIopqo+YOIcZOq1nJwXWBZ9p8xiu5llXlekDbZkca79a/fozEYQXIA4sW6A=="], @@ -299,9 +301,9 @@ "@sinclair/typebox": ["@sinclair/typebox@0.34.41", "", {}, "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g=="], - "@tanstack/query-core": ["@tanstack/query-core@5.90.2", "", {}, "sha512-k/TcR3YalnzibscALLwxeiLUub6jN5EDLwKDiO7q5f4ICEoptJ+n9+7vcEFy5/x/i6Q+Lb/tXrsKCggf5uQJXQ=="], + "@tanstack/query-core": ["@tanstack/query-core@5.90.7", "", {}, "sha512-6PN65csiuTNfBMXqQUxQhCNdtm1rV+9kC9YwWAIKcaxAauq3Wu7p18j3gQY3YIBJU70jT/wzCCZ2uqto/vQgiQ=="], - "@tanstack/react-query": ["@tanstack/react-query@5.90.2", "", { "dependencies": { "@tanstack/query-core": "5.90.2" }, "peerDependencies": { "react": "^18 || ^19" } }, "sha512-CLABiR+h5PYfOWr/z+vWFt5VsOA2ekQeRQBFSKlcoW6Ndx/f8rfyVmq4LbgOM4GG2qtxAxjLYLOpCNTYm4uKzw=="], + "@tanstack/react-query": ["@tanstack/react-query@5.90.7", "", { "dependencies": { "@tanstack/query-core": "5.90.7" }, "peerDependencies": { "react": "^18 || ^19" } }, "sha512-wAHc/cgKzW7LZNFloThyHnV/AX9gTg3w5yAv0gvQHPZoCnepwqCMtzbuPbb2UvfvO32XZ46e8bPOYbfZhzVnnQ=="], "@tokenizer/inflate": ["@tokenizer/inflate@0.2.7", "", { "dependencies": { "debug": "^4.4.0", "fflate": "^0.8.2", "token-types": "^6.0.0" } }, "sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg=="], @@ -321,11 +323,11 @@ "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], - "@types/node": ["@types/node@24.7.2", "", { "dependencies": { "undici-types": "~7.14.0" } }, "sha512-/NbVmcGTP+lj5oa4yiYxxeBjRivKQ5Ns1eSZeB99ExsEQ6rX5XYU1Zy/gGxY/ilqtD4Etx9mKyrPxZRetiahhA=="], + "@types/node": ["@types/node@24.10.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-qzQZRBqkFsYyaSWXuEHc2WR9c0a0CXwiE5FWUvn7ZM+vdy1uZLfCunD38UzhuB7YN/J11ndbDBcTmOdxJo9Q7A=="], "@types/react": ["@types/react@19.2.2", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA=="], - "@types/react-dom": ["@types/react-dom@19.2.1", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-/EEvYBdT3BflCWvTMO7YkYBHVE9Ci6XdqZciZANQgKpaiDRGOLIlRo91jbTNRQjgPFWVaRxcYc0luVNFitz57A=="], + "@types/react-dom": ["@types/react-dom@19.2.2", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-9KQPoO6mZCi7jcIStSnlOWn2nEF3mNmyr3rIAsGnAbQKYbRLyqmeSc39EVgtxXVia+LMT8j3knZLAZAh+xLmrw=="], "@types/shimmer": ["@types/shimmer@1.2.0", "", {}, "sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg=="], @@ -363,11 +365,11 @@ "buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], - "bun": ["bun@1.3.0", "", { "optionalDependencies": { "@oven/bun-darwin-aarch64": "1.3.0", "@oven/bun-darwin-x64": "1.3.0", "@oven/bun-darwin-x64-baseline": "1.3.0", "@oven/bun-linux-aarch64": "1.3.0", "@oven/bun-linux-aarch64-musl": "1.3.0", "@oven/bun-linux-x64": "1.3.0", "@oven/bun-linux-x64-baseline": "1.3.0", "@oven/bun-linux-x64-musl": "1.3.0", "@oven/bun-linux-x64-musl-baseline": "1.3.0", "@oven/bun-windows-x64": "1.3.0", "@oven/bun-windows-x64-baseline": "1.3.0" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ], "bin": { "bun": "bin/bun.exe", "bunx": "bin/bunx.exe" } }, "sha512-YI7mFs7iWc/VsGsh2aw6eAPD2cjzn1j+LKdYVk09x1CrdTWKYIHyd+dG5iQoN9//3hCDoZj8U6vKpZzEf5UARA=="], + "bun": ["bun@1.3.2", "", { "optionalDependencies": { "@oven/bun-darwin-aarch64": "1.3.2", "@oven/bun-darwin-x64": "1.3.2", "@oven/bun-darwin-x64-baseline": "1.3.2", "@oven/bun-linux-aarch64": "1.3.2", "@oven/bun-linux-aarch64-musl": "1.3.2", "@oven/bun-linux-x64": "1.3.2", "@oven/bun-linux-x64-baseline": "1.3.2", "@oven/bun-linux-x64-musl": "1.3.2", "@oven/bun-linux-x64-musl-baseline": "1.3.2", "@oven/bun-windows-x64": "1.3.2", "@oven/bun-windows-x64-baseline": "1.3.2" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ], "bin": { "bun": "bin/bun.exe", "bunx": "bin/bunx.exe" } }, "sha512-x75mPJiEfhO1j4Tfc65+PtW6ZyrAB6yTZInydnjDZXF9u9PRAnr6OK3v0Q9dpDl0dxRHkXlYvJ8tteJxc8t4Sw=="], "bun-plugin-tailwind": ["bun-plugin-tailwind@0.1.2", "", { "peerDependencies": { "bun": ">=1.0.0" } }, "sha512-41jNC1tZRSK3s1o7pTNrLuQG8kL/0vR/JgiTmZAJ1eHwe0w5j6HFPKeqEk0WAD13jfrUC7+ULuewFBBCoADPpg=="], - "bun-types": ["bun-types@1.3.0", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-u8X0thhx+yJ0KmkxuEo9HAtdfgCBaM/aI9K90VQcQioAmkVp3SG3FkwWGibUFz3WdXAdcsqOcbU40lK7tbHdkQ=="], + "bun-types": ["bun-types@1.3.2", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-i/Gln4tbzKNuxP70OWhJRZz1MRfvqExowP7U6JKoI8cntFrtxg7RJK3jvz7wQW54UuvNC8tbKHHri5fy74FVqg=="], "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], @@ -443,7 +445,7 @@ "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], - "elysia": ["elysia@1.4.11", "", { "dependencies": { "cookie": "^1.0.2", "exact-mirror": "0.2.2", "fast-decode-uri-component": "^1.0.1" }, "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "file-type": ">= 20.0.0", "openapi-types": ">= 12.0.0", "typescript": ">= 5.0.0" }, "optionalPeers": ["typescript"] }, "sha512-cphuzQj0fRw1ICRvwHy2H3xQio9bycaZUVHnDHJQnKqBfMNlZ+Hzj6TMmt9lc0Az0mvbCnPXWVF7y1MCRhUuOA=="], + "elysia": ["elysia@1.4.15", "", { "dependencies": { "cookie": "^1.0.2", "exact-mirror": "0.2.2", "fast-decode-uri-component": "^1.0.1", "memoirist": "^0.4.0" }, "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "@types/bun": ">= 1.2.0", "file-type": ">= 20.0.0", "openapi-types": ">= 12.0.0", "typescript": ">= 5.0.0" }, "optionalPeers": ["@types/bun", "typescript"] }, "sha512-RaDqqZdLuC4UJetfVRQ4Z5aVpGgEtQ+pZnsbI4ZzEaf3l/MzuHcqSVoL/Fue3d6qE4RV9HMB2rAZaHyPIxkyzg=="], "emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], @@ -579,6 +581,8 @@ "media-typer": ["media-typer@0.3.0", "", {}, "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ=="], + "memoirist": ["memoirist@0.4.0", "", {}, "sha512-zxTgA0mSYELa66DimuNQDvyLq36AwDlTuVRbnQtB+VuTcKWm5Qc4z3WkSpgsFWHNhexqkIooqpv4hdcqrX5Nmg=="], + "merge-descriptors": ["merge-descriptors@1.0.3", "", {}, "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ=="], "methods": ["methods@1.1.2", "", {}, "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w=="], @@ -777,13 +781,13 @@ "tailwind-merge": ["tailwind-merge@3.3.1", "", {}, "sha512-gBXpgUm/3rp1lMZZrM/w7D8GKqshif0zAymAhbCyIt8KMe+0v9DQ7cdYLR4FHH/cKpdTXb+A/tKKU3eolfsI+g=="], - "tailwindcss": ["tailwindcss@4.1.14", "", {}, "sha512-b7pCxjGO98LnxVkKjaZSDeNuljC4ueKUddjENJOADtubtdo8llTaJy7HwBMeLNSSo2N5QIAgklslK1+Ir8r6CA=="], + "tailwindcss": ["tailwindcss@4.1.17", "", {}, "sha512-j9Ee2YjuQqYT9bbRTfTZht9W/ytp5H+jJpZKiYdP/bpnXARAuELt9ofP0lPnmHjbga7SNQIxdTAXCmtKVYjN+Q=="], "thread-stream": ["thread-stream@2.7.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-qQiRWsU/wvNolI6tbbCKd9iKaTnCXsTwVxhhKM6nctPdujTyztjlbUkUTUymidWcMnZ5pWR0ej4a0tjsW021vw=="], "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], - "tlds": ["tlds@1.260.0", "", { "bin": { "tlds": "bin.js" } }, "sha512-78+28EWBhCEE7qlyaHA9OR3IPvbCLiDh3Ckla593TksfFc9vfTsgvH7eS+dr3o9qr31gwGbogcI16yN91PoRjQ=="], + "tlds": ["tlds@1.261.0", "", { "bin": { "tlds": "bin.js" } }, "sha512-QXqwfEl9ddlGBaRFXIvNKK6OhipSiLXuRuLJX5DErz0o0Q0rYxulWLdFryTkV5PkdZct5iMInwYEGe/eR++1AA=="], "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], @@ -809,7 +813,7 @@ "undici": ["undici@6.22.0", "", {}, "sha512-hU/10obOIu62MGYjdskASR3CUAiYaFTtC9Pa6vHyf//mAipSvSQg6od2CnJswq7fvzNS3zJhxoRkgNVaHurWKw=="], - "undici-types": ["undici-types@7.14.0", "", {}, "sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA=="], + "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], "unist-util-is": ["unist-util-is@6.0.1", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g=="], @@ -853,6 +857,14 @@ "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + "@radix-ui/react-collection/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-label/@radix-ui/react-primitive": ["@radix-ui/react-primitive@2.1.4", "", { "dependencies": { "@radix-ui/react-slot": "1.2.4" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-9hQc4+GNVtJAIEPEqlYqW5RiYdrr8ea5XQ0ZOnD6fgru+83kqT15mq2OCcbe8KnjRZl5vF3ks69AKz3kh1jrhg=="], + + "@radix-ui/react-primitive/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + "@tokenizer/inflate/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "express/cookie": ["cookie@0.7.1", "", {}, "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w=="], diff --git a/package.json b/package.json index f8c07bb..9cb2334 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,8 @@ }, "module": "src/index.js", "trustedDependencies": [ + "bun", + "cbor-extract", "core-js", "protobufjs" ] -- 2.51.0 From 618711a16ba13f919a8d5d438eb9b51b29a0b1eb Mon Sep 17 00:00:00 2001 From: "@nekomimi.pet" Date: Sun, 9 Nov 2025 03:00:28 -0500 Subject: [PATCH] check manifest and calculate CIDs then compare if we need to reupload blobs --- bun.lock | 19 +- package.json | 1 + public/editor/editor.tsx | 4 +- src/lib/db.ts | 7 - src/lib/oauth-client.ts | 1 - src/lib/wisp-utils.test.ts | 360 +++++++++++++++++++++++++++++++++++++ src/lib/wisp-utils.ts | 67 ++++++- src/routes/wisp.ts | 140 +++++++++++++-- 8 files changed, 576 insertions(+), 23 deletions(-) diff --git a/bun.lock b/bun.lock index 7b3f2c7..292986e 100644 --- a/bun.lock +++ b/bun.lock @@ -25,6 +25,7 @@ "elysia": "latest", "iron-session": "^8.0.4", "lucide-react": "^0.546.0", + "multiformats": "^13.4.1", "react": "^19.2.0", "react-dom": "^19.2.0", "react-shiki": "^0.9.0", @@ -641,7 +642,7 @@ "ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], - "multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + "multiformats": ["multiformats@13.4.1", "", {}, "sha512-VqO6OSvLrFVAYYjgsr8tyv62/rCQhPgsZUXLTqoFLSgdkgiUYKYeArbt1uWLlEpkjxQe+P0+sHlbPEte1Bi06Q=="], "negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="], @@ -857,6 +858,20 @@ "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + "@atproto/api/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/common/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/common-web/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/jwk/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/oauth-client/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@ipld/dag-cbor/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + "@radix-ui/react-collection/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], "@radix-ui/react-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], @@ -883,6 +898,8 @@ "send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "uint8arrays/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + "@tokenizer/inflate/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "micromark/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], diff --git a/package.json b/package.json index 9cb2334..0b202fc 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "elysia": "latest", "iron-session": "^8.0.4", "lucide-react": "^0.546.0", + "multiformats": "^13.4.1", "react": "^19.2.0", "react-dom": "^19.2.0", "react-shiki": "^0.9.0", diff --git a/public/editor/editor.tsx b/public/editor/editor.tsx index da7530f..359f1af 100644 --- a/public/editor/editor.tsx +++ b/public/editor/editor.tsx @@ -748,7 +748,7 @@ function Dashboard() {
- +

Note about sites.wisp.place URLs @@ -1120,7 +1120,7 @@ function Dashboard() { {skippedFiles.length > 0 && (

- +
{skippedFiles.length} file{skippedFiles.length > 1 ? 's' : ''} skipped diff --git a/src/lib/db.ts b/src/lib/db.ts index 887f212..b5f82c4 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -244,7 +244,6 @@ const STATE_TIMEOUT = 60 * 60; // 3600 seconds const stateStore = { async set(key: string, data: any) { - console.debug('[stateStore] set', key) const expiresAt = Math.floor(Date.now() / 1000) + STATE_TIMEOUT; await db` INSERT INTO oauth_states (key, data, created_at, expires_at) @@ -253,7 +252,6 @@ const stateStore = { `; }, async get(key: string) { - console.debug('[stateStore] get', key) const now = Math.floor(Date.now() / 1000); const result = await db` SELECT data, expires_at @@ -265,7 +263,6 @@ const stateStore = { // Check if expired const expiresAt = Number(result[0].expires_at); if (expiresAt && now > expiresAt) { - console.debug('[stateStore] State expired, deleting', key); await db`DELETE FROM oauth_states WHERE key = ${key}`; return undefined; } @@ -273,14 +270,12 @@ const stateStore = { return JSON.parse(result[0].data); }, async del(key: string) { - console.debug('[stateStore] del', key) await db`DELETE FROM oauth_states WHERE key = ${key}`; } }; const sessionStore = { async set(sub: string, data: any) { - console.debug('[sessionStore] set', sub) const expiresAt = Math.floor(Date.now() / 1000) + SESSION_TIMEOUT; await db` INSERT INTO oauth_sessions (sub, data, updated_at, expires_at) @@ -292,7 +287,6 @@ const sessionStore = { `; }, async get(sub: string) { - console.debug('[sessionStore] get', sub) const now = Math.floor(Date.now() / 1000); const result = await db` SELECT data, expires_at @@ -312,7 +306,6 @@ const sessionStore = { return JSON.parse(result[0].data); }, async del(sub: string) { - console.debug('[sessionStore] del', sub) await db`DELETE FROM oauth_sessions WHERE sub = ${sub}`; } }; diff --git a/src/lib/oauth-client.ts b/src/lib/oauth-client.ts index a087c80..c73d236 100644 --- a/src/lib/oauth-client.ts +++ b/src/lib/oauth-client.ts @@ -58,7 +58,6 @@ const sessionStore = { `; }, async get(sub: string) { - console.debug('[sessionStore] get', sub) const now = Math.floor(Date.now() / 1000); const result = await db` SELECT data, expires_at diff --git a/src/lib/wisp-utils.test.ts b/src/lib/wisp-utils.test.ts index ff180f0..18c423f 100644 --- a/src/lib/wisp-utils.test.ts +++ b/src/lib/wisp-utils.test.ts @@ -5,6 +5,8 @@ import { processUploadedFiles, createManifest, updateFileBlobs, + computeCID, + extractBlobMap, type UploadedFile, type FileUploadResult, } from './wisp-utils' @@ -637,3 +639,361 @@ describe('updateFileBlobs', () => { } }) }) + +describe('computeCID', () => { + test('should compute CID for gzipped+base64 encoded content', () => { + // This simulates the actual flow: gzip -> base64 -> compute CID + const originalContent = Buffer.from('Hello, World!') + const gzipped = compressFile(originalContent) + const base64Content = Buffer.from(gzipped.toString('base64'), 'binary') + + const cid = computeCID(base64Content) + + // CID should be a valid CIDv1 string starting with 'bafkrei' + expect(cid).toMatch(/^bafkrei[a-z0-9]+$/) + expect(cid.length).toBeGreaterThan(10) + }) + + test('should compute deterministic CIDs for identical content', () => { + const content = Buffer.from('Test content for CID calculation') + const gzipped = compressFile(content) + const base64Content = Buffer.from(gzipped.toString('base64'), 'binary') + + const cid1 = computeCID(base64Content) + const cid2 = computeCID(base64Content) + + expect(cid1).toBe(cid2) + }) + + test('should compute different CIDs for different content', () => { + const content1 = Buffer.from('Content A') + const content2 = Buffer.from('Content B') + + const gzipped1 = compressFile(content1) + const gzipped2 = compressFile(content2) + + const base64Content1 = Buffer.from(gzipped1.toString('base64'), 'binary') + const base64Content2 = Buffer.from(gzipped2.toString('base64'), 'binary') + + const cid1 = computeCID(base64Content1) + const cid2 = computeCID(base64Content2) + + expect(cid1).not.toBe(cid2) + }) + + test('should handle empty content', () => { + const emptyContent = Buffer.from('') + const gzipped = compressFile(emptyContent) + const base64Content = Buffer.from(gzipped.toString('base64'), 'binary') + + const cid = computeCID(base64Content) + + expect(cid).toMatch(/^bafkrei[a-z0-9]+$/) + }) + + test('should compute same CID as PDS for base64-encoded content', () => { + // Test that binary encoding produces correct bytes for CID calculation + const testContent = Buffer.from('Hello') + const gzipped = compressFile(testContent) + const base64Content = Buffer.from(gzipped.toString('base64'), 'binary') + + // Compute CID twice to ensure consistency + const cid1 = computeCID(base64Content) + const cid2 = computeCID(base64Content) + + expect(cid1).toBe(cid2) + expect(cid1).toMatch(/^bafkrei/) + }) + + test('should use binary encoding for base64 strings', () => { + // This test verifies we're using the correct encoding method + // For base64 strings, 'binary' encoding ensures each character becomes exactly one byte + const content = Buffer.from('Test content') + const gzipped = compressFile(content) + const base64String = gzipped.toString('base64') + + // Using binary encoding (what we use in production) + const base64Content = Buffer.from(base64String, 'binary') + + // Verify the length matches the base64 string length + expect(base64Content.length).toBe(base64String.length) + + // Verify CID is computed correctly + const cid = computeCID(base64Content) + expect(cid).toMatch(/^bafkrei/) + }) +}) + +describe('extractBlobMap', () => { + test('should extract blob map from flat directory structure', () => { + const mockCid = CID.parse(TEST_CID_STRING) + const mockBlob = new BlobRef(mockCid, 'text/html', 100) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'index.html', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob, + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(1) + expect(blobMap.has('index.html')).toBe(true) + + const entry = blobMap.get('index.html') + expect(entry?.cid).toBe(TEST_CID_STRING) + expect(entry?.blobRef).toBe(mockBlob) + }) + + test('should extract blob map from nested directory structure', () => { + const mockCid1 = CID.parse(TEST_CID_STRING) + const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi') + + const mockBlob1 = new BlobRef(mockCid1, 'text/html', 100) + const mockBlob2 = new BlobRef(mockCid2, 'text/css', 50) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'index.html', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob1, + }, + }, + { + name: 'assets', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'styles.css', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob2, + }, + }, + ], + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(2) + expect(blobMap.has('index.html')).toBe(true) + expect(blobMap.has('assets/styles.css')).toBe(true) + + expect(blobMap.get('index.html')?.cid).toBe(TEST_CID_STRING) + expect(blobMap.get('assets/styles.css')?.cid).toBe('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi') + }) + + test('should handle deeply nested directory structures', () => { + const mockCid = CID.parse(TEST_CID_STRING) + const mockBlob = new BlobRef(mockCid, 'text/javascript', 200) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'src', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'lib', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'utils.js', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob, + }, + }, + ], + }, + }, + ], + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(1) + expect(blobMap.has('src/lib/utils.js')).toBe(true) + expect(blobMap.get('src/lib/utils.js')?.cid).toBe(TEST_CID_STRING) + }) + + test('should handle empty directory', () => { + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(0) + }) + + test('should correctly extract CID from BlobRef instances (not plain objects)', () => { + // This test verifies the fix: AT Protocol SDK returns BlobRef instances, + // not plain objects with $type and $link properties + const mockCid = CID.parse(TEST_CID_STRING) + const mockBlob = new BlobRef(mockCid, 'application/octet-stream', 500) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'test.bin', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob, + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + // The fix: we call .toString() on the CID instance instead of accessing $link + expect(blobMap.get('test.bin')?.cid).toBe(TEST_CID_STRING) + expect(blobMap.get('test.bin')?.blobRef.ref.toString()).toBe(TEST_CID_STRING) + }) + + test('should handle multiple files in same directory', () => { + const mockCid1 = CID.parse(TEST_CID_STRING) + const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi') + const mockCid3 = CID.parse('bafkreieb3ixgchss44kw7xiavnkns47emdfsqbhcdfluo3p6n3o53fl3vq') + + const mockBlob1 = new BlobRef(mockCid1, 'image/png', 1000) + const mockBlob2 = new BlobRef(mockCid2, 'image/png', 2000) + const mockBlob3 = new BlobRef(mockCid3, 'image/png', 3000) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'images', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'logo.png', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob1, + }, + }, + { + name: 'banner.png', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob2, + }, + }, + { + name: 'icon.png', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob3, + }, + }, + ], + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(3) + expect(blobMap.has('images/logo.png')).toBe(true) + expect(blobMap.has('images/banner.png')).toBe(true) + expect(blobMap.has('images/icon.png')).toBe(true) + }) + + test('should handle mixed directory and file structure', () => { + const mockCid1 = CID.parse(TEST_CID_STRING) + const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi') + const mockCid3 = CID.parse('bafkreieb3ixgchss44kw7xiavnkns47emdfsqbhcdfluo3p6n3o53fl3vq') + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'index.html', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: new BlobRef(mockCid1, 'text/html', 100), + }, + }, + { + name: 'assets', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'styles.css', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: new BlobRef(mockCid2, 'text/css', 50), + }, + }, + ], + }, + }, + { + name: 'README.md', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: new BlobRef(mockCid3, 'text/markdown', 200), + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(3) + expect(blobMap.has('index.html')).toBe(true) + expect(blobMap.has('assets/styles.css')).toBe(true) + expect(blobMap.has('README.md')).toBe(true) + }) +}) diff --git a/src/lib/wisp-utils.ts b/src/lib/wisp-utils.ts index 5e06d5a..5a785ed 100644 --- a/src/lib/wisp-utils.ts +++ b/src/lib/wisp-utils.ts @@ -2,6 +2,11 @@ import type { BlobRef } from "@atproto/api"; import type { Record, Directory, File, Entry } from "../lexicons/types/place/wisp/fs"; import { validateRecord } from "../lexicons/types/place/wisp/fs"; import { gzipSync } from 'zlib'; +import { CID } from 'multiformats/cid'; +import { sha256 } from 'multiformats/hashes/sha2'; +import * as raw from 'multiformats/codecs/raw'; +import { createHash } from 'crypto'; +import * as mf from 'multiformats'; export interface UploadedFile { name: string; @@ -48,10 +53,14 @@ export function shouldCompressFile(mimeType: string): boolean { } /** - * Compress a file using gzip + * Compress a file using gzip with deterministic output + * Sets mtime to 0 to ensure identical content produces identical compressed output */ export function compressFile(content: Buffer): Buffer { - return gzipSync(content, { level: 9 }); + return gzipSync(content, { + level: 9, + mtime: 0 // Fixed timestamp for deterministic compression + }); } /** @@ -65,6 +74,12 @@ export function processUploadedFiles(files: UploadedFile[]): ProcessedDirectory const directoryMap = new Map(); for (const file of files) { + // Skip undefined/null files (defensive) + if (!file || !file.name) { + console.error('Skipping undefined or invalid file in processUploadedFiles'); + continue; + } + // Remove any base folder name from the path const normalizedPath = file.name.replace(/^[^\/]*\//, ''); const parts = normalizedPath.split('/'); @@ -239,3 +254,51 @@ export function updateFileBlobs( return result; } + +/** + * Compute CID (Content Identifier) for blob content + * Uses the same algorithm as AT Protocol: CIDv1 with raw codec and SHA-256 + * Based on @atproto/common/src/ipld.ts sha256RawToCid implementation + */ +export function computeCID(content: Buffer): string { + // Use node crypto to compute sha256 hash (same as AT Protocol) + const hash = createHash('sha256').update(content).digest(); + // Create digest object from hash bytes + const digest = mf.digest.create(sha256.code, hash); + // Create CIDv1 with raw codec + const cid = CID.createV1(raw.code, digest); + return cid.toString(); +} + +/** + * Extract blob information from a directory tree + * Returns a map of file paths to their blob refs and CIDs + */ +export function extractBlobMap( + directory: Directory, + currentPath: string = '' +): Map { + const blobMap = new Map(); + + for (const entry of directory.entries) { + const fullPath = currentPath ? `${currentPath}/${entry.name}` : entry.name; + + if ('type' in entry.node && entry.node.type === 'file') { + const fileNode = entry.node as File; + // AT Protocol SDK returns BlobRef class instances, not plain objects + // The ref is a CID instance that can be converted to string + if (fileNode.blob && fileNode.blob.ref) { + const cidString = fileNode.blob.ref.toString(); + blobMap.set(fullPath, { + blobRef: fileNode.blob, + cid: cidString + }); + } + } else if ('type' in entry.node && entry.node.type === 'directory') { + const subMap = extractBlobMap(entry.node as Directory, fullPath); + subMap.forEach((value, key) => blobMap.set(key, value)); + } + } + + return blobMap; +} diff --git a/src/routes/wisp.ts b/src/routes/wisp.ts index 522717a..39b51f1 100644 --- a/src/routes/wisp.ts +++ b/src/routes/wisp.ts @@ -9,7 +9,9 @@ import { createManifest, updateFileBlobs, shouldCompressFile, - compressFile + compressFile, + computeCID, + extractBlobMap } from '../lib/wisp-utils' import { upsertSite } from '../lib/db' import { logger } from '../lib/observability' @@ -49,6 +51,10 @@ export const wispRoutes = (client: NodeOAuthClient) => files: File | File[] }; + console.log('=== UPLOAD FILES START ==='); + console.log('Site name:', siteName); + console.log('Files received:', Array.isArray(files) ? files.length : 'single file'); + try { if (!siteName) { throw new Error('Site name is required') @@ -106,6 +112,33 @@ export const wispRoutes = (client: NodeOAuthClient) => // Create agent with OAuth session const agent = new Agent((url, init) => auth.session.fetchHandler(url, init)) + console.log('Agent created for DID:', auth.did); + + // Try to fetch existing record to enable incremental updates + let existingBlobMap = new Map(); + console.log('Attempting to fetch existing record...'); + try { + const rkey = siteName; + const existingRecord = await agent.com.atproto.repo.getRecord({ + repo: auth.did, + collection: 'place.wisp.fs', + rkey: rkey + }); + console.log('Existing record found!'); + + if (existingRecord.data.value && typeof existingRecord.data.value === 'object' && 'root' in existingRecord.data.value) { + const manifest = existingRecord.data.value as any; + existingBlobMap = extractBlobMap(manifest.root); + console.log(`Found existing manifest with ${existingBlobMap.size} files for incremental update`); + logger.info(`Found existing manifest with ${existingBlobMap.size} files for incremental update`); + } + } catch (error: any) { + console.log('No existing record found or error:', error?.message || error); + // Record doesn't exist yet, this is a new site + if (error?.status !== 400 && error?.error !== 'RecordNotFound') { + logger.warn('Failed to fetch existing record, proceeding with full upload', error); + } + } // Convert File objects to UploadedFile format // Elysia gives us File objects directly, handle both single file and array @@ -113,10 +146,11 @@ export const wispRoutes = (client: NodeOAuthClient) => const uploadedFiles: UploadedFile[] = []; const skippedFiles: Array<{ name: string; reason: string }> = []; - + console.log('Processing files, count:', fileArray.length); for (let i = 0; i < fileArray.length; i++) { const file = fileArray[i]; + console.log(`Processing file ${i + 1}/${fileArray.length}:`, file.name, file.size, 'bytes'); // Skip files that are too large (limit to 100MB per file) const maxSize = MAX_FILE_SIZE; // 100MB @@ -135,13 +169,16 @@ export const wispRoutes = (client: NodeOAuthClient) => // Compress and base64 encode ALL files const compressedContent = compressFile(originalContent); // Base64 encode the gzipped content to prevent PDS content sniffing - const base64Content = Buffer.from(compressedContent.toString('base64'), 'utf-8'); + // Convert base64 string to bytes using binary encoding (each char becomes exactly one byte) + // This is what PDS receives and computes CID on + const base64Content = Buffer.from(compressedContent.toString('base64'), 'binary'); const compressionRatio = (compressedContent.length / originalContent.length * 100).toFixed(1); + console.log(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`); logger.info(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`); uploadedFiles.push({ name: file.name, - content: base64Content, + content: base64Content, // This is the gzipped+base64 content that will be uploaded and CID-computed mimeType: originalMimeType, size: base64Content.length, compressed: true, @@ -206,13 +243,79 @@ export const wispRoutes = (client: NodeOAuthClient) => } // Process files into directory structure - const { directory, fileCount } = processUploadedFiles(uploadedFiles); + console.log('Processing uploaded files into directory structure...'); + console.log('uploadedFiles array length:', uploadedFiles.length); + console.log('uploadedFiles contents:', uploadedFiles.map((f, i) => `${i}: ${f?.name || 'UNDEFINED'}`)); + + // Filter out any undefined/null/invalid entries (defensive) + const validUploadedFiles = uploadedFiles.filter((f, i) => { + if (!f) { + console.error(`Filtering out undefined/null file at index ${i}`); + return false; + } + if (!f.name) { + console.error(`Filtering out file with no name at index ${i}:`, f); + return false; + } + if (!f.content) { + console.error(`Filtering out file with no content at index ${i}:`, f.name); + return false; + } + return true; + }); + if (validUploadedFiles.length !== uploadedFiles.length) { + console.warn(`Filtered out ${uploadedFiles.length - validUploadedFiles.length} invalid files`); + } + console.log('validUploadedFiles length:', validUploadedFiles.length); - // Upload files as blobs in parallel + const { directory, fileCount } = processUploadedFiles(validUploadedFiles); + console.log('Directory structure created, file count:', fileCount); + + // Upload files as blobs in parallel (or reuse existing blobs with matching CIDs) + console.log('Starting blob upload/reuse phase...'); // For compressed files, we upload as octet-stream and store the original MIME type in metadata // For text/html files, we also use octet-stream as a workaround for PDS image pipeline issues - const uploadPromises = uploadedFiles.map(async (file, i) => { + const uploadPromises = validUploadedFiles.map(async (file, i) => { try { + // Skip undefined files (shouldn't happen after filter, but defensive) + if (!file || !file.name) { + console.error(`ERROR: Undefined file at index ${i} in validUploadedFiles!`); + throw new Error(`Undefined file at index ${i}`); + } + + // Compute CID for this file to check if it already exists + // Note: file.content is already gzipped+base64 encoded + const fileCID = computeCID(file.content); + + // Normalize the file path for comparison (remove base folder prefix like "cobblemon/") + const normalizedPath = file.name.replace(/^[^\/]*\//, ''); + + // Check if we have an existing blob with the same CID + // Try both the normalized path and the full path + const existingBlob = existingBlobMap.get(normalizedPath) || existingBlobMap.get(file.name); + + if (existingBlob && existingBlob.cid === fileCID) { + // Reuse existing blob - no need to upload + logger.info(`[File Upload] Reusing existing blob for: ${file.name} (CID: ${fileCID})`); + + return { + result: { + hash: existingBlob.cid, + blobRef: existingBlob.blobRef, + ...(file.compressed && { + encoding: 'gzip' as const, + mimeType: file.originalMimeType || file.mimeType, + base64: true + }) + }, + filePath: file.name, + sentMimeType: file.mimeType, + returnedMimeType: existingBlob.blobRef.mimeType, + reused: true + }; + } + + // File is new or changed - upload it // If compressed, always upload as octet-stream // Otherwise, workaround: PDS incorrectly processes text/html through image pipeline const uploadMimeType = file.compressed || file.mimeType.startsWith('text/html') @@ -220,7 +323,7 @@ export const wispRoutes = (client: NodeOAuthClient) => : file.mimeType; const compressionInfo = file.compressed ? ' (gzipped)' : ''; - logger.info(`[File Upload] Uploading file: ${file.name} (original: ${file.mimeType}, sending as: ${uploadMimeType}, ${file.size} bytes${compressionInfo})`); + logger.info(`[File Upload] Uploading new/changed file: ${file.name} (original: ${file.mimeType}, sending as: ${uploadMimeType}, ${file.size} bytes${compressionInfo}, CID: ${fileCID})`); const uploadResult = await agent.com.atproto.repo.uploadBlob( file.content, @@ -244,7 +347,8 @@ export const wispRoutes = (client: NodeOAuthClient) => }, filePath: file.name, sentMimeType: file.mimeType, - returnedMimeType: returnedBlobRef.mimeType + returnedMimeType: returnedBlobRef.mimeType, + reused: false }; } catch (uploadError) { logger.error('Upload failed for file', uploadError); @@ -255,28 +359,40 @@ export const wispRoutes = (client: NodeOAuthClient) => // Wait for all uploads to complete const uploadedBlobs = await Promise.all(uploadPromises); + // Count reused vs uploaded blobs + const reusedCount = uploadedBlobs.filter(b => (b as any).reused).length; + const uploadedCount = uploadedBlobs.filter(b => !(b as any).reused).length; + console.log(`Blob statistics: ${reusedCount} reused, ${uploadedCount} uploaded, ${uploadedBlobs.length} total`); + logger.info(`Blob statistics: ${reusedCount} reused, ${uploadedCount} uploaded, ${uploadedBlobs.length} total`); + // Extract results and file paths in correct order const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result); const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath); // Update directory with file blobs + console.log('Updating directory with blob references...'); const updatedDirectory = updateFileBlobs(directory, uploadResults, filePaths); // Create manifest + console.log('Creating manifest...'); const manifest = createManifest(siteName, updatedDirectory, fileCount); + console.log('Manifest created successfully'); // Use site name as rkey const rkey = siteName; let record; try { + console.log('Putting record to PDS with rkey:', rkey); record = await agent.com.atproto.repo.putRecord({ repo: auth.did, collection: 'place.wisp.fs', rkey: rkey, record: manifest }); + console.log('Record successfully created on PDS:', record.data.uri); } catch (putRecordError: any) { + console.error('FAILED to create record on PDS:', putRecordError); logger.error('Failed to create record on PDS', putRecordError); throw putRecordError; @@ -292,11 +408,15 @@ export const wispRoutes = (client: NodeOAuthClient) => fileCount, siteName, skippedFiles, - uploadedCount: uploadedFiles.length + uploadedCount: validUploadedFiles.length }; + console.log('=== UPLOAD FILES COMPLETE ==='); return result; } catch (error) { + console.error('=== UPLOAD ERROR ==='); + console.error('Error details:', error); + console.error('Stack trace:', error instanceof Error ? error.stack : 'N/A'); logger.error('Upload error', error, { message: error instanceof Error ? error.message : 'Unknown error', name: error instanceof Error ? error.name : undefined -- 2.51.0