···
408
+
// Track completed files count for accurate progress
409
+
let completedFilesCount = 0;
// Process file with sliding window concurrency
const processFile = async (file: UploadedFile, index: number) => {
···
if (existingBlob && existingBlob.cid === fileCID) {
logger.info(`[File Upload] ♻️ Reused: ${file.name} (unchanged, CID: ${fileCID})`);
424
+
const reusedCount = (getUploadJob(jobId)?.progress.filesReused || 0) + 1;
425
+
completedFilesCount++;
updateJobProgress(jobId, {
422
-
filesReused: (getUploadJob(jobId)?.progress.filesReused || 0) + 1
427
+
filesReused: reusedCount,
428
+
currentFile: `${completedFilesCount}/${validUploadedFiles.length}: ${file.name} (reused)`
···
const returnedBlobRef = uploadResult.data.blob;
464
+
const uploadedCount = (getUploadJob(jobId)?.progress.filesUploaded || 0) + 1;
465
+
completedFilesCount++;
updateJobProgress(jobId, {
459
-
filesUploaded: (getUploadJob(jobId)?.progress.filesUploaded || 0) + 1
467
+
filesUploaded: uploadedCount,
468
+
currentFile: `${completedFilesCount}/${validUploadedFiles.length}: ${file.name} (uploaded)`
logger.info(`[File Upload] ✅ Uploaded: ${file.name} (CID: ${fileCID})`);
···
logger.error(`Upload failed for file: ${fileName} (${fileSize} bytes) at index ${index}`, errorDetails);
console.error(`Upload failed for file: ${fileName} (${fileSize} bytes) at index ${index}`, errorDetails);
501
+
completedFilesCount++;
502
+
updateJobProgress(jobId, {
503
+
currentFile: `${completedFilesCount}/${validUploadedFiles.length}: ${fileName} (failed)`
// Track failed file but don't throw - continue with other files
···
// Wait for remaining uploads
await Promise.all(executing.keys());
549
+
console.log(`\n✅ Upload complete: ${completedFilesCount}/${validUploadedFiles.length} files processed\n`);
return results.filter(r => r !== undefined && r !== null); // Filter out null (failed) and undefined entries
···
const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result);
const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath);
560
-
// Update directory with file blobs
575
+
// Update directory with file blobs (only for successfully uploaded files)
console.log('Updating directory with blob references...');
updateJobProgress(jobId, { phase: 'creating_manifest' });
563
-
const updatedDirectory = updateFileBlobs(directory, uploadResults, filePaths);
579
+
// Create a set of successfully uploaded paths for quick lookup
580
+
const successfulPaths = new Set(filePaths.map(path => path.replace(/^[^\/]*\//, '')));
582
+
const updatedDirectory = updateFileBlobs(directory, uploadResults, filePaths, '', successfulPaths);
584
+
// Calculate actual file count (only successfully uploaded files)
585
+
const actualFileCount = uploadedBlobs.length;
// Check if we need to split into subfs records
// Split proactively if we have lots of files to avoid hitting manifest size limits
const MAX_MANIFEST_SIZE = 140 * 1024; // 140KB to be safe (PDS limit is 150KB)
568
-
const FILE_COUNT_THRESHOLD = 250; // Start splitting early
590
+
const FILE_COUNT_THRESHOLD = 250; // Start splitting at this many files
591
+
const TARGET_FILE_COUNT = 200; // Try to keep main manifest under this many files
const subfsRecords: Array<{ uri: string; path: string }> = [];
let workingDirectory = updatedDirectory;
571
-
let currentFileCount = fileCount;
594
+
let currentFileCount = actualFileCount;
// Create initial manifest to check size
574
-
let manifest = createManifest(siteName, workingDirectory, fileCount);
597
+
let manifest = createManifest(siteName, workingDirectory, actualFileCount);
let manifestSize = JSON.stringify(manifest).length;
// Split if we have lots of files OR if manifest is already too large
578
-
if (fileCount >= FILE_COUNT_THRESHOLD || manifestSize > MAX_MANIFEST_SIZE) {
579
-
console.log(`⚠️ Large site detected (${fileCount} files, ${(manifestSize / 1024).toFixed(1)}KB), splitting into subfs records...`);
580
-
logger.info(`Large site with ${fileCount} files, splitting into subfs records`);
601
+
if (actualFileCount >= FILE_COUNT_THRESHOLD || manifestSize > MAX_MANIFEST_SIZE) {
602
+
console.log(`⚠️ Large site detected (${actualFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB), splitting into subfs records...`);
603
+
logger.info(`Large site with ${actualFileCount} files, splitting into subfs records`);
582
-
// Keep splitting until manifest fits under limit
605
+
// Keep splitting until manifest fits under limits (both size and file count)
const MAX_ATTEMPTS = 100; // Allow many splits for very large sites
586
-
while (manifestSize > MAX_MANIFEST_SIZE && attempts < MAX_ATTEMPTS) {
609
+
while ((manifestSize > MAX_MANIFEST_SIZE || currentFileCount > TARGET_FILE_COUNT) && attempts < MAX_ATTEMPTS) {
// Find all directories sorted by size (largest first)
const directories = findLargeDirectories(workingDirectory);
directories.sort((a, b) => b.size - a.size);
593
-
if (directories.length === 0) {
594
-
// No more directories to split - this should be very rare
596
-
`Cannot split manifest further - no subdirectories available. ` +
597
-
`Current size: ${(manifestSize / 1024).toFixed(1)}KB. ` +
598
-
`Try organizing files into subdirectories.`
616
+
// Check if we can split subdirectories or need to split flat files
617
+
if (directories.length > 0) {
618
+
// Split the largest subdirectory
619
+
const largestDir = directories[0];
620
+
console.log(` Split #${attempts}: ${largestDir.path} (${largestDir.fileCount} files, ${(largestDir.size / 1024).toFixed(1)}KB)`);
622
+
// Create a subfs record for this directory
623
+
const subfsRkey = TID.nextStr();
624
+
const subfsManifest = {
625
+
$type: 'place.wisp.subfs' as const,
626
+
root: largestDir.directory,
627
+
fileCount: largestDir.fileCount,
628
+
createdAt: new Date().toISOString()
631
+
// Validate subfs record
632
+
const subfsValidation = validateSubfsRecord(subfsManifest);
633
+
if (!subfsValidation.success) {
634
+
throw new Error(`Invalid subfs manifest: ${subfsValidation.error?.message || 'Validation failed'}`);
637
+
// Upload subfs record to PDS
638
+
const subfsRecord = await agent.com.atproto.repo.putRecord({
640
+
collection: 'place.wisp.subfs',
642
+
record: subfsManifest
645
+
const subfsUri = subfsRecord.data.uri;
646
+
subfsRecords.push({ uri: subfsUri, path: largestDir.path });
647
+
console.log(` ✅ Created subfs: ${subfsUri}`);
648
+
logger.info(`Created subfs record for ${largestDir.path}: ${subfsUri}`);
650
+
// Replace directory with subfs node in the main tree
651
+
workingDirectory = replaceDirectoryWithSubfs(workingDirectory, largestDir.path, subfsUri);
652
+
currentFileCount -= largestDir.fileCount;
654
+
// No subdirectories - split flat files at root level
655
+
const rootFiles = workingDirectory.entries.filter(e => 'type' in e.node && e.node.type === 'file');
602
-
// Pick the largest directory
603
-
const largestDir = directories[0];
604
-
console.log(` Split #${attempts}: ${largestDir.path} (${largestDir.fileCount} files, ${(largestDir.size / 1024).toFixed(1)}KB)`);
657
+
if (rootFiles.length === 0) {
659
+
`Cannot split manifest further - no files or directories available. ` +
660
+
`Current: ${currentFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB.`
606
-
// Create a subfs record for this directory
607
-
const subfsRkey = TID.nextStr();
608
-
const subfsManifest = {
609
-
$type: 'place.wisp.subfs' as const,
610
-
root: largestDir.directory,
611
-
fileCount: largestDir.fileCount,
612
-
createdAt: new Date().toISOString()
664
+
// Take a chunk of files (aim for ~100 files per chunk)
665
+
const CHUNK_SIZE = 100;
666
+
const chunkFiles = rootFiles.slice(0, Math.min(CHUNK_SIZE, rootFiles.length));
667
+
console.log(` Split #${attempts}: flat root (${chunkFiles.length} files)`);
669
+
// Create a directory with just these files
670
+
const chunkDirectory: Directory = {
671
+
$type: 'place.wisp.fs#directory' as const,
672
+
type: 'directory' as const,
673
+
entries: chunkFiles
676
+
// Create subfs record for this chunk
677
+
const subfsRkey = TID.nextStr();
678
+
const subfsManifest = {
679
+
$type: 'place.wisp.subfs' as const,
680
+
root: chunkDirectory,
681
+
fileCount: chunkFiles.length,
682
+
createdAt: new Date().toISOString()
685
+
// Validate subfs record
686
+
const subfsValidation = validateSubfsRecord(subfsManifest);
687
+
if (!subfsValidation.success) {
688
+
throw new Error(`Invalid subfs manifest: ${subfsValidation.error?.message || 'Validation failed'}`);
691
+
// Upload subfs record to PDS
692
+
const subfsRecord = await agent.com.atproto.repo.putRecord({
694
+
collection: 'place.wisp.subfs',
696
+
record: subfsManifest
699
+
const subfsUri = subfsRecord.data.uri;
700
+
console.log(` ✅ Created flat subfs: ${subfsUri}`);
701
+
logger.info(`Created flat subfs record with ${chunkFiles.length} files: ${subfsUri}`);
615
-
// Validate subfs record
616
-
const subfsValidation = validateSubfsRecord(subfsManifest);
617
-
if (!subfsValidation.success) {
618
-
throw new Error(`Invalid subfs manifest: ${subfsValidation.error?.message || 'Validation failed'}`);
703
+
// Remove these files from the working directory and add a subfs entry
704
+
const remainingEntries = workingDirectory.entries.filter(
705
+
e => !chunkFiles.some(cf => cf.name === e.name)
621
-
// Upload subfs record to PDS
622
-
const subfsRecord = await agent.com.atproto.repo.putRecord({
624
-
collection: 'place.wisp.subfs',
626
-
record: subfsManifest
708
+
// Add subfs entry (will be merged flat when expanded)
709
+
remainingEntries.push({
710
+
name: `__subfs_${attempts}`, // Placeholder name, will be merged away
712
+
$type: 'place.wisp.fs#subfs' as const,
713
+
type: 'subfs' as const,
629
-
const subfsUri = subfsRecord.data.uri;
630
-
subfsRecords.push({ uri: subfsUri, path: largestDir.path });
631
-
console.log(` ✅ Created subfs: ${subfsUri}`);
632
-
logger.info(`Created subfs record for ${largestDir.path}: ${subfsUri}`);
718
+
workingDirectory = {
719
+
$type: 'place.wisp.fs#directory' as const,
720
+
type: 'directory' as const,
721
+
entries: remainingEntries
634
-
// Replace directory with subfs node in the main tree
635
-
workingDirectory = replaceDirectoryWithSubfs(workingDirectory, largestDir.path, subfsUri);
724
+
subfsRecords.push({ uri: subfsUri, path: `__subfs_${attempts}` });
725
+
currentFileCount -= chunkFiles.length;
// Recreate manifest and check new size
638
-
currentFileCount -= largestDir.fileCount;
639
-
manifest = createManifest(siteName, workingDirectory, fileCount);
729
+
manifest = createManifest(siteName, workingDirectory, currentFileCount);
manifestSize = JSON.stringify(manifest).length;
const newSizeKB = (manifestSize / 1024).toFixed(1);
console.log(` → Manifest now ${newSizeKB}KB with ${currentFileCount} files (${subfsRecords.length} subfs total)`);
644
-
// Check if we're under the limit now
645
-
if (manifestSize <= MAX_MANIFEST_SIZE) {
646
-
console.log(` ✅ Manifest fits! (${newSizeKB}KB < 140KB)`);
734
+
// Check if we're under both limits now
735
+
if (manifestSize <= MAX_MANIFEST_SIZE && currentFileCount <= TARGET_FILE_COUNT) {
736
+
console.log(` ✅ Manifest fits! (${currentFileCount} files, ${newSizeKB}KB)`);
651
-
if (manifestSize > MAX_MANIFEST_SIZE) {
741
+
if (manifestSize > MAX_MANIFEST_SIZE || currentFileCount > TARGET_FILE_COUNT) {
`Failed to fit manifest after splitting ${attempts} directories. ` +
654
-
`Current size: ${(manifestSize / 1024).toFixed(1)}KB. ` +
744
+
`Current: ${currentFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB. ` +
`This should never happen - please report this issue.`
···
const fileArray = Array.isArray(files) ? files : [files];
const jobId = createUploadJob(auth.did, siteName, fileArray.length);
905
-
// Create agent with OAuth session
906
-
const agent = new Agent((url, init) => auth.session.fetchHandler(url, init))
995
+
// Track upload speeds to estimate progress
996
+
const uploadStats = {
997
+
speeds: [] as number[], // MB/s from completed uploads
998
+
getAverageSpeed(): number {
999
+
if (this.speeds.length === 0) return 3; // Default 3 MB/s
1000
+
const sum = this.speeds.reduce((a, b) => a + b, 0);
1001
+
return sum / this.speeds.length;
1005
+
// Create agent with OAuth session and upload progress monitoring
1006
+
const wrappedFetchHandler = async (url: string, init?: RequestInit) => {
1007
+
// Check if this is an uploadBlob request with a body
1008
+
if (url.includes('uploadBlob') && init?.body) {
1009
+
const originalBody = init.body;
1010
+
const bodySize = originalBody instanceof Uint8Array ? originalBody.length :
1011
+
originalBody instanceof ArrayBuffer ? originalBody.byteLength :
1012
+
typeof originalBody === 'string' ? new TextEncoder().encode(originalBody).length : 0;
1014
+
const startTime = Date.now();
1016
+
if (bodySize > 10 * 1024 * 1024) { // Files over 10MB
1017
+
const sizeMB = (bodySize / 1024 / 1024).toFixed(1);
1018
+
const avgSpeed = uploadStats.getAverageSpeed();
1019
+
const estimatedDuration = (bodySize / 1024 / 1024) / avgSpeed;
1021
+
console.log(`[Upload Progress] Starting upload of ${sizeMB}MB file`);
1022
+
console.log(`[Upload Stats] Measured speeds from last ${uploadStats.speeds.length} files:`, uploadStats.speeds.map(s => s.toFixed(2) + ' MB/s').join(', '));
1023
+
console.log(`[Upload Stats] Average speed: ${avgSpeed.toFixed(2)} MB/s, estimated duration: ${estimatedDuration.toFixed(0)}s`);
1025
+
// Log estimated progress every 5 seconds
1026
+
const progressInterval = setInterval(() => {
1027
+
const elapsed = (Date.now() - startTime) / 1000;
1028
+
const estimatedPercent = Math.min(95, Math.round((elapsed / estimatedDuration) * 100));
1029
+
const estimatedMB = Math.min(bodySize / 1024 / 1024, elapsed * avgSpeed).toFixed(1);
1030
+
console.log(`[Upload Progress] ~${estimatedPercent}% (~${estimatedMB}/${sizeMB}MB) - ${elapsed.toFixed(0)}s elapsed`);
1034
+
const result = await auth.session.fetchHandler(url, init);
1035
+
clearInterval(progressInterval);
1036
+
const totalTime = (Date.now() - startTime) / 1000;
1037
+
const actualSpeed = (bodySize / 1024 / 1024) / totalTime;
1038
+
uploadStats.speeds.push(actualSpeed);
1039
+
// Keep only last 10 uploads for rolling average
1040
+
if (uploadStats.speeds.length > 10) uploadStats.speeds.shift();
1041
+
console.log(`[Upload Progress] ✅ Completed ${sizeMB}MB in ${totalTime.toFixed(1)}s (${actualSpeed.toFixed(1)} MB/s)`);
1044
+
clearInterval(progressInterval);
1045
+
const elapsed = (Date.now() - startTime) / 1000;
1046
+
console.error(`[Upload Progress] ❌ Upload failed after ${elapsed.toFixed(1)}s`);
1050
+
// Track small files too for speed calculation
1052
+
const result = await auth.session.fetchHandler(url, init);
1053
+
const totalTime = (Date.now() - startTime) / 1000;
1054
+
if (totalTime > 0.5) { // Only track if > 0.5s
1055
+
const actualSpeed = (bodySize / 1024 / 1024) / totalTime;
1056
+
uploadStats.speeds.push(actualSpeed);
1057
+
if (uploadStats.speeds.length > 10) uploadStats.speeds.shift();
1058
+
console.log(`[Upload Stats] Small file: ${(bodySize / 1024).toFixed(1)}KB in ${totalTime.toFixed(2)}s = ${actualSpeed.toFixed(2)} MB/s`);
1068
+
return auth.session.fetchHandler(url, init);
1071
+
const agent = new Agent(wrappedFetchHandler)
console.log('Agent created for DID:', auth.did);
console.log('Created upload job:', jobId);
nekomimi.pet