Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol. wisp.place

start enforcing limits on hosting service, clean up logging in cli

nekomimi.pet 804d61c0 66c7dfa2

verified
Changed files
+228 -59
apps
hosting-service
main-app
cli
packages
@wisp
atproto-utils
constants
src
lexicons
+4 -2
apps/hosting-service/package.json
···
"dev": "tsx --env-file=.env src/index.ts",
"build": "bun run build.ts",
"start": "tsx src/index.ts",
+
"check": "tsc --noEmit",
"backfill": "tsx src/index.ts --backfill"
},
"dependencies": {
···
"@wisp/safe-fetch": "workspace:*",
"@atproto/api": "^0.17.4",
"@atproto/identity": "^0.4.9",
-
"@atproto/lexicon": "^0.5.1",
+
"@atproto/lexicon": "^0.5.2",
"@atproto/sync": "^0.1.36",
"@atproto/xrpc": "^0.7.5",
"@hono/node-server": "^1.19.6",
···
"@types/bun": "^1.3.1",
"@types/mime-types": "^2.1.4",
"@types/node": "^22.10.5",
-
"tsx": "^4.19.2"
+
"tsx": "^4.19.2",
+
"typescript": "^5.9.3"
}
}
+53 -3
apps/hosting-service/src/lib/utils.ts
···
import { safeFetchJson, safeFetchBlob } from '@wisp/safe-fetch';
import { CID } from 'multiformats';
import { extractBlobCid } from '@wisp/atproto-utils';
-
import { sanitizePath, collectFileCidsFromEntries } from '@wisp/fs-utils';
+
import { sanitizePath, collectFileCidsFromEntries, countFilesInDirectory } from '@wisp/fs-utils';
import { shouldCompressMimeType } from '@wisp/atproto-utils/compression';
+
import { MAX_BLOB_SIZE, MAX_FILE_COUNT, MAX_SITE_SIZE } from '@wisp/constants';
// Re-export shared utilities for local usage and tests
export { extractBlobCid, sanitizePath };
···
}
/**
+
* Calculate total size of all blobs in a directory tree from manifest metadata
+
*/
+
function calculateTotalBlobSize(directory: Directory): number {
+
let totalSize = 0;
+
+
function sumBlobSizes(entries: Entry[]) {
+
for (const entry of entries) {
+
const node = entry.node;
+
+
if ('type' in node && node.type === 'directory' && 'entries' in node) {
+
// Recursively sum subdirectories
+
sumBlobSizes(node.entries);
+
} else if ('type' in node && node.type === 'file' && 'blob' in node) {
+
// Add blob size from manifest
+
const fileNode = node as File;
+
const blobSize = (fileNode.blob as any)?.size || 0;
+
totalSize += blobSize;
+
}
+
}
+
}
+
+
sumBlobSizes(directory.entries);
+
return totalSize;
+
}
+
+
/**
* Extract all subfs URIs from a directory tree with their mount paths
*/
function extractSubfsUris(directory: Directory, currentPath: string = ''): Array<{ uri: string; path: string }> {
···
// Expand subfs nodes before caching
const expandedRoot = await expandSubfsNodes(record.root, pdsEndpoint);
+
// Verify all subfs nodes were expanded (defensive check)
+
const remainingSubfs = extractSubfsUris(expandedRoot);
+
if (remainingSubfs.length > 0) {
+
console.warn(`[Cache] Warning: ${remainingSubfs.length} subfs nodes remain unexpanded after expansion`, remainingSubfs);
+
}
+
+
// ===== VALIDATE LIMITS BEFORE DOWNLOADING ANY BLOBS =====
+
+
// 1. Validate file count limit
+
const fileCount = countFilesInDirectory(expandedRoot);
+
if (fileCount > MAX_FILE_COUNT) {
+
throw new Error(`Site exceeds file count limit: ${fileCount} files (max ${MAX_FILE_COUNT})`);
+
}
+
console.log(`[Cache] File count validation passed: ${fileCount} files (limit: ${MAX_FILE_COUNT})`);
+
+
// 2. Validate total size from blob metadata in manifest (before downloading)
+
const totalBlobSize = calculateTotalBlobSize(expandedRoot);
+
if (totalBlobSize > MAX_SITE_SIZE) {
+
throw new Error(`Site exceeds size limit: ${(totalBlobSize / 1024 / 1024).toFixed(2)}MB (max ${(MAX_SITE_SIZE / 1024 / 1024).toFixed(0)}MB)`);
+
}
+
console.log(`[Cache] Size validation passed: ${(totalBlobSize / 1024 / 1024).toFixed(2)}MB (limit: ${(MAX_SITE_SIZE / 1024 / 1024).toFixed(0)}MB)`);
+
+
// All validations passed, proceed with caching
// Get existing cache metadata to check for incremental updates
const existingMetadata = await getCacheMetadata(did, rkey);
const existingFileCids = existingMetadata?.fileCids || {};
···
console.log(`[Cache] Fetching blob for file: ${filePath}, CID: ${cid}`);
-
// Allow up to 500MB per file blob, with 5 minute timeout
-
let content = await safeFetchBlob(blobUrl, { maxSize: 500 * 1024 * 1024, timeout: 300000 });
+
// Allow up to MAX_BLOB_SIZE per file blob, with 5 minute timeout
+
let content = await safeFetchBlob(blobUrl, { maxSize: MAX_BLOB_SIZE, timeout: 300000 });
// If content is base64-encoded, decode it back to raw binary (gzipped or not)
if (base64) {
+3
apps/main-app/package.json
···
"dev": "bun run --watch src/index.ts",
"start": "bun run src/index.ts",
"build": "bun run build.ts",
+
"check": "tsc --noEmit",
"screenshot": "bun run scripts/screenshot-sites.ts"
},
"dependencies": {
···
"zlib": "^1.0.5"
},
"devDependencies": {
+
"@atproto-labs/handle-resolver": "^0.3.4",
+
"@atproto/did": "^0.2.3",
"@types/react": "^19.2.2",
"@types/react-dom": "^19.2.1",
"bun-types": "latest",
+53 -22
bun.lock
···
"dependencies": {
"@atproto/api": "^0.17.4",
"@atproto/identity": "^0.4.9",
-
"@atproto/lexicon": "^0.5.1",
+
"@atproto/lexicon": "^0.5.2",
"@atproto/sync": "^0.1.36",
"@atproto/xrpc": "^0.7.5",
"@hono/node-server": "^1.19.6",
···
"@types/mime-types": "^2.1.4",
"@types/node": "^22.10.5",
"tsx": "^4.19.2",
+
"typescript": "^5.9.3",
},
},
"apps/main-app": {
···
"zlib": "^1.0.5",
},
"devDependencies": {
+
"@atproto-labs/handle-resolver": "^0.3.4",
+
"@atproto/did": "^0.2.3",
"@types/react": "^19.2.2",
"@types/react-dom": "^19.2.1",
"bun-types": "latest",
···
"@wisp/lexicons": "workspace:*",
"multiformats": "^13.3.1",
},
+
"devDependencies": {
+
"@atproto/lexicon": "^0.5.2",
+
},
},
"packages/@wisp/constants": {
"name": "@wisp/constants",
···
},
"devDependencies": {
"@atproto/lex-cli": "^0.9.5",
+
"multiformats": "^13.4.1",
},
},
"packages/@wisp/observability": {
···
"@atproto-labs/fetch-node": ["@atproto-labs/fetch-node@0.2.0", "", { "dependencies": { "@atproto-labs/fetch": "0.2.3", "@atproto-labs/pipe": "0.1.1", "ipaddr.js": "^2.1.0", "undici": "^6.14.1" } }, ""],
-
"@atproto-labs/handle-resolver": ["@atproto-labs/handle-resolver@0.3.2", "", { "dependencies": { "@atproto-labs/simple-store": "0.3.0", "@atproto-labs/simple-store-memory": "0.1.4", "@atproto/did": "0.2.1", "zod": "^3.23.8" } }, ""],
+
"@atproto-labs/handle-resolver": ["@atproto-labs/handle-resolver@0.3.4", "", { "dependencies": { "@atproto-labs/simple-store": "0.3.0", "@atproto-labs/simple-store-memory": "0.1.4", "@atproto/did": "0.2.3", "zod": "^3.23.8" } }, "sha512-wsNopfzfgO3uPvfnFDgNeXgDufXxSXhjBjp2WEiSzEiLrMy0Jodnqggw4OzD9MJKf0a4Iu2/ydd537qdy91LrQ=="],
"@atproto-labs/handle-resolver-node": ["@atproto-labs/handle-resolver-node@0.1.21", "", { "dependencies": { "@atproto-labs/fetch-node": "0.2.0", "@atproto-labs/handle-resolver": "0.3.2", "@atproto/did": "0.2.1" } }, ""],
···
"@atproto/crypto": ["@atproto/crypto@0.4.4", "", { "dependencies": { "@noble/curves": "^1.7.0", "@noble/hashes": "^1.6.1", "uint8arrays": "3.0.0" } }, ""],
-
"@atproto/did": ["@atproto/did@0.2.1", "", { "dependencies": { "zod": "^3.23.8" } }, ""],
+
"@atproto/did": ["@atproto/did@0.2.3", "", { "dependencies": { "zod": "^3.23.8" } }, "sha512-VI8JJkSizvM2cHYJa37WlbzeCm5tWpojyc1/Zy8q8OOjyoy6X4S4BEfoP941oJcpxpMTObamibQIXQDo7tnIjg=="],
"@atproto/identity": ["@atproto/identity@0.4.10", "", { "dependencies": { "@atproto/common-web": "^0.4.4", "@atproto/crypto": "^0.4.4" } }, "sha512-nQbzDLXOhM8p/wo0cTh5DfMSOSHzj6jizpodX37LJ4S1TZzumSxAjHEZa5Rev3JaoD5uSWMVE0MmKEGWkPPvfQ=="],
···
"@atproto/sync": ["@atproto/sync@0.1.38", "", { "dependencies": { "@atproto/common": "^0.5.0", "@atproto/identity": "^0.4.10", "@atproto/lexicon": "^0.5.2", "@atproto/repo": "^0.8.11", "@atproto/syntax": "^0.4.1", "@atproto/xrpc-server": "^0.10.0", "multiformats": "^9.9.0", "p-queue": "^6.6.2", "ws": "^8.12.0" } }, "sha512-2rE0SM21Nk4hWw/XcIYFnzlWO6/gBg8mrzuWbOvDhD49sA/wW4zyjaHZ5t1gvk28/SLok2VZiIR8nYBdbf7F5Q=="],
-
"@atproto/syntax": ["@atproto/syntax@0.4.1", "", {}, ""],
+
"@atproto/syntax": ["@atproto/syntax@0.4.2", "", {}, "sha512-X9XSRPinBy/0VQ677j8VXlBsYSsUXaiqxWVpGGxJYsAhugdQRb0jqaVKJFtm6RskeNkV6y9xclSUi9UYG/COrA=="],
"@atproto/ws-client": ["@atproto/ws-client@0.0.3", "", { "dependencies": { "@atproto/common": "^0.5.0", "ws": "^8.12.0" } }, "sha512-eKqkTWBk6zuMY+6gs02eT7mS8Btewm8/qaL/Dp00NDCqpNC+U59MWvQsOWT3xkNGfd9Eip+V6VI4oyPvAfsfTA=="],
-
"@atproto/xrpc": ["@atproto/xrpc@0.7.5", "", { "dependencies": { "@atproto/lexicon": "^0.5.1", "zod": "^3.23.8" } }, ""],
+
"@atproto/xrpc": ["@atproto/xrpc@0.7.6", "", { "dependencies": { "@atproto/lexicon": "^0.5.2", "zod": "^3.23.8" } }, "sha512-RvCf4j0JnKYWuz3QzsYCntJi3VuiAAybQsMIUw2wLWcHhchO9F7UaBZINLL2z0qc/cYWPv5NSwcVydMseoCZLA=="],
"@atproto/xrpc-server": ["@atproto/xrpc-server@0.9.5", "", { "dependencies": { "@atproto/common": "^0.4.12", "@atproto/crypto": "^0.4.4", "@atproto/lexicon": "^0.5.1", "@atproto/xrpc": "^0.7.5", "cbor-x": "^1.5.1", "express": "^4.17.2", "http-errors": "^2.0.0", "mime-types": "^2.1.35", "rate-limiter-flexible": "^2.4.1", "uint8arrays": "3.0.0", "ws": "^8.12.0", "zod": "^3.23.8" } }, ""],
···
"ee-first": ["ee-first@1.1.1", "", {}, ""],
-
"elysia": ["elysia@1.4.17", "", { "dependencies": { "cookie": "^1.1.1", "exact-mirror": "0.2.5", "fast-decode-uri-component": "^1.0.1", "memoirist": "^0.4.0" }, "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "@types/bun": ">= 1.2.0", "file-type": ">= 20.0.0", "openapi-types": ">= 12.0.0", "typescript": ">= 5.0.0" }, "optionalPeers": ["@types/bun", "typescript"] }, "sha512-GcR7tgxk0+NgMCEqmXMs/xgND4XpmIzUdSdwchcQbYFeFisBcw9cmsvSpI10i160idwtlVyaRXX9K9IZBqnA7Q=="],
+
"elysia": ["elysia@1.4.18", "", { "dependencies": { "cookie": "^1.1.1", "exact-mirror": "0.2.5", "fast-decode-uri-component": "^1.0.1", "memoirist": "^0.4.0" }, "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "@types/bun": ">= 1.2.0", "file-type": ">= 20.0.0", "openapi-types": ">= 12.0.0", "typescript": ">= 5.0.0" }, "optionalPeers": ["@types/bun", "typescript"] }, "sha512-A6BhlipmSvgCy69SBgWADYZSdDIj3fT2gk8/9iMAC8iD+aGcnCr0fitziX0xr36MFDs/fsvVp8dWqxeq1VCgKg=="],
"emoji-regex": ["emoji-regex@8.0.0", "", {}, ""],
···
"zod": ["zod@3.25.76", "", {}, ""],
+
"@atproto-labs/did-resolver/@atproto/did": ["@atproto/did@0.2.1", "", { "dependencies": { "zod": "^3.23.8" } }, ""],
+
"@atproto-labs/fetch-node/ipaddr.js": ["ipaddr.js@2.2.0", "", {}, ""],
+
"@atproto-labs/handle-resolver-node/@atproto-labs/handle-resolver": ["@atproto-labs/handle-resolver@0.3.2", "", { "dependencies": { "@atproto-labs/simple-store": "0.3.0", "@atproto-labs/simple-store-memory": "0.1.4", "@atproto/did": "0.2.1", "zod": "^3.23.8" } }, ""],
+
+
"@atproto-labs/handle-resolver-node/@atproto/did": ["@atproto/did@0.2.1", "", { "dependencies": { "zod": "^3.23.8" } }, ""],
+
+
"@atproto-labs/identity-resolver/@atproto-labs/handle-resolver": ["@atproto-labs/handle-resolver@0.3.2", "", { "dependencies": { "@atproto-labs/simple-store": "0.3.0", "@atproto-labs/simple-store-memory": "0.1.4", "@atproto/did": "0.2.1", "zod": "^3.23.8" } }, ""],
+
"@atproto/api/@atproto/lexicon": ["@atproto/lexicon@0.4.14", "", { "dependencies": { "@atproto/common-web": "^0.4.2", "@atproto/syntax": "^0.4.0", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, "sha512-jiKpmH1QER3Gvc7JVY5brwrfo+etFoe57tKPQX/SmPwjvUsFnJAow5xLIryuBaJgFAhnTZViXKs41t//pahGHQ=="],
+
+
"@atproto/api/@atproto/syntax": ["@atproto/syntax@0.4.1", "", {}, ""],
"@atproto/api/@atproto/xrpc": ["@atproto/xrpc@0.6.12", "", { "dependencies": { "@atproto/lexicon": "^0.4.10", "zod": "^3.23.8" } }, "sha512-Ut3iISNLujlmY9Gu8sNU+SPDJDvqlVzWddU8qUr0Yae5oD4SguaUFjjhireMGhQ3M5E0KljQgDbTmnBo1kIZ3w=="],
···
"@atproto/lex-cli/@atproto/lexicon": ["@atproto/lexicon@0.5.1", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/syntax": "^0.4.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, ""],
-
"@atproto/lex-data/@atproto/syntax": ["@atproto/syntax@0.4.2", "", {}, "sha512-X9XSRPinBy/0VQ677j8VXlBsYSsUXaiqxWVpGGxJYsAhugdQRb0jqaVKJFtm6RskeNkV6y9xclSUi9UYG/COrA=="],
+
"@atproto/lex-cli/@atproto/syntax": ["@atproto/syntax@0.4.1", "", {}, ""],
"@atproto/lex-data/multiformats": ["multiformats@9.9.0", "", {}, ""],
-
"@atproto/lexicon/@atproto/syntax": ["@atproto/syntax@0.4.2", "", {}, "sha512-X9XSRPinBy/0VQ677j8VXlBsYSsUXaiqxWVpGGxJYsAhugdQRb0jqaVKJFtm6RskeNkV6y9xclSUi9UYG/COrA=="],
+
"@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, ""],
+
+
"@atproto/oauth-client/@atproto-labs/handle-resolver": ["@atproto-labs/handle-resolver@0.3.2", "", { "dependencies": { "@atproto-labs/simple-store": "0.3.0", "@atproto-labs/simple-store-memory": "0.1.4", "@atproto/did": "0.2.1", "zod": "^3.23.8" } }, ""],
-
"@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, ""],
+
"@atproto/oauth-client/@atproto/did": ["@atproto/did@0.2.1", "", { "dependencies": { "zod": "^3.23.8" } }, ""],
+
+
"@atproto/oauth-client/@atproto/xrpc": ["@atproto/xrpc@0.7.5", "", { "dependencies": { "@atproto/lexicon": "^0.5.1", "zod": "^3.23.8" } }, ""],
"@atproto/oauth-client/multiformats": ["multiformats@9.9.0", "", {}, ""],
+
+
"@atproto/oauth-client-node/@atproto/did": ["@atproto/did@0.2.1", "", { "dependencies": { "zod": "^3.23.8" } }, ""],
+
+
"@atproto/oauth-types/@atproto/did": ["@atproto/did@0.2.1", "", { "dependencies": { "zod": "^3.23.8" } }, ""],
"@atproto/repo/@atproto/common": ["@atproto/common@0.5.2", "", { "dependencies": { "@atproto/common-web": "^0.4.6", "@atproto/lex-cbor": "0.0.2", "@atproto/lex-data": "0.0.2", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "pino": "^8.21.0" } }, "sha512-7KdU8FcIfnwS2kmv7M86pKxtw/fLvPY2bSI1rXpG+AmA8O++IUGlSCujBGzbrPwnQvY/z++f6Le4rdBzu8bFaA=="],
···
"@atproto/sync/@atproto/common": ["@atproto/common@0.5.2", "", { "dependencies": { "@atproto/common-web": "^0.4.6", "@atproto/lex-cbor": "0.0.2", "@atproto/lex-data": "0.0.2", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "pino": "^8.21.0" } }, "sha512-7KdU8FcIfnwS2kmv7M86pKxtw/fLvPY2bSI1rXpG+AmA8O++IUGlSCujBGzbrPwnQvY/z++f6Le4rdBzu8bFaA=="],
-
"@atproto/sync/@atproto/syntax": ["@atproto/syntax@0.4.2", "", {}, "sha512-X9XSRPinBy/0VQ677j8VXlBsYSsUXaiqxWVpGGxJYsAhugdQRb0jqaVKJFtm6RskeNkV6y9xclSUi9UYG/COrA=="],
-
"@atproto/sync/@atproto/xrpc-server": ["@atproto/xrpc-server@0.10.2", "", { "dependencies": { "@atproto/common": "^0.5.2", "@atproto/crypto": "^0.4.5", "@atproto/lex-cbor": "0.0.2", "@atproto/lex-data": "0.0.2", "@atproto/lexicon": "^0.5.2", "@atproto/ws-client": "^0.0.3", "@atproto/xrpc": "^0.7.6", "express": "^4.17.2", "http-errors": "^2.0.0", "mime-types": "^2.1.35", "rate-limiter-flexible": "^2.4.1", "ws": "^8.12.0", "zod": "^3.23.8" } }, "sha512-5AzN8xoV8K1Omn45z6qKH414+B3Z35D536rrScwF3aQGDEdpObAS+vya9UoSg+Gvm2+oOtVEbVri7riLTBW3Vg=="],
"@atproto/sync/multiformats": ["multiformats@9.9.0", "", {}, ""],
"@atproto/ws-client/@atproto/common": ["@atproto/common@0.5.2", "", { "dependencies": { "@atproto/common-web": "^0.4.6", "@atproto/lex-cbor": "0.0.2", "@atproto/lex-data": "0.0.2", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "pino": "^8.21.0" } }, "sha512-7KdU8FcIfnwS2kmv7M86pKxtw/fLvPY2bSI1rXpG+AmA8O++IUGlSCujBGzbrPwnQvY/z++f6Le4rdBzu8bFaA=="],
-
"@atproto/xrpc/@atproto/lexicon": ["@atproto/lexicon@0.5.1", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/syntax": "^0.4.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, ""],
-
"@atproto/xrpc-server/@atproto/lexicon": ["@atproto/lexicon@0.5.1", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/syntax": "^0.4.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, ""],
+
+
"@atproto/xrpc-server/@atproto/xrpc": ["@atproto/xrpc@0.7.5", "", { "dependencies": { "@atproto/lexicon": "^0.5.1", "zod": "^3.23.8" } }, ""],
"@ipld/dag-cbor/multiformats": ["multiformats@9.9.0", "", {}, ""],
···
"wisp-hosting-service/@atproto/api": ["@atproto/api@0.17.7", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/lexicon": "^0.5.1", "@atproto/syntax": "^0.4.1", "@atproto/xrpc": "^0.7.5", "await-lock": "^2.2.2", "multiformats": "^9.9.0", "tlds": "^1.234.0", "zod": "^3.23.8" } }, ""],
-
"wisp-hosting-service/@atproto/lexicon": ["@atproto/lexicon@0.5.1", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/syntax": "^0.4.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, ""],
+
"@atproto-labs/identity-resolver/@atproto-labs/handle-resolver/@atproto/did": ["@atproto/did@0.2.1", "", { "dependencies": { "zod": "^3.23.8" } }, ""],
"@atproto/lex-cli/@atproto/lexicon/@atproto/common-web": ["@atproto/common-web@0.4.3", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, ""],
"@atproto/lex-cli/@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, ""],
-
"@atproto/sync/@atproto/xrpc-server/@atproto/crypto": ["@atproto/crypto@0.4.5", "", { "dependencies": { "@noble/curves": "^1.7.0", "@noble/hashes": "^1.6.1", "uint8arrays": "3.0.0" } }, "sha512-n40aKkMoCatP0u9Yvhrdk6fXyOHFDDbkdm4h4HCyWW+KlKl8iXfD5iV+ECq+w5BM+QH25aIpt3/j6EUNerhLxw=="],
+
"@atproto/oauth-client/@atproto/xrpc/@atproto/lexicon": ["@atproto/lexicon@0.5.1", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/syntax": "^0.4.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, ""],
-
"@atproto/sync/@atproto/xrpc-server/@atproto/xrpc": ["@atproto/xrpc@0.7.6", "", { "dependencies": { "@atproto/lexicon": "^0.5.2", "zod": "^3.23.8" } }, "sha512-RvCf4j0JnKYWuz3QzsYCntJi3VuiAAybQsMIUw2wLWcHhchO9F7UaBZINLL2z0qc/cYWPv5NSwcVydMseoCZLA=="],
+
"@atproto/sync/@atproto/xrpc-server/@atproto/crypto": ["@atproto/crypto@0.4.5", "", { "dependencies": { "@noble/curves": "^1.7.0", "@noble/hashes": "^1.6.1", "uint8arrays": "3.0.0" } }, "sha512-n40aKkMoCatP0u9Yvhrdk6fXyOHFDDbkdm4h4HCyWW+KlKl8iXfD5iV+ECq+w5BM+QH25aIpt3/j6EUNerhLxw=="],
"@atproto/ws-client/@atproto/common/multiformats": ["multiformats@9.9.0", "", {}, ""],
"@atproto/xrpc-server/@atproto/lexicon/@atproto/common-web": ["@atproto/common-web@0.4.3", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, ""],
+
+
"@atproto/xrpc-server/@atproto/lexicon/@atproto/syntax": ["@atproto/syntax@0.4.1", "", {}, ""],
"@atproto/xrpc-server/@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, ""],
-
"@atproto/xrpc/@atproto/lexicon/@atproto/common-web": ["@atproto/common-web@0.4.3", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, ""],
-
-
"@atproto/xrpc/@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, ""],
-
"@tokenizer/inflate/debug/ms": ["ms@2.1.3", "", {}, ""],
"@wisp/main-app/@atproto/api/@atproto/common-web": ["@atproto/common-web@0.4.3", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, ""],
"@wisp/main-app/@atproto/api/@atproto/lexicon": ["@atproto/lexicon@0.5.1", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/syntax": "^0.4.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, ""],
+
+
"@wisp/main-app/@atproto/api/@atproto/syntax": ["@atproto/syntax@0.4.1", "", {}, ""],
+
+
"@wisp/main-app/@atproto/api/@atproto/xrpc": ["@atproto/xrpc@0.7.5", "", { "dependencies": { "@atproto/lexicon": "^0.5.1", "zod": "^3.23.8" } }, ""],
"@wisp/main-app/@atproto/api/multiformats": ["multiformats@9.9.0", "", {}, ""],
···
"wisp-hosting-service/@atproto/api/@atproto/common-web": ["@atproto/common-web@0.4.3", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, ""],
+
"wisp-hosting-service/@atproto/api/@atproto/lexicon": ["@atproto/lexicon@0.5.1", "", { "dependencies": { "@atproto/common-web": "^0.4.3", "@atproto/syntax": "^0.4.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, ""],
+
+
"wisp-hosting-service/@atproto/api/@atproto/syntax": ["@atproto/syntax@0.4.1", "", {}, ""],
+
+
"wisp-hosting-service/@atproto/api/@atproto/xrpc": ["@atproto/xrpc@0.7.5", "", { "dependencies": { "@atproto/lexicon": "^0.5.1", "zod": "^3.23.8" } }, ""],
+
"wisp-hosting-service/@atproto/api/multiformats": ["multiformats@9.9.0", "", {}, ""],
-
"wisp-hosting-service/@atproto/lexicon/@atproto/common-web": ["@atproto/common-web@0.4.3", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, ""],
+
"@atproto/oauth-client/@atproto/xrpc/@atproto/lexicon/@atproto/common-web": ["@atproto/common-web@0.4.3", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, ""],
-
"wisp-hosting-service/@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, ""],
+
"@atproto/oauth-client/@atproto/xrpc/@atproto/lexicon/@atproto/syntax": ["@atproto/syntax@0.4.1", "", {}, ""],
+54 -8
cli/Cargo.lock
···
checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d"
[[package]]
+
name = "console"
+
version = "0.15.11"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+
dependencies = [
+
"encode_unicode",
+
"libc",
+
"once_cell",
+
"unicode-width 0.2.2",
+
"windows-sys 0.59.0",
+
]
+
+
[[package]]
name = "const-oid"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
···
"subtle",
"zeroize",
]
+
+
[[package]]
+
name = "encode_unicode"
+
version = "1.0.0"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "encoding_rs"
···
[[package]]
+
name = "indicatif"
+
version = "0.17.11"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
+
dependencies = [
+
"console",
+
"number_prefix",
+
"portable-atomic",
+
"unicode-width 0.2.2",
+
"web-time",
+
]
+
+
[[package]]
name = "indoc"
version = "2.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
···
[[package]]
name = "jacquard"
version = "0.9.3"
-
source = "git+https://tangled.org/nekomimi.pet/jacquard#e1b90160d4026e036ab5b797e56ddd7ae5c5537e"
dependencies = [
"bytes",
"getrandom 0.2.16",
···
[[package]]
name = "jacquard-api"
version = "0.9.2"
-
source = "git+https://tangled.org/nekomimi.pet/jacquard#e1b90160d4026e036ab5b797e56ddd7ae5c5537e"
dependencies = [
"bon",
"bytes",
···
[[package]]
name = "jacquard-common"
version = "0.9.2"
-
source = "git+https://tangled.org/nekomimi.pet/jacquard#e1b90160d4026e036ab5b797e56ddd7ae5c5537e"
dependencies = [
"base64 0.22.1",
"bon",
···
[[package]]
name = "jacquard-derive"
version = "0.9.3"
-
source = "git+https://tangled.org/nekomimi.pet/jacquard#e1b90160d4026e036ab5b797e56ddd7ae5c5537e"
dependencies = [
"heck 0.5.0",
"jacquard-lexicon",
···
[[package]]
name = "jacquard-identity"
version = "0.9.2"
-
source = "git+https://tangled.org/nekomimi.pet/jacquard#e1b90160d4026e036ab5b797e56ddd7ae5c5537e"
dependencies = [
"bon",
"bytes",
···
[[package]]
name = "jacquard-lexicon"
version = "0.9.2"
-
source = "git+https://tangled.org/nekomimi.pet/jacquard#e1b90160d4026e036ab5b797e56ddd7ae5c5537e"
dependencies = [
"cid",
"dashmap",
···
[[package]]
name = "jacquard-oauth"
version = "0.9.2"
-
source = "git+https://tangled.org/nekomimi.pet/jacquard#e1b90160d4026e036ab5b797e56ddd7ae5c5537e"
dependencies = [
"base64 0.22.1",
"bytes",
···
[[package]]
name = "mini-moka"
version = "0.10.99"
-
source = "git+https://tangled.org/nekomimi.pet/jacquard#e1b90160d4026e036ab5b797e56ddd7ae5c5537e"
dependencies = [
"crossbeam-channel",
"crossbeam-utils",
···
[[package]]
+
name = "number_prefix"
+
version = "0.4.0"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
+
[[package]]
name = "objc2"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
···
"der",
"spki",
+
+
[[package]]
+
name = "portable-atomic"
+
version = "1.11.1"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
[[package]]
name = "potential_utf"
···
[[package]]
name = "windows-sys"
+
version = "0.59.0"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+
dependencies = [
+
"windows-targets 0.52.6",
+
]
+
+
[[package]]
+
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
···
"futures",
"globset",
"ignore",
+
"indicatif",
"jacquard",
"jacquard-api",
"jacquard-common",
+15 -7
cli/Cargo.toml
···
place_wisp = []
[dependencies]
-
jacquard = { git = "https://tangled.org/nekomimi.pet/jacquard", features = ["loopback"] }
-
jacquard-oauth = { git = "https://tangled.org/nekomimi.pet/jacquard" }
-
jacquard-api = { git = "https://tangled.org/nekomimi.pet/jacquard", features = ["streaming"] }
-
jacquard-common = { git = "https://tangled.org/nekomimi.pet/jacquard", features = ["websocket"] }
-
jacquard-identity = { git = "https://tangled.org/nekomimi.pet/jacquard", features = ["dns"] }
-
jacquard-derive = { git = "https://tangled.org/nekomimi.pet/jacquard" }
-
jacquard-lexicon = { git = "https://tangled.org/nekomimi.pet/jacquard" }
+
# jacquard = { git = "https://tangled.org/nekomimi.pet/jacquard", features = ["loopback"] }
+
# jacquard-oauth = { git = "https://tangled.org/nekomimi.pet/jacquard" }
+
# jacquard-api = { git = "https://tangled.org/nekomimi.pet/jacquard", features = ["streaming"] }
+
# jacquard-common = { git = "https://tangled.org/nekomimi.pet/jacquard", features = ["websocket"] }
+
# jacquard-identity = { git = "https://tangled.org/nekomimi.pet/jacquard", features = ["dns"] }
+
# jacquard-derive = { git = "https://tangled.org/nekomimi.pet/jacquard" }
+
# jacquard-lexicon = { git = "https://tangled.org/nekomimi.pet/jacquard" }
+
jacquard = { path = "../../jacquard/crates/jacquard", features = ["loopback"] }
+
jacquard-oauth = { path = "../../jacquard/crates/jacquard-oauth" }
+
jacquard-api = { path = "../../jacquard/crates/jacquard-api", features = ["streaming"] }
+
jacquard-common = { path = "../../jacquard/crates/jacquard-common", features = ["websocket"] }
+
jacquard-identity = { path = "../../jacquard/crates/jacquard-identity", features = ["dns"] }
+
jacquard-derive = { path = "../../jacquard/crates/jacquard-derive" }
+
jacquard-lexicon = { path = "../../jacquard/crates/jacquard-lexicon" }
clap = { version = "4.5.51", features = ["derive"] }
tokio = { version = "1.48", features = ["full"] }
miette = { version = "7.6.0", features = ["fancy"] }
···
regex = "1.11"
ignore = "0.4"
globset = "0.4"
+
indicatif = "0.17"
+39 -15
cli/src/main.rs
···
use std::io::Write;
use base64::Engine;
use futures::stream::{self, StreamExt};
+
use indicatif::{ProgressBar, ProgressStyle, MultiProgress};
use place_wisp::fs::*;
use place_wisp::settings::*;
+
+
/// Maximum number of concurrent file uploads to the PDS
+
const MAX_CONCURRENT_UPLOADS: usize = 2;
#[derive(Parser, Debug)]
#[command(author, version, about = "wisp.place CLI tool")]
···
// Build directory tree with ignore patterns
let ignore_matcher = ignore_patterns::IgnoreMatcher::new(&path)?;
-
let (root_dir, total_files, reused_count) = build_directory(agent, &path, &existing_blob_map, String::new(), &ignore_matcher).await?;
+
+
// Create progress tracking (spinner style since we don't know total count upfront)
+
let multi_progress = MultiProgress::new();
+
let progress = multi_progress.add(ProgressBar::new_spinner());
+
progress.set_style(
+
ProgressStyle::default_spinner()
+
.template("[{elapsed_precise}] {spinner:.cyan} {pos} files {msg}")
+
.into_diagnostic()?
+
.tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ ")
+
);
+
progress.set_message("Scanning files...");
+
progress.enable_steady_tick(std::time::Duration::from_millis(100));
+
+
let (root_dir, total_files, reused_count) = build_directory(agent, &path, &existing_blob_map, String::new(), &ignore_matcher, &progress).await?;
let uploaded_count = total_files - reused_count;
+
+
progress.finish_with_message(format!("✓ {} files ({} uploaded, {} reused)", total_files, uploaded_count, reused_count));
// Check if we need to split into subfs records
const MAX_MANIFEST_SIZE: usize = 140 * 1024; // 140KB (PDS limit is 150KB)
···
existing_blobs: &'a HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>,
current_path: String,
ignore_matcher: &'a ignore_patterns::IgnoreMatcher,
+
progress: &'a ProgressBar,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = miette::Result<(Directory<'static>, usize, usize)>> + 'a>>
{
Box::pin(async move {
···
}
}
-
// Process files concurrently with a limit of 5
+
// Process files concurrently with a limit of 2
let file_results: Vec<(Entry<'static>, bool)> = stream::iter(file_tasks)
.map(|(name, path, full_path)| async move {
-
let (file_node, reused) = process_file(agent, &path, &full_path, existing_blobs).await?;
+
let (file_node, reused) = process_file(agent, &path, &full_path, existing_blobs, progress).await?;
+
progress.inc(1);
let entry = Entry::new()
.name(CowStr::from(name))
.node(EntryNode::File(Box::new(file_node)))
.build();
Ok::<_, miette::Report>((entry, reused))
})
-
.buffer_unordered(5)
+
.buffer_unordered(MAX_CONCURRENT_UPLOADS)
.collect::<Vec<_>>()
.await
.into_iter()
···
} else {
format!("{}/{}", current_path, name)
};
-
let (subdir, sub_total, sub_reused) = build_directory(agent, &path, existing_blobs, subdir_path, ignore_matcher).await?;
+
let (subdir, sub_total, sub_reused) = build_directory(agent, &path, existing_blobs, subdir_path, ignore_matcher, progress).await?;
dir_entries.push(Entry::new()
.name(CowStr::from(name))
.node(EntryNode::Directory(Box::new(subdir)))
···
file_path: &Path,
file_path_key: &str,
existing_blobs: &HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>,
+
progress: &ProgressBar,
) -> miette::Result<(File<'static>, bool)>
{
// Read file
···
if let Some((existing_blob_ref, existing_cid)) = existing_blob {
if existing_cid == &file_cid {
// CIDs match - reuse existing blob
-
println!(" ✓ Reusing blob for {} (CID: {})", file_path_key, file_cid);
+
progress.set_message(format!("✓ Reused {}", file_path_key));
let mut file_builder = File::new()
.r#type(CowStr::from("file"))
.blob(existing_blob_ref.clone())
···
}
return Ok((file_builder.build(), true));
-
} else {
-
// CID mismatch - file changed
-
println!(" → File changed: {} (old CID: {}, new CID: {})", file_path_key, existing_cid, file_cid);
-
}
-
} else {
-
// File not in existing blob map
-
if file_path_key.starts_with("imgs/") {
-
println!(" → New file (not in blob map): {}", file_path_key);
}
}
···
MimeType::new_static("application/octet-stream")
};
-
println!(" ↑ Uploading {} ({} bytes, CID: {})", file_path_key, upload_bytes.len(), file_cid);
+
// Format file size nicely
+
let size_str = if upload_bytes.len() < 1024 {
+
format!("{} B", upload_bytes.len())
+
} else if upload_bytes.len() < 1024 * 1024 {
+
format!("{:.1} KB", upload_bytes.len() as f64 / 1024.0)
+
} else {
+
format!("{:.1} MB", upload_bytes.len() as f64 / (1024.0 * 1024.0))
+
};
+
+
progress.set_message(format!("↑ Uploading {} ({})", file_path_key, size_str));
let blob = agent.upload_blob(upload_bytes, mime_type).await?;
+
progress.set_message(format!("✓ Uploaded {}", file_path_key));
let mut file_builder = File::new()
.r#type(CowStr::from("file"))
+1
package.json
···
"build": "cd apps/main-app && bun run build.ts",
"build:hosting": "cd apps/hosting-service && npm run build",
"build:all": "bun run build && npm run build:hosting",
+
"check": "cd apps/main-app && npm run check && cd ../hosting-service && npm run check",
"screenshot": "bun run apps/main-app/scripts/screenshot-sites.ts",
"hosting:dev": "cd apps/hosting-service && npm run dev",
"hosting:start": "cd apps/hosting-service && npm run start"
+3
packages/@wisp/atproto-utils/package.json
···
"@atproto/api": "^0.14.1",
"@wisp/lexicons": "workspace:*",
"multiformats": "^13.3.1"
+
},
+
"devDependencies": {
+
"@atproto/lexicon": "^0.5.2"
}
}
+1 -1
packages/@wisp/constants/src/index.ts
···
// File size limits
export const MAX_SITE_SIZE = 300 * 1024 * 1024; // 300MB
-
export const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+
export const MAX_FILE_SIZE = 200 * 1024 * 1024; // 200MB
export const MAX_FILE_COUNT = 1000;
// Cache configuration
+2 -1
packages/@wisp/lexicons/package.json
···
"@atproto/xrpc-server": "^0.9.5"
},
"devDependencies": {
-
"@atproto/lex-cli": "^0.9.5"
+
"@atproto/lex-cli": "^0.9.5",
+
"multiformats": "^13.4.1"
}
}