Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
nekomimi.pet
1mod builder_types;
2mod place_wisp;
3mod cid;
4mod blob_map;
5mod metadata;
6mod download;
7mod pull;
8mod serve;
9mod subfs_utils;
10
11use clap::{Parser, Subcommand};
12use jacquard::CowStr;
13use jacquard::client::{Agent, FileAuthStore, AgentSessionExt, MemoryCredentialSession, AgentSession};
14use jacquard::oauth::client::OAuthClient;
15use jacquard::oauth::loopback::LoopbackConfig;
16use jacquard::prelude::IdentityResolver;
17use jacquard_common::types::string::{Datetime, Rkey, RecordKey};
18use jacquard_common::types::blob::MimeType;
19use miette::IntoDiagnostic;
20use std::path::{Path, PathBuf};
21use std::collections::HashMap;
22use flate2::Compression;
23use flate2::write::GzEncoder;
24use std::io::Write;
25use base64::Engine;
26use futures::stream::{self, StreamExt};
27
28use place_wisp::fs::*;
29
30#[derive(Parser, Debug)]
31#[command(author, version, about = "wisp.place CLI tool")]
32struct Args {
33 #[command(subcommand)]
34 command: Option<Commands>,
35
36 // Deploy arguments (when no subcommand is specified)
37 /// Handle (e.g., alice.bsky.social), DID, or PDS URL
38 #[arg(global = true, conflicts_with = "command")]
39 input: Option<CowStr<'static>>,
40
41 /// Path to the directory containing your static site
42 #[arg(short, long, global = true, conflicts_with = "command")]
43 path: Option<PathBuf>,
44
45 /// Site name (defaults to directory name)
46 #[arg(short, long, global = true, conflicts_with = "command")]
47 site: Option<String>,
48
49 /// Path to auth store file
50 #[arg(long, global = true, conflicts_with = "command")]
51 store: Option<String>,
52
53 /// App Password for authentication
54 #[arg(long, global = true, conflicts_with = "command")]
55 password: Option<CowStr<'static>>,
56}
57
58#[derive(Subcommand, Debug)]
59enum Commands {
60 /// Deploy a static site to wisp.place (default command)
61 Deploy {
62 /// Handle (e.g., alice.bsky.social), DID, or PDS URL
63 input: CowStr<'static>,
64
65 /// Path to the directory containing your static site
66 #[arg(short, long, default_value = ".")]
67 path: PathBuf,
68
69 /// Site name (defaults to directory name)
70 #[arg(short, long)]
71 site: Option<String>,
72
73 /// Path to auth store file (will be created if missing, only used with OAuth)
74 #[arg(long, default_value = "/tmp/wisp-oauth-session.json")]
75 store: String,
76
77 /// App Password for authentication (alternative to OAuth)
78 #[arg(long)]
79 password: Option<CowStr<'static>>,
80 },
81 /// Pull a site from the PDS to a local directory
82 Pull {
83 /// Handle (e.g., alice.bsky.social) or DID
84 input: CowStr<'static>,
85
86 /// Site name (record key)
87 #[arg(short, long)]
88 site: String,
89
90 /// Output directory for the downloaded site
91 #[arg(short, long, default_value = ".")]
92 output: PathBuf,
93 },
94 /// Serve a site locally with real-time firehose updates
95 Serve {
96 /// Handle (e.g., alice.bsky.social) or DID
97 input: CowStr<'static>,
98
99 /// Site name (record key)
100 #[arg(short, long)]
101 site: String,
102
103 /// Output directory for the site files
104 #[arg(short, long, default_value = ".")]
105 output: PathBuf,
106
107 /// Port to serve on
108 #[arg(short, long, default_value = "8080")]
109 port: u16,
110 },
111}
112
113#[tokio::main]
114async fn main() -> miette::Result<()> {
115 let args = Args::parse();
116
117 match args.command {
118 Some(Commands::Deploy { input, path, site, store, password }) => {
119 // Dispatch to appropriate authentication method
120 if let Some(password) = password {
121 run_with_app_password(input, password, path, site).await
122 } else {
123 run_with_oauth(input, store, path, site).await
124 }
125 }
126 Some(Commands::Pull { input, site, output }) => {
127 pull::pull_site(input, CowStr::from(site), output).await
128 }
129 Some(Commands::Serve { input, site, output, port }) => {
130 serve::serve_site(input, CowStr::from(site), output, port).await
131 }
132 None => {
133 // Legacy mode: if input is provided, assume deploy command
134 if let Some(input) = args.input {
135 let path = args.path.unwrap_or_else(|| PathBuf::from("."));
136 let store = args.store.unwrap_or_else(|| "/tmp/wisp-oauth-session.json".to_string());
137
138 // Dispatch to appropriate authentication method
139 if let Some(password) = args.password {
140 run_with_app_password(input, password, path, args.site).await
141 } else {
142 run_with_oauth(input, store, path, args.site).await
143 }
144 } else {
145 // No command and no input, show help
146 use clap::CommandFactory;
147 Args::command().print_help().into_diagnostic()?;
148 Ok(())
149 }
150 }
151 }
152}
153
154/// Run deployment with app password authentication
155async fn run_with_app_password(
156 input: CowStr<'static>,
157 password: CowStr<'static>,
158 path: PathBuf,
159 site: Option<String>,
160) -> miette::Result<()> {
161 let (session, auth) =
162 MemoryCredentialSession::authenticated(input, password, None).await?;
163 println!("Signed in as {}", auth.handle);
164
165 let agent: Agent<_> = Agent::from(session);
166 deploy_site(&agent, path, site).await
167}
168
169/// Run deployment with OAuth authentication
170async fn run_with_oauth(
171 input: CowStr<'static>,
172 store: String,
173 path: PathBuf,
174 site: Option<String>,
175) -> miette::Result<()> {
176 let oauth = OAuthClient::with_default_config(FileAuthStore::new(&store));
177 let session = oauth
178 .login_with_local_server(input, Default::default(), LoopbackConfig::default())
179 .await?;
180
181 let agent: Agent<_> = Agent::from(session);
182 deploy_site(&agent, path, site).await
183}
184
185/// Deploy the site using the provided agent
186async fn deploy_site(
187 agent: &Agent<impl jacquard::client::AgentSession + IdentityResolver>,
188 path: PathBuf,
189 site: Option<String>,
190) -> miette::Result<()> {
191 // Verify the path exists
192 if !path.exists() {
193 return Err(miette::miette!("Path does not exist: {}", path.display()));
194 }
195
196 // Get site name
197 let site_name = site.unwrap_or_else(|| {
198 path
199 .file_name()
200 .and_then(|n| n.to_str())
201 .unwrap_or("site")
202 .to_string()
203 });
204
205 println!("Deploying site '{}'...", site_name);
206
207 // Try to fetch existing manifest for incremental updates
208 let (existing_blob_map, old_subfs_uris): (HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>, Vec<(String, String)>) = {
209 use jacquard_common::types::string::AtUri;
210
211 // Get the DID for this session
212 let session_info = agent.session_info().await;
213 if let Some((did, _)) = session_info {
214 // Construct the AT URI for the record
215 let uri_string = format!("at://{}/place.wisp.fs/{}", did, site_name);
216 if let Ok(uri) = AtUri::new(&uri_string) {
217 match agent.get_record::<Fs>(&uri).await {
218 Ok(response) => {
219 match response.into_output() {
220 Ok(record_output) => {
221 let existing_manifest = record_output.value;
222 let mut blob_map = blob_map::extract_blob_map(&existing_manifest.root);
223 println!("Found existing manifest with {} files in main record", blob_map.len());
224
225 // Extract subfs URIs from main record
226 let subfs_uris = subfs_utils::extract_subfs_uris(&existing_manifest.root, String::new());
227
228 if !subfs_uris.is_empty() {
229 println!("Found {} subfs records, fetching for blob reuse...", subfs_uris.len());
230
231 // Merge blob maps from all subfs records
232 match subfs_utils::merge_subfs_blob_maps(agent, subfs_uris.clone(), &mut blob_map).await {
233 Ok(merged_count) => {
234 println!("Total blob map: {} files (main + {} from subfs)", blob_map.len(), merged_count);
235 }
236 Err(e) => {
237 eprintln!("⚠️ Failed to merge some subfs blob maps: {}", e);
238 }
239 }
240
241 (blob_map, subfs_uris)
242 } else {
243 (blob_map, Vec::new())
244 }
245 }
246 Err(_) => {
247 println!("No existing manifest found, uploading all files...");
248 (HashMap::new(), Vec::new())
249 }
250 }
251 }
252 Err(_) => {
253 // Record doesn't exist yet - this is a new site
254 println!("No existing manifest found, uploading all files...");
255 (HashMap::new(), Vec::new())
256 }
257 }
258 } else {
259 println!("No existing manifest found (invalid URI), uploading all files...");
260 (HashMap::new(), Vec::new())
261 }
262 } else {
263 println!("No existing manifest found (could not get DID), uploading all files...");
264 (HashMap::new(), Vec::new())
265 }
266 };
267
268 // Build directory tree
269 let (root_dir, total_files, reused_count) = build_directory(agent, &path, &existing_blob_map, String::new()).await?;
270 let uploaded_count = total_files - reused_count;
271
272 // Check if we need to split into subfs records
273 const MAX_MANIFEST_SIZE: usize = 140 * 1024; // 140KB (PDS limit is 150KB)
274 const FILE_COUNT_THRESHOLD: usize = 250; // Start splitting at this many files
275 const TARGET_FILE_COUNT: usize = 200; // Keep main manifest under this
276
277 let mut working_directory = root_dir;
278 let mut current_file_count = total_files;
279 let mut new_subfs_uris: Vec<(String, String)> = Vec::new();
280
281 // Estimate initial manifest size
282 let mut manifest_size = subfs_utils::estimate_directory_size(&working_directory);
283
284 if total_files >= FILE_COUNT_THRESHOLD || manifest_size > MAX_MANIFEST_SIZE {
285 println!("\n⚠️ Large site detected ({} files, {:.1}KB manifest), splitting into subfs records...",
286 total_files, manifest_size as f64 / 1024.0);
287
288 let mut attempts = 0;
289 const MAX_SPLIT_ATTEMPTS: usize = 50;
290
291 while (manifest_size > MAX_MANIFEST_SIZE || current_file_count > TARGET_FILE_COUNT) && attempts < MAX_SPLIT_ATTEMPTS {
292 attempts += 1;
293
294 // Find large directories to split
295 let directories = subfs_utils::find_large_directories(&working_directory, String::new());
296
297 if let Some(largest_dir) = directories.first() {
298 println!(" Split #{}: {} ({} files, {:.1}KB)",
299 attempts, largest_dir.path, largest_dir.file_count, largest_dir.size as f64 / 1024.0);
300
301 // Create a subfs record for this directory
302 use jacquard_common::types::string::Tid;
303 let subfs_tid = Tid::now_0();
304 let subfs_rkey = subfs_tid.to_string();
305
306 let subfs_manifest = crate::place_wisp::subfs::SubfsRecord::new()
307 .root(convert_fs_dir_to_subfs_dir(largest_dir.directory.clone()))
308 .file_count(Some(largest_dir.file_count as i64))
309 .created_at(Datetime::now())
310 .build();
311
312 // Upload subfs record
313 let subfs_output = agent.put_record(
314 RecordKey::from(Rkey::new(&subfs_rkey).into_diagnostic()?),
315 subfs_manifest
316 ).await.into_diagnostic()?;
317
318 let subfs_uri = subfs_output.uri.to_string();
319 println!(" ✅ Created subfs: {}", subfs_uri);
320
321 // Replace directory with subfs node (flat: false to preserve structure)
322 working_directory = subfs_utils::replace_directory_with_subfs(
323 working_directory,
324 &largest_dir.path,
325 &subfs_uri,
326 false // Preserve directory structure
327 )?;
328
329 new_subfs_uris.push((subfs_uri, largest_dir.path.clone()));
330 current_file_count -= largest_dir.file_count;
331
332 // Recalculate manifest size
333 manifest_size = subfs_utils::estimate_directory_size(&working_directory);
334 println!(" → Manifest now {:.1}KB with {} files ({} subfs total)",
335 manifest_size as f64 / 1024.0, current_file_count, new_subfs_uris.len());
336
337 if manifest_size <= MAX_MANIFEST_SIZE && current_file_count <= TARGET_FILE_COUNT {
338 println!("✅ Manifest now fits within limits");
339 break;
340 }
341 } else {
342 println!(" No more subdirectories to split - stopping");
343 break;
344 }
345 }
346
347 if attempts >= MAX_SPLIT_ATTEMPTS {
348 return Err(miette::miette!(
349 "Exceeded maximum split attempts ({}). Manifest still too large: {:.1}KB with {} files",
350 MAX_SPLIT_ATTEMPTS,
351 manifest_size as f64 / 1024.0,
352 current_file_count
353 ));
354 }
355
356 println!("✅ Split complete: {} subfs records, {} files in main manifest, {:.1}KB",
357 new_subfs_uris.len(), current_file_count, manifest_size as f64 / 1024.0);
358 } else {
359 println!("Manifest created ({} files, {:.1}KB) - no splitting needed",
360 total_files, manifest_size as f64 / 1024.0);
361 }
362
363 // Create the final Fs record
364 let fs_record = Fs::new()
365 .site(CowStr::from(site_name.clone()))
366 .root(working_directory)
367 .file_count(current_file_count as i64)
368 .created_at(Datetime::now())
369 .build();
370
371 // Use site name as the record key
372 let rkey = Rkey::new(&site_name).map_err(|e| miette::miette!("Invalid rkey: {}", e))?;
373 let output = agent.put_record(RecordKey::from(rkey), fs_record).await?;
374
375 // Extract DID from the AT URI (format: at://did:plc:xxx/collection/rkey)
376 let uri_str = output.uri.to_string();
377 let did = uri_str
378 .strip_prefix("at://")
379 .and_then(|s| s.split('/').next())
380 .ok_or_else(|| miette::miette!("Failed to parse DID from URI"))?;
381
382 println!("\n✓ Deployed site '{}': {}", site_name, output.uri);
383 println!(" Total files: {} ({} reused, {} uploaded)", total_files, reused_count, uploaded_count);
384 println!(" Available at: https://sites.wisp.place/{}/{}", did, site_name);
385
386 // Clean up old subfs records
387 if !old_subfs_uris.is_empty() {
388 println!("\nCleaning up {} old subfs records...", old_subfs_uris.len());
389
390 let mut deleted_count = 0;
391 let mut failed_count = 0;
392
393 for (uri, _path) in old_subfs_uris {
394 match subfs_utils::delete_subfs_record(agent, &uri).await {
395 Ok(_) => {
396 deleted_count += 1;
397 println!(" 🗑️ Deleted old subfs: {}", uri);
398 }
399 Err(e) => {
400 failed_count += 1;
401 eprintln!(" ⚠️ Failed to delete {}: {}", uri, e);
402 }
403 }
404 }
405
406 if failed_count > 0 {
407 eprintln!("⚠️ Cleanup completed with {} deleted, {} failed", deleted_count, failed_count);
408 } else {
409 println!("✅ Cleanup complete: {} old subfs records deleted", deleted_count);
410 }
411 }
412
413 Ok(())
414}
415
416/// Recursively build a Directory from a filesystem path
417/// current_path is the path from the root of the site (e.g., "" for root, "config" for config dir)
418fn build_directory<'a>(
419 agent: &'a Agent<impl jacquard::client::AgentSession + IdentityResolver + 'a>,
420 dir_path: &'a Path,
421 existing_blobs: &'a HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>,
422 current_path: String,
423) -> std::pin::Pin<Box<dyn std::future::Future<Output = miette::Result<(Directory<'static>, usize, usize)>> + 'a>>
424{
425 Box::pin(async move {
426 // Collect all directory entries first
427 let dir_entries: Vec<_> = std::fs::read_dir(dir_path)
428 .into_diagnostic()?
429 .collect::<Result<Vec<_>, _>>()
430 .into_diagnostic()?;
431
432 // Separate files and directories
433 let mut file_tasks = Vec::new();
434 let mut dir_tasks = Vec::new();
435
436 for entry in dir_entries {
437 let path = entry.path();
438 let name = entry.file_name();
439 let name_str = name.to_str()
440 .ok_or_else(|| miette::miette!("Invalid filename: {:?}", name))?
441 .to_string();
442
443 // Skip .git directories
444 if name_str == ".git" {
445 continue;
446 }
447
448 let metadata = entry.metadata().into_diagnostic()?;
449
450 if metadata.is_file() {
451 // Construct full path for this file (for blob map lookup)
452 let full_path = if current_path.is_empty() {
453 name_str.clone()
454 } else {
455 format!("{}/{}", current_path, name_str)
456 };
457 file_tasks.push((name_str, path, full_path));
458 } else if metadata.is_dir() {
459 dir_tasks.push((name_str, path));
460 }
461 }
462
463 // Process files concurrently with a limit of 5
464 let file_results: Vec<(Entry<'static>, bool)> = stream::iter(file_tasks)
465 .map(|(name, path, full_path)| async move {
466 let (file_node, reused) = process_file(agent, &path, &full_path, existing_blobs).await?;
467 let entry = Entry::new()
468 .name(CowStr::from(name))
469 .node(EntryNode::File(Box::new(file_node)))
470 .build();
471 Ok::<_, miette::Report>((entry, reused))
472 })
473 .buffer_unordered(5)
474 .collect::<Vec<_>>()
475 .await
476 .into_iter()
477 .collect::<miette::Result<Vec<_>>>()?;
478
479 let mut file_entries = Vec::new();
480 let mut reused_count = 0;
481 let mut total_files = 0;
482
483 for (entry, reused) in file_results {
484 file_entries.push(entry);
485 total_files += 1;
486 if reused {
487 reused_count += 1;
488 }
489 }
490
491 // Process directories recursively (sequentially to avoid too much nesting)
492 let mut dir_entries = Vec::new();
493 for (name, path) in dir_tasks {
494 // Construct full path for subdirectory
495 let subdir_path = if current_path.is_empty() {
496 name.clone()
497 } else {
498 format!("{}/{}", current_path, name)
499 };
500 let (subdir, sub_total, sub_reused) = build_directory(agent, &path, existing_blobs, subdir_path).await?;
501 dir_entries.push(Entry::new()
502 .name(CowStr::from(name))
503 .node(EntryNode::Directory(Box::new(subdir)))
504 .build());
505 total_files += sub_total;
506 reused_count += sub_reused;
507 }
508
509 // Combine file and directory entries
510 let mut entries = file_entries;
511 entries.extend(dir_entries);
512
513 let directory = Directory::new()
514 .r#type(CowStr::from("directory"))
515 .entries(entries)
516 .build();
517
518 Ok((directory, total_files, reused_count))
519 })
520}
521
522/// Process a single file: gzip -> base64 -> upload blob (or reuse existing)
523/// Returns (File, reused: bool)
524/// file_path_key is the full path from the site root (e.g., "config/file.json") for blob map lookup
525async fn process_file(
526 agent: &Agent<impl jacquard::client::AgentSession + IdentityResolver>,
527 file_path: &Path,
528 file_path_key: &str,
529 existing_blobs: &HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>,
530) -> miette::Result<(File<'static>, bool)>
531{
532 // Read file
533 let file_data = std::fs::read(file_path).into_diagnostic()?;
534
535 // Detect original MIME type
536 let original_mime = mime_guess::from_path(file_path)
537 .first_or_octet_stream()
538 .to_string();
539
540 // Gzip compress
541 let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
542 encoder.write_all(&file_data).into_diagnostic()?;
543 let gzipped = encoder.finish().into_diagnostic()?;
544
545 // Base64 encode the gzipped data
546 let base64_bytes = base64::prelude::BASE64_STANDARD.encode(&gzipped).into_bytes();
547
548 // Compute CID for this file (CRITICAL: on base64-encoded gzipped content)
549 let file_cid = cid::compute_cid(&base64_bytes);
550
551 // Check if we have an existing blob with the same CID
552 let existing_blob = existing_blobs.get(file_path_key);
553
554 if let Some((existing_blob_ref, existing_cid)) = existing_blob {
555 if existing_cid == &file_cid {
556 // CIDs match - reuse existing blob
557 println!(" ✓ Reusing blob for {} (CID: {})", file_path_key, file_cid);
558 return Ok((
559 File::new()
560 .r#type(CowStr::from("file"))
561 .blob(existing_blob_ref.clone())
562 .encoding(CowStr::from("gzip"))
563 .mime_type(CowStr::from(original_mime))
564 .base64(true)
565 .build(),
566 true
567 ));
568 }
569 }
570
571 // File is new or changed - upload it
572 println!(" ↑ Uploading {} ({} bytes, CID: {})", file_path_key, base64_bytes.len(), file_cid);
573 let blob = agent.upload_blob(
574 base64_bytes,
575 MimeType::new_static("application/octet-stream"),
576 ).await?;
577
578 Ok((
579 File::new()
580 .r#type(CowStr::from("file"))
581 .blob(blob)
582 .encoding(CowStr::from("gzip"))
583 .mime_type(CowStr::from(original_mime))
584 .base64(true)
585 .build(),
586 false
587 ))
588}
589
590/// Convert fs::Directory to subfs::Directory
591/// They have the same structure, but different types
592fn convert_fs_dir_to_subfs_dir(fs_dir: place_wisp::fs::Directory<'static>) -> place_wisp::subfs::Directory<'static> {
593 use place_wisp::subfs::{Directory as SubfsDirectory, Entry as SubfsEntry, EntryNode as SubfsEntryNode, File as SubfsFile};
594
595 let subfs_entries: Vec<SubfsEntry> = fs_dir.entries.into_iter().map(|entry| {
596 let node = match entry.node {
597 place_wisp::fs::EntryNode::File(file) => {
598 SubfsEntryNode::File(Box::new(SubfsFile::new()
599 .r#type(file.r#type)
600 .blob(file.blob)
601 .encoding(file.encoding)
602 .mime_type(file.mime_type)
603 .base64(file.base64)
604 .build()))
605 }
606 place_wisp::fs::EntryNode::Directory(dir) => {
607 SubfsEntryNode::Directory(Box::new(convert_fs_dir_to_subfs_dir(*dir)))
608 }
609 place_wisp::fs::EntryNode::Subfs(subfs) => {
610 // Nested subfs in the directory we're converting
611 // Note: subfs::Subfs doesn't have the 'flat' field - that's only in fs::Subfs
612 SubfsEntryNode::Subfs(Box::new(place_wisp::subfs::Subfs::new()
613 .r#type(subfs.r#type)
614 .subject(subfs.subject)
615 .build()))
616 }
617 place_wisp::fs::EntryNode::Unknown(unknown) => {
618 SubfsEntryNode::Unknown(unknown)
619 }
620 };
621
622 SubfsEntry::new()
623 .name(entry.name)
624 .node(node)
625 .build()
626 }).collect();
627
628 SubfsDirectory::new()
629 .r#type(fs_dir.r#type)
630 .entries(subfs_entries)
631 .build()
632}
633