Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
nekomimi.pet
1mod builder_types;
2mod place_wisp;
3mod cid;
4mod blob_map;
5mod metadata;
6mod download;
7mod pull;
8mod serve;
9
10use clap::{Parser, Subcommand};
11use jacquard::CowStr;
12use jacquard::client::{Agent, FileAuthStore, AgentSessionExt, MemoryCredentialSession, AgentSession};
13use jacquard::oauth::client::OAuthClient;
14use jacquard::oauth::loopback::LoopbackConfig;
15use jacquard::prelude::IdentityResolver;
16use jacquard_common::types::string::{Datetime, Rkey, RecordKey};
17use jacquard_common::types::blob::MimeType;
18use miette::IntoDiagnostic;
19use std::path::{Path, PathBuf};
20use std::collections::HashMap;
21use flate2::Compression;
22use flate2::write::GzEncoder;
23use std::io::Write;
24use base64::Engine;
25use futures::stream::{self, StreamExt};
26
27use place_wisp::fs::*;
28
29#[derive(Parser, Debug)]
30#[command(author, version, about = "wisp.place CLI tool")]
31struct Args {
32 #[command(subcommand)]
33 command: Option<Commands>,
34
35 // Deploy arguments (when no subcommand is specified)
36 /// Handle (e.g., alice.bsky.social), DID, or PDS URL
37 #[arg(global = true, conflicts_with = "command")]
38 input: Option<CowStr<'static>>,
39
40 /// Path to the directory containing your static site
41 #[arg(short, long, global = true, conflicts_with = "command")]
42 path: Option<PathBuf>,
43
44 /// Site name (defaults to directory name)
45 #[arg(short, long, global = true, conflicts_with = "command")]
46 site: Option<String>,
47
48 /// Path to auth store file
49 #[arg(long, global = true, conflicts_with = "command")]
50 store: Option<String>,
51
52 /// App Password for authentication
53 #[arg(long, global = true, conflicts_with = "command")]
54 password: Option<CowStr<'static>>,
55}
56
57#[derive(Subcommand, Debug)]
58enum Commands {
59 /// Deploy a static site to wisp.place (default command)
60 Deploy {
61 /// Handle (e.g., alice.bsky.social), DID, or PDS URL
62 input: CowStr<'static>,
63
64 /// Path to the directory containing your static site
65 #[arg(short, long, default_value = ".")]
66 path: PathBuf,
67
68 /// Site name (defaults to directory name)
69 #[arg(short, long)]
70 site: Option<String>,
71
72 /// Path to auth store file (will be created if missing, only used with OAuth)
73 #[arg(long, default_value = "/tmp/wisp-oauth-session.json")]
74 store: String,
75
76 /// App Password for authentication (alternative to OAuth)
77 #[arg(long)]
78 password: Option<CowStr<'static>>,
79 },
80 /// Pull a site from the PDS to a local directory
81 Pull {
82 /// Handle (e.g., alice.bsky.social) or DID
83 input: CowStr<'static>,
84
85 /// Site name (record key)
86 #[arg(short, long)]
87 site: String,
88
89 /// Output directory for the downloaded site
90 #[arg(short, long, default_value = ".")]
91 output: PathBuf,
92 },
93 /// Serve a site locally with real-time firehose updates
94 Serve {
95 /// Handle (e.g., alice.bsky.social) or DID
96 input: CowStr<'static>,
97
98 /// Site name (record key)
99 #[arg(short, long)]
100 site: String,
101
102 /// Output directory for the site files
103 #[arg(short, long, default_value = ".")]
104 output: PathBuf,
105
106 /// Port to serve on
107 #[arg(short, long, default_value = "8080")]
108 port: u16,
109 },
110}
111
112#[tokio::main]
113async fn main() -> miette::Result<()> {
114 let args = Args::parse();
115
116 match args.command {
117 Some(Commands::Deploy { input, path, site, store, password }) => {
118 // Dispatch to appropriate authentication method
119 if let Some(password) = password {
120 run_with_app_password(input, password, path, site).await
121 } else {
122 run_with_oauth(input, store, path, site).await
123 }
124 }
125 Some(Commands::Pull { input, site, output }) => {
126 pull::pull_site(input, CowStr::from(site), output).await
127 }
128 Some(Commands::Serve { input, site, output, port }) => {
129 serve::serve_site(input, CowStr::from(site), output, port).await
130 }
131 None => {
132 // Legacy mode: if input is provided, assume deploy command
133 if let Some(input) = args.input {
134 let path = args.path.unwrap_or_else(|| PathBuf::from("."));
135 let store = args.store.unwrap_or_else(|| "/tmp/wisp-oauth-session.json".to_string());
136
137 // Dispatch to appropriate authentication method
138 if let Some(password) = args.password {
139 run_with_app_password(input, password, path, args.site).await
140 } else {
141 run_with_oauth(input, store, path, args.site).await
142 }
143 } else {
144 // No command and no input, show help
145 use clap::CommandFactory;
146 Args::command().print_help().into_diagnostic()?;
147 Ok(())
148 }
149 }
150 }
151}
152
153/// Run deployment with app password authentication
154async fn run_with_app_password(
155 input: CowStr<'static>,
156 password: CowStr<'static>,
157 path: PathBuf,
158 site: Option<String>,
159) -> miette::Result<()> {
160 let (session, auth) =
161 MemoryCredentialSession::authenticated(input, password, None).await?;
162 println!("Signed in as {}", auth.handle);
163
164 let agent: Agent<_> = Agent::from(session);
165 deploy_site(&agent, path, site).await
166}
167
168/// Run deployment with OAuth authentication
169async fn run_with_oauth(
170 input: CowStr<'static>,
171 store: String,
172 path: PathBuf,
173 site: Option<String>,
174) -> miette::Result<()> {
175 let oauth = OAuthClient::with_default_config(FileAuthStore::new(&store));
176 let session = oauth
177 .login_with_local_server(input, Default::default(), LoopbackConfig::default())
178 .await?;
179
180 let agent: Agent<_> = Agent::from(session);
181 deploy_site(&agent, path, site).await
182}
183
184/// Deploy the site using the provided agent
185async fn deploy_site(
186 agent: &Agent<impl jacquard::client::AgentSession + IdentityResolver>,
187 path: PathBuf,
188 site: Option<String>,
189) -> miette::Result<()> {
190 // Verify the path exists
191 if !path.exists() {
192 return Err(miette::miette!("Path does not exist: {}", path.display()));
193 }
194
195 // Get site name
196 let site_name = site.unwrap_or_else(|| {
197 path
198 .file_name()
199 .and_then(|n| n.to_str())
200 .unwrap_or("site")
201 .to_string()
202 });
203
204 println!("Deploying site '{}'...", site_name);
205
206 // Try to fetch existing manifest for incremental updates
207 let existing_blob_map: HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)> = {
208 use jacquard_common::types::string::AtUri;
209
210 // Get the DID for this session
211 let session_info = agent.session_info().await;
212 if let Some((did, _)) = session_info {
213 // Construct the AT URI for the record
214 let uri_string = format!("at://{}/place.wisp.fs/{}", did, site_name);
215 if let Ok(uri) = AtUri::new(&uri_string) {
216 match agent.get_record::<Fs>(&uri).await {
217 Ok(response) => {
218 match response.into_output() {
219 Ok(record_output) => {
220 let existing_manifest = record_output.value;
221 let blob_map = blob_map::extract_blob_map(&existing_manifest.root);
222 println!("Found existing manifest with {} files, checking for changes...", blob_map.len());
223 blob_map
224 }
225 Err(_) => {
226 println!("No existing manifest found, uploading all files...");
227 HashMap::new()
228 }
229 }
230 }
231 Err(_) => {
232 // Record doesn't exist yet - this is a new site
233 println!("No existing manifest found, uploading all files...");
234 HashMap::new()
235 }
236 }
237 } else {
238 println!("No existing manifest found (invalid URI), uploading all files...");
239 HashMap::new()
240 }
241 } else {
242 println!("No existing manifest found (could not get DID), uploading all files...");
243 HashMap::new()
244 }
245 };
246
247 // Build directory tree
248 let (root_dir, total_files, reused_count) = build_directory(agent, &path, &existing_blob_map, String::new()).await?;
249 let uploaded_count = total_files - reused_count;
250
251 // Create the Fs record
252 let fs_record = Fs::new()
253 .site(CowStr::from(site_name.clone()))
254 .root(root_dir)
255 .file_count(total_files as i64)
256 .created_at(Datetime::now())
257 .build();
258
259 // Use site name as the record key
260 let rkey = Rkey::new(&site_name).map_err(|e| miette::miette!("Invalid rkey: {}", e))?;
261 let output = agent.put_record(RecordKey::from(rkey), fs_record).await?;
262
263 // Extract DID from the AT URI (format: at://did:plc:xxx/collection/rkey)
264 let uri_str = output.uri.to_string();
265 let did = uri_str
266 .strip_prefix("at://")
267 .and_then(|s| s.split('/').next())
268 .ok_or_else(|| miette::miette!("Failed to parse DID from URI"))?;
269
270 println!("\n✓ Deployed site '{}': {}", site_name, output.uri);
271 println!(" Total files: {} ({} reused, {} uploaded)", total_files, reused_count, uploaded_count);
272 println!(" Available at: https://sites.wisp.place/{}/{}", did, site_name);
273
274 Ok(())
275}
276
277/// Recursively build a Directory from a filesystem path
278/// current_path is the path from the root of the site (e.g., "" for root, "config" for config dir)
279fn build_directory<'a>(
280 agent: &'a Agent<impl jacquard::client::AgentSession + IdentityResolver + 'a>,
281 dir_path: &'a Path,
282 existing_blobs: &'a HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>,
283 current_path: String,
284) -> std::pin::Pin<Box<dyn std::future::Future<Output = miette::Result<(Directory<'static>, usize, usize)>> + 'a>>
285{
286 Box::pin(async move {
287 // Collect all directory entries first
288 let dir_entries: Vec<_> = std::fs::read_dir(dir_path)
289 .into_diagnostic()?
290 .collect::<Result<Vec<_>, _>>()
291 .into_diagnostic()?;
292
293 // Separate files and directories
294 let mut file_tasks = Vec::new();
295 let mut dir_tasks = Vec::new();
296
297 for entry in dir_entries {
298 let path = entry.path();
299 let name = entry.file_name();
300 let name_str = name.to_str()
301 .ok_or_else(|| miette::miette!("Invalid filename: {:?}", name))?
302 .to_string();
303
304 // Skip hidden files
305 if name_str.starts_with('.') {
306 continue;
307 }
308
309 let metadata = entry.metadata().into_diagnostic()?;
310
311 if metadata.is_file() {
312 // Construct full path for this file (for blob map lookup)
313 let full_path = if current_path.is_empty() {
314 name_str.clone()
315 } else {
316 format!("{}/{}", current_path, name_str)
317 };
318 file_tasks.push((name_str, path, full_path));
319 } else if metadata.is_dir() {
320 dir_tasks.push((name_str, path));
321 }
322 }
323
324 // Process files concurrently with a limit of 5
325 let file_results: Vec<(Entry<'static>, bool)> = stream::iter(file_tasks)
326 .map(|(name, path, full_path)| async move {
327 let (file_node, reused) = process_file(agent, &path, &full_path, existing_blobs).await?;
328 let entry = Entry::new()
329 .name(CowStr::from(name))
330 .node(EntryNode::File(Box::new(file_node)))
331 .build();
332 Ok::<_, miette::Report>((entry, reused))
333 })
334 .buffer_unordered(5)
335 .collect::<Vec<_>>()
336 .await
337 .into_iter()
338 .collect::<miette::Result<Vec<_>>>()?;
339
340 let mut file_entries = Vec::new();
341 let mut reused_count = 0;
342 let mut total_files = 0;
343
344 for (entry, reused) in file_results {
345 file_entries.push(entry);
346 total_files += 1;
347 if reused {
348 reused_count += 1;
349 }
350 }
351
352 // Process directories recursively (sequentially to avoid too much nesting)
353 let mut dir_entries = Vec::new();
354 for (name, path) in dir_tasks {
355 // Construct full path for subdirectory
356 let subdir_path = if current_path.is_empty() {
357 name.clone()
358 } else {
359 format!("{}/{}", current_path, name)
360 };
361 let (subdir, sub_total, sub_reused) = build_directory(agent, &path, existing_blobs, subdir_path).await?;
362 dir_entries.push(Entry::new()
363 .name(CowStr::from(name))
364 .node(EntryNode::Directory(Box::new(subdir)))
365 .build());
366 total_files += sub_total;
367 reused_count += sub_reused;
368 }
369
370 // Combine file and directory entries
371 let mut entries = file_entries;
372 entries.extend(dir_entries);
373
374 let directory = Directory::new()
375 .r#type(CowStr::from("directory"))
376 .entries(entries)
377 .build();
378
379 Ok((directory, total_files, reused_count))
380 })
381}
382
383/// Process a single file: gzip -> base64 -> upload blob (or reuse existing)
384/// Returns (File, reused: bool)
385/// file_path_key is the full path from the site root (e.g., "config/file.json") for blob map lookup
386async fn process_file(
387 agent: &Agent<impl jacquard::client::AgentSession + IdentityResolver>,
388 file_path: &Path,
389 file_path_key: &str,
390 existing_blobs: &HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>,
391) -> miette::Result<(File<'static>, bool)>
392{
393 // Read file
394 let file_data = std::fs::read(file_path).into_diagnostic()?;
395
396 // Detect original MIME type
397 let original_mime = mime_guess::from_path(file_path)
398 .first_or_octet_stream()
399 .to_string();
400
401 // Gzip compress
402 let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
403 encoder.write_all(&file_data).into_diagnostic()?;
404 let gzipped = encoder.finish().into_diagnostic()?;
405
406 // Base64 encode the gzipped data
407 let base64_bytes = base64::prelude::BASE64_STANDARD.encode(&gzipped).into_bytes();
408
409 // Compute CID for this file (CRITICAL: on base64-encoded gzipped content)
410 let file_cid = cid::compute_cid(&base64_bytes);
411
412 // Check if we have an existing blob with the same CID
413 let existing_blob = existing_blobs.get(file_path_key);
414
415 if let Some((existing_blob_ref, existing_cid)) = existing_blob {
416 if existing_cid == &file_cid {
417 // CIDs match - reuse existing blob
418 println!(" ✓ Reusing blob for {} (CID: {})", file_path_key, file_cid);
419 return Ok((
420 File::new()
421 .r#type(CowStr::from("file"))
422 .blob(existing_blob_ref.clone())
423 .encoding(CowStr::from("gzip"))
424 .mime_type(CowStr::from(original_mime))
425 .base64(true)
426 .build(),
427 true
428 ));
429 }
430 }
431
432 // File is new or changed - upload it
433 println!(" ↑ Uploading {} ({} bytes, CID: {})", file_path_key, base64_bytes.len(), file_cid);
434 let blob = agent.upload_blob(
435 base64_bytes,
436 MimeType::new_static("application/octet-stream"),
437 ).await?;
438
439 Ok((
440 File::new()
441 .r#type(CowStr::from("file"))
442 .blob(blob)
443 .encoding(CowStr::from("gzip"))
444 .mime_type(CowStr::from(original_mime))
445 .base64(true)
446 .build(),
447 false
448 ))
449}
450