Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
nekomimi.pet
1use crate::blob_map;
2use crate::download;
3use crate::metadata::SiteMetadata;
4use crate::place_wisp::fs::*;
5use jacquard::CowStr;
6use jacquard::prelude::IdentityResolver;
7use jacquard_common::types::string::Did;
8use jacquard_common::xrpc::XrpcExt;
9use jacquard_identity::PublicResolver;
10use miette::IntoDiagnostic;
11use std::collections::HashMap;
12use std::path::{Path, PathBuf};
13use url::Url;
14
15/// Pull a site from the PDS to a local directory
16pub async fn pull_site(
17 input: CowStr<'static>,
18 rkey: CowStr<'static>,
19 output_dir: PathBuf,
20) -> miette::Result<()> {
21 println!("Pulling site {} from {}...", rkey, input);
22
23 // Resolve handle to DID if needed
24 let resolver = PublicResolver::default();
25 let did = if input.starts_with("did:") {
26 Did::new(&input).into_diagnostic()?
27 } else {
28 // It's a handle, resolve it
29 let handle = jacquard_common::types::string::Handle::new(&input).into_diagnostic()?;
30 resolver.resolve_handle(&handle).await.into_diagnostic()?
31 };
32
33 // Resolve PDS endpoint for the DID
34 let pds_url = resolver.pds_for_did(&did).await.into_diagnostic()?;
35 println!("Resolved PDS: {}", pds_url);
36
37 // Fetch the place.wisp.fs record
38
39 println!("Fetching record from PDS...");
40 let client = reqwest::Client::new();
41
42 // Use com.atproto.repo.getRecord
43 use jacquard::api::com_atproto::repo::get_record::GetRecord;
44 use jacquard_common::types::string::Rkey as RkeyType;
45 let rkey_parsed = RkeyType::new(&rkey).into_diagnostic()?;
46
47 use jacquard_common::types::ident::AtIdentifier;
48 use jacquard_common::types::string::RecordKey;
49 let request = GetRecord::new()
50 .repo(AtIdentifier::Did(did.clone()))
51 .collection(CowStr::from("place.wisp.fs"))
52 .rkey(RecordKey::from(rkey_parsed))
53 .build();
54
55 let response = client
56 .xrpc(pds_url.clone())
57 .send(&request)
58 .await
59 .into_diagnostic()?;
60
61 let record_output = response.into_output().into_diagnostic()?;
62 let record_cid = record_output.cid.as_ref().map(|c| c.to_string()).unwrap_or_default();
63
64 // Parse the record value as Fs
65 use jacquard_common::types::value::from_data;
66 let fs_record: Fs = from_data(&record_output.value).into_diagnostic()?;
67
68 let file_count = fs_record.file_count.map(|c| c.to_string()).unwrap_or_else(|| "?".to_string());
69 println!("Found site '{}' with {} files", fs_record.site, file_count);
70
71 // Load existing metadata for incremental updates
72 let existing_metadata = SiteMetadata::load(&output_dir)?;
73 let existing_file_cids = existing_metadata
74 .as_ref()
75 .map(|m| m.file_cids.clone())
76 .unwrap_or_default();
77
78 // Extract blob map from the new manifest
79 let new_blob_map = blob_map::extract_blob_map(&fs_record.root);
80 let new_file_cids: HashMap<String, String> = new_blob_map
81 .iter()
82 .map(|(path, (_blob_ref, cid))| (path.clone(), cid.clone()))
83 .collect();
84
85 // Clean up any leftover temp directories from previous failed attempts
86 let parent = output_dir.parent().unwrap_or_else(|| std::path::Path::new("."));
87 let output_name = output_dir.file_name().unwrap_or_else(|| std::ffi::OsStr::new("site")).to_string_lossy();
88 let temp_prefix = format!(".tmp-{}-", output_name);
89
90 if let Ok(entries) = parent.read_dir() {
91 for entry in entries.flatten() {
92 let name = entry.file_name();
93 if name.to_string_lossy().starts_with(&temp_prefix) {
94 let _ = std::fs::remove_dir_all(entry.path());
95 }
96 }
97 }
98
99 // Check if we need to update (but only if output directory actually exists with files)
100 if let Some(metadata) = &existing_metadata {
101 if metadata.record_cid == record_cid {
102 // Verify that the output directory actually exists and has content
103 let has_content = output_dir.exists() &&
104 output_dir.read_dir()
105 .map(|mut entries| entries.any(|e| {
106 if let Ok(entry) = e {
107 !entry.file_name().to_string_lossy().starts_with(".wisp-metadata")
108 } else {
109 false
110 }
111 }))
112 .unwrap_or(false);
113
114 if has_content {
115 println!("Site is already up to date!");
116 return Ok(());
117 }
118 }
119 }
120
121 // Create temporary directory for atomic update
122 // Place temp dir in parent directory to avoid issues with non-existent output_dir
123 let parent = output_dir.parent().unwrap_or_else(|| std::path::Path::new("."));
124 let temp_dir_name = format!(
125 ".tmp-{}-{}",
126 output_dir.file_name().unwrap_or_else(|| std::ffi::OsStr::new("site")).to_string_lossy(),
127 chrono::Utc::now().timestamp()
128 );
129 let temp_dir = parent.join(temp_dir_name);
130 std::fs::create_dir_all(&temp_dir).into_diagnostic()?;
131
132 println!("Downloading files...");
133 let mut downloaded = 0;
134 let mut reused = 0;
135
136 // Download files recursively
137 let download_result = download_directory(
138 &fs_record.root,
139 &temp_dir,
140 &pds_url,
141 did.as_str(),
142 &new_blob_map,
143 &existing_file_cids,
144 &output_dir,
145 String::new(),
146 &mut downloaded,
147 &mut reused,
148 )
149 .await;
150
151 // If download failed, clean up temp directory
152 if let Err(e) = download_result {
153 let _ = std::fs::remove_dir_all(&temp_dir);
154 return Err(e);
155 }
156
157 println!(
158 "Downloaded {} files, reused {} files",
159 downloaded, reused
160 );
161
162 // Save metadata
163 let metadata = SiteMetadata::new(record_cid, new_file_cids);
164 metadata.save(&temp_dir)?;
165
166 // Move files from temp to output directory
167 let output_abs = std::fs::canonicalize(&output_dir).unwrap_or_else(|_| output_dir.clone());
168 let current_dir = std::env::current_dir().into_diagnostic()?;
169
170 // Special handling for pulling to current directory
171 if output_abs == current_dir {
172 // Move files from temp to current directory
173 for entry in std::fs::read_dir(&temp_dir).into_diagnostic()? {
174 let entry = entry.into_diagnostic()?;
175 let dest = current_dir.join(entry.file_name());
176
177 // Remove existing file/dir if it exists
178 if dest.exists() {
179 if dest.is_dir() {
180 std::fs::remove_dir_all(&dest).into_diagnostic()?;
181 } else {
182 std::fs::remove_file(&dest).into_diagnostic()?;
183 }
184 }
185
186 // Move from temp to current dir
187 std::fs::rename(entry.path(), dest).into_diagnostic()?;
188 }
189
190 // Clean up temp directory
191 std::fs::remove_dir_all(&temp_dir).into_diagnostic()?;
192 } else {
193 // If output directory exists and has content, remove it first
194 if output_dir.exists() {
195 std::fs::remove_dir_all(&output_dir).into_diagnostic()?;
196 }
197
198 // Ensure parent directory exists
199 if let Some(parent) = output_dir.parent() {
200 if !parent.as_os_str().is_empty() && !parent.exists() {
201 std::fs::create_dir_all(parent).into_diagnostic()?;
202 }
203 }
204
205 // Rename temp to final location
206 match std::fs::rename(&temp_dir, &output_dir) {
207 Ok(_) => {},
208 Err(e) => {
209 // Clean up temp directory on failure
210 let _ = std::fs::remove_dir_all(&temp_dir);
211 return Err(miette::miette!("Failed to move temp directory: {}", e));
212 }
213 }
214 }
215
216 println!("✓ Site pulled successfully to {}", output_dir.display());
217
218 Ok(())
219}
220
221/// Recursively download a directory
222fn download_directory<'a>(
223 dir: &'a Directory<'_>,
224 output_dir: &'a Path,
225 pds_url: &'a Url,
226 did: &'a str,
227 new_blob_map: &'a HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>,
228 existing_file_cids: &'a HashMap<String, String>,
229 existing_output_dir: &'a Path,
230 path_prefix: String,
231 downloaded: &'a mut usize,
232 reused: &'a mut usize,
233) -> std::pin::Pin<Box<dyn std::future::Future<Output = miette::Result<()>> + Send + 'a>> {
234 Box::pin(async move {
235 for entry in &dir.entries {
236 let entry_name = entry.name.as_str();
237 let current_path = if path_prefix.is_empty() {
238 entry_name.to_string()
239 } else {
240 format!("{}/{}", path_prefix, entry_name)
241 };
242
243 match &entry.node {
244 EntryNode::File(file) => {
245 let output_path = output_dir.join(entry_name);
246
247 // Check if file CID matches existing
248 if let Some((_blob_ref, new_cid)) = new_blob_map.get(¤t_path) {
249 if let Some(existing_cid) = existing_file_cids.get(¤t_path) {
250 if existing_cid == new_cid {
251 // File unchanged, copy from existing directory
252 let existing_path = existing_output_dir.join(¤t_path);
253 if existing_path.exists() {
254 std::fs::copy(&existing_path, &output_path).into_diagnostic()?;
255 *reused += 1;
256 println!(" ✓ Reused {}", current_path);
257 continue;
258 }
259 }
260 }
261 }
262
263 // File is new or changed, download it
264 println!(" ↓ Downloading {}", current_path);
265 let data = download::download_and_decompress_blob(
266 pds_url,
267 &file.blob,
268 did,
269 file.base64.unwrap_or(false),
270 file.encoding.as_ref().map(|e| e.as_str() == "gzip").unwrap_or(false),
271 )
272 .await?;
273
274 std::fs::write(&output_path, data).into_diagnostic()?;
275 *downloaded += 1;
276 }
277 EntryNode::Directory(subdir) => {
278 let subdir_path = output_dir.join(entry_name);
279 std::fs::create_dir_all(&subdir_path).into_diagnostic()?;
280
281 download_directory(
282 subdir,
283 &subdir_path,
284 pds_url,
285 did,
286 new_blob_map,
287 existing_file_cids,
288 existing_output_dir,
289 current_path,
290 downloaded,
291 reused,
292 )
293 .await?;
294 }
295 EntryNode::Unknown(_) => {
296 // Skip unknown node types
297 println!(" ⚠ Skipping unknown node type for {}", current_path);
298 }
299 }
300 }
301
302 Ok(())
303 })
304}
305