···
Ok(record_output.value.into_static())
+
/// Recursively fetch all subfs records (including nested ones)
+
/// Returns a list of (mount_path, SubfsRecord) tuples
+
/// Note: Multiple records can have the same mount_path (for flat-merged chunks)
+
pub async fn fetch_all_subfs_records_recursive(
agent: &Agent<impl AgentSession + IdentityResolver>,
+
initial_uris: Vec<(String, String)>,
+
) -> miette::Result<Vec<(String, SubfsRecord<'static>)>> {
+
use futures::stream::{self, StreamExt};
+
let mut all_subfs: Vec<(String, SubfsRecord<'static>)> = Vec::new();
+
let mut fetched_uris: std::collections::HashSet<String> = std::collections::HashSet::new();
+
let mut to_fetch = initial_uris;
+
if to_fetch.is_empty() {
+
println!("Found {} subfs records, fetching recursively...", to_fetch.len());
+
const MAX_ITERATIONS: usize = 10;
+
while !to_fetch.is_empty() && iteration < MAX_ITERATIONS {
+
println!(" Iteration {}: fetching {} subfs records...", iteration, to_fetch.len());
+
let subfs_results: Vec<_> = stream::iter(to_fetch.clone())
+
.map(|(uri, mount_path)| async move {
+
match fetch_subfs_record(agent, &uri).await {
+
Ok(record) => Some((mount_path, record, uri)),
+
eprintln!(" ⚠️ Failed to fetch subfs {}: {}", uri, e);
+
// Process results and find nested subfs
+
let mut newly_found_uris = Vec::new();
+
for result in subfs_results {
+
if let Some((mount_path, record, uri)) = result {
+
println!(" ✓ Fetched subfs at {}", mount_path);
+
// Extract nested subfs URIs from this record
+
let nested_uris = extract_subfs_uris_from_subfs_dir(&record.root, mount_path.clone());
+
newly_found_uris.extend(nested_uris);
+
all_subfs.push((mount_path, record));
+
fetched_uris.insert(uri);
+
// Filter out already-fetched URIs (based on URI, not path)
+
to_fetch = newly_found_uris
+
.filter(|(uri, _)| !fetched_uris.contains(uri))
+
if iteration >= MAX_ITERATIONS {
+
eprintln!("⚠️ Max iterations reached while fetching nested subfs");
+
println!(" Total subfs records fetched: {}", all_subfs.len());
+
/// Extract subfs URIs from a subfs::Directory
+
fn extract_subfs_uris_from_subfs_dir(
+
directory: &crate::place_wisp::subfs::Directory,
+
) -> Vec<(String, String)> {
+
let mut uris = Vec::new();
+
for entry in &directory.entries {
+
crate::place_wisp::subfs::EntryNode::Subfs(subfs_node) => {
+
// Check if this is a chunk entry (chunk0, chunk1, etc.)
+
// Chunks should be flat-merged, so use the parent's path
+
let mount_path = if entry.name.starts_with("chunk") &&
+
entry.name.chars().skip(5).all(|c| c.is_ascii_digit()) {
+
// This is a chunk - use parent's path for flat merge
+
println!(" → Found chunk {} at {}, will flat-merge to {}", entry.name, current_path, current_path);
+
// Normal subfs - append name to path
+
if current_path.is_empty() {
+
format!("{}/{}", current_path, entry.name)
+
uris.push((subfs_node.subject.to_string(), mount_path));
+
crate::place_wisp::subfs::EntryNode::Directory(subdir) => {
+
let full_path = if current_path.is_empty() {
+
format!("{}/{}", current_path, entry.name)
+
let nested = extract_subfs_uris_from_subfs_dir(subdir, full_path);
+
/// Merge blob maps from subfs records into the main blob map (RECURSIVE)
+
/// Returns the total number of blobs merged from all subfs records
+
pub async fn merge_subfs_blob_maps(
+
agent: &Agent<impl AgentSession + IdentityResolver>,
+
subfs_uris: Vec<(String, String)>,
+
main_blob_map: &mut HashMap<String, (BlobRef<'static>, String)>,
+
) -> miette::Result<usize> {
+
// Fetch all subfs records recursively
+
let all_subfs = fetch_all_subfs_records_recursive(agent, subfs_uris).await?;
+
let mut total_merged = 0;
+
// Extract blobs from all fetched subfs records
+
// Skip parent records that only contain chunk references (no actual files)
+
for (mount_path, subfs_record) in all_subfs {
+
// Check if this record only contains chunk subfs references (no files)
+
let only_has_chunks = subfs_record.root.entries.iter().all(|e| {
+
matches!(&e.node, crate::place_wisp::subfs::EntryNode::Subfs(_)) &&
+
e.name.starts_with("chunk") &&
+
e.name.chars().skip(5).all(|c| c.is_ascii_digit())
+
if only_has_chunks && !subfs_record.root.entries.is_empty() {
+
// This is a parent containing only chunks - skip it, blobs are in the chunks
+
println!(" → Skipping parent subfs at {} ({} chunks, no files)", mount_path, subfs_record.root.entries.len());
+
let subfs_blob_map = extract_subfs_blobs(&subfs_record.root, mount_path.clone());
+
let count = subfs_blob_map.len();
+
for (path, blob_info) in subfs_blob_map {
+
main_blob_map.insert(path, blob_info);
+
println!(" ✓ Merged {} blobs from subfs at {}", count, mount_path);
···
+
/// Split a large directory into multiple smaller chunks
+
/// Returns a list of chunk directories, each small enough to fit in a subfs record
+
pub fn split_directory_into_chunks(
+
directory: &FsDirectory,
+
) -> Vec<FsDirectory<'static>> {
+
use jacquard_common::CowStr;
+
let mut chunks = Vec::new();
+
let mut current_chunk_entries = Vec::new();
+
let mut current_chunk_size = 100; // Base size for directory structure
+
for entry in &directory.entries {
+
// Estimate the size of this entry
+
let entry_size = estimate_entry_size(entry);
+
// If adding this entry would exceed the max size, start a new chunk
+
if !current_chunk_entries.is_empty() && (current_chunk_size + entry_size > max_size) {
+
// Create a chunk from current entries
+
let chunk = FsDirectory::new()
+
.r#type(CowStr::from("directory"))
+
.entries(current_chunk_entries.clone())
+
current_chunk_entries.clear();
+
current_chunk_size = 100;
+
current_chunk_entries.push(entry.clone().into_static());
+
current_chunk_size += entry_size;
+
// Add the last chunk if it has any entries
+
if !current_chunk_entries.is_empty() {
+
let chunk = FsDirectory::new()
+
.r#type(CowStr::from("directory"))
+
.entries(current_chunk_entries)
+
/// Estimate the JSON size of a single entry
+
fn estimate_entry_size(entry: &crate::place_wisp::fs::Entry) -> usize {
+
match serde_json::to_string(entry) {
+
Ok(json) => json.len(),
+
Err(_) => 500, // Conservative estimate if serialization fails
nekomimi.pet