···
+
This file is a merged representation of the entire codebase, combined into a single document by Repomix.
+
This section contains a summary of this file.
+
This file contains a packed representation of the entire repository's contents.
+
It is designed to be easily consumable by AI systems for analysis, code review,
+
or other automated processes.
+
The content is organized as follows:
+
1. This summary section
+
2. Repository information
+
4. Repository files (if enabled)
+
5. Multiple file entries, each consisting of:
+
- File path as an attribute
+
- Full contents of the file
+
- This file should be treated as read-only. Any changes should be made to the
+
original repository files, not this packed version.
+
- When processing this file, use the file path to distinguish
+
between different files in the repository.
+
- Be aware that this file may contain sensitive information. Handle it with
+
the same level of security as you would the original repository.
+
- Some files may have been excluded based on .gitignore rules and Repomix's configuration
+
- Binary files are not included in this packed representation. Please refer to the Repository Structure section for a complete list of file paths, including binary files
+
- Files matching patterns in .gitignore are excluded
+
- Files matching default ignore patterns are excluded
+
- Files are sorted by Git change count (files with more changes are at the bottom)
+
This section contains the contents of the repository's files.
+
<file path=".claude/settings.local.json">
+
"enableAllProjectMcpServers": false
+
<file path="src/thicket/cli/commands/generate.py">
+
"""Generate static HTML website from thicket data."""
+
from datetime import datetime
+
from pathlib import Path
+
from typing import Any, Optional, TypedDict, Union
+
from jinja2 import Environment, FileSystemLoader, select_autoescape
+
from rich.progress import Progress, SpinnerColumn, TextColumn
+
from ...core.git_store import GitStore
+
from ...models.feed import AtomEntry
+
from ...models.user import GitStoreIndex, UserMetadata
+
from ..utils import console, load_config
+
class UserData(TypedDict):
+
"""Type definition for user data structure."""
+
recent_entries: list[tuple[str, AtomEntry]]
+
def safe_anchor_id(atom_id: str) -> str:
+
"""Convert an Atom ID to a safe HTML anchor ID."""
+
# Use base64 URL-safe encoding without padding
+
encoded = base64.urlsafe_b64encode(atom_id.encode('utf-8')).decode('ascii').rstrip('=')
+
# Prefix with 'id' to ensure it starts with a letter (HTML requirement)
+
class WebsiteGenerator:
+
"""Generate static HTML website from thicket data."""
+
def __init__(self, git_store: GitStore, output_dir: Path):
+
self.git_store = git_store
+
self.output_dir = output_dir
+
self.template_dir = Path(__file__).parent.parent.parent / "templates"
+
# Initialize Jinja2 environment
+
self.env = Environment(
+
loader=FileSystemLoader(self.template_dir),
+
autoescape=select_autoescape(["html", "xml"]),
+
self.index: Optional[GitStoreIndex] = None
+
self.entries: list[tuple[str, AtomEntry]] = [] # (username, entry)
+
self.links_data: Optional[dict[str, Any]] = None
+
self.threads: list[list[dict[str, Any]]] = [] # List of threads with metadata
+
def get_display_name(self, username: str) -> str:
+
"""Get display name for a user, falling back to username."""
+
if self.index and username in self.index.users:
+
user = self.index.users[username]
+
return user.display_name or username
+
def get_user_homepage(self, username: str) -> Optional[str]:
+
"""Get homepage URL for a user."""
+
if self.index and username in self.index.users:
+
user = self.index.users[username]
+
return str(user.homepage) if user.homepage else None
+
def clean_html_summary(self, content: Optional[str], max_length: int = 200) -> str:
+
"""Clean HTML content and truncate for display in timeline."""
+
clean_text = re.sub(r"<[^>]+>", " ", content)
+
# Replace multiple whitespace with single space
+
clean_text = re.sub(r"\s+", " ", clean_text)
+
# Strip leading/trailing whitespace
+
clean_text = clean_text.strip()
+
# Truncate with ellipsis if needed
+
if len(clean_text) > max_length:
+
# Try to break at word boundary
+
truncated = clean_text[:max_length]
+
last_space = truncated.rfind(" ")
+
last_space > max_length * 0.8
+
): # If we can break reasonably close to the limit
+
clean_text = truncated[:last_space] + "..."
+
clean_text = truncated + "..."
+
def load_data(self) -> None:
+
"""Load all data from the git repository."""
+
TextColumn("[progress.description]{task.description}"),
+
task = progress.add_task("Loading repository index...", total=None)
+
self.index = self.git_store._load_index()
+
raise ValueError("No index found in repository")
+
progress.update(task, completed=True)
+
task = progress.add_task("Loading entries...", total=None)
+
for username, user_metadata in self.index.users.items():
+
user_dir = self.git_store.repo_path / user_metadata.directory
+
for entry_file in user_dir.glob("*.json"):
+
if entry_file.name not in ["index.json", "duplicates.json"]:
+
with open(entry_file) as f:
+
entry_data = json.load(f)
+
entry = AtomEntry(**entry_data)
+
self.entries.append((username, entry))
+
f"[yellow]Warning: Failed to load {entry_file}: {e}[/yellow]"
+
progress.update(task, completed=True)
+
# Sort entries by date (newest first) - prioritize updated over published
+
key=lambda x: x[1].updated or x[1].published or datetime.min, reverse=True
+
task = progress.add_task("Loading links and references...", total=None)
+
links_file = self.git_store.repo_path / "links.json"
+
if links_file.exists():
+
with open(links_file) as f:
+
self.links_data = json.load(f)
+
progress.update(task, completed=True)
+
def build_threads(self) -> None:
+
"""Build threaded conversations from references."""
+
if not self.links_data or "references" not in self.links_data:
+
# Map entry IDs to (username, entry) tuples
+
entry_map: dict[str, tuple[str, AtomEntry]] = {}
+
for username, entry in self.entries:
+
entry_map[entry.id] = (username, entry)
+
# Build adjacency lists for references
+
self.outbound_refs: dict[str, set[str]] = {}
+
self.inbound_refs: dict[str, set[str]] = {}
+
self.reference_details: dict[
+
str, list[dict[str, Any]]
+
] = {} # Store full reference info
+
for ref in self.links_data["references"]:
+
source_id = ref["source_entry_id"]
+
target_id = ref.get("target_entry_id")
+
if target_id and source_id in entry_map and target_id in entry_map:
+
self.outbound_refs.setdefault(source_id, set()).add(target_id)
+
self.inbound_refs.setdefault(target_id, set()).add(source_id)
+
# Store reference details for UI
+
self.reference_details.setdefault(source_id, []).append(
+
"target_id": target_id,
+
"target_username": ref.get("target_username"),
+
self.reference_details.setdefault(target_id, []).append(
+
"source_id": source_id,
+
"source_username": ref.get("source_username"),
+
# Find conversation threads (multi-post discussions)
+
for entry_id, (_username, _entry) in entry_map.items():
+
if entry_id in processed:
+
# Build thread starting from this entry
+
level_map: dict[str, int] = {} # Track levels for this thread
+
# First, traverse up to find the root
+
while current in self.inbound_refs:
+
parents = self.inbound_refs[current] - {
+
} # Exclude self-references
+
# Take the first parent
+
parent = next(iter(parents))
+
if parent in thread_ids: # Avoid cycles
+
to_visit.insert(0, current)
+
# Now traverse down from the root
+
current = to_visit.pop(0)
+
if current in thread_ids or current not in entry_map:
+
thread_ids.add(current)
+
username, entry = entry_map[current]
+
# Calculate thread level
+
thread_level = self._calculate_thread_level(current, level_map)
+
# Add threading metadata
+
"display_name": self.get_display_name(username),
+
"references_to": list(self.outbound_refs.get(current, [])),
+
"referenced_by": list(self.inbound_refs.get(current, [])),
+
"thread_level": thread_level,
+
thread.append(thread_entry)
+
if current in self.outbound_refs:
+
children = self.outbound_refs[current] - thread_ids # Avoid cycles
+
to_visit.extend(sorted(children))
+
if len(thread) > 1: # Only keep actual threads
+
# Sort thread by date (newest first) - prioritize updated over published
+
thread.sort(key=lambda x: x["entry"].updated or x["entry"].published or datetime.min, reverse=True) # type: ignore
+
self.threads.append(thread)
+
# Sort threads by the date of their most recent entry - prioritize updated over published
+
item["entry"].updated or item["entry"].published or datetime.min for item in t
+
def _calculate_thread_level(
+
self, entry_id: str, processed_entries: dict[str, int]
+
"""Calculate indentation level for threaded display."""
+
if entry_id in processed_entries:
+
return processed_entries[entry_id]
+
if entry_id not in self.inbound_refs:
+
processed_entries[entry_id] = 0
+
parents_in_thread = self.inbound_refs[entry_id] & set(processed_entries.keys())
+
if not parents_in_thread:
+
processed_entries[entry_id] = 0
+
# Find the deepest parent level + 1
+
for parent_id in parents_in_thread:
+
parent_level = self._calculate_thread_level(parent_id, processed_entries)
+
max_parent_level = max(max_parent_level, parent_level)
+
level = min(max_parent_level + 1, 4) # Cap at level 4
+
processed_entries[entry_id] = level
+
def get_standalone_references(self) -> list[dict[str, Any]]:
+
"""Get posts that have references but aren't part of multi-post threads."""
+
if not hasattr(self, "reference_details"):
+
threaded_entry_ids = set()
+
for thread in self.threads:
+
threaded_entry_ids.add(item["entry_id"])
+
for username, entry in self.entries:
+
entry.id in self.reference_details
+
and entry.id not in threaded_entry_ids
+
refs = self.reference_details[entry.id]
+
# Only include if it has meaningful references (not just self-references)
+
if r.get("target_id") != entry.id and r.get("source_id") != entry.id
+
standalone_refs.append(
+
"display_name": self.get_display_name(username),
+
"references": meaningful_refs,
+
def _add_cross_thread_links(self, timeline_items: list[dict[str, Any]]) -> None:
+
"""Add cross-thread linking for entries that appear in multiple threads."""
+
# Map entry IDs to their positions in the timeline
+
entry_positions: dict[str, list[int]] = {}
+
# Map URLs referenced by entries to the entries that reference them
+
url_references: dict[str, list[tuple[str, int]]] = {} # url -> [(entry_id, position)]
+
# First pass: collect all entry IDs, their positions, and referenced URLs
+
for i, item in enumerate(timeline_items):
+
if item["type"] == "post":
+
entry_id = item["content"]["entry"].id
+
entry_positions.setdefault(entry_id, []).append(i)
+
# Track URLs this entry references
+
if entry_id in self.reference_details:
+
for ref in self.reference_details[entry_id]:
+
if ref["type"] == "outbound" and "target_id" in ref:
+
# Find the target entry's URL if available
+
target_entry = self._find_entry_by_id(ref["target_id"])
+
if target_entry and target_entry.link:
+
url = str(target_entry.link)
+
url_references.setdefault(url, []).append((entry_id, i))
+
elif item["type"] == "thread":
+
for thread_item in item["content"]:
+
entry_id = thread_item["entry"].id
+
entry_positions.setdefault(entry_id, []).append(i)
+
# Track URLs this entry references
+
if entry_id in self.reference_details:
+
for ref in self.reference_details[entry_id]:
+
if ref["type"] == "outbound" and "target_id" in ref:
+
target_entry = self._find_entry_by_id(ref["target_id"])
+
if target_entry and target_entry.link:
+
url = str(target_entry.link)
+
url_references.setdefault(url, []).append((entry_id, i))
+
# Build cross-thread connections - only for entries that actually appear multiple times
+
cross_thread_connections: dict[str, set[int]] = {} # entry_id -> set of timeline positions
+
# Add connections ONLY for entries that appear multiple times in the timeline
+
for entry_id, positions in entry_positions.items():
+
cross_thread_connections[entry_id] = set(positions)
+
# Debug: uncomment to see which entries have multiple appearances
+
# print(f"Entry {entry_id[:50]}... appears at positions: {positions}")
+
# Apply cross-thread links to timeline items
+
for entry_id, positions_set in cross_thread_connections.items():
+
positions_list = list(positions_set)
+
for pos in positions_list:
+
item = timeline_items[pos]
+
other_positions = sorted([p for p in positions_list if p != pos])
+
if item["type"] == "post":
+
# Add cross-thread info to individual posts
+
item["content"]["cross_thread_links"] = self._build_cross_thread_link_data(entry_id, other_positions, timeline_items)
+
# Add info about shared references
+
item["content"]["shared_references"] = self._get_shared_references(entry_id, positions_set, timeline_items)
+
elif item["type"] == "thread":
+
# Add cross-thread info to thread items
+
for thread_item in item["content"]:
+
if thread_item["entry"].id == entry_id:
+
thread_item["cross_thread_links"] = self._build_cross_thread_link_data(entry_id, other_positions, timeline_items)
+
thread_item["shared_references"] = self._get_shared_references(entry_id, positions_set, timeline_items)
+
def _build_cross_thread_link_data(self, entry_id: str, other_positions: list[int], timeline_items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+
"""Build detailed cross-thread link data with anchor information."""
+
cross_thread_links = []
+
for pos in other_positions:
+
item = timeline_items[pos]
+
if item["type"] == "post":
+
safe_id = safe_anchor_id(entry_id)
+
cross_thread_links.append({
+
"anchor_id": f"post-{pos}-{safe_id}",
+
"context": "individual post",
+
"title": item["content"]["entry"].title
+
elif item["type"] == "thread":
+
# For thread items, find the specific thread item
+
for thread_idx, thread_item in enumerate(item["content"]):
+
if thread_item["entry"].id == entry_id:
+
safe_id = safe_anchor_id(entry_id)
+
cross_thread_links.append({
+
"anchor_id": f"post-{pos}-{thread_idx}-{safe_id}",
+
"context": f"thread (level {thread_item.get('thread_level', 0)})",
+
"title": thread_item["entry"].title
+
return cross_thread_links
+
def _find_entry_by_id(self, entry_id: str) -> Optional[AtomEntry]:
+
"""Find an entry by its ID."""
+
for _username, entry in self.entries:
+
if entry.id == entry_id:
+
def _get_shared_references(self, entry_id: str, positions: Union[set[int], list[int]], timeline_items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+
"""Get information about shared references between cross-thread entries."""
+
# Collect all referenced URLs from entries at these positions
+
url_counts: dict[str, int] = {}
+
referencing_entries: dict[str, list[str]] = {} # url -> [entry_ids]
+
item = timeline_items[pos]
+
if item["type"] == "post":
+
entries_to_check.append(item["content"]["entry"])
+
elif item["type"] == "thread":
+
entries_to_check.extend([ti["entry"] for ti in item["content"]])
+
for entry in entries_to_check:
+
if entry.id in self.reference_details:
+
for ref in self.reference_details[entry.id]:
+
if ref["type"] == "outbound" and "target_id" in ref:
+
target_entry = self._find_entry_by_id(ref["target_id"])
+
if target_entry and target_entry.link:
+
url = str(target_entry.link)
+
url_counts[url] = url_counts.get(url, 0) + 1
+
if url not in referencing_entries:
+
referencing_entries[url] = []
+
if entry.id not in referencing_entries[url]:
+
referencing_entries[url].append(entry.id)
+
# Find URLs referenced by multiple entries
+
for url, count in url_counts.items():
+
if count > 1 and len(referencing_entries[url]) > 1:
+
# Get the target entry info
+
for ref in (self.links_data or {}).get("references", []):
+
if ref.get("target_url") == url:
+
target_username = ref.get("target_username")
+
if ref.get("target_entry_id"):
+
target_entry = self._find_entry_by_id(ref["target_entry_id"])
+
"referencing_entries": referencing_entries[url],
+
"target_username": target_username,
+
"target_title": target_entry.title if target_entry else None
+
return sorted(shared_refs, key=lambda x: x["count"], reverse=True)
+
def generate_site(self) -> None:
+
"""Generate the static website."""
+
# Create output directory
+
self.output_dir.mkdir(parents=True, exist_ok=True)
+
# Create static directories
+
(self.output_dir / "css").mkdir(exist_ok=True)
+
(self.output_dir / "js").mkdir(exist_ok=True)
+
css_template = self.env.get_template("style.css")
+
css_content = css_template.render()
+
with open(self.output_dir / "css" / "style.css", "w") as f:
+
js_template = self.env.get_template("script.js")
+
js_content = js_template.render()
+
with open(self.output_dir / "js" / "script.js", "w") as f:
+
# Prepare common template data
+
"title": "Energy & Environment Group",
+
"generated_at": datetime.now().isoformat(),
+
"get_display_name": self.get_display_name,
+
"get_user_homepage": self.get_user_homepage,
+
"clean_html_summary": self.clean_html_summary,
+
"safe_anchor_id": safe_anchor_id,
+
# Build unified timeline
+
# Only consider the threads that will actually be displayed
+
displayed_threads = self.threads[:20] # Limit to 20 threads
+
# Track which entries are part of displayed threads
+
threaded_entry_ids = set()
+
for thread in displayed_threads:
+
threaded_entry_ids.add(item["entry_id"])
+
# Add threads to timeline (using the date of the most recent post)
+
for thread in displayed_threads:
+
most_recent_date = max(
+
item["entry"].updated or item["entry"].published or datetime.min
+
timeline_items.append({
+
"date": most_recent_date,
+
# Add individual posts (not in threads)
+
for username, entry in self.entries[:50]:
+
if entry.id not in threaded_entry_ids:
+
# Check if this entry has references
+
entry.id in self.reference_details
+
if hasattr(self, "reference_details")
+
refs = self.reference_details.get(entry.id, [])
+
if r.get("target_id") != entry.id
+
and r.get("source_id") != entry.id
+
timeline_items.append({
+
"date": entry.updated or entry.published or datetime.min,
+
"display_name": self.get_display_name(username),
+
"references": refs if refs else None
+
# Sort unified timeline by date (newest first)
+
timeline_items.sort(key=lambda x: x["date"], reverse=True)
+
# Limit timeline to what will actually be rendered
+
timeline_items = timeline_items[:50] # Limit to 50 items total
+
# Add cross-thread linking for repeat blog references
+
self._add_cross_thread_links(timeline_items)
+
# Prepare outgoing links data
+
if self.links_data and "links" in self.links_data:
+
for url, link_info in self.links_data["links"].items():
+
referencing_entries = []
+
for entry_id in link_info.get("referencing_entries", []):
+
for username, entry in self.entries:
+
if entry.id == entry_id:
+
referencing_entries.append(
+
(self.get_display_name(username), entry)
+
if referencing_entries:
+
# Sort by date - prioritize updated over published
+
referencing_entries.sort(
+
key=lambda x: x[1].updated or x[1].published or datetime.min, reverse=True
+
"target_username": link_info.get("target_username"),
+
"entries": referencing_entries,
+
# Sort links by most recent reference - prioritize updated over published
+
key=lambda x: x["entries"][0][1].updated
+
or x["entries"][0][1].published or datetime.min,
+
users: list[UserData] = []
+
for username, user_metadata in self.index.users.items():
+
# Get recent entries for this user with display names
+
(self.get_display_name(u), e)
+
for u, e in self.entries
+
{"metadata": user_metadata, "recent_entries": user_entries}
+
users.sort(key=lambda x: x["metadata"].entry_count, reverse=True)
+
# Generate timeline page
+
timeline_template = self.env.get_template("timeline.html")
+
timeline_content = timeline_template.render(
+
timeline_items=timeline_items, # Already limited above
+
with open(self.output_dir / "timeline.html", "w") as f:
+
f.write(timeline_content)
+
links_template = self.env.get_template("links.html")
+
links_content = links_template.render(
+
outgoing_links=outgoing_links[:100],
+
with open(self.output_dir / "links.html", "w") as f:
+
users_template = self.env.get_template("users.html")
+
users_content = users_template.render(
+
with open(self.output_dir / "users.html", "w") as f:
+
# Generate main index page (redirect to timeline)
+
index_template = self.env.get_template("index.html")
+
index_content = index_template.render(**base_data)
+
with open(self.output_dir / "index.html", "w") as f:
+
console.print(f"[green]✓[/green] Generated website at {self.output_dir}")
+
console.print(f" - {len(self.entries)} entries")
+
console.print(f" - {len(self.threads)} conversation threads")
+
console.print(f" - {len(outgoing_links)} outgoing links")
+
console.print(f" - {len(users)} users")
+
" - Generated pages: index.html, timeline.html, links.html, users.html"
+
output: Path = typer.Option(
+
Path("./thicket-site"),
+
help="Output directory for the generated website",
+
force: bool = typer.Option(
+
False, "--force", "-f", help="Overwrite existing output directory"
+
config_file: Path = typer.Option(
+
Path("thicket.yaml"), "--config", help="Configuration file path"
+
"""Generate a static HTML website from thicket data."""
+
config = load_config(config_file)
+
if not config.git_store:
+
console.print("[red]No git store path configured[/red]")
+
git_store = GitStore(config.git_store)
+
# Check if output directory exists
+
if output.exists() and not force:
+
f"[red]Output directory {output} already exists. Use --force to overwrite.[/red]"
+
# Clean output directory if forcing
+
if output.exists() and force:
+
generator = WebsiteGenerator(git_store, output)
+
console.print("[bold]Generating static website...[/bold]")
+
generator.build_threads()
+
generator.generate_site()
+
console.print(f"[red]Error generating website: {e}[/red]")
+
raise typer.Exit(1) from e
+
<file path="src/thicket/templates/base.html">
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
+
<title>{% block page_title %}{{ title }}{% endblock %}</title>
+
<link rel="stylesheet" href="css/style.css">
+
<header class="site-header">
+
<div class="header-content">
+
<h1 class="site-title">{{ title }}</h1>
+
<a href="timeline.html" class="nav-link {% if page == 'timeline' %}active{% endif %}">Timeline</a>
+
<a href="links.html" class="nav-link {% if page == 'links' %}active{% endif %}">Links</a>
+
<a href="users.html" class="nav-link {% if page == 'users' %}active{% endif %}">Users</a>
+
<main class="main-content">
+
{% block content %}{% endblock %}
+
<footer class="site-footer">
+
<p>Generated on {{ generated_at }} by <a href="https://github.com/avsm/thicket">Thicket</a></p>
+
<script src="js/script.js"></script>
+
<file path="src/thicket/templates/index.html">
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
+
<title>{{ title }}</title>
+
<meta http-equiv="refresh" content="0; url=timeline.html">
+
<link rel="canonical" href="timeline.html">
+
<p>Redirecting to <a href="timeline.html">Timeline</a>...</p>
+
<file path="src/thicket/templates/links.html">
+
{% extends "base.html" %}
+
{% block page_title %}Outgoing Links - {{ title }}{% endblock %}
+
<div class="page-content">
+
<h2>Outgoing Links</h2>
+
<p class="page-description">External links referenced in blog posts, ordered by most recent reference.</p>
+
{% for link in outgoing_links %}
+
<article class="link-group">
+
<a href="{{ link.url }}" target="_blank">{{ link.url|truncate(80) }}</a>
+
{% if link.target_username %}
+
<span class="target-user">({{ link.target_username }})</span>
+
<div class="referencing-entries">
+
<span class="ref-count">Referenced in {{ link.entries|length }} post(s):</span>
+
{% for display_name, entry in link.entries[:5] %}
+
<span class="author">{{ display_name }}</span> -
+
<a href="{{ entry.link }}" target="_blank">{{ entry.title }}</a>
+
<time datetime="{{ entry.updated or entry.published }}">
+
({{ (entry.updated or entry.published).strftime('%Y-%m-%d') }})
+
{% if link.entries|length > 5 %}
+
<li class="more">... and {{ link.entries|length - 5 }} more</li>
+
<file path="src/thicket/templates/script.js">
+
// Enhanced functionality for thicket website
+
document.addEventListener('DOMContentLoaded', function() {
+
// Enhance thread collapsing (optional feature)
+
const threadHeaders = document.querySelectorAll('.thread-header');
+
threadHeaders.forEach(header => {
+
header.style.cursor = 'pointer';
+
header.addEventListener('click', function() {
+
const thread = this.parentElement;
+
const entries = thread.querySelectorAll('.thread-entry');
+
// Toggle visibility of all but the first entry
+
for (let i = 1; i < entries.length; i++) {
+
entries[i].style.display = entries[i].style.display === 'none' ? 'block' : 'none';
+
// Update thread count text
+
const count = this.querySelector('.thread-count');
+
if (entries[1] && entries[1].style.display === 'none') {
+
count.textContent = count.textContent.replace('posts', 'posts (collapsed)');
+
count.textContent = count.textContent.replace(' (collapsed)', '');
+
// Add relative time display
+
const timeElements = document.querySelectorAll('time');
+
timeElements.forEach(timeEl => {
+
const datetime = new Date(timeEl.getAttribute('datetime'));
+
const now = new Date();
+
const diffMs = now - datetime;
+
const diffDays = Math.floor(diffMs / (1000 * 60 * 60 * 24));
+
const diffHours = Math.floor(diffMs / (1000 * 60 * 60));
+
const diffMinutes = Math.floor(diffMs / (1000 * 60));
+
relativeTime = diffMinutes === 0 ? 'just now' : `${diffMinutes}m ago`;
+
relativeTime = `${diffHours}h ago`;
+
} else if (diffDays === 1) {
+
relativeTime = 'yesterday';
+
} else if (diffDays < 7) {
+
relativeTime = `${diffDays}d ago`;
+
} else if (diffDays < 30) {
+
const weeks = Math.floor(diffDays / 7);
+
relativeTime = weeks === 1 ? '1w ago' : `${weeks}w ago`;
+
} else if (diffDays < 365) {
+
const months = Math.floor(diffDays / 30);
+
relativeTime = months === 1 ? '1mo ago' : `${months}mo ago`;
+
const years = Math.floor(diffDays / 365);
+
relativeTime = years === 1 ? '1y ago' : `${years}y ago`;
+
// Add relative time as title attribute
+
timeEl.setAttribute('title', timeEl.textContent);
+
timeEl.textContent = relativeTime;
+
// Enhanced anchor link scrolling for shared references
+
document.querySelectorAll('a[href^="#"]').forEach(anchor => {
+
anchor.addEventListener('click', function (e) {
+
const target = document.querySelector(this.getAttribute('href'));
+
target.scrollIntoView({
+
// Highlight the target briefly
+
const timelineEntry = target.closest('.timeline-entry');
+
timelineEntry.style.outline = '2px solid var(--primary-color)';
+
timelineEntry.style.borderRadius = '8px';
+
timelineEntry.style.outline = '';
+
timelineEntry.style.borderRadius = '';
+
<file path="src/thicket/templates/style.css">
+
/* Modern, clean design with high-density text and readable theme */
+
--primary-color: #2c3e50;
+
--secondary-color: #3498db;
+
--accent-color: #e74c3c;
+
--text-primary: #2c3e50;
+
--text-secondary: #7f8c8d;
+
--border-color: #e0e0e0;
+
box-sizing: border-box;
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Helvetica Neue', Arial, sans-serif;
+
color: var(--text-primary);
+
background-color: var(--background);
+
background-color: var(--surface);
+
border-bottom: 1px solid var(--border-color);
+
max-width: var(--max-width);
+
justify-content: space-between;
+
color: var(--primary-color);
+
color: var(--text-secondary);
+
padding: 0.5rem 0.75rem;
+
transition: all 0.2s ease;
+
color: var(--primary-color);
+
background-color: var(--background);
+
color: var(--secondary-color);
+
background-color: var(--background);
+
max-width: var(--max-width);
+
color: var(--text-secondary);
+
margin-bottom: 0.75rem;
+
color: var(--primary-color);
+
margin-bottom: 0.75rem;
+
color: var(--primary-color);
+
/* Entries and Threads */
+
background-color: var(--surface);
+
border: 1px solid var(--border-color);
+
/* Timeline-style entries */
+
padding: 0.5rem 0.75rem;
+
background: transparent;
+
transition: background-color 0.2s ease;
+
.timeline-entry:hover {
+
background-color: var(--surface);
+
color: var(--text-secondary);
+
margin-bottom: 0.25rem;
+
font-family: 'SF Mono', Monaco, Consolas, 'Courier New', monospace;
+
color: var(--text-secondary);
+
color: var(--primary-color);
+
.timeline-author:hover {
+
color: var(--secondary-color);
+
text-decoration: underline;
+
color: var(--primary-color);
+
.timeline-title a:hover {
+
color: var(--secondary-color);
+
text-decoration: underline;
+
color: var(--text-secondary);
+
/* Legacy styles for other sections */
+
.entry-meta, .thread-header {
+
color: var(--text-secondary);
+
color: var(--primary-color);
+
color: var(--primary-color);
+
color: var(--secondary-color);
+
text-decoration: underline;
+
color: var(--text-primary);
+
/* Enhanced Threading Styles */
+
/* Conversation Clusters */
+
.conversation-cluster {
+
background-color: var(--background);
+
border: 2px solid var(--border-color);
+
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
+
background: linear-gradient(135deg, var(--surface) 0%, #f1f3f4 100%);
+
border-bottom: 1px solid var(--border-color);
+
justify-content: space-between;
+
color: var(--secondary-color);
+
.conversation-participants {
+
color: var(--text-secondary);
+
/* Threaded Conversation Entries */
+
margin-bottom: 0.75rem;
+
align-items: flex-start;
+
.conversation-entry.level-0 {
+
.conversation-entry.level-1 {
+
.conversation-entry.level-2 {
+
.conversation-entry.level-3 {
+
.conversation-entry.level-4 {
+
background-color: var(--secondary-color);
+
.conversation-entry.level-0 .entry-connector {
+
background-color: var(--accent-color);
+
background-color: var(--surface);
+
border: 1px solid var(--border-color);
+
transition: all 0.2s ease;
+
border-color: var(--secondary-color);
+
box-shadow: 0 2px 8px rgba(52, 152, 219, 0.1);
+
/* Reference Indicators */
+
.reference-indicators {
+
background-color: #e8f5e8;
+
background-color: #e8f0ff;
+
/* Reference Badges for Individual Posts */
+
.timeline-entry.with-references {
+
background-color: var(--surface);
+
/* Conversation posts in unified timeline */
+
.timeline-entry.conversation-post {
+
background: transparent;
+
padding: 0.5rem 0.75rem;
+
.timeline-entry.conversation-post.level-0 {
+
border-left: 2px solid var(--accent-color);
+
.timeline-entry.conversation-post.level-1 {
+
border-left: 2px solid var(--secondary-color);
+
.timeline-entry.conversation-post.level-2 {
+
border-left: 2px solid var(--text-secondary);
+
.timeline-entry.conversation-post.level-3 {
+
border-left: 2px solid var(--text-secondary);
+
.timeline-entry.conversation-post.level-4 {
+
border-left: 2px solid var(--text-secondary);
+
/* Cross-thread linking */
+
border-top: 1px solid var(--border-color);
+
.cross-thread-indicator {
+
color: var(--text-secondary);
+
background-color: var(--surface);
+
padding: 0.25rem 0.5rem;
+
border: 1px solid var(--border-color);
+
/* Inline shared references styling */
+
color: var(--text-secondary);
+
color: var(--primary-color);
+
transition: color 0.2s ease;
+
.shared-ref-link:hover {
+
color: var(--secondary-color);
+
text-decoration: underline;
+
color: var(--text-secondary);
+
.user-anchor, .post-anchor {
+
margin-top: -60px; /* Offset for fixed header */
+
color: var(--primary-color);
+
transition: color 0.2s ease;
+
.cross-thread-link:hover {
+
color: var(--secondary-color);
+
text-decoration: underline;
+
padding: 0.1rem 0.4rem;
+
text-transform: uppercase;
+
letter-spacing: 0.05em;
+
.ref-badge.ref-outbound {
+
background-color: #e8f5e8;
+
border: 1px solid #c3e6c3;
+
.ref-badge.ref-inbound {
+
background-color: #e8f0ff;
+
border: 1px solid #b3d9ff;
+
/* Author Color Coding */
+
.timeline-author::before {
+
background-color: var(--secondary-color);
+
/* Generate consistent colors for authors */
+
.author-avsm::before { background-color: #e74c3c; }
+
.author-mort::before { background-color: #3498db; }
+
.author-mte::before { background-color: #2ecc71; }
+
.author-ryan::before { background-color: #f39c12; }
+
.author-mwd::before { background-color: #9b59b6; }
+
.author-dra::before { background-color: #1abc9c; }
+
.author-pf341::before { background-color: #34495e; }
+
.author-sadiqj::before { background-color: #e67e22; }
+
.author-martinkl::before { background-color: #8e44ad; }
+
.author-jonsterling::before { background-color: #27ae60; }
+
.author-jon::before { background-color: #f1c40f; }
+
.author-onkar::before { background-color: #e91e63; }
+
.author-gabriel::before { background-color: #00bcd4; }
+
.author-jess::before { background-color: #ff5722; }
+
.author-ibrahim::before { background-color: #607d8b; }
+
.author-andres::before { background-color: #795548; }
+
.author-eeg::before { background-color: #ff9800; }
+
.conversations-section h3,
+
.referenced-posts-section h3,
+
.individual-posts-section h3 {
+
border-bottom: 2px solid var(--border-color);
+
padding-bottom: 0.5rem;
+
.conversations-section h3::before {
+
.referenced-posts-section h3::before {
+
.individual-posts-section h3::before {
+
/* Legacy thread styles (for backward compatibility) */
+
background-color: var(--background);
+
border: 1px solid var(--border-color);
+
background-color: var(--surface);
+
padding: 0.5rem 0.75rem;
+
border-bottom: 1px solid var(--border-color);
+
color: var(--secondary-color);
+
padding: 0.5rem 0.75rem;
+
border-bottom: 1px solid var(--border-color);
+
.thread-entry:last-child {
+
margin-left: var(--thread-indent);
+
border-left: 3px solid var(--secondary-color);
+
background-color: var(--surface);
+
background-color: var(--background);
+
word-break: break-word;
+
color: var(--secondary-color);
+
text-decoration: underline;
+
color: var(--text-secondary);
+
color: var(--text-secondary);
+
.referencing-entries ul {
+
.referencing-entries li {
+
margin-bottom: 0.25rem;
+
.referencing-entries .more {
+
color: var(--text-secondary);
+
background-color: var(--background);
+
margin-bottom: 0.25rem;
+
color: var(--text-secondary);
+
color: var(--text-secondary);
+
color: var(--secondary-color);
+
text-decoration: underline;
+
color: var(--text-secondary);
+
margin-bottom: 0.25rem;
+
max-width: var(--max-width);
+
margin: 3rem auto 2rem;
+
color: var(--text-secondary);
+
border-top: 1px solid var(--border-color);
+
color: var(--secondary-color);
+
text-decoration: underline;
+
@media (max-width: 768px) {
+
flex-direction: column;
+
align-items: flex-start;
+
margin-left: calc(var(--thread-indent) / 2);
+
flex-direction: column;
+
<file path="src/thicket/templates/timeline.html">
+
{% extends "base.html" %}
+
{% block page_title %}Timeline - {{ title }}{% endblock %}
+
{% set seen_users = [] %}
+
<div class="page-content">
+
<h2>Recent Posts & Conversations</h2>
+
<section class="unified-timeline">
+
{% for item in timeline_items %}
+
{% if item.type == "post" %}
+
<!-- Individual Post -->
+
<article class="timeline-entry {% if item.content.references %}with-references{% endif %}">
+
<div class="timeline-meta">
+
<time datetime="{{ item.content.entry.updated or item.content.entry.published }}" class="timeline-time">
+
{{ (item.content.entry.updated or item.content.entry.published).strftime('%Y-%m-%d %H:%M') }}
+
{% set homepage = get_user_homepage(item.content.username) %}
+
{% if item.content.username not in seen_users %}
+
<a id="{{ item.content.username }}" class="user-anchor"></a>
+
{% set _ = seen_users.append(item.content.username) %}
+
<a id="post-{{ loop.index0 }}-{{ safe_anchor_id(item.content.entry.id) }}" class="post-anchor"></a>
+
<a href="{{ homepage }}" target="_blank" class="timeline-author">{{ item.content.display_name }}</a>
+
<span class="timeline-author">{{ item.content.display_name }}</span>
+
{% if item.content.references %}
+
<div class="reference-badges">
+
{% for ref in item.content.references %}
+
{% if ref.type == 'outbound' %}
+
<span class="ref-badge ref-outbound" title="References {{ ref.target_username or 'external post' }}">
+
→ {{ ref.target_username or 'ext' }}
+
{% elif ref.type == 'inbound' %}
+
<span class="ref-badge ref-inbound" title="Referenced by {{ ref.source_username or 'external post' }}">
+
← {{ ref.source_username or 'ext' }}
+
<div class="timeline-content">
+
<strong class="timeline-title">
+
<a href="{{ item.content.entry.link }}" target="_blank">{{ item.content.entry.title }}</a>
+
{% if item.content.entry.summary %}
+
<span class="timeline-summary">— {{ clean_html_summary(item.content.entry.summary, 250) }}</span>
+
{% if item.content.shared_references %}
+
<span class="inline-shared-refs">
+
{% for ref in item.content.shared_references[:3] %}
+
{% if ref.target_username %}
+
<a href="#{{ ref.target_username }}" class="shared-ref-link" title="Referenced by {{ ref.count }} entries">@{{ ref.target_username }}</a>{% if not loop.last %}, {% endif %}
+
{% if item.content.shared_references|length > 3 %}
+
<span class="shared-ref-more">+{{ item.content.shared_references|length - 3 }} more</span>
+
{% if item.content.cross_thread_links %}
+
<div class="cross-thread-links">
+
<span class="cross-thread-indicator">🔗 Also appears: </span>
+
{% for link in item.content.cross_thread_links %}
+
<a href="#{{ link.anchor_id }}" class="cross-thread-link" title="{{ link.title }}">{{ link.context }}</a>{% if not loop.last %}, {% endif %}
+
{% elif item.type == "thread" %}
+
<!-- Conversation Thread -->
+
{% set outer_loop_index = loop.index0 %}
+
{% for thread_item in item.content %}
+
<article class="timeline-entry conversation-post level-{{ thread_item.thread_level }}">
+
<div class="timeline-meta">
+
<time datetime="{{ thread_item.entry.updated or thread_item.entry.published }}" class="timeline-time">
+
{{ (thread_item.entry.updated or thread_item.entry.published).strftime('%Y-%m-%d %H:%M') }}
+
{% set homepage = get_user_homepage(thread_item.username) %}
+
{% if thread_item.username not in seen_users %}
+
<a id="{{ thread_item.username }}" class="user-anchor"></a>
+
{% set _ = seen_users.append(thread_item.username) %}
+
<a id="post-{{ outer_loop_index }}-{{ loop.index0 }}-{{ safe_anchor_id(thread_item.entry.id) }}" class="post-anchor"></a>
+
<a href="{{ homepage }}" target="_blank" class="timeline-author author-{{ thread_item.username }}">{{ thread_item.display_name }}</a>
+
<span class="timeline-author author-{{ thread_item.username }}">{{ thread_item.display_name }}</span>
+
{% if thread_item.references_to or thread_item.referenced_by %}
+
<span class="reference-indicators">
+
{% if thread_item.references_to %}
+
<span class="ref-out" title="References other posts">→</span>
+
{% if thread_item.referenced_by %}
+
<span class="ref-in" title="Referenced by other posts">←</span>
+
<div class="timeline-content">
+
<strong class="timeline-title">
+
<a href="{{ thread_item.entry.link }}" target="_blank">{{ thread_item.entry.title }}</a>
+
{% if thread_item.entry.summary %}
+
<span class="timeline-summary">— {{ clean_html_summary(thread_item.entry.summary, 300) }}</span>
+
{% if thread_item.shared_references %}
+
<span class="inline-shared-refs">
+
{% for ref in thread_item.shared_references[:3] %}
+
{% if ref.target_username %}
+
<a href="#{{ ref.target_username }}" class="shared-ref-link" title="Referenced by {{ ref.count }} entries">@{{ ref.target_username }}</a>{% if not loop.last %}, {% endif %}
+
{% if thread_item.shared_references|length > 3 %}
+
<span class="shared-ref-more">+{{ thread_item.shared_references|length - 3 }} more</span>
+
{% if thread_item.cross_thread_links %}
+
<div class="cross-thread-links">
+
<span class="cross-thread-indicator">🔗 Also appears: </span>
+
{% for link in thread_item.cross_thread_links %}
+
<a href="#{{ link.anchor_id }}" class="cross-thread-link" title="{{ link.title }}">{{ link.context }}</a>{% if not loop.last %}, {% endif %}
+
<file path="src/thicket/templates/users.html">
+
{% extends "base.html" %}
+
{% block page_title %}Users - {{ title }}{% endblock %}
+
<div class="page-content">
+
<p class="page-description">All users contributing to this thicket, ordered by post count.</p>
+
{% for user_info in users %}
+
<article class="user-card">
+
<div class="user-header">
+
{% if user_info.metadata.icon and user_info.metadata.icon != "None" %}
+
<img src="{{ user_info.metadata.icon }}" alt="{{ user_info.metadata.username }}" class="user-icon">
+
<div class="user-info">
+
{% if user_info.metadata.display_name %}
+
{{ user_info.metadata.display_name }}
+
<span class="username">({{ user_info.metadata.username }})</span>
+
{{ user_info.metadata.username }}
+
<div class="user-meta">
+
{% if user_info.metadata.homepage %}
+
<a href="{{ user_info.metadata.homepage }}" target="_blank">{{ user_info.metadata.homepage }}</a>
+
{% if user_info.metadata.email %}
+
<span class="separator">•</span>
+
<a href="mailto:{{ user_info.metadata.email }}">{{ user_info.metadata.email }}</a>
+
<span class="separator">•</span>
+
<span class="post-count">{{ user_info.metadata.entry_count }} posts</span>
+
{% if user_info.recent_entries %}
+
<div class="user-recent">
+
{% for display_name, entry in user_info.recent_entries %}
+
<a href="{{ entry.link }}" target="_blank">{{ entry.title }}</a>
+
<time datetime="{{ entry.updated or entry.published }}">
+
({{ (entry.updated or entry.published).strftime('%Y-%m-%d') }})
+
<file path="README.md">
+
A modern CLI tool for persisting Atom/RSS feeds in Git repositories, designed to enable distributed webblog comment structures.
+
- **Feed Auto-Discovery**: Automatically extracts user metadata from Atom/RSS feeds
+
- **Git Storage**: Stores feed entries in a Git repository with full history
+
- **Duplicate Management**: Manual curation of duplicate entries across feeds
+
- **Modern CLI**: Built with Typer and Rich for beautiful terminal output
+
- **Comprehensive Parsing**: Supports RSS 0.9x, RSS 1.0, RSS 2.0, and Atom feeds
+
- **Cron-Friendly**: Designed for scheduled execution
+
# Or install with dev dependencies
+
1. **Initialize a new thicket repository:**
+
thicket init ./my-feeds
+
2. **Add a user with their feed:**
+
thicket add user "alice" --feed "https://alice.example.com/feed.xml"
+
3. **Sync feeds to download entries:**
+
4. **List users and feeds:**
+
thicket init <git-store-path> [--cache-dir <path>] [--config <config-file>]
+
### Add Users and Feeds
+
# Add user with auto-discovery
+
thicket add user "username" --feed "https://example.com/feed.xml"
+
# Add user with manual metadata
+
thicket add user "username" \
+
--feed "https://example.com/feed.xml" \
+
--email "user@example.com" \
+
--homepage "https://example.com" \
+
--display-name "User Name"
+
# Add additional feed to existing user
+
thicket add feed "username" "https://example.com/other-feed.xml"
+
thicket sync --user "username"
+
# Dry run (preview changes)
+
thicket sync --all --dry-run
+
# List feeds for specific user
+
thicket list feeds --user "username"
+
thicket list entries --limit 20
+
# List entries for specific user
+
thicket list entries --user "username"
+
# List duplicate mappings
+
thicket duplicates list
+
# Mark entries as duplicates
+
thicket duplicates add "https://example.com/dup" "https://example.com/canonical"
+
# Remove duplicate mapping
+
thicket duplicates remove "https://example.com/dup"
+
Thicket uses a YAML configuration file (default: `thicket.yaml`):
+
git_store: ./feeds-repo
+
cache_dir: ~/.cache/thicket
+
- https://alice.example.com/feed.xml
+
email: alice@example.com
+
homepage: https://alice.example.com
+
## Git Repository Structure
+
├── index.json # User directory index
+
├── duplicates.json # Duplicate entry mappings
+
│ ├── metadata.json # User metadata
+
│ ├── entry_id_1.json # Feed entries
+
# Install in development mode
+
- **CLI**: Modern interface with Typer and Rich
+
- **Feed Processing**: Universal parsing with feedparser
+
- **Git Storage**: Structured storage with GitPython
+
- **Data Models**: Pydantic for validation and serialization
+
- **Async HTTP**: httpx for efficient feed fetching
+
- **Blog Aggregation**: Collect and archive blog posts from multiple sources
+
- **Comment Networks**: Enable distributed commenting systems
+
- **Feed Archival**: Preserve feed history beyond typical feed depth limits
+
- **Content Curation**: Manage and deduplicate content across feeds
+
MIT License - see LICENSE file for details.
+
<file path="src/thicket/cli/commands/index_cmd.py">
+
"""CLI command for building reference index from blog entries."""
+
from pathlib import Path
+
from typing import Optional
+
from rich.console import Console
+
from rich.progress import (
+
from rich.table import Table
+
from ...core.git_store import GitStore
+
from ...core.reference_parser import ReferenceIndex, ReferenceParser
+
from ..utils import get_tsv_mode, load_config
+
config_file: Optional[Path] = typer.Option(
+
help="Path to configuration file",
+
output_file: Optional[Path] = typer.Option(
+
help="Path to output index file (default: updates links.json in git store)",
+
verbose: bool = typer.Option(
+
help="Show detailed progress information",
+
"""Build a reference index showing which blog entries reference others.
+
This command analyzes all blog entries to detect cross-references between
+
different blogs, creating an index that can be used to build threaded
+
views of related content.
+
Updates the unified links.json file with reference data.
+
config = load_config(config_file)
+
git_store = GitStore(config.git_store)
+
# Initialize reference parser
+
parser = ReferenceParser()
+
# Build user domain mapping
+
console.print("Building user domain mapping...")
+
user_domains = parser.build_user_domain_mapping(git_store)
+
console.print(f"Found {len(user_domains)} users with {sum(len(d) for d in user_domains.values())} total domains")
+
# Initialize reference index
+
ref_index = ReferenceIndex()
+
ref_index.user_domains = user_domains
+
index = git_store._load_index()
+
users = list(index.users.keys())
+
console.print("[yellow]No users found in Git store[/yellow]")
+
TextColumn("[progress.description]{task.description}"),
+
# Count total entries first
+
counting_task = progress.add_task("Counting entries...", total=len(users))
+
entries = git_store.list_entries(username)
+
entry_counts[username] = len(entries)
+
total_entries += len(entries)
+
progress.advance(counting_task)
+
progress.remove_task(counting_task)
+
# Process entries - extract references
+
processing_task = progress.add_task(
+
f"Extracting references from {total_entries} entries...",
+
entries = git_store.list_entries(username)
+
# Extract references from this entry
+
references = parser.extract_references(entry, username, user_domains)
+
all_references.extend(references)
+
progress.advance(processing_task)
+
if verbose and references:
+
console.print(f" Found {len(references)} references in {username}:{entry.title[:50]}...")
+
progress.remove_task(processing_task)
+
# Resolve target_entry_ids for references
+
resolve_task = progress.add_task(
+
f"Resolving {len(all_references)} references...",
+
total=len(all_references)
+
console.print(f"Resolving target entry IDs for {len(all_references)} references...")
+
resolved_references = parser.resolve_target_entry_ids(all_references, git_store)
+
# Count resolved references
+
resolved_count = sum(1 for ref in resolved_references if ref.target_entry_id is not None)
+
console.print(f"Resolved {resolved_count} out of {len(all_references)} references")
+
# Add resolved references to index
+
for ref in resolved_references:
+
ref_index.add_reference(ref)
+
progress.advance(resolve_task)
+
progress.remove_task(resolve_task)
+
# Determine output path
+
output_path = output_file
+
output_path = config.git_store / "links.json"
+
# Load existing links data or create new structure
+
if output_path.exists() and not output_file:
+
# Load existing unified structure
+
with open(output_path) as f:
+
existing_data = json.load(f)
+
# Update with reference data
+
existing_data["references"] = ref_index.to_dict()["references"]
+
existing_data["user_domains"] = {k: list(v) for k, v in user_domains.items()}
+
# Save updated structure
+
with open(output_path, "w") as f:
+
json.dump(existing_data, f, indent=2, default=str)
+
console.print("\n[green]✓ Reference index built successfully[/green]")
+
# Create summary table or TSV output
+
print(f"Total Users\t{len(users)}")
+
print(f"Total Entries\t{total_entries}")
+
print(f"Total References\t{total_references}")
+
print(f"Outbound Refs\t{len(ref_index.outbound_refs)}")
+
print(f"Inbound Refs\t{len(ref_index.inbound_refs)}")
+
print(f"Output File\t{output_path}")
+
table = Table(title="Reference Index Summary")
+
table.add_column("Metric", style="cyan")
+
table.add_column("Count", style="green")
+
table.add_row("Total Users", str(len(users)))
+
table.add_row("Total Entries", str(total_entries))
+
table.add_row("Total References", str(total_references))
+
table.add_row("Outbound Refs", str(len(ref_index.outbound_refs)))
+
table.add_row("Inbound Refs", str(len(ref_index.inbound_refs)))
+
table.add_row("Output File", str(output_path))
+
# Show some interesting statistics
+
if total_references > 0:
+
console.print("\n[bold]Reference Statistics:[/bold]")
+
# Most referenced users
+
unresolved_domains = set()
+
for ref in ref_index.references:
+
if ref.target_username:
+
target_counts[ref.target_username] = target_counts.get(ref.target_username, 0) + 1
+
# Track unresolved domains
+
from urllib.parse import urlparse
+
domain = urlparse(ref.target_url).netloc.lower()
+
unresolved_domains.add(domain)
+
print("Referenced User\tReference Count")
+
for username, count in sorted(target_counts.items(), key=lambda x: x[1], reverse=True)[:5]:
+
print(f"{username}\t{count}")
+
console.print("\nMost referenced users:")
+
for username, count in sorted(target_counts.items(), key=lambda x: x[1], reverse=True)[:5]:
+
console.print(f" {username}: {count} references")
+
if unresolved_domains and verbose:
+
print("Unresolved Domain\tCount")
+
for domain in sorted(list(unresolved_domains)[:10]):
+
if len(unresolved_domains) > 10:
+
print(f"... and {len(unresolved_domains) - 10} more\t...")
+
console.print(f"\nUnresolved domains: {len(unresolved_domains)}")
+
for domain in sorted(list(unresolved_domains)[:10]):
+
console.print(f" {domain}")
+
if len(unresolved_domains) > 10:
+
console.print(f" ... and {len(unresolved_domains) - 10} more")
+
console.print(f"[red]Error building reference index: {e}[/red]")
+
console.print_exception()
+
config_file: Optional[Path] = typer.Option(
+
help="Path to configuration file",
+
index_file: Optional[Path] = typer.Option(
+
help="Path to reference index file (default: links.json in git store)",
+
username: Optional[str] = typer.Option(
+
help="Show threads for specific username only",
+
entry_id: Optional[str] = typer.Option(
+
help="Show thread for specific entry ID",
+
min_size: int = typer.Option(
+
help="Minimum thread size to display",
+
"""Show threaded view of related blog entries.
+
This command uses the reference index to show which blog entries
+
are connected through cross-references, creating an email-style
+
threaded view of the conversation.
+
Reads reference data from the unified links.json file.
+
config = load_config(config_file)
+
# Determine index file path
+
index_path = index_file
+
index_path = config.git_store / "links.json"
+
if not index_path.exists():
+
console.print(f"[red]Links file not found: {index_path}[/red]")
+
console.print("Run 'thicket links' and 'thicket index' first to build the reference index")
+
with open(index_path) as f:
+
unified_data = json.load(f)
+
# Check if references exist in the unified structure
+
if "references" not in unified_data:
+
console.print(f"[red]No references found in {index_path}[/red]")
+
console.print("Run 'thicket index' first to build the reference index")
+
# Extract reference data and reconstruct ReferenceIndex
+
ref_index = ReferenceIndex.from_dict({
+
"references": unified_data["references"],
+
"user_domains": unified_data.get("user_domains", {})
+
# Initialize Git store to get entry details
+
git_store = GitStore(config.git_store)
+
if entry_id and username:
+
thread_members = ref_index.get_thread_members(username, entry_id)
+
_display_thread(thread_members, ref_index, git_store, f"Thread for {username}:{entry_id}")
+
# Show all threads involving this user
+
user_index = git_store._load_index()
+
user = user_index.get_user(username)
+
console.print(f"[red]User not found: {username}[/red]")
+
entries = git_store.list_entries(username)
+
console.print(f"[bold]Threads involving {username}:[/bold]\n")
+
thread_members = ref_index.get_thread_members(username, entry.id)
+
if len(thread_members) >= min_size:
+
thread_key = tuple(sorted(thread_members))
+
if thread_key not in threads_found:
+
threads_found.add(thread_key)
+
_display_thread(thread_members, ref_index, git_store, f"Thread #{len(threads_found)}")
+
console.print("[bold]All conversation threads:[/bold]\n")
+
processed_entries = set()
+
user_index = git_store._load_index()
+
for username in user_index.users.keys():
+
entries = git_store.list_entries(username)
+
entry_key = (username, entry.id)
+
if entry_key in processed_entries:
+
thread_members = ref_index.get_thread_members(username, entry.id)
+
if len(thread_members) >= min_size:
+
thread_key = tuple(sorted(thread_members))
+
if thread_key not in all_threads:
+
all_threads.add(thread_key)
+
_display_thread(thread_members, ref_index, git_store, f"Thread #{len(all_threads)}")
+
# Mark all members as processed
+
for member in thread_members:
+
processed_entries.add(member)
+
console.print("[yellow]No conversation threads found[/yellow]")
+
console.print(f"(minimum thread size: {min_size})")
+
console.print(f"[red]Error showing threads: {e}[/red]")
+
def _display_thread(thread_members, ref_index, git_store, title):
+
"""Display a single conversation thread."""
+
console.print(f"[bold cyan]{title}[/bold cyan]")
+
console.print(f"Thread size: {len(thread_members)} entries")
+
# Get entry details for each member
+
for username, entry_id in thread_members:
+
entry = git_store.get_entry(username, entry_id)
+
thread_entries.append((username, entry))
+
# Sort by publication date
+
thread_entries.sort(key=lambda x: x[1].published or x[1].updated)
+
for i, (username, entry) in enumerate(thread_entries):
+
prefix = "├─" if i < len(thread_entries) - 1 else "└─"
+
# Get references for this entry
+
outbound = ref_index.get_outbound_refs(username, entry.id)
+
inbound = ref_index.get_inbound_refs(username, entry.id)
+
if outbound or inbound:
+
ref_info = f" ({len(outbound)} out, {len(inbound)} in)"
+
console.print(f" {prefix} [{username}] {entry.title[:60]}...{ref_info}")
+
console.print(f" Published: {entry.published.strftime('%Y-%m-%d')}")
+
console.print() # Empty line after each thread
+
<file path="src/thicket/cli/commands/info_cmd.py">
+
"""CLI command for displaying detailed information about a specific atom entry."""
+
from pathlib import Path
+
from typing import Optional
+
from rich.console import Console
+
from rich.panel import Panel
+
from rich.table import Table
+
from rich.text import Text
+
from ...core.git_store import GitStore
+
from ...core.reference_parser import ReferenceIndex
+
from ..utils import load_config, get_tsv_mode
+
identifier: str = typer.Argument(
+
help="The atom ID or URL of the entry to display information about"
+
username: Optional[str] = typer.Option(
+
help="Username to search for the entry (if not provided, searches all users)"
+
config_file: Optional[Path] = typer.Option(
+
help="Path to configuration file",
+
show_content: bool = typer.Option(
+
help="Include the full content of the entry in the output"
+
"""Display detailed information about a specific atom entry.
+
You can specify the entry using either its atom ID or URL.
+
Shows all metadata for the given entry, including title, dates, categories,
+
and summarizes all inbound and outbound links to/from other posts.
+
config = load_config(config_file)
+
git_store = GitStore(config.git_store)
+
# Check if identifier looks like a URL
+
is_url = identifier.startswith(('http://', 'https://'))
+
# Search specific username
+
entries = git_store.list_entries(username)
+
if str(e.link) == identifier:
+
found_username = username
+
entry = git_store.get_entry(username, identifier)
+
found_username = username
+
index = git_store._load_index()
+
for user in index.users.keys():
+
entries = git_store.list_entries(user)
+
if str(e.link) == identifier:
+
entry = git_store.get_entry(user, identifier)
+
if not entry or not found_username:
+
console.print(f"[red]Entry with {'URL' if is_url else 'atom ID'} '{identifier}' not found for user '{username}'[/red]")
+
console.print(f"[red]Entry with {'URL' if is_url else 'atom ID'} '{identifier}' not found in any user's entries[/red]")
+
# Load reference index if available
+
links_path = config.git_store / "links.json"
+
if links_path.exists():
+
with open(links_path) as f:
+
unified_data = json.load(f)
+
# Check if references exist in the unified structure
+
if "references" in unified_data:
+
ref_index = ReferenceIndex.from_dict({
+
"references": unified_data["references"],
+
"user_domains": unified_data.get("user_domains", {})
+
_display_entry_info_tsv(entry, found_username, ref_index, show_content)
+
_display_entry_info(entry, found_username)
+
_display_link_info(entry, found_username, ref_index)
+
console.print("\n[yellow]No reference index found. Run 'thicket links' and 'thicket index' to build cross-reference data.[/yellow]")
+
# Optionally display content
+
if show_content and entry.content:
+
_display_content(entry.content)
+
console.print(f"[red]Error displaying entry info: {e}[/red]")
+
def _display_entry_info(entry, username: str) -> None:
+
"""Display basic entry information in a structured format."""
+
# Create main info panel
+
info_table = Table.grid(padding=(0, 2))
+
info_table.add_column("Field", style="cyan bold", width=15)
+
info_table.add_column("Value", style="white")
+
info_table.add_row("User", f"[green]{username}[/green]")
+
info_table.add_row("Atom ID", f"[blue]{entry.id}[/blue]")
+
info_table.add_row("Title", entry.title)
+
info_table.add_row("Link", str(entry.link))
+
info_table.add_row("Published", entry.published.strftime("%Y-%m-%d %H:%M:%S UTC"))
+
info_table.add_row("Updated", entry.updated.strftime("%Y-%m-%d %H:%M:%S UTC"))
+
# Truncate long summaries
+
summary = entry.summary[:200] + "..." if len(entry.summary) > 200 else entry.summary
+
info_table.add_row("Summary", summary)
+
categories_text = ", ".join(entry.categories)
+
info_table.add_row("Categories", categories_text)
+
if "name" in entry.author:
+
author_info.append(entry.author["name"])
+
if "email" in entry.author:
+
author_info.append(f"<{entry.author['email']}>")
+
info_table.add_row("Author", " ".join(author_info))
+
info_table.add_row("Content Type", entry.content_type)
+
info_table.add_row("Rights", entry.rights)
+
info_table.add_row("Source Feed", entry.source)
+
title=f"[bold]Entry Information[/bold]",
+
def _display_link_info(entry, username: str, ref_index: ReferenceIndex) -> None:
+
"""Display inbound and outbound link information."""
+
outbound_refs = ref_index.get_outbound_refs(username, entry.id)
+
inbound_refs = ref_index.get_inbound_refs(username, entry.id)
+
if not outbound_refs and not inbound_refs:
+
console.print("\n[dim]No cross-references found for this entry.[/dim]")
+
links_table = Table(title="Cross-References")
+
links_table.add_column("Direction", style="cyan", width=10)
+
links_table.add_column("Target/Source", style="green", width=20)
+
links_table.add_column("URL", style="blue", width=50)
+
# Add outbound references
+
for ref in outbound_refs:
+
target_info = f"{ref.target_username}:{ref.target_entry_id}" if ref.target_username and ref.target_entry_id else "External"
+
links_table.add_row("→ Out", target_info, ref.target_url)
+
# Add inbound references
+
for ref in inbound_refs:
+
source_info = f"{ref.source_username}:{ref.source_entry_id}"
+
links_table.add_row("← In", source_info, ref.target_url)
+
console.print(links_table)
+
console.print(f"\n[bold]Summary:[/bold] {len(outbound_refs)} outbound, {len(inbound_refs)} inbound references")
+
def _display_content(content: str) -> None:
+
"""Display the full content of the entry."""
+
# Truncate very long content
+
display_content = content
+
if len(content) > 5000:
+
display_content = content[:5000] + "\n\n[... content truncated ...]"
+
title="[bold]Entry Content[/bold]",
+
def _display_entry_info_tsv(entry, username: str, ref_index: Optional[ReferenceIndex], show_content: bool) -> None:
+
"""Display entry information in TSV format."""
+
print(f"User\t{username}")
+
print(f"Atom ID\t{entry.id}")
+
print(f"Title\t{entry.title.replace(chr(9), ' ').replace(chr(10), ' ').replace(chr(13), ' ')}")
+
print(f"Link\t{entry.link}")
+
print(f"Published\t{entry.published.strftime('%Y-%m-%d %H:%M:%S UTC')}")
+
print(f"Updated\t{entry.updated.strftime('%Y-%m-%d %H:%M:%S UTC')}")
+
# Escape tabs and newlines in summary
+
summary = entry.summary.replace('\t', ' ').replace('\n', ' ').replace('\r', ' ')
+
print(f"Summary\t{summary}")
+
print(f"Categories\t{', '.join(entry.categories)}")
+
if "name" in entry.author:
+
author_info.append(entry.author["name"])
+
if "email" in entry.author:
+
author_info.append(f"<{entry.author['email']}>")
+
print(f"Author\t{' '.join(author_info)}")
+
print(f"Content Type\t{entry.content_type}")
+
print(f"Rights\t{entry.rights}")
+
print(f"Source Feed\t{entry.source}")
+
# Add reference info if available
+
outbound_refs = ref_index.get_outbound_refs(username, entry.id)
+
inbound_refs = ref_index.get_inbound_refs(username, entry.id)
+
print(f"Outbound References\t{len(outbound_refs)}")
+
print(f"Inbound References\t{len(inbound_refs)}")
+
for ref in outbound_refs:
+
target_info = f"{ref.target_username}:{ref.target_entry_id}" if ref.target_username and ref.target_entry_id else "External"
+
print(f"Outbound Reference\t{target_info}\t{ref.target_url}")
+
for ref in inbound_refs:
+
source_info = f"{ref.source_username}:{ref.source_entry_id}"
+
print(f"Inbound Reference\t{source_info}\t{ref.target_url}")
+
# Show content if requested
+
if show_content and entry.content:
+
# Escape tabs and newlines in content
+
content = entry.content.replace('\t', ' ').replace('\n', ' ').replace('\r', ' ')
+
print(f"Content\t{content}")
+
<file path="src/thicket/cli/commands/init.py">
+
"""Initialize command for thicket."""
+
from pathlib import Path
+
from typing import Optional
+
from pydantic import ValidationError
+
from ...core.git_store import GitStore
+
from ...models import ThicketConfig
+
from ..utils import print_error, print_success, save_config
+
git_store: Path = typer.Argument(..., help="Path to Git repository for storing feeds"),
+
cache_dir: Optional[Path] = typer.Option(
+
None, "--cache-dir", "-c", help="Cache directory (default: ~/.cache/thicket)"
+
config_file: Optional[Path] = typer.Option(
+
None, "--config", help="Configuration file path (default: thicket.yaml)"
+
force: bool = typer.Option(
+
False, "--force", "-f", help="Overwrite existing configuration"
+
"""Initialize a new thicket configuration and Git store."""
+
from platformdirs import user_cache_dir
+
cache_dir = Path(user_cache_dir("thicket"))
+
if config_file is None:
+
config_file = Path("thicket.yaml")
+
# Check if config already exists
+
if config_file.exists() and not force:
+
print_error(f"Configuration file already exists: {config_file}")
+
print_error("Use --force to overwrite")
+
# Create cache directory
+
cache_dir.mkdir(parents=True, exist_ok=True)
+
print_success(f"Initialized Git store at: {git_store}")
+
print_error(f"Failed to initialize Git store: {e}")
+
raise typer.Exit(1) from e
+
config = ThicketConfig(
+
save_config(config, config_file)
+
print_success(f"Created configuration file: {config_file}")
+
except ValidationError as e:
+
print_error(f"Invalid configuration: {e}")
+
raise typer.Exit(1) from e
+
print_error(f"Failed to create configuration: {e}")
+
raise typer.Exit(1) from e
+
print_success("Thicket initialized successfully!")
+
print_success(f"Git store: {git_store}")
+
print_success(f"Cache directory: {cache_dir}")
+
print_success(f"Configuration: {config_file}")
+
print_success("Run 'thicket add user' to add your first user and feed.")
+
<file path="src/thicket/cli/__init__.py">
+
"""CLI interface for thicket."""
+
<file path="src/thicket/core/__init__.py">
+
"""Core business logic for thicket."""
+
from .feed_parser import FeedParser
+
from .git_store import GitStore
+
__all__ = ["FeedParser", "GitStore"]
+
<file path="src/thicket/core/feed_parser.py">
+
"""Feed parsing and normalization with auto-discovery."""
+
from datetime import datetime
+
from typing import Optional
+
from urllib.parse import urlparse
+
from pydantic import HttpUrl, ValidationError
+
from ..models import AtomEntry, FeedMetadata
+
"""Parser for RSS/Atom feeds with normalization and auto-discovery."""
+
def __init__(self, user_agent: str = "thicket/0.1.0"):
+
"""Initialize the feed parser."""
+
self.user_agent = user_agent
+
"a", "abbr", "acronym", "b", "blockquote", "br", "code", "em",
+
"i", "li", "ol", "p", "pre", "strong", "ul", "h1", "h2", "h3",
+
"h4", "h5", "h6", "img", "div", "span",
+
self.allowed_attributes = {
+
"a": ["href", "title"],
+
"img": ["src", "alt", "title", "width", "height"],
+
"blockquote": ["cite"],
+
async def fetch_feed(self, url: HttpUrl) -> str:
+
"""Fetch feed content from URL."""
+
async with httpx.AsyncClient() as client:
+
response = await client.get(
+
headers={"User-Agent": self.user_agent},
+
response.raise_for_status()
+
def parse_feed(self, content: str, source_url: Optional[HttpUrl] = None) -> tuple[FeedMetadata, list[AtomEntry]]:
+
"""Parse feed content and return metadata and entries."""
+
parsed = feedparser.parse(content)
+
if parsed.bozo and parsed.bozo_exception:
+
# Try to continue with potentially malformed feed
+
# Extract feed metadata
+
feed_meta = self._extract_feed_metadata(parsed.feed)
+
# Extract and normalize entries
+
for entry in parsed.entries:
+
atom_entry = self._normalize_entry(entry, source_url)
+
entries.append(atom_entry)
+
# Log error but continue processing other entries
+
print(f"Error processing entry {getattr(entry, 'id', 'unknown')}: {e}")
+
return feed_meta, entries
+
def _extract_feed_metadata(self, feed: feedparser.FeedParserDict) -> FeedMetadata:
+
"""Extract metadata from feed for auto-discovery."""
+
# Parse author information
+
if hasattr(feed, 'author_detail'):
+
author_name = feed.author_detail.get('name')
+
author_email = feed.author_detail.get('email')
+
author_uri = feed.author_detail.get('href')
+
elif hasattr(feed, 'author'):
+
author_name = feed.author
+
# Parse managing editor for RSS feeds
+
if not author_email and hasattr(feed, 'managingEditor'):
+
author_email = feed.managingEditor
+
if hasattr(feed, 'link'):
+
feed_link = HttpUrl(feed.link)
+
except ValidationError:
+
# Parse image/icon/logo
+
if hasattr(feed, 'image'):
+
image_url = HttpUrl(feed.image.get('href', feed.image.get('url', '')))
+
except (ValidationError, AttributeError):
+
if hasattr(feed, 'icon'):
+
icon = HttpUrl(feed.icon)
+
except ValidationError:
+
if hasattr(feed, 'logo'):
+
logo = HttpUrl(feed.logo)
+
except ValidationError:
+
title=getattr(feed, 'title', None),
+
author_name=author_name,
+
author_email=author_email,
+
author_uri=HttpUrl(author_uri) if author_uri else None,
+
description=getattr(feed, 'description', None),
+
def _normalize_entry(self, entry: feedparser.FeedParserDict, source_url: Optional[HttpUrl] = None) -> AtomEntry:
+
"""Normalize an entry to Atom format."""
+
updated = self._parse_timestamp(entry.get('updated_parsed') or entry.get('published_parsed'))
+
published = self._parse_timestamp(entry.get('published_parsed'))
+
content = self._extract_content(entry)
+
content_type = self._extract_content_type(entry)
+
author = self._extract_author(entry)
+
# Parse categories/tags
+
if hasattr(entry, 'tags'):
+
categories = [tag.get('term', '') for tag in entry.tags if tag.get('term')]
+
# Sanitize HTML content
+
content = self._sanitize_html(content)
+
summary = entry.get('summary', '')
+
summary = self._sanitize_html(summary)
+
id=entry.get('id', entry.get('link', '')),
+
title=entry.get('title', ''),
+
link=HttpUrl(entry.get('link', '')),
+
summary=summary or None,
+
content=content or None,
+
content_type=content_type,
+
rights=entry.get('rights', None),
+
source=str(source_url) if source_url else None,
+
def _parse_timestamp(self, time_struct) -> datetime:
+
"""Parse feedparser time struct to datetime."""
+
return datetime(*time_struct[:6])
+
def _extract_content(self, entry: feedparser.FeedParserDict) -> Optional[str]:
+
"""Extract the best content from an entry."""
+
# Prefer content over summary
+
if hasattr(entry, 'content') and entry.content:
+
# Find the best content (prefer text/html, then text/plain)
+
for content_item in entry.content:
+
if content_item.get('type') in ['text/html', 'html']:
+
return content_item.get('value', '')
+
elif content_item.get('type') in ['text/plain', 'text']:
+
return content_item.get('value', '')
+
# Fallback to first content item
+
return entry.content[0].get('value', '')
+
return entry.get('summary', '')
+
def _extract_content_type(self, entry: feedparser.FeedParserDict) -> str:
+
"""Extract content type from entry."""
+
if hasattr(entry, 'content') and entry.content:
+
content_type = entry.content[0].get('type', 'html')
+
# Normalize content type
+
if content_type in ['text/html', 'html']:
+
elif content_type in ['text/plain', 'text']:
+
elif content_type == 'xhtml':
+
def _extract_author(self, entry: feedparser.FeedParserDict) -> Optional[dict]:
+
"""Extract author information from entry."""
+
if hasattr(entry, 'author_detail'):
+
'name': entry.author_detail.get('name'),
+
'email': entry.author_detail.get('email'),
+
'uri': entry.author_detail.get('href'),
+
elif hasattr(entry, 'author'):
+
author['name'] = entry.author
+
return author if author else None
+
def _sanitize_html(self, html: str) -> str:
+
"""Sanitize HTML content to prevent XSS."""
+
tags=self.allowed_tags,
+
attributes=self.allowed_attributes,
+
def sanitize_entry_id(self, entry_id: str) -> str:
+
"""Sanitize entry ID to be a safe filename."""
+
# Parse URL to get meaningful parts
+
parsed = urlparse(entry_id)
+
# Start with the path component
+
# Remove leading slash and replace problematic characters
+
safe_id = parsed.path.lstrip('/').replace('/', '_').replace('\\', '_')
+
# Use the entire ID as fallback
+
# Replace problematic characters
+
if char.isalnum() or char in '-_.':
+
safe_chars.append(char)
+
safe_id = ''.join(safe_chars)
+
# Ensure it's not too long (max 200 chars)
+
safe_id = safe_id[:200]
+
# Ensure it's not empty
+
<file path="src/thicket/core/reference_parser.py">
+
"""Reference detection and parsing for blog entries."""
+
from typing import Optional
+
from urllib.parse import urlparse
+
from ..models import AtomEntry
+
"""Represents a reference from one blog entry to another."""
+
target_username: Optional[str] = None,
+
target_entry_id: Optional[str] = None,
+
self.source_entry_id = source_entry_id
+
self.source_username = source_username
+
self.target_url = target_url
+
self.target_username = target_username
+
self.target_entry_id = target_entry_id
+
def to_dict(self) -> dict:
+
"""Convert to dictionary for JSON serialization."""
+
"source_entry_id": self.source_entry_id,
+
"source_username": self.source_username,
+
"target_url": self.target_url,
+
# Only include optional fields if they are not None
+
if self.target_username is not None:
+
result["target_username"] = self.target_username
+
if self.target_entry_id is not None:
+
result["target_entry_id"] = self.target_entry_id
+
def from_dict(cls, data: dict) -> "BlogReference":
+
"""Create from dictionary."""
+
source_entry_id=data["source_entry_id"],
+
source_username=data["source_username"],
+
target_url=data["target_url"],
+
target_username=data.get("target_username"),
+
target_entry_id=data.get("target_entry_id"),
+
"""Index of blog-to-blog references for creating threaded views."""
+
self.references: list[BlogReference] = []
+
self.outbound_refs: dict[
+
str, list[BlogReference]
+
] = {} # entry_id -> outbound refs
+
self.inbound_refs: dict[
+
str, list[BlogReference]
+
] = {} # entry_id -> inbound refs
+
self.user_domains: dict[str, set[str]] = {} # username -> set of domains
+
def add_reference(self, ref: BlogReference) -> None:
+
"""Add a reference to the index."""
+
self.references.append(ref)
+
# Update outbound references
+
source_key = f"{ref.source_username}:{ref.source_entry_id}"
+
if source_key not in self.outbound_refs:
+
self.outbound_refs[source_key] = []
+
self.outbound_refs[source_key].append(ref)
+
# Update inbound references if we can identify the target
+
if ref.target_username and ref.target_entry_id:
+
target_key = f"{ref.target_username}:{ref.target_entry_id}"
+
if target_key not in self.inbound_refs:
+
self.inbound_refs[target_key] = []
+
self.inbound_refs[target_key].append(ref)
+
def get_outbound_refs(self, username: str, entry_id: str) -> list[BlogReference]:
+
"""Get all outbound references from an entry."""
+
key = f"{username}:{entry_id}"
+
return self.outbound_refs.get(key, [])
+
def get_inbound_refs(self, username: str, entry_id: str) -> list[BlogReference]:
+
"""Get all inbound references to an entry."""
+
key = f"{username}:{entry_id}"
+
return self.inbound_refs.get(key, [])
+
def get_thread_members(self, username: str, entry_id: str) -> set[tuple[str, str]]:
+
"""Get all entries that are part of the same thread."""
+
to_visit = [(username, entry_id)]
+
current_user, current_entry = to_visit.pop()
+
if (current_user, current_entry) in visited:
+
visited.add((current_user, current_entry))
+
thread_members.add((current_user, current_entry))
+
# Add outbound references
+
for ref in self.get_outbound_refs(current_user, current_entry):
+
if ref.target_username and ref.target_entry_id:
+
to_visit.append((ref.target_username, ref.target_entry_id))
+
# Add inbound references
+
for ref in self.get_inbound_refs(current_user, current_entry):
+
to_visit.append((ref.source_username, ref.source_entry_id))
+
def to_dict(self) -> dict:
+
"""Convert to dictionary for JSON serialization."""
+
"references": [ref.to_dict() for ref in self.references],
+
"user_domains": {k: list(v) for k, v in self.user_domains.items()},
+
def from_dict(cls, data: dict) -> "ReferenceIndex":
+
"""Create from dictionary."""
+
for ref_data in data.get("references", []):
+
ref = BlogReference.from_dict(ref_data)
+
index.add_reference(ref)
+
for username, domains in data.get("user_domains", {}).items():
+
index.user_domains[username] = set(domains)
+
"""Parses blog entries to detect references to other blogs."""
+
# Common blog platforms and patterns
+
r"https?://[^/]+\.(?:org|com|net|io|dev|me|co\.uk)/.*", # Common blog domains
+
r"https?://[^/]+\.github\.io/.*", # GitHub Pages
+
r"https?://[^/]+\.substack\.com/.*", # Substack
+
r"https?://medium\.com/.*", # Medium
+
r"https?://[^/]+\.wordpress\.com/.*", # WordPress.com
+
r"https?://[^/]+\.blogspot\.com/.*", # Blogger
+
# Compile regex patterns
+
self.link_pattern = re.compile(
+
r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', re.IGNORECASE | re.DOTALL
+
self.url_pattern = re.compile(r'https?://[^\s<>"]+')
+
def extract_links_from_html(self, html_content: str) -> list[tuple[str, str]]:
+
"""Extract all links from HTML content."""
+
# Extract links from <a> tags
+
for match in self.link_pattern.finditer(html_content):
+
r"<[^>]+>", "", match.group(2)
+
).strip() # Remove HTML tags from link text
+
links.append((url, text))
+
def is_blog_url(self, url: str) -> bool:
+
"""Check if a URL likely points to a blog post."""
+
for pattern in self.blog_patterns:
+
if re.match(pattern, url):
+
def _is_likely_blog_post_url(self, url: str) -> bool:
+
"""Check if a same-domain URL likely points to a blog post (not CSS, images, etc.)."""
+
parsed_url = urlparse(url)
+
path = parsed_url.path.lower()
+
# Skip obvious non-blog content
+
if any(path.endswith(ext) for ext in ['.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.pdf', '.xml', '.json']):
+
# Skip common non-blog paths
+
if any(segment in path for segment in ['/static/', '/assets/', '/css/', '/js/', '/images/', '/img/', '/media/', '/uploads/']):
+
# Skip fragment-only links (same page anchors)
+
if not path or path == '/':
+
# Look for positive indicators of blog posts
+
# Common blog post patterns: dates, slugs, post indicators
+
r'/\d{4}/', # Year in path
+
r'/\d{4}/\d{2}/', # Year/month in path
+
for pattern in blog_indicators:
+
if re.search(pattern, path):
+
# If it has a reasonable path depth and doesn't match exclusions, likely a blog post
+
path_segments = [seg for seg in path.split('/') if seg]
+
return len(path_segments) >= 1 # At least one meaningful path segment
+
def resolve_target_user(
+
self, url: str, user_domains: dict[str, set[str]]
+
"""Try to resolve a URL to a known user based on domain mapping."""
+
parsed_url = urlparse(url)
+
domain = parsed_url.netloc.lower()
+
for username, domains in user_domains.items():
+
def extract_references(
+
self, entry: AtomEntry, username: str, user_domains: dict[str, set[str]]
+
) -> list[BlogReference]:
+
"""Extract all blog references from an entry."""
+
# Combine all text content for analysis
+
content_to_search.append(entry.content)
+
content_to_search.append(entry.summary)
+
for content in content_to_search:
+
links = self.extract_links_from_html(content)
+
for url, _link_text in links:
+
urlparse(str(entry.link)).netloc.lower() if entry.link else ""
+
link_domain = urlparse(url).netloc.lower()
+
# Check if this looks like a blog URL
+
if not self.is_blog_url(url):
+
# For same-domain links, apply additional filtering to avoid non-blog content
+
if link_domain == entry_domain:
+
# Only include same-domain links that look like blog posts
+
if not self._is_likely_blog_post_url(url):
+
# Try to resolve to a known user
+
if link_domain == entry_domain:
+
# Same domain - target user is the same as source user
+
target_username: Optional[str] = username
+
# Different domain - try to resolve
+
target_username = self.resolve_target_user(url, user_domains)
+
source_entry_id=entry.id,
+
source_username=username,
+
target_username=target_username,
+
target_entry_id=None, # Will be resolved later if possible
+
def build_user_domain_mapping(self, git_store: "GitStore") -> dict[str, set[str]]:
+
"""Build mapping of usernames to their known domains."""
+
index = git_store._load_index()
+
for username, user_metadata in index.users.items():
+
# Add domains from feeds
+
for feed_url in user_metadata.feeds:
+
domain = urlparse(feed_url).netloc.lower()
+
# Add domain from homepage
+
if user_metadata.homepage:
+
domain = urlparse(str(user_metadata.homepage)).netloc.lower()
+
user_domains[username] = domains
+
def _build_url_to_entry_mapping(self, git_store: "GitStore") -> dict[str, str]:
+
"""Build a comprehensive mapping from URLs to entry IDs using git store data.
+
This creates a bidirectional mapping that handles:
+
- Entry link URLs -> Entry IDs
+
- URL variations (with/without www, http/https)
+
- Multiple URLs pointing to the same entry
+
url_to_entry: dict[str, str] = {}
+
# Load index to get all users
+
index = git_store._load_index()
+
for username in index.users.keys():
+
entries = git_store.list_entries(username)
+
link_url = str(entry.link)
+
# Map the canonical link URL
+
url_to_entry[link_url] = entry_id
+
# Handle common URL variations
+
parsed = urlparse(link_url)
+
if parsed.netloc and parsed.path:
+
# Add version without www
+
if parsed.netloc.startswith('www.'):
+
no_www_url = f"{parsed.scheme}://{parsed.netloc[4:]}{parsed.path}"
+
no_www_url += f"?{parsed.query}"
+
no_www_url += f"#{parsed.fragment}"
+
url_to_entry[no_www_url] = entry_id
+
# Add version with www if not present
+
elif not parsed.netloc.startswith('www.'):
+
www_url = f"{parsed.scheme}://www.{parsed.netloc}{parsed.path}"
+
www_url += f"?{parsed.query}"
+
www_url += f"#{parsed.fragment}"
+
url_to_entry[www_url] = entry_id
+
# Add http/https variations
+
if parsed.scheme == 'https':
+
http_url = link_url.replace('https://', 'http://', 1)
+
url_to_entry[http_url] = entry_id
+
elif parsed.scheme == 'http':
+
https_url = link_url.replace('http://', 'https://', 1)
+
url_to_entry[https_url] = entry_id
+
def _normalize_url(self, url: str) -> str:
+
"""Normalize URL for consistent matching.
+
Handles common variations like trailing slashes, fragments, etc.
+
# Remove trailing slash from path
+
path = parsed.path.rstrip('/') if parsed.path != '/' else parsed.path
+
# Reconstruct without fragment for consistent matching
+
normalized = f"{parsed.scheme}://{parsed.netloc}{path}"
+
normalized += f"?{parsed.query}"
+
def resolve_target_entry_ids(
+
self, references: list[BlogReference], git_store: "GitStore"
+
) -> list[BlogReference]:
+
"""Resolve target_entry_id for references using comprehensive URL mapping."""
+
# Build comprehensive URL to entry ID mapping
+
url_to_entry = self._build_url_to_entry_mapping(git_store)
+
# If we already have a target_entry_id, keep the reference as-is
+
if ref.target_entry_id is not None:
+
resolved_refs.append(ref)
+
# If we don't have a target_username, we can't resolve it
+
if ref.target_username is None:
+
resolved_refs.append(ref)
+
# Try to resolve using URL mapping
+
resolved_entry_id = None
+
# First, try exact match
+
if ref.target_url in url_to_entry:
+
resolved_entry_id = url_to_entry[ref.target_url]
+
# Try normalized URL matching
+
normalized_target = self._normalize_url(ref.target_url)
+
if normalized_target in url_to_entry:
+
resolved_entry_id = url_to_entry[normalized_target]
+
for mapped_url, entry_id in url_to_entry.items():
+
if self._normalize_url(mapped_url) == normalized_target:
+
resolved_entry_id = entry_id
+
# Verify the resolved entry belongs to the target username
+
# Double-check by loading the actual entry
+
entries = git_store.list_entries(ref.target_username)
+
entry_found = any(entry.id == resolved_entry_id for entry in entries)
+
resolved_entry_id = None
+
# Create a new reference with the resolved target_entry_id
+
resolved_ref = BlogReference(
+
source_entry_id=ref.source_entry_id,
+
source_username=ref.source_username,
+
target_url=ref.target_url,
+
target_username=ref.target_username,
+
target_entry_id=resolved_entry_id,
+
resolved_refs.append(resolved_ref)
+
<file path="src/thicket/models/__init__.py">
+
"""Data models for thicket."""
+
from .config import ThicketConfig, UserConfig
+
from .feed import AtomEntry, DuplicateMap, FeedMetadata
+
from .user import GitStoreIndex, UserMetadata
+
<file path="src/thicket/models/feed.py">
+
"""Feed and entry models for thicket."""
+
from datetime import datetime
+
from typing import TYPE_CHECKING, Optional
+
from pydantic import BaseModel, ConfigDict, EmailStr, HttpUrl
+
from .config import UserConfig
+
class AtomEntry(BaseModel):
+
"""Represents an Atom feed entry stored in the Git repository."""
+
model_config = ConfigDict(
+
json_encoders={datetime: lambda v: v.isoformat()},
+
str_strip_whitespace=True,
+
id: str # Original Atom ID
+
published: Optional[datetime] = None
+
summary: Optional[str] = None
+
content: Optional[str] = None # Full body content from Atom entry
+
content_type: Optional[str] = "html" # text, html, xhtml
+
author: Optional[dict] = None
+
categories: list[str] = []
+
rights: Optional[str] = None # Copyright info
+
source: Optional[str] = None # Source feed URL
+
class FeedMetadata(BaseModel):
+
"""Metadata extracted from a feed for auto-discovery."""
+
title: Optional[str] = None
+
author_name: Optional[str] = None
+
author_email: Optional[EmailStr] = None
+
author_uri: Optional[HttpUrl] = None
+
link: Optional[HttpUrl] = None
+
logo: Optional[HttpUrl] = None
+
icon: Optional[HttpUrl] = None
+
image_url: Optional[HttpUrl] = None
+
description: Optional[str] = None
+
def to_user_config(self, username: str, feed_url: HttpUrl) -> "UserConfig":
+
"""Convert discovered metadata to UserConfig with fallbacks."""
+
from .config import UserConfig
+
display_name=self.author_name or self.title,
+
email=self.author_email,
+
homepage=self.author_uri or self.link,
+
icon=self.logo or self.icon or self.image_url,
+
class DuplicateMap(BaseModel):
+
"""Maps duplicate entry IDs to canonical entry IDs."""
+
duplicates: dict[str, str] = {} # duplicate_id -> canonical_id
+
comment: str = "Entry IDs that map to the same canonical content"
+
def add_duplicate(self, duplicate_id: str, canonical_id: str) -> None:
+
"""Add a duplicate mapping."""
+
self.duplicates[duplicate_id] = canonical_id
+
def remove_duplicate(self, duplicate_id: str) -> bool:
+
"""Remove a duplicate mapping. Returns True if existed."""
+
return self.duplicates.pop(duplicate_id, None) is not None
+
def get_canonical(self, entry_id: str) -> str:
+
"""Get canonical ID for an entry (returns original if not duplicate)."""
+
return self.duplicates.get(entry_id, entry_id)
+
def is_duplicate(self, entry_id: str) -> bool:
+
"""Check if entry ID is marked as duplicate."""
+
return entry_id in self.duplicates
+
def get_duplicates_for_canonical(self, canonical_id: str) -> list[str]:
+
"""Get all duplicate IDs that map to a canonical ID."""
+
for duplicate_id, canonical in self.duplicates.items()
+
if canonical == canonical_id
+
<file path="src/thicket/models/user.py">
+
"""User metadata models for thicket."""
+
from datetime import datetime
+
from typing import Optional
+
from pydantic import BaseModel, ConfigDict
+
class UserMetadata(BaseModel):
+
"""Metadata about a user stored in the Git repository."""
+
model_config = ConfigDict(
+
json_encoders={datetime: lambda v: v.isoformat()},
+
str_strip_whitespace=True,
+
display_name: Optional[str] = None
+
email: Optional[str] = None
+
homepage: Optional[str] = None
+
icon: Optional[str] = None
+
directory: str # Directory name in Git store
+
def update_timestamp(self) -> None:
+
"""Update the last_updated timestamp to now."""
+
self.last_updated = datetime.now()
+
def increment_entry_count(self, count: int = 1) -> None:
+
"""Increment the entry count by the given amount."""
+
self.entry_count += count
+
self.update_timestamp()
+
class GitStoreIndex(BaseModel):
+
"""Index of all users and their directories in the Git store."""
+
model_config = ConfigDict(
+
json_encoders={datetime: lambda v: v.isoformat()}
+
users: dict[str, UserMetadata] = {} # username -> UserMetadata
+
def add_user(self, user_metadata: UserMetadata) -> None:
+
"""Add or update a user in the index."""
+
self.users[user_metadata.username] = user_metadata
+
self.last_updated = datetime.now()
+
def remove_user(self, username: str) -> bool:
+
"""Remove a user from the index. Returns True if user existed."""
+
if username in self.users:
+
del self.users[username]
+
self.last_updated = datetime.now()
+
def get_user(self, username: str) -> Optional[UserMetadata]:
+
"""Get user metadata by username."""
+
return self.users.get(username)
+
def update_entry_count(self, username: str, count: int) -> None:
+
"""Update entry count for a user and total."""
+
user = self.get_user(username)
+
user.increment_entry_count(count)
+
self.total_entries += count
+
self.last_updated = datetime.now()
+
def recalculate_totals(self) -> None:
+
"""Recalculate total entries from all users."""
+
self.total_entries = sum(user.entry_count for user in self.users.values())
+
self.last_updated = datetime.now()
+
<file path="src/thicket/utils/__init__.py">
+
"""Utility modules for thicket."""
+
# This module will contain shared utilities
+
# For now, it's empty but can be expanded with common functions
+
<file path="src/thicket/__init__.py">
+
"""Thicket: A CLI tool for persisting Atom/RSS feeds in Git repositories."""
+
__email__ = "thicket@example.com"
+
<file path="src/thicket/__main__.py">
+
"""Entry point for running thicket as a module."""
+
from .cli.main import app
+
if __name__ == "__main__":
+
<file path=".gitignore">
+
# Byte-compiled / optimized / DLL files
+
# Distribution / packaging
+
# Usually these files are written by a python script from a template
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
+
pip-delete-this-directory.txt
+
# Unit test / coverage reports
+
# For a library or package, you might want to ignore these files since the code is
+
# intended to run in multiple environments; otherwise, check them in:
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
+
# install all needed dependencies.
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
+
# commonly ignored for libraries.
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
+
# commonly ignored for libraries.
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+
# in the .venv directory. It is recommended not to include this directory in version control.
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+
# SageMath parsed files
+
# Spyder project settings
+
# Rope project settings
+
# pytype static type analyzer
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+
# Abstra is an AI-powered process automation framework.
+
# Ignore directories containing user credentials, local state, and settings.
+
# Learn more at https://abstra.io/docs
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
+
# you could uncomment the following to ignore the entire vscode folder
+
# PyPI configuration file
+
.streamlit/secrets.toml
+
<file path="CLAUDE.md">
+
My goal is to build a CLI tool called thicket in Python that maintains a Git repository within which Atom feeds can be persisted, including their contents.
+
# Python Environment and Package Management
+
This project uses `uv` for Python package management and virtual environment handling.
+
ALWAYS use `uv run` to execute Python commands:
+
- Run the CLI: `uv run -m thicket`
+
- Run tests: `uv run pytest`
+
- Type checking: `uv run mypy src/`
+
- Linting: `uv run ruff check src/`
+
- Format code: `uv run ruff format src/`
+
- Compile check: `uv run python -m py_compile <file>`
+
- Add dependencies: `uv add <package>`
+
- Add dev dependencies: `uv add --dev <package>`
+
- Install dependencies: `uv sync`
+
- Update dependencies: `uv lock --upgrade`
+
The configuration file specifies:
+
- the location of a git store
+
- a list of usernames and target Atom/RSS feed(s) and optional metadata about the username such as their email, homepage, icon and display name
+
- a cache directory to store temporary results such as feed downloads and their last modification date that speed up operations across runs of the tool
+
The Git data store should:
+
- have a subdirectory per user
+
- within that directory, an entry per Atom entry indexed by the Atom id for that entry. The id should be sanitised consistently to be a safe filename. RSS feed should be normalized to Atom before storing it.
+
- within each entry file, the metadata of the Atom feed converted into a JSON format that preserves as much metadata as possible.
+
- have a JSON file in the Git repository that indexes the users, their associated directories within the Git repository, and any other metadata about that user from the config file
+
The CLI should be modern and use cool progress bars and any otfrom ecosystem libraries.
+
The intention behind the Git repository is that it can be queried by other websites in order to build a webblog structure of comments that link to other blogs.
+
<file path="pyproject.toml">
+
requires = ["hatchling"]
+
build-backend = "hatchling.build"
+
description = "A CLI tool for persisting Atom/RSS feeds in Git repositories"
+
requires-python = ">=3.9"
+
{name = "thicket", email = "thicket@example.com"},
+
"Development Status :: 3 - Alpha",
+
"Intended Audience :: Developers",
+
"License :: OSI Approved :: MIT License",
+
"Operating System :: OS Independent",
+
"Programming Language :: Python :: 3",
+
"Programming Language :: Python :: 3.9",
+
"Programming Language :: Python :: 3.10",
+
"Programming Language :: Python :: 3.11",
+
"Programming Language :: Python :: 3.12",
+
"Programming Language :: Python :: 3.13",
+
"Topic :: Internet :: WWW/HTTP :: Dynamic Content :: News/Diary",
+
"Topic :: Software Development :: Version Control :: Git",
+
"Topic :: Text Processing :: Markup :: XML",
+
"pydantic-settings>=2.10.0",
+
[project.optional-dependencies]
+
"pytest-asyncio>=0.24.0",
+
Homepage = "https://github.com/example/thicket"
+
Documentation = "https://github.com/example/thicket"
+
Repository = "https://github.com/example/thicket"
+
"Bug Tracker" = "https://github.com/example/thicket/issues"
+
thicket = "thicket.cli.main:app"
+
path = "src/thicket/__init__.py"
+
[tool.hatch.build.targets.wheel]
+
packages = ["src/thicket"]
+
target-version = ['py39']
+
target-version = "py39"
+
"E", # pycodestyle errors
+
"W", # pycodestyle warnings
+
"C4", # flake8-comprehensions
+
"E501", # line too long, handled by black
+
"B008", # do not perform function calls in argument defaults
+
[tool.ruff.lint.per-file-ignores]
+
"__init__.py" = ["F401"]
+
check_untyped_defs = true
+
disallow_any_generics = true
+
disallow_incomplete_defs = true
+
disallow_untyped_defs = true
+
no_implicit_optional = true
+
warn_redundant_casts = true
+
warn_unused_ignores = true
+
[[tool.mypy.overrides]]
+
ignore_missing_imports = true
+
[tool.pytest.ini_options]
+
python_files = ["test_*.py"]
+
python_classes = ["Test*"]
+
python_functions = ["test_*"]
+
"--cov-report=term-missing",
+
"ignore::DeprecationWarning",
+
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
+
"integration: marks tests as integration tests",
+
"raise AssertionError",
+
"raise NotImplementedError",
+
"if __name__ == .__main__.:",
+
"class .*\\bProtocol\\):",
+
"@(abc\\.)?abstractmethod",
+
<file path="src/thicket/cli/commands/__init__.py">
+
"""CLI commands for thicket."""
+
# Import all commands to register them with the main app
+
from . import add, duplicates, generate, index_cmd, info_cmd, init, links_cmd, list_cmd, sync
+
__all__ = ["add", "duplicates", "generate", "index_cmd", "info_cmd", "init", "links_cmd", "list_cmd", "sync"]
+
<file path="src/thicket/cli/commands/add.py">
+
"""Add command for thicket."""
+
from pathlib import Path
+
from typing import Optional
+
from pydantic import HttpUrl, ValidationError
+
from ...core.feed_parser import FeedParser
+
from ...core.git_store import GitStore
+
subcommand: str = typer.Argument(..., help="Subcommand: 'user' or 'feed'"),
+
username: str = typer.Argument(..., help="Username"),
+
feed_url: Optional[str] = typer.Argument(None, help="Feed URL (required for 'user' command)"),
+
email: Optional[str] = typer.Option(None, "--email", "-e", help="User email"),
+
homepage: Optional[str] = typer.Option(None, "--homepage", "-h", help="User homepage"),
+
icon: Optional[str] = typer.Option(None, "--icon", "-i", help="User icon URL"),
+
display_name: Optional[str] = typer.Option(None, "--display-name", "-d", help="User display name"),
+
config_file: Optional[Path] = typer.Option(
+
Path("thicket.yaml"), "--config", help="Configuration file path"
+
auto_discover: bool = typer.Option(
+
True, "--auto-discover/--no-auto-discover", help="Auto-discover user metadata from feed"
+
"""Add a user or feed to thicket."""
+
if subcommand == "user":
+
add_user(username, feed_url, email, homepage, icon, display_name, config_file, auto_discover)
+
elif subcommand == "feed":
+
add_feed(username, feed_url, config_file)
+
print_error(f"Unknown subcommand: {subcommand}")
+
print_error("Use 'user' or 'feed'")
+
feed_url: Optional[str],
+
homepage: Optional[str],
+
display_name: Optional[str],
+
"""Add a new user with feed."""
+
print_error("Feed URL is required when adding a user")
+
validated_feed_url = HttpUrl(feed_url)
+
except ValidationError:
+
print_error(f"Invalid feed URL: {feed_url}")
+
raise typer.Exit(1) from None
+
config = load_config(config_file)
+
git_store = GitStore(config.git_store)
+
# Check if user already exists
+
existing_user = git_store.get_user(username)
+
print_error(f"User '{username}' already exists")
+
print_error("Use 'thicket add feed' to add additional feeds")
+
# Auto-discover metadata if enabled
+
discovered_metadata = None
+
discovered_metadata = asyncio.run(discover_feed_metadata(validated_feed_url))
+
# Prepare user data with manual overrides taking precedence
+
user_display_name = display_name or (discovered_metadata.author_name or discovered_metadata.title if discovered_metadata else None)
+
user_email = email or (discovered_metadata.author_email if discovered_metadata else None)
+
user_homepage = homepage or (str(discovered_metadata.author_uri or discovered_metadata.link) if discovered_metadata else None)
+
user_icon = icon or (str(discovered_metadata.logo or discovered_metadata.icon or discovered_metadata.image_url) if discovered_metadata else None)
+
# Add user to Git store
+
display_name=user_display_name,
+
homepage=user_homepage,
+
feeds=[str(validated_feed_url)],
+
git_store.commit_changes(f"Add user: {username}")
+
print_success(f"Added user '{username}' with feed: {feed_url}")
+
if discovered_metadata and auto_discover:
+
print_info("Auto-discovered metadata:")
+
print_info(f" Display name: {user_display_name}")
+
print_info(f" Email: {user_email}")
+
print_info(f" Homepage: {user_homepage}")
+
print_info(f" Icon: {user_icon}")
+
def add_feed(username: str, feed_url: Optional[str], config_file: Path) -> None:
+
"""Add a feed to an existing user."""
+
print_error("Feed URL is required")
+
validated_feed_url = HttpUrl(feed_url)
+
except ValidationError:
+
print_error(f"Invalid feed URL: {feed_url}")
+
raise typer.Exit(1) from None
+
config = load_config(config_file)
+
git_store = GitStore(config.git_store)
+
user = git_store.get_user(username)
+
print_error(f"User '{username}' not found")
+
print_error("Use 'thicket add user' to add a new user")
+
# Check if feed already exists
+
if str(validated_feed_url) in user.feeds:
+
print_error(f"Feed already exists for user '{username}': {feed_url}")
+
updated_feeds = user.feeds + [str(validated_feed_url)]
+
if git_store.update_user(username, feeds=updated_feeds):
+
git_store.commit_changes(f"Add feed to user {username}: {feed_url}")
+
print_success(f"Added feed to user '{username}': {feed_url}")
+
print_error(f"Failed to add feed to user '{username}'")
+
async def discover_feed_metadata(feed_url: HttpUrl):
+
"""Discover metadata from a feed URL."""
+
with create_progress() as progress:
+
task = progress.add_task("Discovering feed metadata...", total=None)
+
content = await parser.fetch_feed(feed_url)
+
metadata, _ = parser.parse_feed(content, feed_url)
+
progress.update(task, completed=True)
+
print_error(f"Failed to discover feed metadata: {e}")
+
<file path="src/thicket/cli/commands/duplicates.py">
+
"""Duplicates command for thicket."""
+
from pathlib import Path
+
from typing import Optional
+
from rich.table import Table
+
from ...core.git_store import GitStore
+
@app.command("duplicates")
+
def duplicates_command(
+
action: str = typer.Argument(..., help="Action: 'list', 'add', 'remove'"),
+
duplicate_id: Optional[str] = typer.Argument(None, help="Duplicate entry ID"),
+
canonical_id: Optional[str] = typer.Argument(None, help="Canonical entry ID"),
+
config_file: Optional[Path] = typer.Option(
+
Path("thicket.yaml"), "--config", help="Configuration file path"
+
"""Manage duplicate entry mappings."""
+
config = load_config(config_file)
+
git_store = GitStore(config.git_store)
+
list_duplicates(git_store)
+
add_duplicate(git_store, duplicate_id, canonical_id)
+
elif action == "remove":
+
remove_duplicate(git_store, duplicate_id)
+
print_error(f"Unknown action: {action}")
+
print_error("Use 'list', 'add', or 'remove'")
+
def list_duplicates(git_store: GitStore) -> None:
+
"""List all duplicate mappings."""
+
duplicates = git_store.get_duplicates()
+
if not duplicates.duplicates:
+
print("No duplicate mappings found")
+
print_info("No duplicate mappings found")
+
print("Duplicate ID\tCanonical ID")
+
for duplicate_id, canonical_id in duplicates.duplicates.items():
+
print(f"{duplicate_id}\t{canonical_id}")
+
print(f"Total duplicates: {len(duplicates.duplicates)}")
+
table = Table(title="Duplicate Entry Mappings")
+
table.add_column("Duplicate ID", style="red")
+
table.add_column("Canonical ID", style="green")
+
for duplicate_id, canonical_id in duplicates.duplicates.items():
+
table.add_row(duplicate_id, canonical_id)
+
print_info(f"Total duplicates: {len(duplicates.duplicates)}")
+
def add_duplicate(git_store: GitStore, duplicate_id: Optional[str], canonical_id: Optional[str]) -> None:
+
"""Add a duplicate mapping."""
+
print_error("Duplicate ID is required")
+
print_error("Canonical ID is required")
+
# Check if duplicate_id already exists
+
duplicates = git_store.get_duplicates()
+
if duplicates.is_duplicate(duplicate_id):
+
existing_canonical = duplicates.get_canonical(duplicate_id)
+
print_error(f"Duplicate ID already mapped to: {existing_canonical}")
+
print_error("Use 'remove' first to change the mapping")
+
# Check if we're trying to make a canonical ID point to itself
+
if duplicate_id == canonical_id:
+
print_error("Duplicate ID cannot be the same as canonical ID")
+
git_store.add_duplicate(duplicate_id, canonical_id)
+
git_store.commit_changes(f"Add duplicate mapping: {duplicate_id} -> {canonical_id}")
+
print_success(f"Added duplicate mapping: {duplicate_id} -> {canonical_id}")
+
def remove_duplicate(git_store: GitStore, duplicate_id: Optional[str]) -> None:
+
"""Remove a duplicate mapping."""
+
print_error("Duplicate ID is required")
+
# Check if mapping exists
+
duplicates = git_store.get_duplicates()
+
if not duplicates.is_duplicate(duplicate_id):
+
print_error(f"No duplicate mapping found for: {duplicate_id}")
+
canonical_id = duplicates.get_canonical(duplicate_id)
+
if git_store.remove_duplicate(duplicate_id):
+
git_store.commit_changes(f"Remove duplicate mapping: {duplicate_id} -> {canonical_id}")
+
print_success(f"Removed duplicate mapping: {duplicate_id} -> {canonical_id}")
+
print_error(f"Failed to remove duplicate mapping: {duplicate_id}")
+
<file path="src/thicket/cli/commands/sync.py">
+
"""Sync command for thicket."""
+
from pathlib import Path
+
from typing import Optional
+
from rich.progress import track
+
from ...core.feed_parser import FeedParser
+
from ...core.git_store import GitStore
+
all_users: bool = typer.Option(
+
False, "--all", "-a", help="Sync all users and feeds"
+
user: Optional[str] = typer.Option(
+
None, "--user", "-u", help="Sync specific user only"
+
config_file: Optional[Path] = typer.Option(
+
Path("thicket.yaml"), "--config", help="Configuration file path"
+
dry_run: bool = typer.Option(
+
False, "--dry-run", help="Show what would be synced without making changes"
+
"""Sync feeds and store entries in Git repository."""
+
config = load_config(config_file)
+
git_store = GitStore(config.git_store)
+
# Determine which users to sync from git repository
+
index = git_store._load_index()
+
users_to_sync = list(index.users.values())
+
user_metadata = git_store.get_user(user)
+
print_error(f"User '{user}' not found in git repository")
+
users_to_sync = [user_metadata]
+
print_error("Specify --all to sync all users or --user to sync a specific user")
+
print_info("No users configured to sync")
+
total_updated_entries = 0
+
for user_metadata in users_to_sync:
+
print_info(f"Syncing user: {user_metadata.username}")
+
user_updated_entries = 0
+
# Sync each feed for the user
+
for feed_url in track(user_metadata.feeds, description=f"Syncing {user_metadata.username}'s feeds"):
+
new_entries, updated_entries = asyncio.run(
+
sync_feed(git_store, user_metadata.username, feed_url, dry_run)
+
user_new_entries += new_entries
+
user_updated_entries += updated_entries
+
print_error(f"Failed to sync feed {feed_url}: {e}")
+
print_info(f"User {user_metadata.username}: {user_new_entries} new, {user_updated_entries} updated")
+
total_new_entries += user_new_entries
+
total_updated_entries += user_updated_entries
+
# Commit changes if not dry run
+
if not dry_run and (total_new_entries > 0 or total_updated_entries > 0):
+
commit_message = f"Sync feeds: {total_new_entries} new entries, {total_updated_entries} updated"
+
git_store.commit_changes(commit_message)
+
print_success(f"Committed changes: {commit_message}")
+
print_info(f"Dry run complete: would sync {total_new_entries} new entries, {total_updated_entries} updated")
+
print_success(f"Sync complete: {total_new_entries} new entries, {total_updated_entries} updated")
+
async def sync_feed(git_store: GitStore, username: str, feed_url, dry_run: bool) -> tuple[int, int]:
+
"""Sync a single feed for a user."""
+
content = await parser.fetch_feed(feed_url)
+
metadata, entries = parser.parse_feed(content, feed_url)
+
# Check if entry already exists
+
existing_entry = git_store.get_entry(username, entry.id)
+
# Check if entry has been updated
+
if existing_entry.updated != entry.updated:
+
git_store.store_entry(username, entry)
+
git_store.store_entry(username, entry)
+
print_error(f"Failed to process entry {entry.id}: {e}")
+
return new_entries, updated_entries
+
print_error(f"Failed to sync feed {feed_url}: {e}")
+
<file path="src/thicket/models/config.py">
+
"""Configuration models for thicket."""
+
from pathlib import Path
+
from typing import Optional
+
from pydantic import BaseModel, EmailStr, HttpUrl
+
from pydantic_settings import BaseSettings, SettingsConfigDict
+
class UserConfig(BaseModel):
+
"""Configuration for a single user and their feeds."""
+
email: Optional[EmailStr] = None
+
homepage: Optional[HttpUrl] = None
+
icon: Optional[HttpUrl] = None
+
display_name: Optional[str] = None
+
class ThicketConfig(BaseSettings):
+
"""Main configuration for thicket."""
+
model_config = SettingsConfigDict(
+
yaml_file="thicket.yaml",
+
users: list[UserConfig] = []
+
<file path="src/thicket/cli/commands/links_cmd.py">
+
"""CLI command for extracting and categorizing all outbound links from blog entries."""
+
from pathlib import Path
+
from typing import Dict, List, Optional, Set
+
from urllib.parse import urljoin, urlparse
+
from rich.console import Console
+
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
+
from rich.table import Table
+
from ...core.git_store import GitStore
+
from ..utils import load_config, get_tsv_mode
+
"""Represents a link found in a blog entry."""
+
def __init__(self, url: str, entry_id: str, username: str):
+
self.entry_id = entry_id
+
self.username = username
+
def to_dict(self) -> dict:
+
"""Convert to dictionary for JSON serialization."""
+
"entry_id": self.entry_id,
+
"username": self.username
+
def from_dict(cls, data: dict) -> "LinkData":
+
"""Create from dictionary."""
+
entry_id=data["entry_id"],
+
username=data["username"]
+
"""Categorizes links as internal, user, or unknown."""
+
def __init__(self, user_domains: Dict[str, Set[str]]):
+
self.user_domains = user_domains
+
# Create reverse mapping of domain -> username
+
self.domain_to_user = {}
+
for username, domains in user_domains.items():
+
self.domain_to_user[domain] = username
+
def categorize_url(self, url: str, source_username: str) -> tuple[str, Optional[str]]:
+
Categorize a URL as 'internal', 'user', or 'unknown'.
+
Returns (category, target_username).
+
domain = parsed.netloc.lower()
+
# Check if it's a link to the same user's domain (internal)
+
if domain in self.user_domains.get(source_username, set()):
+
return "internal", source_username
+
# Check if it's a link to another user's domain
+
if domain in self.domain_to_user:
+
return "user", self.domain_to_user[domain]
+
# Everything else is unknown
+
"""Extracts and resolves links from blog entries."""
+
# Pattern for extracting links from HTML
+
self.link_pattern = re.compile(r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', re.IGNORECASE | re.DOTALL)
+
self.url_pattern = re.compile(r'https?://[^\s<>"]+')
+
def extract_links_from_html(self, html_content: str, base_url: str) -> List[tuple[str, str]]:
+
"""Extract all links from HTML content and resolve them against base URL."""
+
# Extract links from <a> tags
+
for match in self.link_pattern.finditer(html_content):
+
text = re.sub(r'<[^>]+>', '', match.group(2)).strip() # Remove HTML tags from link text
+
# Resolve relative URLs against base URL
+
resolved_url = urljoin(base_url, url)
+
links.append((resolved_url, text))
+
def extract_links_from_entry(self, entry, username: str, base_url: str) -> List[LinkData]:
+
"""Extract all links from a blog entry."""
+
# Combine all text content for analysis
+
content_to_search.append(entry.content)
+
content_to_search.append(entry.summary)
+
for content in content_to_search:
+
extracted_links = self.extract_links_from_html(content, base_url)
+
for url, link_text in extracted_links:
+
if not url or url.startswith('#'):
+
links.append(link_data)
+
config_file: Optional[Path] = typer.Option(
+
help="Path to configuration file",
+
output_file: Optional[Path] = typer.Option(
+
help="Path to output unified links file (default: links.json in git store)",
+
verbose: bool = typer.Option(
+
help="Show detailed progress information",
+
"""Extract and categorize all outbound links from blog entries.
+
This command analyzes all blog entries to extract outbound links,
+
resolve them properly with respect to the feed's base URL, and
+
categorize them as internal, user, or unknown links.
+
Creates a unified links.json file containing all link data.
+
config = load_config(config_file)
+
git_store = GitStore(config.git_store)
+
# Build user domain mapping
+
console.print("Building user domain mapping...")
+
index = git_store._load_index()
+
for username, user_metadata in index.users.items():
+
# Add domains from feeds
+
for feed_url in user_metadata.feeds:
+
domain = urlparse(feed_url).netloc.lower()
+
# Add domain from homepage
+
if user_metadata.homepage:
+
domain = urlparse(str(user_metadata.homepage)).netloc.lower()
+
user_domains[username] = domains
+
console.print(f"Found {len(user_domains)} users with {sum(len(d) for d in user_domains.values())} total domains")
+
# Initialize components
+
link_extractor = LinkExtractor()
+
categorizer = LinkCategorizer(user_domains)
+
users = list(index.users.keys())
+
console.print("[yellow]No users found in Git store[/yellow]")
+
link_categories = {"internal": [], "user": [], "unknown": []}
+
link_dict = {} # Dictionary with link URL as key, maps to list of atom IDs
+
reverse_dict = {} # Dictionary with atom ID as key, maps to list of URLs
+
TextColumn("[progress.description]{task.description}"),
+
# Count total entries first
+
counting_task = progress.add_task("Counting entries...", total=len(users))
+
entries = git_store.list_entries(username)
+
total_entries += len(entries)
+
progress.advance(counting_task)
+
progress.remove_task(counting_task)
+
processing_task = progress.add_task(
+
f"Processing {total_entries} entries...",
+
entries = git_store.list_entries(username)
+
user_metadata = index.users[username]
+
# Get base URL for this user (use first feed URL)
+
base_url = str(user_metadata.feeds[0]) if user_metadata.feeds else "https://example.com"
+
# Extract links from this entry
+
entry_links = link_extractor.extract_links_from_entry(entry, username, base_url)
+
# Track unique links per entry
+
entry_urls_seen = set()
+
for link_data in entry_links:
+
# Skip if we've already seen this URL in this entry
+
if link_data.url in entry_urls_seen:
+
entry_urls_seen.add(link_data.url)
+
category, target_username = categorizer.categorize_url(link_data.url, username)
+
# Add to link dictionary (URL as key, maps to list of atom IDs)
+
if link_data.url not in link_dict:
+
link_dict[link_data.url] = []
+
if link_data.entry_id not in link_dict[link_data.url]:
+
link_dict[link_data.url].append(link_data.entry_id)
+
# Also add to reverse mapping (atom ID -> list of URLs)
+
if link_data.entry_id not in reverse_dict:
+
reverse_dict[link_data.entry_id] = []
+
if link_data.url not in reverse_dict[link_data.entry_id]:
+
reverse_dict[link_data.entry_id].append(link_data.url)
+
# Add category info to link data for categories tracking
+
link_info = link_data.to_dict()
+
link_info["category"] = category
+
link_info["target_username"] = target_username
+
all_links.append(link_info)
+
link_categories[category].append(link_info)
+
progress.advance(processing_task)
+
if verbose and entry_links:
+
console.print(f" Found {len(entry_links)} links in {username}:{entry.title[:50]}...")
+
# Determine output path
+
output_path = output_file
+
output_path = config.git_store / "links.json"
+
# Save all extracted links (not just filtered ones)
+
console.print("Preparing output data...")
+
# Build a set of all URLs that correspond to posts in the git database
+
registered_urls = set()
+
# Get all entries from all users and build URL mappings
+
entries = git_store.list_entries(username)
+
user_metadata = index.users[username]
+
# Try to match entry URLs with extracted links
+
if hasattr(entry, 'link') and entry.link:
+
registered_urls.add(str(entry.link))
+
# Also check entry alternate links if they exist
+
if hasattr(entry, 'links') and entry.links:
+
for link in entry.links:
+
if hasattr(link, 'href') and link.href:
+
registered_urls.add(str(link.href))
+
# Build unified structure with metadata
+
for url, entry_ids in link_dict.items():
+
"referencing_entries": entry_ids
+
# Find target username if this is a tracked post
+
if url in registered_urls:
+
user_domains_set = {domain for domain in user_domains.get(username, [])}
+
if any(domain in url for domain in user_domains_set):
+
unified_links[url]["target_username"] = username
+
# Build reverse mapping
+
for entry_id in entry_ids:
+
if entry_id not in reverse_mapping:
+
reverse_mapping[entry_id] = []
+
if url not in reverse_mapping[entry_id]:
+
reverse_mapping[entry_id].append(url)
+
# Create unified output data
+
"links": unified_links,
+
"reverse_mapping": reverse_mapping,
+
"user_domains": {k: list(v) for k, v in user_domains.items()}
+
console.print(f"Found {len(registered_urls)} registered post URLs")
+
console.print(f"Found {len(link_dict)} total links, {sum(1 for link in unified_links.values() if 'target_username' in link)} tracked posts")
+
with open(output_path, "w") as f:
+
json.dump(output_data, f, indent=2, default=str)
+
console.print("\n[green]✓ Links extraction completed successfully[/green]")
+
# Create summary table or TSV output
+
print("Category\tCount\tDescription")
+
print(f"Internal\t{len(link_categories['internal'])}\tLinks to same user's domain")
+
print(f"User\t{len(link_categories['user'])}\tLinks to other tracked users")
+
print(f"Unknown\t{len(link_categories['unknown'])}\tLinks to external sites")
+
print(f"Total Extracted\t{len(all_links)}\tAll extracted links")
+
print(f"Saved to Output\t{len(output_data['links'])}\tLinks saved to output file")
+
print(f"Cross-references\t{sum(1 for link in unified_links.values() if 'target_username' in link)}\tLinks to registered posts only")
+
table = Table(title="Links Summary")
+
table.add_column("Category", style="cyan")
+
table.add_column("Count", style="green")
+
table.add_column("Description", style="white")
+
table.add_row("Internal", str(len(link_categories["internal"])), "Links to same user's domain")
+
table.add_row("User", str(len(link_categories["user"])), "Links to other tracked users")
+
table.add_row("Unknown", str(len(link_categories["unknown"])), "Links to external sites")
+
table.add_row("Total Extracted", str(len(all_links)), "All extracted links")
+
table.add_row("Saved to Output", str(len(output_data['links'])), "Links saved to output file")
+
table.add_row("Cross-references", str(sum(1 for link in unified_links.values() if 'target_username' in link)), "Links to registered posts only")
+
# Show user links if verbose
+
if verbose and link_categories["user"]:
+
print("User Link Source\tUser Link Target\tLink Count")
+
for link in link_categories["user"]:
+
key = f"{link['username']} -> {link['target_username']}"
+
user_link_counts[key] = user_link_counts.get(key, 0) + 1
+
for link_pair, count in sorted(user_link_counts.items(), key=lambda x: x[1], reverse=True)[:10]:
+
source, target = link_pair.split(" -> ")
+
print(f"{source}\t{target}\t{count}")
+
console.print("\n[bold]User-to-user links:[/bold]")
+
for link in link_categories["user"]:
+
key = f"{link['username']} -> {link['target_username']}"
+
user_link_counts[key] = user_link_counts.get(key, 0) + 1
+
for link_pair, count in sorted(user_link_counts.items(), key=lambda x: x[1], reverse=True)[:10]:
+
console.print(f" {link_pair}: {count} links")
+
console.print(f"\nUnified links data saved to: {output_path}")
+
console.print(f"[red]Error extracting links: {e}[/red]")
+
console.print_exception()
+
<file path="src/thicket/cli/commands/list_cmd.py">
+
"""List command for thicket."""
+
from pathlib import Path
+
from typing import Optional
+
from rich.table import Table
+
from ...core.git_store import GitStore
+
print_feeds_table_from_git,
+
print_users_table_from_git,
+
what: str = typer.Argument(..., help="What to list: 'users', 'feeds', 'entries'"),
+
user: Optional[str] = typer.Option(
+
None, "--user", "-u", help="Filter by specific user"
+
limit: Optional[int] = typer.Option(
+
None, "--limit", "-l", help="Limit number of results"
+
config_file: Optional[Path] = typer.Option(
+
Path("thicket.yaml"), "--config", help="Configuration file path"
+
"""List users, feeds, or entries."""
+
config = load_config(config_file)
+
git_store = GitStore(config.git_store)
+
list_feeds(git_store, user)
+
elif what == "entries":
+
list_entries(git_store, user, limit)
+
print_error(f"Unknown list type: {what}")
+
print_error("Use 'users', 'feeds', or 'entries'")
+
def list_users(git_store: GitStore) -> None:
+
index = git_store._load_index()
+
users = list(index.users.values())
+
print_info("No users configured")
+
print_users_table_from_git(users)
+
def list_feeds(git_store: GitStore, username: Optional[str] = None) -> None:
+
"""List feeds, optionally filtered by user."""
+
user = git_store.get_user(username)
+
print_error(f"User '{username}' not found")
+
print_info(f"No feeds configured for user '{username}'")
+
print_feeds_table_from_git(git_store, username)
+
def list_entries(git_store: GitStore, username: Optional[str] = None, limit: Optional[int] = None) -> None:
+
"""List entries, optionally filtered by user."""
+
# List entries for specific user
+
user = git_store.get_user(username)
+
print_error(f"User '{username}' not found")
+
entries = git_store.list_entries(username, limit)
+
print_info(f"No entries found for user '{username}'")
+
print_entries_table([entries], [username])
+
# List entries for all users
+
index = git_store._load_index()
+
for user in index.users.values():
+
entries = git_store.list_entries(user.username, limit)
+
all_entries.append(entries)
+
all_usernames.append(user.username)
+
print_info("No entries found")
+
print_entries_table(all_entries, all_usernames)
+
def _clean_html_content(content: Optional[str]) -> str:
+
"""Clean HTML content for display in table."""
+
clean_text = re.sub(r'<[^>]+>', ' ', content)
+
# Replace multiple whitespace with single space
+
clean_text = re.sub(r'\s+', ' ', clean_text)
+
# Strip and limit length
+
clean_text = clean_text.strip()
+
if len(clean_text) > 100:
+
clean_text = clean_text[:97] + "..."
+
def print_entries_table(entries_by_user: list[list], usernames: list[str]) -> None:
+
"""Print a table of entries."""
+
print_entries_tsv(entries_by_user, usernames)
+
table = Table(title="Feed Entries")
+
table.add_column("User", style="cyan", no_wrap=True)
+
table.add_column("Title", style="bold")
+
table.add_column("Updated", style="blue")
+
table.add_column("URL", style="green")
+
# Combine all entries with usernames
+
for entries, username in zip(entries_by_user, usernames):
+
all_entries.append((username, entry))
+
# Sort by updated time (newest first)
+
all_entries.sort(key=lambda x: x[1].updated, reverse=True)
+
for username, entry in all_entries:
+
updated_str = entry.updated.strftime("%Y-%m-%d %H:%M")
+
# Truncate title if too long
+
title = title[:47] + "..."
+
<file path="src/thicket/cli/main.py">
+
"""Main CLI application using Typer."""
+
from rich.console import Console
+
from .. import __version__
+
help="A CLI tool for persisting Atom/RSS feeds in Git repositories",
+
rich_markup_mode="rich",
+
# Global state for TSV output mode
+
def version_callback(value: bool) -> None:
+
"""Show version and exit."""
+
console.print(f"thicket version {__version__}")
+
version: bool = typer.Option(
+
help="Show the version and exit",
+
callback=version_callback,
+
tsv: bool = typer.Option(
+
help="Output in tab-separated values format without truncation",
+
"""Thicket: A CLI tool for persisting Atom/RSS feeds in Git repositories."""
+
# Import commands to register them
+
from .commands import add, duplicates, generate, index_cmd, info_cmd, init, links_cmd, list_cmd, sync
+
if __name__ == "__main__":
+
<file path="src/thicket/core/git_store.py">
+
"""Git repository operations for thicket."""
+
from datetime import datetime
+
from pathlib import Path
+
from typing import Optional
+
from ..models import AtomEntry, DuplicateMap, GitStoreIndex, UserMetadata
+
"""Manages the Git repository for storing feed entries."""
+
def __init__(self, repo_path: Path):
+
"""Initialize the Git store."""
+
self.repo_path = repo_path
+
self.repo: Optional[Repo] = None
+
def _ensure_repo(self) -> None:
+
"""Ensure the Git repository exists and is initialized."""
+
if not self.repo_path.exists():
+
self.repo_path.mkdir(parents=True, exist_ok=True)
+
self.repo = Repo(self.repo_path)
+
except git.InvalidGitRepositoryError:
+
# Initialize new repository
+
self.repo = Repo.init(self.repo_path)
+
self._create_initial_structure()
+
def _create_initial_structure(self) -> None:
+
"""Create initial Git store structure."""
+
created=datetime.now(),
+
last_updated=datetime.now(),
+
self._save_index(index)
+
# Create duplicates.json
+
duplicates = DuplicateMap()
+
self._save_duplicates(duplicates)
+
# Create initial commit
+
self.repo.index.add(["index.json", "duplicates.json"])
+
self.repo.index.commit("Initial thicket repository structure")
+
def _save_index(self, index: GitStoreIndex) -> None:
+
"""Save the index to index.json."""
+
index_path = self.repo_path / "index.json"
+
with open(index_path, "w") as f:
+
json.dump(index.model_dump(mode="json", exclude_none=True), f, indent=2, default=str)
+
def _load_index(self) -> GitStoreIndex:
+
"""Load the index from index.json."""
+
index_path = self.repo_path / "index.json"
+
if not index_path.exists():
+
created=datetime.now(),
+
last_updated=datetime.now(),
+
with open(index_path) as f:
+
return GitStoreIndex(**data)
+
def _save_duplicates(self, duplicates: DuplicateMap) -> None:
+
"""Save duplicates map to duplicates.json."""
+
duplicates_path = self.repo_path / "duplicates.json"
+
with open(duplicates_path, "w") as f:
+
json.dump(duplicates.model_dump(exclude_none=True), f, indent=2)
+
def _load_duplicates(self) -> DuplicateMap:
+
"""Load duplicates map from duplicates.json."""
+
duplicates_path = self.repo_path / "duplicates.json"
+
if not duplicates_path.exists():
+
with open(duplicates_path) as f:
+
return DuplicateMap(**data)
+
def add_user(self, username: str, display_name: Optional[str] = None,
+
email: Optional[str] = None, homepage: Optional[str] = None,
+
icon: Optional[str] = None, feeds: Optional[list[str]] = None) -> UserMetadata:
+
"""Add a new user to the Git store."""
+
index = self._load_index()
+
# Create user directory
+
user_dir = self.repo_path / username
+
user_dir.mkdir(exist_ok=True)
+
user_metadata = UserMetadata(
+
display_name=display_name,
+
created=datetime.now(),
+
last_updated=datetime.now(),
+
index.add_user(user_metadata)
+
self._save_index(index)
+
def get_user(self, username: str) -> Optional[UserMetadata]:
+
"""Get user metadata by username."""
+
index = self._load_index()
+
return index.get_user(username)
+
def update_user(self, username: str, **kwargs) -> bool:
+
"""Update user metadata."""
+
index = self._load_index()
+
user = index.get_user(username)
+
for key, value in kwargs.items():
+
if hasattr(user, key) and value is not None:
+
setattr(user, key, value)
+
user.update_timestamp()
+
self._save_index(index)
+
def store_entry(self, username: str, entry: AtomEntry) -> bool:
+
"""Store an entry in the user's directory."""
+
user = self.get_user(username)
+
# Sanitize entry ID for filename
+
from .feed_parser import FeedParser
+
safe_id = parser.sanitize_entry_id(entry.id)
+
user_dir = self.repo_path / user.directory
+
entry_path = user_dir / f"{safe_id}.json"
+
# Check if entry already exists
+
entry_exists = entry_path.exists()
+
with open(entry_path, "w") as f:
+
json.dump(entry.model_dump(mode="json", exclude_none=True), f, indent=2, default=str)
+
# Update user metadata if new entry
+
index = self._load_index()
+
index.update_entry_count(username, 1)
+
self._save_index(index)
+
def get_entry(self, username: str, entry_id: str) -> Optional[AtomEntry]:
+
"""Get an entry by username and entry ID."""
+
user = self.get_user(username)
+
from .feed_parser import FeedParser
+
safe_id = parser.sanitize_entry_id(entry_id)
+
entry_path = self.repo_path / user.directory / f"{safe_id}.json"
+
if not entry_path.exists():
+
with open(entry_path) as f:
+
return AtomEntry(**data)
+
def list_entries(self, username: str, limit: Optional[int] = None) -> list[AtomEntry]:
+
"""List entries for a user."""
+
user = self.get_user(username)
+
user_dir = self.repo_path / user.directory
+
if not user_dir.exists():
+
entry_files = sorted(user_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+
entry_files = entry_files[:limit]
+
for entry_file in entry_files:
+
with open(entry_file) as f:
+
entries.append(AtomEntry(**data))
+
def get_duplicates(self) -> DuplicateMap:
+
"""Get the duplicates map."""
+
return self._load_duplicates()
+
def add_duplicate(self, duplicate_id: str, canonical_id: str) -> None:
+
"""Add a duplicate mapping."""
+
duplicates = self._load_duplicates()
+
duplicates.add_duplicate(duplicate_id, canonical_id)
+
self._save_duplicates(duplicates)
+
def remove_duplicate(self, duplicate_id: str) -> bool:
+
"""Remove a duplicate mapping."""
+
duplicates = self._load_duplicates()
+
result = duplicates.remove_duplicate(duplicate_id)
+
self._save_duplicates(duplicates)
+
def commit_changes(self, message: str) -> None:
+
"""Commit all changes to the Git repository."""
+
self.repo.git.add(A=True)
+
# Check if there are changes to commit
+
if self.repo.index.diff("HEAD"):
+
self.repo.index.commit(message)
+
def get_stats(self) -> dict:
+
"""Get statistics about the Git store."""
+
index = self._load_index()
+
duplicates = self._load_duplicates()
+
"total_users": len(index.users),
+
"total_entries": index.total_entries,
+
"total_duplicates": len(duplicates.duplicates),
+
"last_updated": index.last_updated,
+
"repository_size": sum(f.stat().st_size for f in self.repo_path.rglob("*") if f.is_file()),
+
def search_entries(self, query: str, username: Optional[str] = None,
+
limit: Optional[int] = None) -> list[tuple[str, AtomEntry]]:
+
"""Search entries by content."""
+
index = self._load_index()
+
users = [index.get_user(username)] if username else list(index.users.values())
+
users = [u for u in users if u is not None]
+
user_dir = self.repo_path / user.directory
+
if not user_dir.exists():
+
entry_files = user_dir.glob("*.json")
+
for entry_file in entry_files:
+
with open(entry_file) as f:
+
entry = AtomEntry(**data)
+
# Simple text search in title, summary, and content
+
searchable_text = " ".join(filter(None, [
+
if query.lower() in searchable_text:
+
results.append((user.username, entry))
+
if limit and len(results) >= limit:
+
# Sort by updated time (newest first)
+
results.sort(key=lambda x: x[1].updated, reverse=True)
+
return results[:limit] if limit else results
+
# Thicket Architecture Design
+
Thicket is a modern CLI tool for persisting Atom/RSS feeds in a Git repository, designed to enable distributed webblog comment structures.
+
- **Typer** (0.15.x) - Modern CLI framework with type hints
+
- **Rich** (13.x) - Beautiful terminal output, progress bars, and tables
+
- **prompt-toolkit** - Interactive prompts when needed
+
- **feedparser** (6.0.11) - Universal feed parser supporting RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0
+
- Alternative: **atoma** for stricter Atom/RSS parsing with JSON feed support
+
- Alternative: **fastfeedparser** for high-performance parsing (10x faster)
+
- **GitPython** (3.1.44) - High-level git operations, requires git CLI
+
- Alternative: **pygit2** (1.18.0) - Direct libgit2 bindings, better for authentication
+
- **httpx** (0.28.x) - Modern async/sync HTTP client with connection pooling
+
- **aiohttp** (3.11.x) - For async-only operations if needed
+
#### Configuration & Data Models
+
- **pydantic** (2.11.x) - Data validation and settings management
+
- **pydantic-settings** (2.10.x) - Configuration file handling with env var support
+
- **pendulum** (3.x) - Better datetime handling
+
- **bleach** (6.x) - HTML sanitization for feed content
+
- **platformdirs** (4.x) - Cross-platform directory paths
+
├── pyproject.toml # Modern Python packaging
+
├── README.md # Project documentation
+
├── ARCH.md # This file
+
├── CLAUDE.md # Project instructions
+
│ ├── __main__.py # Entry point for `python -m thicket`
+
│ ├── cli/ # CLI commands and interface
+
│ │ ├── main.py # Main CLI app with Typer
+
│ │ ├── commands/ # Subcommands
+
│ │ │ ├── init.py # Initialize git store
+
│ │ │ ├── add.py # Add users and feeds
+
│ │ │ ├── sync.py # Sync feeds
+
│ │ │ ├── list_cmd.py # List users/feeds
+
│ │ │ ├── duplicates.py # Manage duplicate entries
+
│ │ │ ├── links_cmd.py # Extract and categorize links
+
│ │ │ └── index_cmd.py # Build reference index and show threads
+
│ │ └── utils.py # CLI utilities (progress, formatting)
+
│ ├── core/ # Core business logic
+
│ │ ├── feed_parser.py # Feed parsing and normalization
+
│ │ ├── git_store.py # Git repository operations
+
│ │ └── reference_parser.py # Link extraction and threading
+
│ ├── models/ # Pydantic data models
+
│ │ ├── config.py # Configuration models
+
│ │ ├── feed.py # Feed/Entry models
+
│ │ └── user.py # User metadata models
+
│ └── utils/ # Shared utilities
+
│ ├── conftest.py # pytest configuration
+
│ ├── test_feed_parser.py
+
│ ├── test_git_store.py
+
│ └── fixtures/ # Test data
+
└── examples/ # Example configurations
+
### Configuration File (YAML/TOML)
+
class ThicketConfig(BaseSettings):
+
git_store: Path # Git repository location
+
cache_dir: Path # Cache directory
+
users: list[UserConfig]
+
model_config = SettingsConfigDict(
+
yaml_file="thicket.yaml"
+
class UserConfig(BaseModel):
+
email: Optional[EmailStr] = None
+
homepage: Optional[HttpUrl] = None
+
icon: Optional[HttpUrl] = None
+
display_name: Optional[str] = None
+
### Feed Storage Format
+
class AtomEntry(BaseModel):
+
id: str # Original Atom ID
+
published: Optional[datetime]
+
content: Optional[str] # Full body content from Atom entry
+
content_type: Optional[str] = "html" # text, html, xhtml
+
categories: list[str] = []
+
rights: Optional[str] = None # Copyright info
+
source: Optional[str] = None # Source feed URL
+
# Additional Atom fields preserved during RSS->Atom conversion
+
model_config = ConfigDict(
+
datetime: lambda v: v.isoformat()
+
class DuplicateMap(BaseModel):
+
"""Maps duplicate entry IDs to canonical entry IDs"""
+
duplicates: dict[str, str] = {} # duplicate_id -> canonical_id
+
comment: str = "Entry IDs that map to the same canonical content"
+
def add_duplicate(self, duplicate_id: str, canonical_id: str) -> None:
+
"""Add a duplicate mapping"""
+
self.duplicates[duplicate_id] = canonical_id
+
def remove_duplicate(self, duplicate_id: str) -> bool:
+
"""Remove a duplicate mapping. Returns True if existed."""
+
return self.duplicates.pop(duplicate_id, None) is not None
+
def get_canonical(self, entry_id: str) -> str:
+
"""Get canonical ID for an entry (returns original if not duplicate)"""
+
return self.duplicates.get(entry_id, entry_id)
+
def is_duplicate(self, entry_id: str) -> bool:
+
"""Check if entry ID is marked as duplicate"""
+
return entry_id in self.duplicates
+
## Git Repository Structure
+
├── index.json # User directory index
+
├── duplicates.json # Manual curation of duplicate entries
+
├── links.json # Unified links, references, and mapping data
+
│ ├── entry_id_1.json # Sanitized entry files
+
## Key Design Decisions
+
### 1. Feed Normalization & Auto-Discovery
+
- All RSS feeds converted to Atom format before storage
+
- Preserves maximum metadata during conversion
+
- Sanitizes HTML content to prevent XSS
+
- **Auto-discovery**: Extracts user metadata from feed during `add user` command
+
- Consistent algorithm to convert Atom IDs to safe filenames
+
- Handles edge cases (very long IDs, special characters)
+
- Maintains reversibility where possible
+
- Uses GitPython for simplicity (no authentication required)
+
- Single main branch for all users and entries
+
- Atomic commits per sync operation
+
- Meaningful commit messages with feed update summaries
+
- Preserves complete history - never delete entries even if they disappear from feeds
+
### 4. Caching Strategy
+
- HTTP caching with Last-Modified/ETag support
+
- Local cache of parsed feeds with TTL
+
- Cache invalidation on configuration changes
+
- Git store serves as permanent historical archive beyond feed depth limits
+
- Graceful handling of feed parsing errors
+
- Retry logic for network failures
+
- Clear error messages with recovery suggestions
+
## CLI Command Structure
+
# Initialize a new git store
+
thicket init /path/to/store
+
# Add a user with feeds (auto-discovers metadata from feed)
+
thicket add user "alyssa" \
+
--feed "https://example.com/feed.atom"
+
# Auto-populates: email, homepage, icon, display_name from feed metadata
+
# Add a user with manual overrides
+
thicket add user "alyssa" \
+
--feed "https://example.com/feed.atom" \
+
--email "alyssa@example.com" \
+
--homepage "https://alyssa.example.com" \
+
--icon "https://example.com/avatar.png" \
+
--display-name "Alyssa P. Hacker"
+
# Add additional feed to existing user
+
thicket add feed "alyssa" "https://example.com/other-feed.rss"
+
# Sync all feeds (designed for cron usage)
+
thicket sync --user alyssa
+
# List users and their feeds
+
thicket list feeds --user alyssa
+
# Manage duplicate entries
+
thicket duplicates list
+
thicket duplicates add <entry_id_1> <entry_id_2> # Mark as duplicates
+
thicket duplicates remove <entry_id_1> <entry_id_2> # Unmark duplicates
+
# Link processing and threading
+
thicket links --verbose # Extract and categorize all links
+
thicket index --verbose # Build reference index for threading
+
thicket threads # Show conversation threads
+
thicket threads --username user1 # Show threads for specific user
+
thicket threads --min-size 3 # Show threads with minimum size
+
## Performance Considerations
+
1. **Concurrent Feed Fetching**: Use httpx with asyncio for parallel downloads
+
2. **Incremental Updates**: Only fetch/parse feeds that have changed
+
3. **Efficient Git Operations**: Batch commits, use shallow clones where appropriate
+
4. **Progress Feedback**: Rich progress bars for long operations
+
## Security Considerations
+
1. **HTML Sanitization**: Use bleach to clean feed content
+
2. **URL Validation**: Strict validation of feed URLs
+
3. **Git Security**: No credentials stored in repository
+
4. **Path Traversal**: Careful sanitization of filenames
+
1. **Web Interface**: Optional web UI for browsing the git store
+
2. **Webhooks**: Notify external services on feed updates
+
3. **Feed Discovery**: Auto-discover feeds from HTML pages
+
4. **Export Formats**: Generate static sites, OPML exports
+
5. **Federation**: P2P sync between thicket instances
+
## Requirements Clarification
+
**✓ Resolved Requirements:**
+
1. **Feed Update Frequency**: Designed for cron usage - no built-in scheduling needed
+
2. **Duplicate Handling**: Manual curation via `duplicates.json` file with CLI commands
+
3. **Git Branching**: Single main branch for all users and entries
+
4. **Authentication**: No feeds require authentication currently
+
5. **Content Storage**: Store complete Atom entry body content as provided
+
6. **Deleted Entries**: Preserve all entries in Git store permanently (historical archive)
+
7. **History Depth**: Git store maintains full history beyond feed depth limits
+
8. **Feed Auto-Discovery**: Extract user metadata from feed during `add user` command
+
## Duplicate Entry Management
+
### Duplicate Detection Strategy
+
- **Manual Curation**: Duplicates identified and managed manually via CLI
+
- **Storage**: `duplicates.json` file in Git root maps entry IDs to canonical entries
+
- **Structure**: `{"duplicate_id": "canonical_id", ...}`
+
- **CLI Commands**: Add/remove duplicate mappings with validation
+
- **Query Resolution**: Search/list commands resolve duplicates to canonical entries
+
### Duplicate File Format
+
"https://example.com/feed/entry/123": "https://canonical.com/posts/same-post",
+
"https://mirror.com/articles/456": "https://canonical.com/posts/same-post",
+
"comment": "Entry IDs that map to the same canonical content"
+
## Feed Metadata Auto-Discovery
+
### Extraction Strategy
+
When adding a new user with `thicket add user`, the system fetches and parses the feed to extract:
+
- **Display Name**: From `feed.title` or `feed.author.name`
+
- **Email**: From `feed.author.email` or `feed.managingEditor`
+
- **Homepage**: From `feed.link` or `feed.author.uri`
+
- **Icon**: From `feed.logo`, `feed.icon`, or `feed.image.url`
+
### Discovery Priority Order
+
1. **Author Information**: Prefer `feed.author.*` fields (more specific to person)
+
2. **Feed-Level**: Fall back to feed-level metadata
+
3. **Manual Override**: CLI flags always take precedence over discovered values
+
4. **Update Behavior**: Auto-discovery only runs during initial `add user`, not on sync
+
### Extracted Metadata Format
+
class FeedMetadata(BaseModel):
+
title: Optional[str] = None
+
author_name: Optional[str] = None
+
author_email: Optional[EmailStr] = None
+
author_uri: Optional[HttpUrl] = None
+
link: Optional[HttpUrl] = None
+
logo: Optional[HttpUrl] = None
+
icon: Optional[HttpUrl] = None
+
image_url: Optional[HttpUrl] = None
+
def to_user_config(self, username: str, feed_url: HttpUrl) -> UserConfig:
+
"""Convert discovered metadata to UserConfig with fallbacks"""
+
display_name=self.author_name or self.title,
+
email=self.author_email,
+
homepage=self.author_uri or self.link,
+
icon=self.logo or self.icon or self.image_url
+
## Link Processing and Threading Architecture
+
The thicket system implements a sophisticated link processing and threading system to create email-style threaded views of blog entries by tracking cross-references between different blogs.
+
### Link Processing Pipeline
+
#### 1. Link Extraction (`thicket links`)
+
The `links` command systematically extracts all outbound links from blog entries and categorizes them:
+
class LinkData(BaseModel):
+
url: str # Fully resolved URL
+
entry_id: str # Source entry ID
+
username: str # Source username
+
context: str # Surrounding text context
+
category: str # "internal", "user", or "unknown"
+
target_username: Optional[str] # Target user if applicable
+
- **Internal**: Links to the same user's domain (self-references)
+
- **User**: Links to other tracked users' domains
+
- **Unknown**: Links to external sites not tracked by thicket
+
All links are properly resolved using the Atom feed's base URL to handle:
+
- Relative URLs (converted to absolute)
+
- Protocol-relative URLs
+
- Redirects and canonical URLs
+
The system builds a comprehensive domain mapping from user configuration:
+
- Feed URLs → domain extraction
+
- Homepage URLs → domain extraction
+
- Reverse mapping: domain → username
+
#### 1. Reference Index Generation (`thicket index`)
+
Creates a bidirectional reference index from the categorized links:
+
class BlogReference(BaseModel):
+
target_username: Optional[str]
+
target_entry_id: Optional[str]
+
#### 2. Thread Detection Algorithm
+
Uses graph traversal to find connected blog entries:
+
- **Outbound references**: Links from an entry to other entries
+
- **Inbound references**: Links to an entry from other entries
+
- **Thread members**: All entries connected through references
+
#### 3. Threading Display (`thicket threads`)
+
Creates email-style threaded views:
+
- Chronological ordering within threads
+
- Reference counts (outbound/inbound)
+
- Filtering options (user, entry, minimum size)
+
#### links.json Format (Unified Structure)
+
"https://example.com/post/123": {
+
"referencing_entries": ["https://blog.user.com/entry/456"],
+
"target_username": "user2"
+
"https://external-site.com/article": {
+
"referencing_entries": ["https://blog.user.com/entry/789"]
+
"https://blog.user.com/entry/456": ["https://example.com/post/123"],
+
"https://blog.user.com/entry/789": ["https://external-site.com/article"]
+
"source_entry_id": "https://blog.user.com/entry/456",
+
"source_username": "user1",
+
"target_url": "https://example.com/post/123",
+
"target_username": "user2",
+
"target_entry_id": "https://example.com/post/123",
+
"context": "As mentioned in this post..."
+
"user1": ["blog.user.com"],
+
"user2": ["example.com"]
+
This unified structure eliminates duplication by:
+
- Storing each URL only once with minimal metadata
+
- Including all link data, reference data, and mappings in one file
+
- Using presence of `target_username` to identify tracked vs external links
+
- Providing bidirectional mappings for efficient queries
+
### Unified Structure Benefits
+
- **Eliminates Duplication**: Each URL appears only once with metadata
+
- **Single Source of Truth**: All link-related data in one file
+
- **Efficient Queries**: Fast lookups for both directions (URL→entries, entry→URLs)
+
- **Atomic Updates**: All link data changes together
+
- **Reduced I/O**: Fewer file operations
+
### Implementation Benefits
+
1. **Systematic Link Processing**: All links are extracted and categorized consistently
+
2. **Proper URL Resolution**: Handles relative URLs and base URL resolution correctly
+
3. **Domain-based Categorization**: Automatically identifies user-to-user references
+
4. **Bidirectional Indexing**: Supports both "who links to whom" and "who is linked by whom"
+
5. **Thread Discovery**: Finds conversation threads automatically
+
6. **Rich Context**: Preserves surrounding text for each link
+
7. **Performance**: Pre-computed indexes for fast threading queries
+
# Extract and categorize all links
+
thicket links --verbose
+
# Build reference index for threading
+
thicket index --verbose
+
# Show all conversation threads
+
# Show threads for specific user
+
thicket threads --username user1
+
# Show threads with minimum size
+
thicket threads --min-size 3
+
### Integration with Existing Commands
+
The link processing system integrates seamlessly with existing thicket commands:
+
- `thicket sync` updates entries, requiring `thicket links` to be run afterward
+
- `thicket index` uses the output from `thicket links` for improved accuracy
+
- `thicket threads` provides the user-facing threading interface
+
## Current Implementation Status
+
### ✅ Completed Features
+
1. **Core Infrastructure**
+
- Modern CLI with Typer and Rich
+
- Pydantic data models for type safety
+
- Git repository operations with GitPython
+
- Feed parsing and normalization with feedparser
+
2. **User and Feed Management**
+
- `thicket init` - Initialize git store
+
- `thicket add` - Add users and feeds with auto-discovery
+
- `thicket sync` - Sync feeds with progress tracking
+
- `thicket list` - List users, feeds, and entries
+
- `thicket duplicates` - Manage duplicate entries
+
3. **Link Processing and Threading**
+
- `thicket links` - Extract and categorize all outbound links
+
- `thicket index` - Build reference index from links
+
- `thicket threads` - Display threaded conversation views
+
- Proper URL resolution with base URL handling
+
- Domain-based link categorization
+
- Context preservation for links
+
### 📊 System Performance
+
- **Link Extraction**: Successfully processes thousands of blog entries
+
- **Categorization**: Identifies internal, user, and unknown links
+
- **Threading**: Creates email-style threaded views of conversations
+
- **Storage**: Efficient JSON-based data structures for links and references
+
### 🔧 Current Architecture Highlights
+
- **Modular Design**: Clear separation between CLI, core logic, and models
+
- **Type Safety**: Comprehensive Pydantic models for data validation
+
- **Rich CLI**: Beautiful progress bars, tables, and error handling
+
- **Extensible**: Easy to add new commands and features
+
- **Git Integration**: All data stored in version-controlled JSON files
+
### 🎯 Proven Functionality
+
The system has been tested with real blog data and successfully:
+
- Extracted 14,396 total links from blog entries
+
- Categorized 3,994 internal links, 363 user-to-user links, and 10,039 unknown links
+
- Built comprehensive domain mappings for 16 users across 20 domains
+
- Generated threaded views showing blog conversation patterns
+
The thicket system is now fully functional for:
+
- Maintaining Git repositories of blog feeds
+
- Tracking cross-references between blogs
+
- Creating threaded views of blog conversations
+
- Discovering blog interaction patterns
+
- Building distributed comment systems
+
<file path="src/thicket/cli/utils.py">
+
"""CLI utilities and helpers."""
+
from pathlib import Path
+
from typing import Optional
+
from rich.console import Console
+
from rich.progress import Progress, SpinnerColumn, TextColumn
+
from rich.table import Table
+
from ..models import ThicketConfig, UserMetadata
+
from ..core.git_store import GitStore
+
def get_tsv_mode() -> bool:
+
"""Get the global TSV mode setting."""
+
from .main import tsv_mode
+
def load_config(config_path: Optional[Path] = None) -> ThicketConfig:
+
"""Load thicket configuration from file or environment."""
+
if config_path and config_path.exists():
+
with open(config_path) as f:
+
config_data = yaml.safe_load(f)
+
# Convert to ThicketConfig
+
return ThicketConfig(**config_data)
+
# Try to load from default locations or environment
+
# First try to find thicket.yaml in current directory
+
default_config = Path("thicket.yaml")
+
if default_config.exists():
+
with open(default_config) as f:
+
config_data = yaml.safe_load(f)
+
return ThicketConfig(**config_data)
+
# Fall back to environment variables
+
console.print(f"[red]Error loading configuration: {e}[/red]")
+
console.print("[yellow]Run 'thicket init' to create a new configuration.[/yellow]")
+
raise typer.Exit(1) from e
+
def save_config(config: ThicketConfig, config_path: Path) -> None:
+
"""Save thicket configuration to file."""
+
config_data = config.model_dump(mode="json", exclude_none=True)
+
# Convert Path objects to strings for YAML serialization
+
config_data["git_store"] = str(config_data["git_store"])
+
config_data["cache_dir"] = str(config_data["cache_dir"])
+
with open(config_path, "w") as f:
+
yaml.dump(config_data, f, default_flow_style=False, sort_keys=False)
+
def create_progress() -> Progress:
+
"""Create a Rich progress display."""
+
TextColumn("[progress.description]{task.description}"),
+
def print_users_table(config: ThicketConfig) -> None:
+
"""Print a table of users and their feeds."""
+
print_users_tsv(config)
+
table = Table(title="Users and Feeds")
+
table.add_column("Username", style="cyan", no_wrap=True)
+
table.add_column("Display Name", style="magenta")
+
table.add_column("Email", style="blue")
+
table.add_column("Homepage", style="green")
+
table.add_column("Feeds", style="yellow")
+
for user in config.users:
+
feeds_str = "\n".join(str(feed) for feed in user.feeds)
+
user.display_name or "",
+
str(user.homepage) if user.homepage else "",
+
def print_feeds_table(config: ThicketConfig, username: Optional[str] = None) -> None:
+
"""Print a table of feeds, optionally filtered by username."""
+
print_feeds_tsv(config, username)
+
table = Table(title=f"Feeds{f' for {username}' if username else ''}")
+
table.add_column("Username", style="cyan", no_wrap=True)
+
table.add_column("Feed URL", style="blue")
+
table.add_column("Status", style="green")
+
users = [config.find_user(username)] if username else config.users
+
users = [u for u in users if u is not None]
+
for feed in user.feeds:
+
"Active", # TODO: Add actual status checking
+
def confirm_action(message: str, default: bool = False) -> bool:
+
"""Prompt for confirmation."""
+
return typer.confirm(message, default=default)
+
def print_success(message: str) -> None:
+
"""Print a success message."""
+
console.print(f"[green]✓[/green] {message}")
+
def print_error(message: str) -> None:
+
"""Print an error message."""
+
console.print(f"[red]✗[/red] {message}")
+
def print_warning(message: str) -> None:
+
"""Print a warning message."""
+
console.print(f"[yellow]⚠[/yellow] {message}")
+
def print_info(message: str) -> None:
+
"""Print an info message."""
+
console.print(f"[blue]ℹ[/blue] {message}")
+
def print_users_table_from_git(users: list[UserMetadata]) -> None:
+
"""Print a table of users from git repository."""
+
print_users_tsv_from_git(users)
+
table = Table(title="Users and Feeds")
+
table.add_column("Username", style="cyan", no_wrap=True)
+
table.add_column("Display Name", style="magenta")
+
table.add_column("Email", style="blue")
+
table.add_column("Homepage", style="green")
+
table.add_column("Feeds", style="yellow")
+
feeds_str = "\n".join(user.feeds)
+
user.display_name or "",
+
def print_feeds_table_from_git(git_store: GitStore, username: Optional[str] = None) -> None:
+
"""Print a table of feeds from git repository."""
+
print_feeds_tsv_from_git(git_store, username)
+
table = Table(title=f"Feeds{f' for {username}' if username else ''}")
+
table.add_column("Username", style="cyan", no_wrap=True)
+
table.add_column("Feed URL", style="blue")
+
table.add_column("Status", style="green")
+
user = git_store.get_user(username)
+
users = [user] if user else []
+
index = git_store._load_index()
+
users = list(index.users.values())
+
for feed in user.feeds:
+
"Active", # TODO: Add actual status checking
+
def print_users_tsv(config: ThicketConfig) -> None:
+
"""Print users in TSV format."""
+
print("Username\tDisplay Name\tEmail\tHomepage\tFeeds")
+
for user in config.users:
+
feeds_str = ",".join(str(feed) for feed in user.feeds)
+
print(f"{user.username}\t{user.display_name or ''}\t{user.email or ''}\t{user.homepage or ''}\t{feeds_str}")
+
def print_users_tsv_from_git(users: list[UserMetadata]) -> None:
+
"""Print users from git repository in TSV format."""
+
print("Username\tDisplay Name\tEmail\tHomepage\tFeeds")
+
feeds_str = ",".join(user.feeds)
+
print(f"{user.username}\t{user.display_name or ''}\t{user.email or ''}\t{user.homepage or ''}\t{feeds_str}")
+
def print_feeds_tsv(config: ThicketConfig, username: Optional[str] = None) -> None:
+
"""Print feeds in TSV format."""
+
print("Username\tFeed URL\tStatus")
+
users = [config.find_user(username)] if username else config.users
+
users = [u for u in users if u is not None]
+
for feed in user.feeds:
+
print(f"{user.username}\t{feed}\tActive")
+
def print_feeds_tsv_from_git(git_store: GitStore, username: Optional[str] = None) -> None:
+
"""Print feeds from git repository in TSV format."""
+
print("Username\tFeed URL\tStatus")
+
user = git_store.get_user(username)
+
users = [user] if user else []
+
index = git_store._load_index()
+
users = list(index.users.values())
+
for feed in user.feeds:
+
print(f"{user.username}\t{feed}\tActive")
+
def print_entries_tsv(entries_by_user: list[list], usernames: list[str]) -> None:
+
"""Print entries in TSV format."""
+
print("User\tAtom ID\tTitle\tUpdated\tURL")
+
# Combine all entries with usernames
+
for entries, username in zip(entries_by_user, usernames):
+
all_entries.append((username, entry))
+
# Sort by updated time (newest first)
+
all_entries.sort(key=lambda x: x[1].updated, reverse=True)
+
for username, entry in all_entries:
+
updated_str = entry.updated.strftime("%Y-%m-%d %H:%M")
+
# Escape tabs and newlines in title to preserve TSV format
+
title = entry.title.replace('\t', ' ').replace('\n', ' ').replace('\r', ' ')
+
print(f"{username}\t{entry.id}\t{title}\t{updated_str}\t{entry.link}")
anil.recoil.org