···
1
+
This file is a merged representation of the entire codebase, combined into a single document by Repomix.
4
+
This section contains a summary of this file.
7
+
This file contains a packed representation of the entire repository's contents.
8
+
It is designed to be easily consumable by AI systems for analysis, code review,
9
+
or other automated processes.
13
+
The content is organized as follows:
14
+
1. This summary section
15
+
2. Repository information
16
+
3. Directory structure
17
+
4. Repository files (if enabled)
18
+
5. Multiple file entries, each consisting of:
19
+
- File path as an attribute
20
+
- Full contents of the file
24
+
- This file should be treated as read-only. Any changes should be made to the
25
+
original repository files, not this packed version.
26
+
- When processing this file, use the file path to distinguish
27
+
between different files in the repository.
28
+
- Be aware that this file may contain sensitive information. Handle it with
29
+
the same level of security as you would the original repository.
33
+
- Some files may have been excluded based on .gitignore rules and Repomix's configuration
34
+
- Binary files are not included in this packed representation. Please refer to the Repository Structure section for a complete list of file paths, including binary files
35
+
- Files matching patterns in .gitignore are excluded
36
+
- Files matching default ignore patterns are excluded
37
+
- Files are sorted by Git change count (files with more changes are at the bottom)
42
+
<directory_structure>
89
+
</directory_structure>
92
+
This section contains the contents of the repository's files.
94
+
<file path=".claude/settings.local.json">
106
+
"enableAllProjectMcpServers": false
110
+
<file path="src/thicket/cli/commands/generate.py">
111
+
"""Generate static HTML website from thicket data."""
117
+
from datetime import datetime
118
+
from pathlib import Path
119
+
from typing import Any, Optional, TypedDict, Union
122
+
from jinja2 import Environment, FileSystemLoader, select_autoescape
123
+
from rich.progress import Progress, SpinnerColumn, TextColumn
125
+
from ...core.git_store import GitStore
126
+
from ...models.feed import AtomEntry
127
+
from ...models.user import GitStoreIndex, UserMetadata
128
+
from ..main import app
129
+
from ..utils import console, load_config
132
+
class UserData(TypedDict):
133
+
"""Type definition for user data structure."""
135
+
metadata: UserMetadata
136
+
recent_entries: list[tuple[str, AtomEntry]]
139
+
def safe_anchor_id(atom_id: str) -> str:
140
+
"""Convert an Atom ID to a safe HTML anchor ID."""
141
+
# Use base64 URL-safe encoding without padding
142
+
encoded = base64.urlsafe_b64encode(atom_id.encode('utf-8')).decode('ascii').rstrip('=')
143
+
# Prefix with 'id' to ensure it starts with a letter (HTML requirement)
144
+
return f"id{encoded}"
147
+
class WebsiteGenerator:
148
+
"""Generate static HTML website from thicket data."""
150
+
def __init__(self, git_store: GitStore, output_dir: Path):
151
+
self.git_store = git_store
152
+
self.output_dir = output_dir
153
+
self.template_dir = Path(__file__).parent.parent.parent / "templates"
155
+
# Initialize Jinja2 environment
156
+
self.env = Environment(
157
+
loader=FileSystemLoader(self.template_dir),
158
+
autoescape=select_autoescape(["html", "xml"]),
162
+
self.index: Optional[GitStoreIndex] = None
163
+
self.entries: list[tuple[str, AtomEntry]] = [] # (username, entry)
164
+
self.links_data: Optional[dict[str, Any]] = None
165
+
self.threads: list[list[dict[str, Any]]] = [] # List of threads with metadata
167
+
def get_display_name(self, username: str) -> str:
168
+
"""Get display name for a user, falling back to username."""
169
+
if self.index and username in self.index.users:
170
+
user = self.index.users[username]
171
+
return user.display_name or username
174
+
def get_user_homepage(self, username: str) -> Optional[str]:
175
+
"""Get homepage URL for a user."""
176
+
if self.index and username in self.index.users:
177
+
user = self.index.users[username]
178
+
return str(user.homepage) if user.homepage else None
181
+
def clean_html_summary(self, content: Optional[str], max_length: int = 200) -> str:
182
+
"""Clean HTML content and truncate for display in timeline."""
187
+
clean_text = re.sub(r"<[^>]+>", " ", content)
188
+
# Replace multiple whitespace with single space
189
+
clean_text = re.sub(r"\s+", " ", clean_text)
190
+
# Strip leading/trailing whitespace
191
+
clean_text = clean_text.strip()
193
+
# Truncate with ellipsis if needed
194
+
if len(clean_text) > max_length:
195
+
# Try to break at word boundary
196
+
truncated = clean_text[:max_length]
197
+
last_space = truncated.rfind(" ")
199
+
last_space > max_length * 0.8
200
+
): # If we can break reasonably close to the limit
201
+
clean_text = truncated[:last_space] + "..."
203
+
clean_text = truncated + "..."
207
+
def load_data(self) -> None:
208
+
"""Load all data from the git repository."""
211
+
TextColumn("[progress.description]{task.description}"),
215
+
task = progress.add_task("Loading repository index...", total=None)
216
+
self.index = self.git_store._load_index()
218
+
raise ValueError("No index found in repository")
219
+
progress.update(task, completed=True)
222
+
task = progress.add_task("Loading entries...", total=None)
223
+
for username, user_metadata in self.index.users.items():
224
+
user_dir = self.git_store.repo_path / user_metadata.directory
225
+
if user_dir.exists():
226
+
for entry_file in user_dir.glob("*.json"):
227
+
if entry_file.name not in ["index.json", "duplicates.json"]:
229
+
with open(entry_file) as f:
230
+
entry_data = json.load(f)
231
+
entry = AtomEntry(**entry_data)
232
+
self.entries.append((username, entry))
233
+
except Exception as e:
235
+
f"[yellow]Warning: Failed to load {entry_file}: {e}[/yellow]"
237
+
progress.update(task, completed=True)
239
+
# Sort entries by date (newest first) - prioritize updated over published
241
+
key=lambda x: x[1].updated or x[1].published or datetime.min, reverse=True
245
+
task = progress.add_task("Loading links and references...", total=None)
246
+
links_file = self.git_store.repo_path / "links.json"
247
+
if links_file.exists():
248
+
with open(links_file) as f:
249
+
self.links_data = json.load(f)
250
+
progress.update(task, completed=True)
252
+
def build_threads(self) -> None:
253
+
"""Build threaded conversations from references."""
254
+
if not self.links_data or "references" not in self.links_data:
257
+
# Map entry IDs to (username, entry) tuples
258
+
entry_map: dict[str, tuple[str, AtomEntry]] = {}
259
+
for username, entry in self.entries:
260
+
entry_map[entry.id] = (username, entry)
262
+
# Build adjacency lists for references
263
+
self.outbound_refs: dict[str, set[str]] = {}
264
+
self.inbound_refs: dict[str, set[str]] = {}
265
+
self.reference_details: dict[
266
+
str, list[dict[str, Any]]
267
+
] = {} # Store full reference info
269
+
for ref in self.links_data["references"]:
270
+
source_id = ref["source_entry_id"]
271
+
target_id = ref.get("target_entry_id")
273
+
if target_id and source_id in entry_map and target_id in entry_map:
274
+
self.outbound_refs.setdefault(source_id, set()).add(target_id)
275
+
self.inbound_refs.setdefault(target_id, set()).add(source_id)
277
+
# Store reference details for UI
278
+
self.reference_details.setdefault(source_id, []).append(
280
+
"target_id": target_id,
281
+
"target_username": ref.get("target_username"),
282
+
"type": "outbound",
285
+
self.reference_details.setdefault(target_id, []).append(
287
+
"source_id": source_id,
288
+
"source_username": ref.get("source_username"),
293
+
# Find conversation threads (multi-post discussions)
296
+
for entry_id, (_username, _entry) in entry_map.items():
297
+
if entry_id in processed:
300
+
# Build thread starting from this entry
302
+
to_visit = [entry_id]
304
+
level_map: dict[str, int] = {} # Track levels for this thread
306
+
# First, traverse up to find the root
308
+
while current in self.inbound_refs:
309
+
parents = self.inbound_refs[current] - {
311
+
} # Exclude self-references
314
+
# Take the first parent
315
+
parent = next(iter(parents))
316
+
if parent in thread_ids: # Avoid cycles
319
+
to_visit.insert(0, current)
321
+
# Now traverse down from the root
323
+
current = to_visit.pop(0)
324
+
if current in thread_ids or current not in entry_map:
327
+
thread_ids.add(current)
328
+
username, entry = entry_map[current]
330
+
# Calculate thread level
331
+
thread_level = self._calculate_thread_level(current, level_map)
333
+
# Add threading metadata
335
+
"username": username,
336
+
"display_name": self.get_display_name(username),
338
+
"entry_id": current,
339
+
"references_to": list(self.outbound_refs.get(current, [])),
340
+
"referenced_by": list(self.inbound_refs.get(current, [])),
341
+
"thread_level": thread_level,
343
+
thread.append(thread_entry)
344
+
processed.add(current)
347
+
if current in self.outbound_refs:
348
+
children = self.outbound_refs[current] - thread_ids # Avoid cycles
349
+
to_visit.extend(sorted(children))
351
+
if len(thread) > 1: # Only keep actual threads
352
+
# Sort thread by date (newest first) - prioritize updated over published
353
+
thread.sort(key=lambda x: x["entry"].updated or x["entry"].published or datetime.min, reverse=True) # type: ignore
354
+
self.threads.append(thread)
356
+
# Sort threads by the date of their most recent entry - prioritize updated over published
359
+
item["entry"].updated or item["entry"].published or datetime.min for item in t
364
+
def _calculate_thread_level(
365
+
self, entry_id: str, processed_entries: dict[str, int]
367
+
"""Calculate indentation level for threaded display."""
368
+
if entry_id in processed_entries:
369
+
return processed_entries[entry_id]
371
+
if entry_id not in self.inbound_refs:
372
+
processed_entries[entry_id] = 0
375
+
parents_in_thread = self.inbound_refs[entry_id] & set(processed_entries.keys())
376
+
if not parents_in_thread:
377
+
processed_entries[entry_id] = 0
380
+
# Find the deepest parent level + 1
381
+
max_parent_level = 0
382
+
for parent_id in parents_in_thread:
383
+
parent_level = self._calculate_thread_level(parent_id, processed_entries)
384
+
max_parent_level = max(max_parent_level, parent_level)
386
+
level = min(max_parent_level + 1, 4) # Cap at level 4
387
+
processed_entries[entry_id] = level
390
+
def get_standalone_references(self) -> list[dict[str, Any]]:
391
+
"""Get posts that have references but aren't part of multi-post threads."""
392
+
if not hasattr(self, "reference_details"):
395
+
threaded_entry_ids = set()
396
+
for thread in self.threads:
397
+
for item in thread:
398
+
threaded_entry_ids.add(item["entry_id"])
400
+
standalone_refs = []
401
+
for username, entry in self.entries:
403
+
entry.id in self.reference_details
404
+
and entry.id not in threaded_entry_ids
406
+
refs = self.reference_details[entry.id]
407
+
# Only include if it has meaningful references (not just self-references)
408
+
meaningful_refs = [
411
+
if r.get("target_id") != entry.id and r.get("source_id") != entry.id
413
+
if meaningful_refs:
414
+
standalone_refs.append(
416
+
"username": username,
417
+
"display_name": self.get_display_name(username),
419
+
"references": meaningful_refs,
423
+
return standalone_refs
425
+
def _add_cross_thread_links(self, timeline_items: list[dict[str, Any]]) -> None:
426
+
"""Add cross-thread linking for entries that appear in multiple threads."""
427
+
# Map entry IDs to their positions in the timeline
428
+
entry_positions: dict[str, list[int]] = {}
429
+
# Map URLs referenced by entries to the entries that reference them
430
+
url_references: dict[str, list[tuple[str, int]]] = {} # url -> [(entry_id, position)]
432
+
# First pass: collect all entry IDs, their positions, and referenced URLs
433
+
for i, item in enumerate(timeline_items):
434
+
if item["type"] == "post":
435
+
entry_id = item["content"]["entry"].id
436
+
entry_positions.setdefault(entry_id, []).append(i)
437
+
# Track URLs this entry references
438
+
if entry_id in self.reference_details:
439
+
for ref in self.reference_details[entry_id]:
440
+
if ref["type"] == "outbound" and "target_id" in ref:
441
+
# Find the target entry's URL if available
442
+
target_entry = self._find_entry_by_id(ref["target_id"])
443
+
if target_entry and target_entry.link:
444
+
url = str(target_entry.link)
445
+
url_references.setdefault(url, []).append((entry_id, i))
446
+
elif item["type"] == "thread":
447
+
for thread_item in item["content"]:
448
+
entry_id = thread_item["entry"].id
449
+
entry_positions.setdefault(entry_id, []).append(i)
450
+
# Track URLs this entry references
451
+
if entry_id in self.reference_details:
452
+
for ref in self.reference_details[entry_id]:
453
+
if ref["type"] == "outbound" and "target_id" in ref:
454
+
target_entry = self._find_entry_by_id(ref["target_id"])
455
+
if target_entry and target_entry.link:
456
+
url = str(target_entry.link)
457
+
url_references.setdefault(url, []).append((entry_id, i))
459
+
# Build cross-thread connections - only for entries that actually appear multiple times
460
+
cross_thread_connections: dict[str, set[int]] = {} # entry_id -> set of timeline positions
462
+
# Add connections ONLY for entries that appear multiple times in the timeline
463
+
for entry_id, positions in entry_positions.items():
464
+
if len(positions) > 1:
465
+
cross_thread_connections[entry_id] = set(positions)
466
+
# Debug: uncomment to see which entries have multiple appearances
467
+
# print(f"Entry {entry_id[:50]}... appears at positions: {positions}")
469
+
# Apply cross-thread links to timeline items
470
+
for entry_id, positions_set in cross_thread_connections.items():
471
+
positions_list = list(positions_set)
472
+
for pos in positions_list:
473
+
item = timeline_items[pos]
474
+
other_positions = sorted([p for p in positions_list if p != pos])
476
+
if item["type"] == "post":
477
+
# Add cross-thread info to individual posts
478
+
item["content"]["cross_thread_links"] = self._build_cross_thread_link_data(entry_id, other_positions, timeline_items)
479
+
# Add info about shared references
480
+
item["content"]["shared_references"] = self._get_shared_references(entry_id, positions_set, timeline_items)
481
+
elif item["type"] == "thread":
482
+
# Add cross-thread info to thread items
483
+
for thread_item in item["content"]:
484
+
if thread_item["entry"].id == entry_id:
485
+
thread_item["cross_thread_links"] = self._build_cross_thread_link_data(entry_id, other_positions, timeline_items)
486
+
thread_item["shared_references"] = self._get_shared_references(entry_id, positions_set, timeline_items)
489
+
def _build_cross_thread_link_data(self, entry_id: str, other_positions: list[int], timeline_items: list[dict[str, Any]]) -> list[dict[str, Any]]:
490
+
"""Build detailed cross-thread link data with anchor information."""
491
+
cross_thread_links = []
493
+
for pos in other_positions:
494
+
item = timeline_items[pos]
495
+
if item["type"] == "post":
496
+
# For individual posts
497
+
safe_id = safe_anchor_id(entry_id)
498
+
cross_thread_links.append({
500
+
"anchor_id": f"post-{pos}-{safe_id}",
501
+
"context": "individual post",
502
+
"title": item["content"]["entry"].title
504
+
elif item["type"] == "thread":
505
+
# For thread items, find the specific thread item
506
+
for thread_idx, thread_item in enumerate(item["content"]):
507
+
if thread_item["entry"].id == entry_id:
508
+
safe_id = safe_anchor_id(entry_id)
509
+
cross_thread_links.append({
511
+
"anchor_id": f"post-{pos}-{thread_idx}-{safe_id}",
512
+
"context": f"thread (level {thread_item.get('thread_level', 0)})",
513
+
"title": thread_item["entry"].title
517
+
return cross_thread_links
519
+
def _find_entry_by_id(self, entry_id: str) -> Optional[AtomEntry]:
520
+
"""Find an entry by its ID."""
521
+
for _username, entry in self.entries:
522
+
if entry.id == entry_id:
526
+
def _get_shared_references(self, entry_id: str, positions: Union[set[int], list[int]], timeline_items: list[dict[str, Any]]) -> list[dict[str, Any]]:
527
+
"""Get information about shared references between cross-thread entries."""
530
+
# Collect all referenced URLs from entries at these positions
531
+
url_counts: dict[str, int] = {}
532
+
referencing_entries: dict[str, list[str]] = {} # url -> [entry_ids]
534
+
for pos in positions:
535
+
item = timeline_items[pos]
536
+
entries_to_check = []
538
+
if item["type"] == "post":
539
+
entries_to_check.append(item["content"]["entry"])
540
+
elif item["type"] == "thread":
541
+
entries_to_check.extend([ti["entry"] for ti in item["content"]])
543
+
for entry in entries_to_check:
544
+
if entry.id in self.reference_details:
545
+
for ref in self.reference_details[entry.id]:
546
+
if ref["type"] == "outbound" and "target_id" in ref:
547
+
target_entry = self._find_entry_by_id(ref["target_id"])
548
+
if target_entry and target_entry.link:
549
+
url = str(target_entry.link)
550
+
url_counts[url] = url_counts.get(url, 0) + 1
551
+
if url not in referencing_entries:
552
+
referencing_entries[url] = []
553
+
if entry.id not in referencing_entries[url]:
554
+
referencing_entries[url].append(entry.id)
556
+
# Find URLs referenced by multiple entries
557
+
for url, count in url_counts.items():
558
+
if count > 1 and len(referencing_entries[url]) > 1:
559
+
# Get the target entry info
560
+
target_entry = None
561
+
target_username = None
562
+
for ref in (self.links_data or {}).get("references", []):
563
+
if ref.get("target_url") == url:
564
+
target_username = ref.get("target_username")
565
+
if ref.get("target_entry_id"):
566
+
target_entry = self._find_entry_by_id(ref["target_entry_id"])
569
+
shared_refs.append({
572
+
"referencing_entries": referencing_entries[url],
573
+
"target_username": target_username,
574
+
"target_title": target_entry.title if target_entry else None
577
+
return sorted(shared_refs, key=lambda x: x["count"], reverse=True)
579
+
def generate_site(self) -> None:
580
+
"""Generate the static website."""
581
+
# Create output directory
582
+
self.output_dir.mkdir(parents=True, exist_ok=True)
584
+
# Create static directories
585
+
(self.output_dir / "css").mkdir(exist_ok=True)
586
+
(self.output_dir / "js").mkdir(exist_ok=True)
589
+
css_template = self.env.get_template("style.css")
590
+
css_content = css_template.render()
591
+
with open(self.output_dir / "css" / "style.css", "w") as f:
592
+
f.write(css_content)
594
+
# Generate JavaScript
595
+
js_template = self.env.get_template("script.js")
596
+
js_content = js_template.render()
597
+
with open(self.output_dir / "js" / "script.js", "w") as f:
598
+
f.write(js_content)
600
+
# Prepare common template data
602
+
"title": "Energy & Environment Group",
603
+
"generated_at": datetime.now().isoformat(),
604
+
"get_display_name": self.get_display_name,
605
+
"get_user_homepage": self.get_user_homepage,
606
+
"clean_html_summary": self.clean_html_summary,
607
+
"safe_anchor_id": safe_anchor_id,
610
+
# Build unified timeline
611
+
timeline_items = []
613
+
# Only consider the threads that will actually be displayed
614
+
displayed_threads = self.threads[:20] # Limit to 20 threads
616
+
# Track which entries are part of displayed threads
617
+
threaded_entry_ids = set()
618
+
for thread in displayed_threads:
619
+
for item in thread:
620
+
threaded_entry_ids.add(item["entry_id"])
622
+
# Add threads to timeline (using the date of the most recent post)
623
+
for thread in displayed_threads:
624
+
most_recent_date = max(
625
+
item["entry"].updated or item["entry"].published or datetime.min
628
+
timeline_items.append({
630
+
"date": most_recent_date,
634
+
# Add individual posts (not in threads)
635
+
for username, entry in self.entries[:50]:
636
+
if entry.id not in threaded_entry_ids:
637
+
# Check if this entry has references
639
+
entry.id in self.reference_details
640
+
if hasattr(self, "reference_details")
646
+
refs = self.reference_details.get(entry.id, [])
649
+
if r.get("target_id") != entry.id
650
+
and r.get("source_id") != entry.id
653
+
timeline_items.append({
655
+
"date": entry.updated or entry.published or datetime.min,
657
+
"username": username,
658
+
"display_name": self.get_display_name(username),
660
+
"references": refs if refs else None
664
+
# Sort unified timeline by date (newest first)
665
+
timeline_items.sort(key=lambda x: x["date"], reverse=True)
667
+
# Limit timeline to what will actually be rendered
668
+
timeline_items = timeline_items[:50] # Limit to 50 items total
670
+
# Add cross-thread linking for repeat blog references
671
+
self._add_cross_thread_links(timeline_items)
673
+
# Prepare outgoing links data
674
+
outgoing_links = []
675
+
if self.links_data and "links" in self.links_data:
676
+
for url, link_info in self.links_data["links"].items():
677
+
referencing_entries = []
678
+
for entry_id in link_info.get("referencing_entries", []):
679
+
for username, entry in self.entries:
680
+
if entry.id == entry_id:
681
+
referencing_entries.append(
682
+
(self.get_display_name(username), entry)
686
+
if referencing_entries:
687
+
# Sort by date - prioritize updated over published
688
+
referencing_entries.sort(
689
+
key=lambda x: x[1].updated or x[1].published or datetime.min, reverse=True
691
+
outgoing_links.append(
694
+
"target_username": link_info.get("target_username"),
695
+
"entries": referencing_entries,
699
+
# Sort links by most recent reference - prioritize updated over published
700
+
outgoing_links.sort(
701
+
key=lambda x: x["entries"][0][1].updated
702
+
or x["entries"][0][1].published or datetime.min,
706
+
# Prepare users data
707
+
users: list[UserData] = []
709
+
for username, user_metadata in self.index.users.items():
710
+
# Get recent entries for this user with display names
712
+
(self.get_display_name(u), e)
713
+
for u, e in self.entries
717
+
{"metadata": user_metadata, "recent_entries": user_entries}
719
+
# Sort by entry count
720
+
users.sort(key=lambda x: x["metadata"].entry_count, reverse=True)
722
+
# Generate timeline page
723
+
timeline_template = self.env.get_template("timeline.html")
724
+
timeline_content = timeline_template.render(
727
+
timeline_items=timeline_items, # Already limited above
729
+
with open(self.output_dir / "timeline.html", "w") as f:
730
+
f.write(timeline_content)
732
+
# Generate links page
733
+
links_template = self.env.get_template("links.html")
734
+
links_content = links_template.render(
737
+
outgoing_links=outgoing_links[:100],
739
+
with open(self.output_dir / "links.html", "w") as f:
740
+
f.write(links_content)
742
+
# Generate users page
743
+
users_template = self.env.get_template("users.html")
744
+
users_content = users_template.render(
749
+
with open(self.output_dir / "users.html", "w") as f:
750
+
f.write(users_content)
752
+
# Generate main index page (redirect to timeline)
753
+
index_template = self.env.get_template("index.html")
754
+
index_content = index_template.render(**base_data)
755
+
with open(self.output_dir / "index.html", "w") as f:
756
+
f.write(index_content)
758
+
console.print(f"[green]✓[/green] Generated website at {self.output_dir}")
759
+
console.print(f" - {len(self.entries)} entries")
760
+
console.print(f" - {len(self.threads)} conversation threads")
761
+
console.print(f" - {len(outgoing_links)} outgoing links")
762
+
console.print(f" - {len(users)} users")
764
+
" - Generated pages: index.html, timeline.html, links.html, users.html"
770
+
output: Path = typer.Option(
771
+
Path("./thicket-site"),
774
+
help="Output directory for the generated website",
776
+
force: bool = typer.Option(
777
+
False, "--force", "-f", help="Overwrite existing output directory"
779
+
config_file: Path = typer.Option(
780
+
Path("thicket.yaml"), "--config", help="Configuration file path"
783
+
"""Generate a static HTML website from thicket data."""
784
+
config = load_config(config_file)
786
+
if not config.git_store:
787
+
console.print("[red]No git store path configured[/red]")
788
+
raise typer.Exit(1)
790
+
git_store = GitStore(config.git_store)
792
+
# Check if output directory exists
793
+
if output.exists() and not force:
795
+
f"[red]Output directory {output} already exists. Use --force to overwrite.[/red]"
797
+
raise typer.Exit(1)
799
+
# Clean output directory if forcing
800
+
if output.exists() and force:
801
+
shutil.rmtree(output)
804
+
generator = WebsiteGenerator(git_store, output)
806
+
console.print("[bold]Generating static website...[/bold]")
807
+
generator.load_data()
808
+
generator.build_threads()
809
+
generator.generate_site()
811
+
except Exception as e:
812
+
console.print(f"[red]Error generating website: {e}[/red]")
813
+
raise typer.Exit(1) from e
816
+
<file path="src/thicket/templates/base.html">
820
+
<meta charset="UTF-8">
821
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
822
+
<title>{% block page_title %}{{ title }}{% endblock %}</title>
823
+
<link rel="stylesheet" href="css/style.css">
826
+
<header class="site-header">
827
+
<div class="header-content">
828
+
<h1 class="site-title">{{ title }}</h1>
829
+
<nav class="site-nav">
830
+
<a href="timeline.html" class="nav-link {% if page == 'timeline' %}active{% endif %}">Timeline</a>
831
+
<a href="links.html" class="nav-link {% if page == 'links' %}active{% endif %}">Links</a>
832
+
<a href="users.html" class="nav-link {% if page == 'users' %}active{% endif %}">Users</a>
837
+
<main class="main-content">
838
+
{% block content %}{% endblock %}
841
+
<footer class="site-footer">
842
+
<p>Generated on {{ generated_at }} by <a href="https://github.com/avsm/thicket">Thicket</a></p>
845
+
<script src="js/script.js"></script>
850
+
<file path="src/thicket/templates/index.html">
854
+
<meta charset="UTF-8">
855
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
856
+
<title>{{ title }}</title>
857
+
<meta http-equiv="refresh" content="0; url=timeline.html">
858
+
<link rel="canonical" href="timeline.html">
861
+
<p>Redirecting to <a href="timeline.html">Timeline</a>...</p>
866
+
<file path="src/thicket/templates/links.html">
867
+
{% extends "base.html" %}
869
+
{% block page_title %}Outgoing Links - {{ title }}{% endblock %}
871
+
{% block content %}
872
+
<div class="page-content">
873
+
<h2>Outgoing Links</h2>
874
+
<p class="page-description">External links referenced in blog posts, ordered by most recent reference.</p>
876
+
{% for link in outgoing_links %}
877
+
<article class="link-group">
878
+
<h3 class="link-url">
879
+
<a href="{{ link.url }}" target="_blank">{{ link.url|truncate(80) }}</a>
880
+
{% if link.target_username %}
881
+
<span class="target-user">({{ link.target_username }})</span>
884
+
<div class="referencing-entries">
885
+
<span class="ref-count">Referenced in {{ link.entries|length }} post(s):</span>
887
+
{% for display_name, entry in link.entries[:5] %}
889
+
<span class="author">{{ display_name }}</span> -
890
+
<a href="{{ entry.link }}" target="_blank">{{ entry.title }}</a>
891
+
<time datetime="{{ entry.updated or entry.published }}">
892
+
({{ (entry.updated or entry.published).strftime('%Y-%m-%d') }})
896
+
{% if link.entries|length > 5 %}
897
+
<li class="more">... and {{ link.entries|length - 5 }} more</li>
907
+
<file path="src/thicket/templates/script.js">
908
+
// Enhanced functionality for thicket website
909
+
document.addEventListener('DOMContentLoaded', function() {
911
+
// Enhance thread collapsing (optional feature)
912
+
const threadHeaders = document.querySelectorAll('.thread-header');
913
+
threadHeaders.forEach(header => {
914
+
header.style.cursor = 'pointer';
915
+
header.addEventListener('click', function() {
916
+
const thread = this.parentElement;
917
+
const entries = thread.querySelectorAll('.thread-entry');
919
+
// Toggle visibility of all but the first entry
920
+
for (let i = 1; i < entries.length; i++) {
921
+
entries[i].style.display = entries[i].style.display === 'none' ? 'block' : 'none';
924
+
// Update thread count text
925
+
const count = this.querySelector('.thread-count');
926
+
if (entries[1] && entries[1].style.display === 'none') {
927
+
count.textContent = count.textContent.replace('posts', 'posts (collapsed)');
929
+
count.textContent = count.textContent.replace(' (collapsed)', '');
934
+
// Add relative time display
935
+
const timeElements = document.querySelectorAll('time');
936
+
timeElements.forEach(timeEl => {
937
+
const datetime = new Date(timeEl.getAttribute('datetime'));
938
+
const now = new Date();
939
+
const diffMs = now - datetime;
940
+
const diffDays = Math.floor(diffMs / (1000 * 60 * 60 * 24));
943
+
if (diffDays === 0) {
944
+
const diffHours = Math.floor(diffMs / (1000 * 60 * 60));
945
+
if (diffHours === 0) {
946
+
const diffMinutes = Math.floor(diffMs / (1000 * 60));
947
+
relativeTime = diffMinutes === 0 ? 'just now' : `${diffMinutes}m ago`;
949
+
relativeTime = `${diffHours}h ago`;
951
+
} else if (diffDays === 1) {
952
+
relativeTime = 'yesterday';
953
+
} else if (diffDays < 7) {
954
+
relativeTime = `${diffDays}d ago`;
955
+
} else if (diffDays < 30) {
956
+
const weeks = Math.floor(diffDays / 7);
957
+
relativeTime = weeks === 1 ? '1w ago' : `${weeks}w ago`;
958
+
} else if (diffDays < 365) {
959
+
const months = Math.floor(diffDays / 30);
960
+
relativeTime = months === 1 ? '1mo ago' : `${months}mo ago`;
962
+
const years = Math.floor(diffDays / 365);
963
+
relativeTime = years === 1 ? '1y ago' : `${years}y ago`;
966
+
// Add relative time as title attribute
967
+
timeEl.setAttribute('title', timeEl.textContent);
968
+
timeEl.textContent = relativeTime;
971
+
// Enhanced anchor link scrolling for shared references
972
+
document.querySelectorAll('a[href^="#"]').forEach(anchor => {
973
+
anchor.addEventListener('click', function (e) {
974
+
e.preventDefault();
975
+
const target = document.querySelector(this.getAttribute('href'));
977
+
target.scrollIntoView({
978
+
behavior: 'smooth',
982
+
// Highlight the target briefly
983
+
const timelineEntry = target.closest('.timeline-entry');
984
+
if (timelineEntry) {
985
+
timelineEntry.style.outline = '2px solid var(--primary-color)';
986
+
timelineEntry.style.borderRadius = '8px';
988
+
timelineEntry.style.outline = '';
989
+
timelineEntry.style.borderRadius = '';
998
+
<file path="src/thicket/templates/style.css">
999
+
/* Modern, clean design with high-density text and readable theme */
1002
+
--primary-color: #2c3e50;
1003
+
--secondary-color: #3498db;
1004
+
--accent-color: #e74c3c;
1005
+
--background: #ffffff;
1006
+
--surface: #f8f9fa;
1007
+
--text-primary: #2c3e50;
1008
+
--text-secondary: #7f8c8d;
1009
+
--border-color: #e0e0e0;
1010
+
--thread-indent: 20px;
1011
+
--max-width: 1200px;
1017
+
box-sizing: border-box;
1021
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Helvetica Neue', Arial, sans-serif;
1024
+
color: var(--text-primary);
1025
+
background-color: var(--background);
1030
+
background-color: var(--surface);
1031
+
border-bottom: 1px solid var(--border-color);
1032
+
padding: 0.75rem 0;
1039
+
max-width: var(--max-width);
1043
+
justify-content: space-between;
1044
+
align-items: center;
1048
+
font-size: 1.5rem;
1050
+
color: var(--primary-color);
1061
+
text-decoration: none;
1062
+
color: var(--text-secondary);
1064
+
font-size: 0.95rem;
1065
+
padding: 0.5rem 0.75rem;
1066
+
border-radius: 4px;
1067
+
transition: all 0.2s ease;
1071
+
color: var(--primary-color);
1072
+
background-color: var(--background);
1075
+
.nav-link.active {
1076
+
color: var(--secondary-color);
1077
+
background-color: var(--background);
1081
+
/* Main Content */
1083
+
max-width: var(--max-width);
1084
+
margin: 2rem auto;
1092
+
.page-description {
1093
+
color: var(--text-secondary);
1094
+
margin-bottom: 1.5rem;
1095
+
font-style: italic;
1100
+
margin-bottom: 2rem;
1104
+
font-size: 1.3rem;
1106
+
margin-bottom: 0.75rem;
1107
+
color: var(--primary-color);
1111
+
font-size: 1.1rem;
1113
+
margin-bottom: 0.75rem;
1114
+
color: var(--primary-color);
1117
+
/* Entries and Threads */
1119
+
margin-bottom: 1.5rem;
1121
+
background-color: var(--surface);
1122
+
border-radius: 4px;
1123
+
border: 1px solid var(--border-color);
1126
+
/* Timeline-style entries */
1128
+
margin-bottom: 0.5rem;
1129
+
padding: 0.5rem 0.75rem;
1131
+
background: transparent;
1132
+
transition: background-color 0.2s ease;
1135
+
.timeline-entry:hover {
1136
+
background-color: var(--surface);
1140
+
display: inline-flex;
1142
+
align-items: center;
1143
+
font-size: 0.75rem;
1144
+
color: var(--text-secondary);
1145
+
margin-bottom: 0.25rem;
1149
+
font-family: 'SF Mono', Monaco, Consolas, 'Courier New', monospace;
1150
+
font-size: 0.75rem;
1151
+
color: var(--text-secondary);
1154
+
.timeline-author {
1156
+
color: var(--primary-color);
1157
+
font-size: 0.8rem;
1158
+
text-decoration: none;
1161
+
.timeline-author:hover {
1162
+
color: var(--secondary-color);
1163
+
text-decoration: underline;
1166
+
.timeline-content {
1171
+
font-size: 0.95rem;
1175
+
.timeline-title a {
1176
+
color: var(--primary-color);
1177
+
text-decoration: none;
1180
+
.timeline-title a:hover {
1181
+
color: var(--secondary-color);
1182
+
text-decoration: underline;
1185
+
.timeline-summary {
1186
+
color: var(--text-secondary);
1187
+
font-size: 0.9rem;
1191
+
/* Legacy styles for other sections */
1192
+
.entry-meta, .thread-header {
1195
+
align-items: center;
1196
+
margin-bottom: 0.5rem;
1197
+
font-size: 0.85rem;
1198
+
color: var(--text-secondary);
1203
+
color: var(--primary-color);
1207
+
font-size: 0.85rem;
1211
+
font-size: 1.1rem;
1213
+
margin-bottom: 0.5rem;
1217
+
color: var(--primary-color);
1218
+
text-decoration: none;
1222
+
color: var(--secondary-color);
1223
+
text-decoration: underline;
1227
+
color: var(--text-primary);
1229
+
margin-top: 0.5rem;
1232
+
/* Enhanced Threading Styles */
1234
+
/* Conversation Clusters */
1235
+
.conversation-cluster {
1236
+
background-color: var(--background);
1237
+
border: 2px solid var(--border-color);
1238
+
border-radius: 8px;
1239
+
margin-bottom: 2rem;
1241
+
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
1244
+
.conversation-header {
1245
+
background: linear-gradient(135deg, var(--surface) 0%, #f1f3f4 100%);
1246
+
padding: 0.75rem 1rem;
1247
+
border-bottom: 1px solid var(--border-color);
1250
+
.conversation-meta {
1252
+
justify-content: space-between;
1253
+
align-items: center;
1258
+
.conversation-count {
1260
+
color: var(--secondary-color);
1261
+
font-size: 0.9rem;
1264
+
.conversation-participants {
1265
+
font-size: 0.8rem;
1266
+
color: var(--text-secondary);
1268
+
text-align: right;
1271
+
.conversation-flow {
1275
+
/* Threaded Conversation Entries */
1276
+
.conversation-entry {
1277
+
position: relative;
1278
+
margin-bottom: 0.75rem;
1280
+
align-items: flex-start;
1283
+
.conversation-entry.level-0 {
1287
+
.conversation-entry.level-1 {
1288
+
margin-left: 1.5rem;
1291
+
.conversation-entry.level-2 {
1292
+
margin-left: 3rem;
1295
+
.conversation-entry.level-3 {
1296
+
margin-left: 4.5rem;
1299
+
.conversation-entry.level-4 {
1300
+
margin-left: 6rem;
1303
+
.entry-connector {
1305
+
background-color: var(--secondary-color);
1306
+
margin-right: 0.75rem;
1307
+
margin-top: 0.25rem;
1309
+
border-radius: 2px;
1313
+
.conversation-entry.level-0 .entry-connector {
1314
+
background-color: var(--accent-color);
1320
+
background-color: var(--surface);
1322
+
border-radius: 6px;
1323
+
border: 1px solid var(--border-color);
1324
+
transition: all 0.2s ease;
1327
+
.entry-content:hover {
1328
+
border-color: var(--secondary-color);
1329
+
box-shadow: 0 2px 8px rgba(52, 152, 219, 0.1);
1332
+
/* Reference Indicators */
1333
+
.reference-indicators {
1334
+
display: inline-flex;
1336
+
margin-left: 0.5rem;
1339
+
.ref-out, .ref-in {
1340
+
display: inline-block;
1343
+
border-radius: 50%;
1344
+
text-align: center;
1345
+
line-height: 1rem;
1346
+
font-size: 0.7rem;
1347
+
font-weight: bold;
1351
+
background-color: #e8f5e8;
1356
+
background-color: #e8f0ff;
1360
+
/* Reference Badges for Individual Posts */
1361
+
.timeline-entry.with-references {
1362
+
background-color: var(--surface);
1365
+
/* Conversation posts in unified timeline */
1366
+
.timeline-entry.conversation-post {
1367
+
background: transparent;
1369
+
margin-bottom: 0.5rem;
1370
+
padding: 0.5rem 0.75rem;
1373
+
.timeline-entry.conversation-post.level-0 {
1375
+
border-left: 2px solid var(--accent-color);
1376
+
padding-left: 0.75rem;
1379
+
.timeline-entry.conversation-post.level-1 {
1380
+
margin-left: 1.5rem;
1381
+
border-left: 2px solid var(--secondary-color);
1382
+
padding-left: 0.75rem;
1385
+
.timeline-entry.conversation-post.level-2 {
1386
+
margin-left: 3rem;
1387
+
border-left: 2px solid var(--text-secondary);
1388
+
padding-left: 0.75rem;
1391
+
.timeline-entry.conversation-post.level-3 {
1392
+
margin-left: 4.5rem;
1393
+
border-left: 2px solid var(--text-secondary);
1394
+
padding-left: 0.75rem;
1397
+
.timeline-entry.conversation-post.level-4 {
1398
+
margin-left: 6rem;
1399
+
border-left: 2px solid var(--text-secondary);
1400
+
padding-left: 0.75rem;
1403
+
/* Cross-thread linking */
1404
+
.cross-thread-links {
1405
+
margin-top: 0.5rem;
1406
+
padding-top: 0.5rem;
1407
+
border-top: 1px solid var(--border-color);
1410
+
.cross-thread-indicator {
1411
+
font-size: 0.75rem;
1412
+
color: var(--text-secondary);
1413
+
background-color: var(--surface);
1414
+
padding: 0.25rem 0.5rem;
1415
+
border-radius: 12px;
1416
+
border: 1px solid var(--border-color);
1417
+
display: inline-block;
1420
+
/* Inline shared references styling */
1421
+
.inline-shared-refs {
1422
+
margin-left: 0.5rem;
1423
+
font-size: 0.85rem;
1424
+
color: var(--text-secondary);
1427
+
.shared-ref-link {
1428
+
color: var(--primary-color);
1429
+
text-decoration: none;
1431
+
transition: color 0.2s ease;
1434
+
.shared-ref-link:hover {
1435
+
color: var(--secondary-color);
1436
+
text-decoration: underline;
1439
+
.shared-ref-more {
1440
+
font-style: italic;
1441
+
color: var(--text-secondary);
1442
+
font-size: 0.8rem;
1443
+
margin-left: 0.25rem;
1446
+
.user-anchor, .post-anchor {
1447
+
position: absolute;
1448
+
margin-top: -60px; /* Offset for fixed header */
1449
+
pointer-events: none;
1452
+
.cross-thread-link {
1453
+
color: var(--primary-color);
1454
+
text-decoration: none;
1456
+
transition: color 0.2s ease;
1459
+
.cross-thread-link:hover {
1460
+
color: var(--secondary-color);
1461
+
text-decoration: underline;
1464
+
.reference-badges {
1467
+
margin-left: 0.5rem;
1472
+
display: inline-block;
1473
+
padding: 0.1rem 0.4rem;
1474
+
border-radius: 12px;
1475
+
font-size: 0.7rem;
1477
+
text-transform: uppercase;
1478
+
letter-spacing: 0.05em;
1481
+
.ref-badge.ref-outbound {
1482
+
background-color: #e8f5e8;
1484
+
border: 1px solid #c3e6c3;
1487
+
.ref-badge.ref-inbound {
1488
+
background-color: #e8f0ff;
1490
+
border: 1px solid #b3d9ff;
1493
+
/* Author Color Coding */
1494
+
.timeline-author {
1495
+
position: relative;
1498
+
.timeline-author::before {
1500
+
display: inline-block;
1503
+
border-radius: 50%;
1504
+
margin-right: 0.5rem;
1505
+
background-color: var(--secondary-color);
1508
+
/* Generate consistent colors for authors */
1509
+
.author-avsm::before { background-color: #e74c3c; }
1510
+
.author-mort::before { background-color: #3498db; }
1511
+
.author-mte::before { background-color: #2ecc71; }
1512
+
.author-ryan::before { background-color: #f39c12; }
1513
+
.author-mwd::before { background-color: #9b59b6; }
1514
+
.author-dra::before { background-color: #1abc9c; }
1515
+
.author-pf341::before { background-color: #34495e; }
1516
+
.author-sadiqj::before { background-color: #e67e22; }
1517
+
.author-martinkl::before { background-color: #8e44ad; }
1518
+
.author-jonsterling::before { background-color: #27ae60; }
1519
+
.author-jon::before { background-color: #f1c40f; }
1520
+
.author-onkar::before { background-color: #e91e63; }
1521
+
.author-gabriel::before { background-color: #00bcd4; }
1522
+
.author-jess::before { background-color: #ff5722; }
1523
+
.author-ibrahim::before { background-color: #607d8b; }
1524
+
.author-andres::before { background-color: #795548; }
1525
+
.author-eeg::before { background-color: #ff9800; }
1527
+
/* Section Headers */
1528
+
.conversations-section h3,
1529
+
.referenced-posts-section h3,
1530
+
.individual-posts-section h3 {
1531
+
border-bottom: 2px solid var(--border-color);
1532
+
padding-bottom: 0.5rem;
1533
+
margin-bottom: 1.5rem;
1534
+
position: relative;
1537
+
.conversations-section h3::before {
1539
+
margin-right: 0.5rem;
1542
+
.referenced-posts-section h3::before {
1544
+
margin-right: 0.5rem;
1547
+
.individual-posts-section h3::before {
1549
+
margin-right: 0.5rem;
1552
+
/* Legacy thread styles (for backward compatibility) */
1554
+
background-color: var(--background);
1555
+
border: 1px solid var(--border-color);
1558
+
margin-bottom: 1rem;
1562
+
background-color: var(--surface);
1563
+
padding: 0.5rem 0.75rem;
1564
+
border-bottom: 1px solid var(--border-color);
1569
+
color: var(--secondary-color);
1573
+
padding: 0.5rem 0.75rem;
1574
+
border-bottom: 1px solid var(--border-color);
1577
+
.thread-entry:last-child {
1578
+
border-bottom: none;
1581
+
.thread-entry.reply {
1582
+
margin-left: var(--thread-indent);
1583
+
border-left: 3px solid var(--secondary-color);
1584
+
background-color: var(--surface);
1587
+
/* Links Section */
1589
+
background-color: var(--background);
1594
+
word-break: break-word;
1598
+
color: var(--secondary-color);
1599
+
text-decoration: none;
1602
+
.link-url a:hover {
1603
+
text-decoration: underline;
1607
+
font-size: 0.9rem;
1608
+
color: var(--text-secondary);
1609
+
font-weight: normal;
1612
+
.referencing-entries {
1613
+
margin-top: 0.75rem;
1618
+
color: var(--text-secondary);
1619
+
font-size: 0.9rem;
1622
+
.referencing-entries ul {
1624
+
margin-top: 0.5rem;
1625
+
padding-left: 1rem;
1628
+
.referencing-entries li {
1629
+
margin-bottom: 0.25rem;
1630
+
font-size: 0.9rem;
1633
+
.referencing-entries .more {
1634
+
font-style: italic;
1635
+
color: var(--text-secondary);
1638
+
/* Users Section */
1640
+
background-color: var(--background);
1646
+
align-items: start;
1647
+
margin-bottom: 1rem;
1653
+
border-radius: 50%;
1654
+
object-fit: cover;
1658
+
margin-bottom: 0.25rem;
1662
+
font-size: 0.9rem;
1663
+
color: var(--text-secondary);
1664
+
font-weight: normal;
1668
+
font-size: 0.9rem;
1669
+
color: var(--text-secondary);
1673
+
color: var(--secondary-color);
1674
+
text-decoration: none;
1677
+
.user-meta a:hover {
1678
+
text-decoration: underline;
1690
+
font-size: 0.95rem;
1691
+
margin-bottom: 0.5rem;
1692
+
color: var(--text-secondary);
1701
+
margin-bottom: 0.25rem;
1702
+
font-size: 0.9rem;
1707
+
max-width: var(--max-width);
1708
+
margin: 3rem auto 2rem;
1709
+
padding: 1rem 2rem;
1710
+
text-align: center;
1711
+
color: var(--text-secondary);
1712
+
font-size: 0.85rem;
1713
+
border-top: 1px solid var(--border-color);
1717
+
color: var(--secondary-color);
1718
+
text-decoration: none;
1721
+
.site-footer a:hover {
1722
+
text-decoration: underline;
1726
+
@media (max-width: 768px) {
1728
+
font-size: 1.3rem;
1732
+
flex-direction: column;
1734
+
align-items: flex-start;
1745
+
.thread-entry.reply {
1746
+
margin-left: calc(var(--thread-indent) / 2);
1750
+
flex-direction: column;
1755
+
<file path="src/thicket/templates/timeline.html">
1756
+
{% extends "base.html" %}
1758
+
{% block page_title %}Timeline - {{ title }}{% endblock %}
1760
+
{% block content %}
1761
+
{% set seen_users = [] %}
1762
+
<div class="page-content">
1763
+
<h2>Recent Posts & Conversations</h2>
1765
+
<section class="unified-timeline">
1766
+
{% for item in timeline_items %}
1767
+
{% if item.type == "post" %}
1768
+
<!-- Individual Post -->
1769
+
<article class="timeline-entry {% if item.content.references %}with-references{% endif %}">
1770
+
<div class="timeline-meta">
1771
+
<time datetime="{{ item.content.entry.updated or item.content.entry.published }}" class="timeline-time">
1772
+
{{ (item.content.entry.updated or item.content.entry.published).strftime('%Y-%m-%d %H:%M') }}
1774
+
{% set homepage = get_user_homepage(item.content.username) %}
1775
+
{% if item.content.username not in seen_users %}
1776
+
<a id="{{ item.content.username }}" class="user-anchor"></a>
1777
+
{% set _ = seen_users.append(item.content.username) %}
1779
+
<a id="post-{{ loop.index0 }}-{{ safe_anchor_id(item.content.entry.id) }}" class="post-anchor"></a>
1781
+
<a href="{{ homepage }}" target="_blank" class="timeline-author">{{ item.content.display_name }}</a>
1783
+
<span class="timeline-author">{{ item.content.display_name }}</span>
1785
+
{% if item.content.references %}
1786
+
<div class="reference-badges">
1787
+
{% for ref in item.content.references %}
1788
+
{% if ref.type == 'outbound' %}
1789
+
<span class="ref-badge ref-outbound" title="References {{ ref.target_username or 'external post' }}">
1790
+
→ {{ ref.target_username or 'ext' }}
1792
+
{% elif ref.type == 'inbound' %}
1793
+
<span class="ref-badge ref-inbound" title="Referenced by {{ ref.source_username or 'external post' }}">
1794
+
← {{ ref.source_username or 'ext' }}
1801
+
<div class="timeline-content">
1802
+
<strong class="timeline-title">
1803
+
<a href="{{ item.content.entry.link }}" target="_blank">{{ item.content.entry.title }}</a>
1805
+
{% if item.content.entry.summary %}
1806
+
<span class="timeline-summary">— {{ clean_html_summary(item.content.entry.summary, 250) }}</span>
1808
+
{% if item.content.shared_references %}
1809
+
<span class="inline-shared-refs">
1810
+
{% for ref in item.content.shared_references[:3] %}
1811
+
{% if ref.target_username %}
1812
+
<a href="#{{ ref.target_username }}" class="shared-ref-link" title="Referenced by {{ ref.count }} entries">@{{ ref.target_username }}</a>{% if not loop.last %}, {% endif %}
1815
+
{% if item.content.shared_references|length > 3 %}
1816
+
<span class="shared-ref-more">+{{ item.content.shared_references|length - 3 }} more</span>
1820
+
{% if item.content.cross_thread_links %}
1821
+
<div class="cross-thread-links">
1822
+
<span class="cross-thread-indicator">🔗 Also appears: </span>
1823
+
{% for link in item.content.cross_thread_links %}
1824
+
<a href="#{{ link.anchor_id }}" class="cross-thread-link" title="{{ link.title }}">{{ link.context }}</a>{% if not loop.last %}, {% endif %}
1831
+
{% elif item.type == "thread" %}
1832
+
<!-- Conversation Thread -->
1833
+
{% set outer_loop_index = loop.index0 %}
1834
+
{% for thread_item in item.content %}
1835
+
<article class="timeline-entry conversation-post level-{{ thread_item.thread_level }}">
1836
+
<div class="timeline-meta">
1837
+
<time datetime="{{ thread_item.entry.updated or thread_item.entry.published }}" class="timeline-time">
1838
+
{{ (thread_item.entry.updated or thread_item.entry.published).strftime('%Y-%m-%d %H:%M') }}
1840
+
{% set homepage = get_user_homepage(thread_item.username) %}
1841
+
{% if thread_item.username not in seen_users %}
1842
+
<a id="{{ thread_item.username }}" class="user-anchor"></a>
1843
+
{% set _ = seen_users.append(thread_item.username) %}
1845
+
<a id="post-{{ outer_loop_index }}-{{ loop.index0 }}-{{ safe_anchor_id(thread_item.entry.id) }}" class="post-anchor"></a>
1847
+
<a href="{{ homepage }}" target="_blank" class="timeline-author author-{{ thread_item.username }}">{{ thread_item.display_name }}</a>
1849
+
<span class="timeline-author author-{{ thread_item.username }}">{{ thread_item.display_name }}</span>
1851
+
{% if thread_item.references_to or thread_item.referenced_by %}
1852
+
<span class="reference-indicators">
1853
+
{% if thread_item.references_to %}
1854
+
<span class="ref-out" title="References other posts">→</span>
1856
+
{% if thread_item.referenced_by %}
1857
+
<span class="ref-in" title="Referenced by other posts">←</span>
1862
+
<div class="timeline-content">
1863
+
<strong class="timeline-title">
1864
+
<a href="{{ thread_item.entry.link }}" target="_blank">{{ thread_item.entry.title }}</a>
1866
+
{% if thread_item.entry.summary %}
1867
+
<span class="timeline-summary">— {{ clean_html_summary(thread_item.entry.summary, 300) }}</span>
1869
+
{% if thread_item.shared_references %}
1870
+
<span class="inline-shared-refs">
1871
+
{% for ref in thread_item.shared_references[:3] %}
1872
+
{% if ref.target_username %}
1873
+
<a href="#{{ ref.target_username }}" class="shared-ref-link" title="Referenced by {{ ref.count }} entries">@{{ ref.target_username }}</a>{% if not loop.last %}, {% endif %}
1876
+
{% if thread_item.shared_references|length > 3 %}
1877
+
<span class="shared-ref-more">+{{ thread_item.shared_references|length - 3 }} more</span>
1881
+
{% if thread_item.cross_thread_links %}
1882
+
<div class="cross-thread-links">
1883
+
<span class="cross-thread-indicator">🔗 Also appears: </span>
1884
+
{% for link in thread_item.cross_thread_links %}
1885
+
<a href="#{{ link.anchor_id }}" class="cross-thread-link" title="{{ link.title }}">{{ link.context }}</a>{% if not loop.last %}, {% endif %}
1899
+
<file path="src/thicket/templates/users.html">
1900
+
{% extends "base.html" %}
1902
+
{% block page_title %}Users - {{ title }}{% endblock %}
1904
+
{% block content %}
1905
+
<div class="page-content">
1907
+
<p class="page-description">All users contributing to this thicket, ordered by post count.</p>
1909
+
{% for user_info in users %}
1910
+
<article class="user-card">
1911
+
<div class="user-header">
1912
+
{% if user_info.metadata.icon and user_info.metadata.icon != "None" %}
1913
+
<img src="{{ user_info.metadata.icon }}" alt="{{ user_info.metadata.username }}" class="user-icon">
1915
+
<div class="user-info">
1917
+
{% if user_info.metadata.display_name %}
1918
+
{{ user_info.metadata.display_name }}
1919
+
<span class="username">({{ user_info.metadata.username }})</span>
1921
+
{{ user_info.metadata.username }}
1924
+
<div class="user-meta">
1925
+
{% if user_info.metadata.homepage %}
1926
+
<a href="{{ user_info.metadata.homepage }}" target="_blank">{{ user_info.metadata.homepage }}</a>
1928
+
{% if user_info.metadata.email %}
1929
+
<span class="separator">•</span>
1930
+
<a href="mailto:{{ user_info.metadata.email }}">{{ user_info.metadata.email }}</a>
1932
+
<span class="separator">•</span>
1933
+
<span class="post-count">{{ user_info.metadata.entry_count }} posts</span>
1938
+
{% if user_info.recent_entries %}
1939
+
<div class="user-recent">
1940
+
<h4>Recent posts:</h4>
1942
+
{% for display_name, entry in user_info.recent_entries %}
1944
+
<a href="{{ entry.link }}" target="_blank">{{ entry.title }}</a>
1945
+
<time datetime="{{ entry.updated or entry.published }}">
1946
+
({{ (entry.updated or entry.published).strftime('%Y-%m-%d') }})
1959
+
<file path="README.md">
1962
+
A modern CLI tool for persisting Atom/RSS feeds in Git repositories, designed to enable distributed webblog comment structures.
1966
+
- **Feed Auto-Discovery**: Automatically extracts user metadata from Atom/RSS feeds
1967
+
- **Git Storage**: Stores feed entries in a Git repository with full history
1968
+
- **Duplicate Management**: Manual curation of duplicate entries across feeds
1969
+
- **Modern CLI**: Built with Typer and Rich for beautiful terminal output
1970
+
- **Comprehensive Parsing**: Supports RSS 0.9x, RSS 1.0, RSS 2.0, and Atom feeds
1971
+
- **Cron-Friendly**: Designed for scheduled execution
1976
+
# Install from source
1979
+
# Or install with dev dependencies
1980
+
pip install -e .[dev]
1985
+
1. **Initialize a new thicket repository:**
1987
+
thicket init ./my-feeds
1990
+
2. **Add a user with their feed:**
1992
+
thicket add user "alice" --feed "https://alice.example.com/feed.xml"
1995
+
3. **Sync feeds to download entries:**
1997
+
thicket sync --all
2000
+
4. **List users and feeds:**
2002
+
thicket list users
2003
+
thicket list feeds
2004
+
thicket list entries
2011
+
thicket init <git-store-path> [--cache-dir <path>] [--config <config-file>]
2014
+
### Add Users and Feeds
2016
+
# Add user with auto-discovery
2017
+
thicket add user "username" --feed "https://example.com/feed.xml"
2019
+
# Add user with manual metadata
2020
+
thicket add user "username" \
2021
+
--feed "https://example.com/feed.xml" \
2022
+
--email "user@example.com" \
2023
+
--homepage "https://example.com" \
2024
+
--display-name "User Name"
2026
+
# Add additional feed to existing user
2027
+
thicket add feed "username" "https://example.com/other-feed.xml"
2033
+
thicket sync --all
2035
+
# Sync specific user
2036
+
thicket sync --user "username"
2038
+
# Dry run (preview changes)
2039
+
thicket sync --all --dry-run
2042
+
### List Information
2045
+
thicket list users
2048
+
thicket list feeds
2050
+
# List feeds for specific user
2051
+
thicket list feeds --user "username"
2053
+
# List recent entries
2054
+
thicket list entries --limit 20
2056
+
# List entries for specific user
2057
+
thicket list entries --user "username"
2060
+
### Manage Duplicates
2062
+
# List duplicate mappings
2063
+
thicket duplicates list
2065
+
# Mark entries as duplicates
2066
+
thicket duplicates add "https://example.com/dup" "https://example.com/canonical"
2068
+
# Remove duplicate mapping
2069
+
thicket duplicates remove "https://example.com/dup"
2074
+
Thicket uses a YAML configuration file (default: `thicket.yaml`):
2077
+
git_store: ./feeds-repo
2078
+
cache_dir: ~/.cache/thicket
2082
+
- https://alice.example.com/feed.xml
2083
+
email: alice@example.com
2084
+
homepage: https://alice.example.com
2085
+
display_name: Alice
2088
+
## Git Repository Structure
2092
+
├── index.json # User directory index
2093
+
├── duplicates.json # Duplicate entry mappings
2095
+
│ ├── metadata.json # User metadata
2096
+
│ ├── entry_id_1.json # Feed entries
2097
+
│ └── entry_id_2.json
2106
+
# Install in development mode
2107
+
pip install -e .[dev]
2114
+
black --check src/
2116
+
# Run type checking
2122
+
- **CLI**: Modern interface with Typer and Rich
2123
+
- **Feed Processing**: Universal parsing with feedparser
2124
+
- **Git Storage**: Structured storage with GitPython
2125
+
- **Data Models**: Pydantic for validation and serialization
2126
+
- **Async HTTP**: httpx for efficient feed fetching
2130
+
- **Blog Aggregation**: Collect and archive blog posts from multiple sources
2131
+
- **Comment Networks**: Enable distributed commenting systems
2132
+
- **Feed Archival**: Preserve feed history beyond typical feed depth limits
2133
+
- **Content Curation**: Manage and deduplicate content across feeds
2137
+
MIT License - see LICENSE file for details.
2140
+
<file path="src/thicket/cli/commands/index_cmd.py">
2141
+
"""CLI command for building reference index from blog entries."""
2144
+
from pathlib import Path
2145
+
from typing import Optional
2148
+
from rich.console import Console
2149
+
from rich.progress import (
2153
+
TaskProgressColumn,
2156
+
from rich.table import Table
2158
+
from ...core.git_store import GitStore
2159
+
from ...core.reference_parser import ReferenceIndex, ReferenceParser
2160
+
from ..main import app
2161
+
from ..utils import get_tsv_mode, load_config
2163
+
console = Console()
2168
+
config_file: Optional[Path] = typer.Option(
2172
+
help="Path to configuration file",
2174
+
output_file: Optional[Path] = typer.Option(
2178
+
help="Path to output index file (default: updates links.json in git store)",
2180
+
verbose: bool = typer.Option(
2184
+
help="Show detailed progress information",
2187
+
"""Build a reference index showing which blog entries reference others.
2189
+
This command analyzes all blog entries to detect cross-references between
2190
+
different blogs, creating an index that can be used to build threaded
2191
+
views of related content.
2193
+
Updates the unified links.json file with reference data.
2196
+
# Load configuration
2197
+
config = load_config(config_file)
2199
+
# Initialize Git store
2200
+
git_store = GitStore(config.git_store)
2202
+
# Initialize reference parser
2203
+
parser = ReferenceParser()
2205
+
# Build user domain mapping
2207
+
console.print("Building user domain mapping...")
2208
+
user_domains = parser.build_user_domain_mapping(git_store)
2211
+
console.print(f"Found {len(user_domains)} users with {sum(len(d) for d in user_domains.values())} total domains")
2213
+
# Initialize reference index
2214
+
ref_index = ReferenceIndex()
2215
+
ref_index.user_domains = user_domains
2218
+
index = git_store._load_index()
2219
+
users = list(index.users.keys())
2222
+
console.print("[yellow]No users found in Git store[/yellow]")
2223
+
raise typer.Exit(0)
2225
+
# Process all entries
2227
+
total_references = 0
2228
+
all_references = []
2232
+
TextColumn("[progress.description]{task.description}"),
2234
+
TaskProgressColumn(),
2238
+
# Count total entries first
2239
+
counting_task = progress.add_task("Counting entries...", total=len(users))
2241
+
for username in users:
2242
+
entries = git_store.list_entries(username)
2243
+
entry_counts[username] = len(entries)
2244
+
total_entries += len(entries)
2245
+
progress.advance(counting_task)
2247
+
progress.remove_task(counting_task)
2249
+
# Process entries - extract references
2250
+
processing_task = progress.add_task(
2251
+
f"Extracting references from {total_entries} entries...",
2252
+
total=total_entries
2255
+
for username in users:
2256
+
entries = git_store.list_entries(username)
2258
+
for entry in entries:
2259
+
# Extract references from this entry
2260
+
references = parser.extract_references(entry, username, user_domains)
2261
+
all_references.extend(references)
2263
+
progress.advance(processing_task)
2265
+
if verbose and references:
2266
+
console.print(f" Found {len(references)} references in {username}:{entry.title[:50]}...")
2268
+
progress.remove_task(processing_task)
2270
+
# Resolve target_entry_ids for references
2271
+
if all_references:
2272
+
resolve_task = progress.add_task(
2273
+
f"Resolving {len(all_references)} references...",
2274
+
total=len(all_references)
2278
+
console.print(f"Resolving target entry IDs for {len(all_references)} references...")
2280
+
resolved_references = parser.resolve_target_entry_ids(all_references, git_store)
2282
+
# Count resolved references
2283
+
resolved_count = sum(1 for ref in resolved_references if ref.target_entry_id is not None)
2285
+
console.print(f"Resolved {resolved_count} out of {len(all_references)} references")
2287
+
# Add resolved references to index
2288
+
for ref in resolved_references:
2289
+
ref_index.add_reference(ref)
2290
+
total_references += 1
2291
+
progress.advance(resolve_task)
2293
+
progress.remove_task(resolve_task)
2295
+
# Determine output path
2297
+
output_path = output_file
2299
+
output_path = config.git_store / "links.json"
2301
+
# Load existing links data or create new structure
2302
+
if output_path.exists() and not output_file:
2303
+
# Load existing unified structure
2304
+
with open(output_path) as f:
2305
+
existing_data = json.load(f)
2307
+
# Create new structure
2310
+
"reverse_mapping": {},
2311
+
"user_domains": {}
2314
+
# Update with reference data
2315
+
existing_data["references"] = ref_index.to_dict()["references"]
2316
+
existing_data["user_domains"] = {k: list(v) for k, v in user_domains.items()}
2318
+
# Save updated structure
2319
+
with open(output_path, "w") as f:
2320
+
json.dump(existing_data, f, indent=2, default=str)
2323
+
if not get_tsv_mode():
2324
+
console.print("\n[green]✓ Reference index built successfully[/green]")
2326
+
# Create summary table or TSV output
2327
+
if get_tsv_mode():
2328
+
print("Metric\tCount")
2329
+
print(f"Total Users\t{len(users)}")
2330
+
print(f"Total Entries\t{total_entries}")
2331
+
print(f"Total References\t{total_references}")
2332
+
print(f"Outbound Refs\t{len(ref_index.outbound_refs)}")
2333
+
print(f"Inbound Refs\t{len(ref_index.inbound_refs)}")
2334
+
print(f"Output File\t{output_path}")
2336
+
table = Table(title="Reference Index Summary")
2337
+
table.add_column("Metric", style="cyan")
2338
+
table.add_column("Count", style="green")
2340
+
table.add_row("Total Users", str(len(users)))
2341
+
table.add_row("Total Entries", str(total_entries))
2342
+
table.add_row("Total References", str(total_references))
2343
+
table.add_row("Outbound Refs", str(len(ref_index.outbound_refs)))
2344
+
table.add_row("Inbound Refs", str(len(ref_index.inbound_refs)))
2345
+
table.add_row("Output File", str(output_path))
2347
+
console.print(table)
2349
+
# Show some interesting statistics
2350
+
if total_references > 0:
2351
+
if not get_tsv_mode():
2352
+
console.print("\n[bold]Reference Statistics:[/bold]")
2354
+
# Most referenced users
2355
+
target_counts = {}
2356
+
unresolved_domains = set()
2358
+
for ref in ref_index.references:
2359
+
if ref.target_username:
2360
+
target_counts[ref.target_username] = target_counts.get(ref.target_username, 0) + 1
2362
+
# Track unresolved domains
2363
+
from urllib.parse import urlparse
2364
+
domain = urlparse(ref.target_url).netloc.lower()
2365
+
unresolved_domains.add(domain)
2368
+
if get_tsv_mode():
2369
+
print("Referenced User\tReference Count")
2370
+
for username, count in sorted(target_counts.items(), key=lambda x: x[1], reverse=True)[:5]:
2371
+
print(f"{username}\t{count}")
2373
+
console.print("\nMost referenced users:")
2374
+
for username, count in sorted(target_counts.items(), key=lambda x: x[1], reverse=True)[:5]:
2375
+
console.print(f" {username}: {count} references")
2377
+
if unresolved_domains and verbose:
2378
+
if get_tsv_mode():
2379
+
print("Unresolved Domain\tCount")
2380
+
for domain in sorted(list(unresolved_domains)[:10]):
2381
+
print(f"{domain}\t1")
2382
+
if len(unresolved_domains) > 10:
2383
+
print(f"... and {len(unresolved_domains) - 10} more\t...")
2385
+
console.print(f"\nUnresolved domains: {len(unresolved_domains)}")
2386
+
for domain in sorted(list(unresolved_domains)[:10]):
2387
+
console.print(f" {domain}")
2388
+
if len(unresolved_domains) > 10:
2389
+
console.print(f" ... and {len(unresolved_domains) - 10} more")
2391
+
except Exception as e:
2392
+
console.print(f"[red]Error building reference index: {e}[/red]")
2394
+
console.print_exception()
2395
+
raise typer.Exit(1)
2400
+
config_file: Optional[Path] = typer.Option(
2404
+
help="Path to configuration file",
2406
+
index_file: Optional[Path] = typer.Option(
2410
+
help="Path to reference index file (default: links.json in git store)",
2412
+
username: Optional[str] = typer.Option(
2416
+
help="Show threads for specific username only",
2418
+
entry_id: Optional[str] = typer.Option(
2422
+
help="Show thread for specific entry ID",
2424
+
min_size: int = typer.Option(
2428
+
help="Minimum thread size to display",
2431
+
"""Show threaded view of related blog entries.
2433
+
This command uses the reference index to show which blog entries
2434
+
are connected through cross-references, creating an email-style
2435
+
threaded view of the conversation.
2437
+
Reads reference data from the unified links.json file.
2440
+
# Load configuration
2441
+
config = load_config(config_file)
2443
+
# Determine index file path
2445
+
index_path = index_file
2447
+
index_path = config.git_store / "links.json"
2449
+
if not index_path.exists():
2450
+
console.print(f"[red]Links file not found: {index_path}[/red]")
2451
+
console.print("Run 'thicket links' and 'thicket index' first to build the reference index")
2452
+
raise typer.Exit(1)
2454
+
# Load unified data
2455
+
with open(index_path) as f:
2456
+
unified_data = json.load(f)
2458
+
# Check if references exist in the unified structure
2459
+
if "references" not in unified_data:
2460
+
console.print(f"[red]No references found in {index_path}[/red]")
2461
+
console.print("Run 'thicket index' first to build the reference index")
2462
+
raise typer.Exit(1)
2464
+
# Extract reference data and reconstruct ReferenceIndex
2465
+
ref_index = ReferenceIndex.from_dict({
2466
+
"references": unified_data["references"],
2467
+
"user_domains": unified_data.get("user_domains", {})
2470
+
# Initialize Git store to get entry details
2471
+
git_store = GitStore(config.git_store)
2473
+
if entry_id and username:
2474
+
# Show specific thread
2475
+
thread_members = ref_index.get_thread_members(username, entry_id)
2476
+
_display_thread(thread_members, ref_index, git_store, f"Thread for {username}:{entry_id}")
2479
+
# Show all threads involving this user
2480
+
user_index = git_store._load_index()
2481
+
user = user_index.get_user(username)
2483
+
console.print(f"[red]User not found: {username}[/red]")
2484
+
raise typer.Exit(1)
2486
+
entries = git_store.list_entries(username)
2487
+
threads_found = set()
2489
+
console.print(f"[bold]Threads involving {username}:[/bold]\n")
2491
+
for entry in entries:
2492
+
thread_members = ref_index.get_thread_members(username, entry.id)
2493
+
if len(thread_members) >= min_size:
2494
+
thread_key = tuple(sorted(thread_members))
2495
+
if thread_key not in threads_found:
2496
+
threads_found.add(thread_key)
2497
+
_display_thread(thread_members, ref_index, git_store, f"Thread #{len(threads_found)}")
2500
+
# Show all threads
2501
+
console.print("[bold]All conversation threads:[/bold]\n")
2503
+
all_threads = set()
2504
+
processed_entries = set()
2507
+
user_index = git_store._load_index()
2508
+
for username in user_index.users.keys():
2509
+
entries = git_store.list_entries(username)
2510
+
for entry in entries:
2511
+
entry_key = (username, entry.id)
2512
+
if entry_key in processed_entries:
2515
+
thread_members = ref_index.get_thread_members(username, entry.id)
2516
+
if len(thread_members) >= min_size:
2517
+
thread_key = tuple(sorted(thread_members))
2518
+
if thread_key not in all_threads:
2519
+
all_threads.add(thread_key)
2520
+
_display_thread(thread_members, ref_index, git_store, f"Thread #{len(all_threads)}")
2522
+
# Mark all members as processed
2523
+
for member in thread_members:
2524
+
processed_entries.add(member)
2526
+
if not all_threads:
2527
+
console.print("[yellow]No conversation threads found[/yellow]")
2528
+
console.print(f"(minimum thread size: {min_size})")
2530
+
except Exception as e:
2531
+
console.print(f"[red]Error showing threads: {e}[/red]")
2532
+
raise typer.Exit(1)
2535
+
def _display_thread(thread_members, ref_index, git_store, title):
2536
+
"""Display a single conversation thread."""
2537
+
console.print(f"[bold cyan]{title}[/bold cyan]")
2538
+
console.print(f"Thread size: {len(thread_members)} entries")
2540
+
# Get entry details for each member
2541
+
thread_entries = []
2542
+
for username, entry_id in thread_members:
2543
+
entry = git_store.get_entry(username, entry_id)
2545
+
thread_entries.append((username, entry))
2547
+
# Sort by publication date
2548
+
thread_entries.sort(key=lambda x: x[1].published or x[1].updated)
2551
+
for i, (username, entry) in enumerate(thread_entries):
2552
+
prefix = "├─" if i < len(thread_entries) - 1 else "└─"
2554
+
# Get references for this entry
2555
+
outbound = ref_index.get_outbound_refs(username, entry.id)
2556
+
inbound = ref_index.get_inbound_refs(username, entry.id)
2559
+
if outbound or inbound:
2560
+
ref_info = f" ({len(outbound)} out, {len(inbound)} in)"
2562
+
console.print(f" {prefix} [{username}] {entry.title[:60]}...{ref_info}")
2564
+
if entry.published:
2565
+
console.print(f" Published: {entry.published.strftime('%Y-%m-%d')}")
2567
+
console.print() # Empty line after each thread
2570
+
<file path="src/thicket/cli/commands/info_cmd.py">
2571
+
"""CLI command for displaying detailed information about a specific atom entry."""
2574
+
from pathlib import Path
2575
+
from typing import Optional
2578
+
from rich.console import Console
2579
+
from rich.panel import Panel
2580
+
from rich.table import Table
2581
+
from rich.text import Text
2583
+
from ...core.git_store import GitStore
2584
+
from ...core.reference_parser import ReferenceIndex
2585
+
from ..main import app
2586
+
from ..utils import load_config, get_tsv_mode
2588
+
console = Console()
2593
+
identifier: str = typer.Argument(
2595
+
help="The atom ID or URL of the entry to display information about"
2597
+
username: Optional[str] = typer.Option(
2601
+
help="Username to search for the entry (if not provided, searches all users)"
2603
+
config_file: Optional[Path] = typer.Option(
2604
+
Path("thicket.yaml"),
2607
+
help="Path to configuration file",
2609
+
show_content: bool = typer.Option(
2612
+
help="Include the full content of the entry in the output"
2615
+
"""Display detailed information about a specific atom entry.
2617
+
You can specify the entry using either its atom ID or URL.
2618
+
Shows all metadata for the given entry, including title, dates, categories,
2619
+
and summarizes all inbound and outbound links to/from other posts.
2622
+
# Load configuration
2623
+
config = load_config(config_file)
2625
+
# Initialize Git store
2626
+
git_store = GitStore(config.git_store)
2630
+
found_username = None
2632
+
# Check if identifier looks like a URL
2633
+
is_url = identifier.startswith(('http://', 'https://'))
2636
+
# Search specific username
2639
+
entries = git_store.list_entries(username)
2641
+
if str(e.link) == identifier:
2643
+
found_username = username
2646
+
# Search by atom ID
2647
+
entry = git_store.get_entry(username, identifier)
2649
+
found_username = username
2651
+
# Search all users
2652
+
index = git_store._load_index()
2653
+
for user in index.users.keys():
2656
+
entries = git_store.list_entries(user)
2658
+
if str(e.link) == identifier:
2660
+
found_username = user
2665
+
# Search by atom ID
2666
+
entry = git_store.get_entry(user, identifier)
2668
+
found_username = user
2671
+
if not entry or not found_username:
2673
+
console.print(f"[red]Entry with {'URL' if is_url else 'atom ID'} '{identifier}' not found for user '{username}'[/red]")
2675
+
console.print(f"[red]Entry with {'URL' if is_url else 'atom ID'} '{identifier}' not found in any user's entries[/red]")
2676
+
raise typer.Exit(1)
2678
+
# Load reference index if available
2679
+
links_path = config.git_store / "links.json"
2681
+
if links_path.exists():
2682
+
with open(links_path) as f:
2683
+
unified_data = json.load(f)
2685
+
# Check if references exist in the unified structure
2686
+
if "references" in unified_data:
2687
+
ref_index = ReferenceIndex.from_dict({
2688
+
"references": unified_data["references"],
2689
+
"user_domains": unified_data.get("user_domains", {})
2692
+
# Display information
2693
+
if get_tsv_mode():
2694
+
_display_entry_info_tsv(entry, found_username, ref_index, show_content)
2696
+
_display_entry_info(entry, found_username)
2699
+
_display_link_info(entry, found_username, ref_index)
2701
+
console.print("\n[yellow]No reference index found. Run 'thicket links' and 'thicket index' to build cross-reference data.[/yellow]")
2703
+
# Optionally display content
2704
+
if show_content and entry.content:
2705
+
_display_content(entry.content)
2707
+
except Exception as e:
2708
+
console.print(f"[red]Error displaying entry info: {e}[/red]")
2709
+
raise typer.Exit(1)
2712
+
def _display_entry_info(entry, username: str) -> None:
2713
+
"""Display basic entry information in a structured format."""
2715
+
# Create main info panel
2716
+
info_table = Table.grid(padding=(0, 2))
2717
+
info_table.add_column("Field", style="cyan bold", width=15)
2718
+
info_table.add_column("Value", style="white")
2720
+
info_table.add_row("User", f"[green]{username}[/green]")
2721
+
info_table.add_row("Atom ID", f"[blue]{entry.id}[/blue]")
2722
+
info_table.add_row("Title", entry.title)
2723
+
info_table.add_row("Link", str(entry.link))
2725
+
if entry.published:
2726
+
info_table.add_row("Published", entry.published.strftime("%Y-%m-%d %H:%M:%S UTC"))
2728
+
info_table.add_row("Updated", entry.updated.strftime("%Y-%m-%d %H:%M:%S UTC"))
2731
+
# Truncate long summaries
2732
+
summary = entry.summary[:200] + "..." if len(entry.summary) > 200 else entry.summary
2733
+
info_table.add_row("Summary", summary)
2735
+
if entry.categories:
2736
+
categories_text = ", ".join(entry.categories)
2737
+
info_table.add_row("Categories", categories_text)
2741
+
if "name" in entry.author:
2742
+
author_info.append(entry.author["name"])
2743
+
if "email" in entry.author:
2744
+
author_info.append(f"<{entry.author['email']}>")
2746
+
info_table.add_row("Author", " ".join(author_info))
2748
+
if entry.content_type:
2749
+
info_table.add_row("Content Type", entry.content_type)
2752
+
info_table.add_row("Rights", entry.rights)
2755
+
info_table.add_row("Source Feed", entry.source)
2759
+
title=f"[bold]Entry Information[/bold]",
2760
+
border_style="blue"
2763
+
console.print(panel)
2766
+
def _display_link_info(entry, username: str, ref_index: ReferenceIndex) -> None:
2767
+
"""Display inbound and outbound link information."""
2770
+
outbound_refs = ref_index.get_outbound_refs(username, entry.id)
2771
+
inbound_refs = ref_index.get_inbound_refs(username, entry.id)
2773
+
if not outbound_refs and not inbound_refs:
2774
+
console.print("\n[dim]No cross-references found for this entry.[/dim]")
2777
+
# Create links table
2778
+
links_table = Table(title="Cross-References")
2779
+
links_table.add_column("Direction", style="cyan", width=10)
2780
+
links_table.add_column("Target/Source", style="green", width=20)
2781
+
links_table.add_column("URL", style="blue", width=50)
2783
+
# Add outbound references
2784
+
for ref in outbound_refs:
2785
+
target_info = f"{ref.target_username}:{ref.target_entry_id}" if ref.target_username and ref.target_entry_id else "External"
2786
+
links_table.add_row("→ Out", target_info, ref.target_url)
2788
+
# Add inbound references
2789
+
for ref in inbound_refs:
2790
+
source_info = f"{ref.source_username}:{ref.source_entry_id}"
2791
+
links_table.add_row("← In", source_info, ref.target_url)
2794
+
console.print(links_table)
2797
+
console.print(f"\n[bold]Summary:[/bold] {len(outbound_refs)} outbound, {len(inbound_refs)} inbound references")
2800
+
def _display_content(content: str) -> None:
2801
+
"""Display the full content of the entry."""
2803
+
# Truncate very long content
2804
+
display_content = content
2805
+
if len(content) > 5000:
2806
+
display_content = content[:5000] + "\n\n[... content truncated ...]"
2810
+
title="[bold]Entry Content[/bold]",
2811
+
border_style="green",
2816
+
console.print(panel)
2819
+
def _display_entry_info_tsv(entry, username: str, ref_index: Optional[ReferenceIndex], show_content: bool) -> None:
2820
+
"""Display entry information in TSV format."""
2823
+
print("Field\tValue")
2824
+
print(f"User\t{username}")
2825
+
print(f"Atom ID\t{entry.id}")
2826
+
print(f"Title\t{entry.title.replace(chr(9), ' ').replace(chr(10), ' ').replace(chr(13), ' ')}")
2827
+
print(f"Link\t{entry.link}")
2829
+
if entry.published:
2830
+
print(f"Published\t{entry.published.strftime('%Y-%m-%d %H:%M:%S UTC')}")
2832
+
print(f"Updated\t{entry.updated.strftime('%Y-%m-%d %H:%M:%S UTC')}")
2835
+
# Escape tabs and newlines in summary
2836
+
summary = entry.summary.replace('\t', ' ').replace('\n', ' ').replace('\r', ' ')
2837
+
print(f"Summary\t{summary}")
2839
+
if entry.categories:
2840
+
print(f"Categories\t{', '.join(entry.categories)}")
2844
+
if "name" in entry.author:
2845
+
author_info.append(entry.author["name"])
2846
+
if "email" in entry.author:
2847
+
author_info.append(f"<{entry.author['email']}>")
2849
+
print(f"Author\t{' '.join(author_info)}")
2851
+
if entry.content_type:
2852
+
print(f"Content Type\t{entry.content_type}")
2855
+
print(f"Rights\t{entry.rights}")
2858
+
print(f"Source Feed\t{entry.source}")
2860
+
# Add reference info if available
2862
+
outbound_refs = ref_index.get_outbound_refs(username, entry.id)
2863
+
inbound_refs = ref_index.get_inbound_refs(username, entry.id)
2865
+
print(f"Outbound References\t{len(outbound_refs)}")
2866
+
print(f"Inbound References\t{len(inbound_refs)}")
2868
+
# Show each reference
2869
+
for ref in outbound_refs:
2870
+
target_info = f"{ref.target_username}:{ref.target_entry_id}" if ref.target_username and ref.target_entry_id else "External"
2871
+
print(f"Outbound Reference\t{target_info}\t{ref.target_url}")
2873
+
for ref in inbound_refs:
2874
+
source_info = f"{ref.source_username}:{ref.source_entry_id}"
2875
+
print(f"Inbound Reference\t{source_info}\t{ref.target_url}")
2877
+
# Show content if requested
2878
+
if show_content and entry.content:
2879
+
# Escape tabs and newlines in content
2880
+
content = entry.content.replace('\t', ' ').replace('\n', ' ').replace('\r', ' ')
2881
+
print(f"Content\t{content}")
2884
+
<file path="src/thicket/cli/commands/init.py">
2885
+
"""Initialize command for thicket."""
2887
+
from pathlib import Path
2888
+
from typing import Optional
2891
+
from pydantic import ValidationError
2893
+
from ...core.git_store import GitStore
2894
+
from ...models import ThicketConfig
2895
+
from ..main import app
2896
+
from ..utils import print_error, print_success, save_config
2901
+
git_store: Path = typer.Argument(..., help="Path to Git repository for storing feeds"),
2902
+
cache_dir: Optional[Path] = typer.Option(
2903
+
None, "--cache-dir", "-c", help="Cache directory (default: ~/.cache/thicket)"
2905
+
config_file: Optional[Path] = typer.Option(
2906
+
None, "--config", help="Configuration file path (default: thicket.yaml)"
2908
+
force: bool = typer.Option(
2909
+
False, "--force", "-f", help="Overwrite existing configuration"
2912
+
"""Initialize a new thicket configuration and Git store."""
2914
+
# Set default paths
2915
+
if cache_dir is None:
2916
+
from platformdirs import user_cache_dir
2917
+
cache_dir = Path(user_cache_dir("thicket"))
2919
+
if config_file is None:
2920
+
config_file = Path("thicket.yaml")
2922
+
# Check if config already exists
2923
+
if config_file.exists() and not force:
2924
+
print_error(f"Configuration file already exists: {config_file}")
2925
+
print_error("Use --force to overwrite")
2926
+
raise typer.Exit(1)
2928
+
# Create cache directory
2929
+
cache_dir.mkdir(parents=True, exist_ok=True)
2931
+
# Create Git store
2933
+
GitStore(git_store)
2934
+
print_success(f"Initialized Git store at: {git_store}")
2935
+
except Exception as e:
2936
+
print_error(f"Failed to initialize Git store: {e}")
2937
+
raise typer.Exit(1) from e
2939
+
# Create configuration
2941
+
config = ThicketConfig(
2942
+
git_store=git_store,
2943
+
cache_dir=cache_dir,
2947
+
save_config(config, config_file)
2948
+
print_success(f"Created configuration file: {config_file}")
2950
+
except ValidationError as e:
2951
+
print_error(f"Invalid configuration: {e}")
2952
+
raise typer.Exit(1) from e
2953
+
except Exception as e:
2954
+
print_error(f"Failed to create configuration: {e}")
2955
+
raise typer.Exit(1) from e
2957
+
print_success("Thicket initialized successfully!")
2958
+
print_success(f"Git store: {git_store}")
2959
+
print_success(f"Cache directory: {cache_dir}")
2960
+
print_success(f"Configuration: {config_file}")
2961
+
print_success("Run 'thicket add user' to add your first user and feed.")
2964
+
<file path="src/thicket/cli/__init__.py">
2965
+
"""CLI interface for thicket."""
2967
+
from .main import app
2972
+
<file path="src/thicket/core/__init__.py">
2973
+
"""Core business logic for thicket."""
2975
+
from .feed_parser import FeedParser
2976
+
from .git_store import GitStore
2978
+
__all__ = ["FeedParser", "GitStore"]
2981
+
<file path="src/thicket/core/feed_parser.py">
2982
+
"""Feed parsing and normalization with auto-discovery."""
2984
+
from datetime import datetime
2985
+
from typing import Optional
2986
+
from urllib.parse import urlparse
2991
+
from pydantic import HttpUrl, ValidationError
2993
+
from ..models import AtomEntry, FeedMetadata
2997
+
"""Parser for RSS/Atom feeds with normalization and auto-discovery."""
2999
+
def __init__(self, user_agent: str = "thicket/0.1.0"):
3000
+
"""Initialize the feed parser."""
3001
+
self.user_agent = user_agent
3002
+
self.allowed_tags = [
3003
+
"a", "abbr", "acronym", "b", "blockquote", "br", "code", "em",
3004
+
"i", "li", "ol", "p", "pre", "strong", "ul", "h1", "h2", "h3",
3005
+
"h4", "h5", "h6", "img", "div", "span",
3007
+
self.allowed_attributes = {
3008
+
"a": ["href", "title"],
3009
+
"abbr": ["title"],
3010
+
"acronym": ["title"],
3011
+
"img": ["src", "alt", "title", "width", "height"],
3012
+
"blockquote": ["cite"],
3015
+
async def fetch_feed(self, url: HttpUrl) -> str:
3016
+
"""Fetch feed content from URL."""
3017
+
async with httpx.AsyncClient() as client:
3018
+
response = await client.get(
3020
+
headers={"User-Agent": self.user_agent},
3022
+
follow_redirects=True,
3024
+
response.raise_for_status()
3025
+
return response.text
3027
+
def parse_feed(self, content: str, source_url: Optional[HttpUrl] = None) -> tuple[FeedMetadata, list[AtomEntry]]:
3028
+
"""Parse feed content and return metadata and entries."""
3029
+
parsed = feedparser.parse(content)
3031
+
if parsed.bozo and parsed.bozo_exception:
3032
+
# Try to continue with potentially malformed feed
3035
+
# Extract feed metadata
3036
+
feed_meta = self._extract_feed_metadata(parsed.feed)
3038
+
# Extract and normalize entries
3040
+
for entry in parsed.entries:
3042
+
atom_entry = self._normalize_entry(entry, source_url)
3043
+
entries.append(atom_entry)
3044
+
except Exception as e:
3045
+
# Log error but continue processing other entries
3046
+
print(f"Error processing entry {getattr(entry, 'id', 'unknown')}: {e}")
3049
+
return feed_meta, entries
3051
+
def _extract_feed_metadata(self, feed: feedparser.FeedParserDict) -> FeedMetadata:
3052
+
"""Extract metadata from feed for auto-discovery."""
3053
+
# Parse author information
3054
+
author_name = None
3055
+
author_email = None
3058
+
if hasattr(feed, 'author_detail'):
3059
+
author_name = feed.author_detail.get('name')
3060
+
author_email = feed.author_detail.get('email')
3061
+
author_uri = feed.author_detail.get('href')
3062
+
elif hasattr(feed, 'author'):
3063
+
author_name = feed.author
3065
+
# Parse managing editor for RSS feeds
3066
+
if not author_email and hasattr(feed, 'managingEditor'):
3067
+
author_email = feed.managingEditor
3071
+
if hasattr(feed, 'link'):
3073
+
feed_link = HttpUrl(feed.link)
3074
+
except ValidationError:
3077
+
# Parse image/icon/logo
3082
+
if hasattr(feed, 'image'):
3084
+
image_url = HttpUrl(feed.image.get('href', feed.image.get('url', '')))
3085
+
except (ValidationError, AttributeError):
3088
+
if hasattr(feed, 'icon'):
3090
+
icon = HttpUrl(feed.icon)
3091
+
except ValidationError:
3094
+
if hasattr(feed, 'logo'):
3096
+
logo = HttpUrl(feed.logo)
3097
+
except ValidationError:
3100
+
return FeedMetadata(
3101
+
title=getattr(feed, 'title', None),
3102
+
author_name=author_name,
3103
+
author_email=author_email,
3104
+
author_uri=HttpUrl(author_uri) if author_uri else None,
3108
+
image_url=image_url,
3109
+
description=getattr(feed, 'description', None),
3112
+
def _normalize_entry(self, entry: feedparser.FeedParserDict, source_url: Optional[HttpUrl] = None) -> AtomEntry:
3113
+
"""Normalize an entry to Atom format."""
3114
+
# Parse timestamps
3115
+
updated = self._parse_timestamp(entry.get('updated_parsed') or entry.get('published_parsed'))
3116
+
published = self._parse_timestamp(entry.get('published_parsed'))
3119
+
content = self._extract_content(entry)
3120
+
content_type = self._extract_content_type(entry)
3123
+
author = self._extract_author(entry)
3125
+
# Parse categories/tags
3127
+
if hasattr(entry, 'tags'):
3128
+
categories = [tag.get('term', '') for tag in entry.tags if tag.get('term')]
3130
+
# Sanitize HTML content
3132
+
content = self._sanitize_html(content)
3134
+
summary = entry.get('summary', '')
3136
+
summary = self._sanitize_html(summary)
3139
+
id=entry.get('id', entry.get('link', '')),
3140
+
title=entry.get('title', ''),
3141
+
link=HttpUrl(entry.get('link', '')),
3143
+
published=published,
3144
+
summary=summary or None,
3145
+
content=content or None,
3146
+
content_type=content_type,
3148
+
categories=categories,
3149
+
rights=entry.get('rights', None),
3150
+
source=str(source_url) if source_url else None,
3153
+
def _parse_timestamp(self, time_struct) -> datetime:
3154
+
"""Parse feedparser time struct to datetime."""
3156
+
return datetime(*time_struct[:6])
3157
+
return datetime.now()
3159
+
def _extract_content(self, entry: feedparser.FeedParserDict) -> Optional[str]:
3160
+
"""Extract the best content from an entry."""
3161
+
# Prefer content over summary
3162
+
if hasattr(entry, 'content') and entry.content:
3163
+
# Find the best content (prefer text/html, then text/plain)
3164
+
for content_item in entry.content:
3165
+
if content_item.get('type') in ['text/html', 'html']:
3166
+
return content_item.get('value', '')
3167
+
elif content_item.get('type') in ['text/plain', 'text']:
3168
+
return content_item.get('value', '')
3169
+
# Fallback to first content item
3170
+
return entry.content[0].get('value', '')
3172
+
# Fallback to summary
3173
+
return entry.get('summary', '')
3175
+
def _extract_content_type(self, entry: feedparser.FeedParserDict) -> str:
3176
+
"""Extract content type from entry."""
3177
+
if hasattr(entry, 'content') and entry.content:
3178
+
content_type = entry.content[0].get('type', 'html')
3179
+
# Normalize content type
3180
+
if content_type in ['text/html', 'html']:
3182
+
elif content_type in ['text/plain', 'text']:
3184
+
elif content_type == 'xhtml':
3188
+
def _extract_author(self, entry: feedparser.FeedParserDict) -> Optional[dict]:
3189
+
"""Extract author information from entry."""
3192
+
if hasattr(entry, 'author_detail'):
3194
+
'name': entry.author_detail.get('name'),
3195
+
'email': entry.author_detail.get('email'),
3196
+
'uri': entry.author_detail.get('href'),
3198
+
elif hasattr(entry, 'author'):
3199
+
author['name'] = entry.author
3201
+
return author if author else None
3203
+
def _sanitize_html(self, html: str) -> str:
3204
+
"""Sanitize HTML content to prevent XSS."""
3205
+
return bleach.clean(
3207
+
tags=self.allowed_tags,
3208
+
attributes=self.allowed_attributes,
3212
+
def sanitize_entry_id(self, entry_id: str) -> str:
3213
+
"""Sanitize entry ID to be a safe filename."""
3214
+
# Parse URL to get meaningful parts
3215
+
parsed = urlparse(entry_id)
3217
+
# Start with the path component
3219
+
# Remove leading slash and replace problematic characters
3220
+
safe_id = parsed.path.lstrip('/').replace('/', '_').replace('\\', '_')
3222
+
# Use the entire ID as fallback
3223
+
safe_id = entry_id
3225
+
# Replace problematic characters
3227
+
for char in safe_id:
3228
+
if char.isalnum() or char in '-_.':
3229
+
safe_chars.append(char)
3231
+
safe_chars.append('_')
3233
+
safe_id = ''.join(safe_chars)
3235
+
# Ensure it's not too long (max 200 chars)
3236
+
if len(safe_id) > 200:
3237
+
safe_id = safe_id[:200]
3239
+
# Ensure it's not empty
3246
+
<file path="src/thicket/core/reference_parser.py">
3247
+
"""Reference detection and parsing for blog entries."""
3250
+
from typing import Optional
3251
+
from urllib.parse import urlparse
3253
+
from ..models import AtomEntry
3256
+
class BlogReference:
3257
+
"""Represents a reference from one blog entry to another."""
3261
+
source_entry_id: str,
3262
+
source_username: str,
3264
+
target_username: Optional[str] = None,
3265
+
target_entry_id: Optional[str] = None,
3267
+
self.source_entry_id = source_entry_id
3268
+
self.source_username = source_username
3269
+
self.target_url = target_url
3270
+
self.target_username = target_username
3271
+
self.target_entry_id = target_entry_id
3273
+
def to_dict(self) -> dict:
3274
+
"""Convert to dictionary for JSON serialization."""
3276
+
"source_entry_id": self.source_entry_id,
3277
+
"source_username": self.source_username,
3278
+
"target_url": self.target_url,
3281
+
# Only include optional fields if they are not None
3282
+
if self.target_username is not None:
3283
+
result["target_username"] = self.target_username
3284
+
if self.target_entry_id is not None:
3285
+
result["target_entry_id"] = self.target_entry_id
3290
+
def from_dict(cls, data: dict) -> "BlogReference":
3291
+
"""Create from dictionary."""
3293
+
source_entry_id=data["source_entry_id"],
3294
+
source_username=data["source_username"],
3295
+
target_url=data["target_url"],
3296
+
target_username=data.get("target_username"),
3297
+
target_entry_id=data.get("target_entry_id"),
3301
+
class ReferenceIndex:
3302
+
"""Index of blog-to-blog references for creating threaded views."""
3304
+
def __init__(self):
3305
+
self.references: list[BlogReference] = []
3306
+
self.outbound_refs: dict[
3307
+
str, list[BlogReference]
3308
+
] = {} # entry_id -> outbound refs
3309
+
self.inbound_refs: dict[
3310
+
str, list[BlogReference]
3311
+
] = {} # entry_id -> inbound refs
3312
+
self.user_domains: dict[str, set[str]] = {} # username -> set of domains
3314
+
def add_reference(self, ref: BlogReference) -> None:
3315
+
"""Add a reference to the index."""
3316
+
self.references.append(ref)
3318
+
# Update outbound references
3319
+
source_key = f"{ref.source_username}:{ref.source_entry_id}"
3320
+
if source_key not in self.outbound_refs:
3321
+
self.outbound_refs[source_key] = []
3322
+
self.outbound_refs[source_key].append(ref)
3324
+
# Update inbound references if we can identify the target
3325
+
if ref.target_username and ref.target_entry_id:
3326
+
target_key = f"{ref.target_username}:{ref.target_entry_id}"
3327
+
if target_key not in self.inbound_refs:
3328
+
self.inbound_refs[target_key] = []
3329
+
self.inbound_refs[target_key].append(ref)
3331
+
def get_outbound_refs(self, username: str, entry_id: str) -> list[BlogReference]:
3332
+
"""Get all outbound references from an entry."""
3333
+
key = f"{username}:{entry_id}"
3334
+
return self.outbound_refs.get(key, [])
3336
+
def get_inbound_refs(self, username: str, entry_id: str) -> list[BlogReference]:
3337
+
"""Get all inbound references to an entry."""
3338
+
key = f"{username}:{entry_id}"
3339
+
return self.inbound_refs.get(key, [])
3341
+
def get_thread_members(self, username: str, entry_id: str) -> set[tuple[str, str]]:
3342
+
"""Get all entries that are part of the same thread."""
3344
+
to_visit = [(username, entry_id)]
3345
+
thread_members = set()
3348
+
current_user, current_entry = to_visit.pop()
3349
+
if (current_user, current_entry) in visited:
3352
+
visited.add((current_user, current_entry))
3353
+
thread_members.add((current_user, current_entry))
3355
+
# Add outbound references
3356
+
for ref in self.get_outbound_refs(current_user, current_entry):
3357
+
if ref.target_username and ref.target_entry_id:
3358
+
to_visit.append((ref.target_username, ref.target_entry_id))
3360
+
# Add inbound references
3361
+
for ref in self.get_inbound_refs(current_user, current_entry):
3362
+
to_visit.append((ref.source_username, ref.source_entry_id))
3364
+
return thread_members
3366
+
def to_dict(self) -> dict:
3367
+
"""Convert to dictionary for JSON serialization."""
3369
+
"references": [ref.to_dict() for ref in self.references],
3370
+
"user_domains": {k: list(v) for k, v in self.user_domains.items()},
3374
+
def from_dict(cls, data: dict) -> "ReferenceIndex":
3375
+
"""Create from dictionary."""
3377
+
for ref_data in data.get("references", []):
3378
+
ref = BlogReference.from_dict(ref_data)
3379
+
index.add_reference(ref)
3381
+
for username, domains in data.get("user_domains", {}).items():
3382
+
index.user_domains[username] = set(domains)
3387
+
class ReferenceParser:
3388
+
"""Parses blog entries to detect references to other blogs."""
3390
+
def __init__(self):
3391
+
# Common blog platforms and patterns
3392
+
self.blog_patterns = [
3393
+
r"https?://[^/]+\.(?:org|com|net|io|dev|me|co\.uk)/.*", # Common blog domains
3394
+
r"https?://[^/]+\.github\.io/.*", # GitHub Pages
3395
+
r"https?://[^/]+\.substack\.com/.*", # Substack
3396
+
r"https?://medium\.com/.*", # Medium
3397
+
r"https?://[^/]+\.wordpress\.com/.*", # WordPress.com
3398
+
r"https?://[^/]+\.blogspot\.com/.*", # Blogger
3401
+
# Compile regex patterns
3402
+
self.link_pattern = re.compile(
3403
+
r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', re.IGNORECASE | re.DOTALL
3405
+
self.url_pattern = re.compile(r'https?://[^\s<>"]+')
3407
+
def extract_links_from_html(self, html_content: str) -> list[tuple[str, str]]:
3408
+
"""Extract all links from HTML content."""
3411
+
# Extract links from <a> tags
3412
+
for match in self.link_pattern.finditer(html_content):
3413
+
url = match.group(1)
3415
+
r"<[^>]+>", "", match.group(2)
3416
+
).strip() # Remove HTML tags from link text
3417
+
links.append((url, text))
3421
+
def is_blog_url(self, url: str) -> bool:
3422
+
"""Check if a URL likely points to a blog post."""
3423
+
for pattern in self.blog_patterns:
3424
+
if re.match(pattern, url):
3428
+
def _is_likely_blog_post_url(self, url: str) -> bool:
3429
+
"""Check if a same-domain URL likely points to a blog post (not CSS, images, etc.)."""
3430
+
parsed_url = urlparse(url)
3431
+
path = parsed_url.path.lower()
3433
+
# Skip obvious non-blog content
3434
+
if any(path.endswith(ext) for ext in ['.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.pdf', '.xml', '.json']):
3437
+
# Skip common non-blog paths
3438
+
if any(segment in path for segment in ['/static/', '/assets/', '/css/', '/js/', '/images/', '/img/', '/media/', '/uploads/']):
3441
+
# Skip fragment-only links (same page anchors)
3442
+
if not path or path == '/':
3445
+
# Look for positive indicators of blog posts
3446
+
# Common blog post patterns: dates, slugs, post indicators
3447
+
blog_indicators = [
3448
+
r'/\d{4}/', # Year in path
3449
+
r'/\d{4}/\d{2}/', # Year/month in path
3459
+
for pattern in blog_indicators:
3460
+
if re.search(pattern, path):
3463
+
# If it has a reasonable path depth and doesn't match exclusions, likely a blog post
3464
+
path_segments = [seg for seg in path.split('/') if seg]
3465
+
return len(path_segments) >= 1 # At least one meaningful path segment
3467
+
def resolve_target_user(
3468
+
self, url: str, user_domains: dict[str, set[str]]
3469
+
) -> Optional[str]:
3470
+
"""Try to resolve a URL to a known user based on domain mapping."""
3471
+
parsed_url = urlparse(url)
3472
+
domain = parsed_url.netloc.lower()
3474
+
for username, domains in user_domains.items():
3475
+
if domain in domains:
3480
+
def extract_references(
3481
+
self, entry: AtomEntry, username: str, user_domains: dict[str, set[str]]
3482
+
) -> list[BlogReference]:
3483
+
"""Extract all blog references from an entry."""
3486
+
# Combine all text content for analysis
3487
+
content_to_search = []
3489
+
content_to_search.append(entry.content)
3491
+
content_to_search.append(entry.summary)
3493
+
for content in content_to_search:
3494
+
links = self.extract_links_from_html(content)
3496
+
for url, _link_text in links:
3498
+
urlparse(str(entry.link)).netloc.lower() if entry.link else ""
3500
+
link_domain = urlparse(url).netloc.lower()
3502
+
# Check if this looks like a blog URL
3503
+
if not self.is_blog_url(url):
3506
+
# For same-domain links, apply additional filtering to avoid non-blog content
3507
+
if link_domain == entry_domain:
3508
+
# Only include same-domain links that look like blog posts
3509
+
if not self._is_likely_blog_post_url(url):
3512
+
# Try to resolve to a known user
3513
+
if link_domain == entry_domain:
3514
+
# Same domain - target user is the same as source user
3515
+
target_username: Optional[str] = username
3517
+
# Different domain - try to resolve
3518
+
target_username = self.resolve_target_user(url, user_domains)
3520
+
ref = BlogReference(
3521
+
source_entry_id=entry.id,
3522
+
source_username=username,
3524
+
target_username=target_username,
3525
+
target_entry_id=None, # Will be resolved later if possible
3528
+
references.append(ref)
3532
+
def build_user_domain_mapping(self, git_store: "GitStore") -> dict[str, set[str]]:
3533
+
"""Build mapping of usernames to their known domains."""
3535
+
index = git_store._load_index()
3537
+
for username, user_metadata in index.users.items():
3540
+
# Add domains from feeds
3541
+
for feed_url in user_metadata.feeds:
3542
+
domain = urlparse(feed_url).netloc.lower()
3544
+
domains.add(domain)
3546
+
# Add domain from homepage
3547
+
if user_metadata.homepage:
3548
+
domain = urlparse(str(user_metadata.homepage)).netloc.lower()
3550
+
domains.add(domain)
3552
+
user_domains[username] = domains
3554
+
return user_domains
3556
+
def _build_url_to_entry_mapping(self, git_store: "GitStore") -> dict[str, str]:
3557
+
"""Build a comprehensive mapping from URLs to entry IDs using git store data.
3559
+
This creates a bidirectional mapping that handles:
3560
+
- Entry link URLs -> Entry IDs
3561
+
- URL variations (with/without www, http/https)
3562
+
- Multiple URLs pointing to the same entry
3564
+
url_to_entry: dict[str, str] = {}
3566
+
# Load index to get all users
3567
+
index = git_store._load_index()
3569
+
for username in index.users.keys():
3570
+
entries = git_store.list_entries(username)
3572
+
for entry in entries:
3574
+
link_url = str(entry.link)
3575
+
entry_id = entry.id
3577
+
# Map the canonical link URL
3578
+
url_to_entry[link_url] = entry_id
3580
+
# Handle common URL variations
3581
+
parsed = urlparse(link_url)
3582
+
if parsed.netloc and parsed.path:
3583
+
# Add version without www
3584
+
if parsed.netloc.startswith('www.'):
3585
+
no_www_url = f"{parsed.scheme}://{parsed.netloc[4:]}{parsed.path}"
3587
+
no_www_url += f"?{parsed.query}"
3588
+
if parsed.fragment:
3589
+
no_www_url += f"#{parsed.fragment}"
3590
+
url_to_entry[no_www_url] = entry_id
3592
+
# Add version with www if not present
3593
+
elif not parsed.netloc.startswith('www.'):
3594
+
www_url = f"{parsed.scheme}://www.{parsed.netloc}{parsed.path}"
3596
+
www_url += f"?{parsed.query}"
3597
+
if parsed.fragment:
3598
+
www_url += f"#{parsed.fragment}"
3599
+
url_to_entry[www_url] = entry_id
3601
+
# Add http/https variations
3602
+
if parsed.scheme == 'https':
3603
+
http_url = link_url.replace('https://', 'http://', 1)
3604
+
url_to_entry[http_url] = entry_id
3605
+
elif parsed.scheme == 'http':
3606
+
https_url = link_url.replace('http://', 'https://', 1)
3607
+
url_to_entry[https_url] = entry_id
3609
+
return url_to_entry
3611
+
def _normalize_url(self, url: str) -> str:
3612
+
"""Normalize URL for consistent matching.
3614
+
Handles common variations like trailing slashes, fragments, etc.
3616
+
parsed = urlparse(url)
3618
+
# Remove trailing slash from path
3619
+
path = parsed.path.rstrip('/') if parsed.path != '/' else parsed.path
3621
+
# Reconstruct without fragment for consistent matching
3622
+
normalized = f"{parsed.scheme}://{parsed.netloc}{path}"
3624
+
normalized += f"?{parsed.query}"
3628
+
def resolve_target_entry_ids(
3629
+
self, references: list[BlogReference], git_store: "GitStore"
3630
+
) -> list[BlogReference]:
3631
+
"""Resolve target_entry_id for references using comprehensive URL mapping."""
3632
+
resolved_refs = []
3634
+
# Build comprehensive URL to entry ID mapping
3635
+
url_to_entry = self._build_url_to_entry_mapping(git_store)
3637
+
for ref in references:
3638
+
# If we already have a target_entry_id, keep the reference as-is
3639
+
if ref.target_entry_id is not None:
3640
+
resolved_refs.append(ref)
3643
+
# If we don't have a target_username, we can't resolve it
3644
+
if ref.target_username is None:
3645
+
resolved_refs.append(ref)
3648
+
# Try to resolve using URL mapping
3649
+
resolved_entry_id = None
3651
+
# First, try exact match
3652
+
if ref.target_url in url_to_entry:
3653
+
resolved_entry_id = url_to_entry[ref.target_url]
3655
+
# Try normalized URL matching
3656
+
normalized_target = self._normalize_url(ref.target_url)
3657
+
if normalized_target in url_to_entry:
3658
+
resolved_entry_id = url_to_entry[normalized_target]
3660
+
# Try URL variations
3661
+
for mapped_url, entry_id in url_to_entry.items():
3662
+
if self._normalize_url(mapped_url) == normalized_target:
3663
+
resolved_entry_id = entry_id
3666
+
# Verify the resolved entry belongs to the target username
3667
+
if resolved_entry_id:
3668
+
# Double-check by loading the actual entry
3669
+
entries = git_store.list_entries(ref.target_username)
3670
+
entry_found = any(entry.id == resolved_entry_id for entry in entries)
3671
+
if not entry_found:
3672
+
resolved_entry_id = None
3674
+
# Create a new reference with the resolved target_entry_id
3675
+
resolved_ref = BlogReference(
3676
+
source_entry_id=ref.source_entry_id,
3677
+
source_username=ref.source_username,
3678
+
target_url=ref.target_url,
3679
+
target_username=ref.target_username,
3680
+
target_entry_id=resolved_entry_id,
3682
+
resolved_refs.append(resolved_ref)
3684
+
return resolved_refs
3687
+
<file path="src/thicket/models/__init__.py">
3688
+
"""Data models for thicket."""
3690
+
from .config import ThicketConfig, UserConfig
3691
+
from .feed import AtomEntry, DuplicateMap, FeedMetadata
3692
+
from .user import GitStoreIndex, UserMetadata
3705
+
<file path="src/thicket/models/feed.py">
3706
+
"""Feed and entry models for thicket."""
3708
+
from datetime import datetime
3709
+
from typing import TYPE_CHECKING, Optional
3711
+
from pydantic import BaseModel, ConfigDict, EmailStr, HttpUrl
3714
+
from .config import UserConfig
3717
+
class AtomEntry(BaseModel):
3718
+
"""Represents an Atom feed entry stored in the Git repository."""
3720
+
model_config = ConfigDict(
3721
+
json_encoders={datetime: lambda v: v.isoformat()},
3722
+
str_strip_whitespace=True,
3725
+
id: str # Original Atom ID
3729
+
published: Optional[datetime] = None
3730
+
summary: Optional[str] = None
3731
+
content: Optional[str] = None # Full body content from Atom entry
3732
+
content_type: Optional[str] = "html" # text, html, xhtml
3733
+
author: Optional[dict] = None
3734
+
categories: list[str] = []
3735
+
rights: Optional[str] = None # Copyright info
3736
+
source: Optional[str] = None # Source feed URL
3739
+
class FeedMetadata(BaseModel):
3740
+
"""Metadata extracted from a feed for auto-discovery."""
3742
+
title: Optional[str] = None
3743
+
author_name: Optional[str] = None
3744
+
author_email: Optional[EmailStr] = None
3745
+
author_uri: Optional[HttpUrl] = None
3746
+
link: Optional[HttpUrl] = None
3747
+
logo: Optional[HttpUrl] = None
3748
+
icon: Optional[HttpUrl] = None
3749
+
image_url: Optional[HttpUrl] = None
3750
+
description: Optional[str] = None
3752
+
def to_user_config(self, username: str, feed_url: HttpUrl) -> "UserConfig":
3753
+
"""Convert discovered metadata to UserConfig with fallbacks."""
3754
+
from .config import UserConfig
3756
+
return UserConfig(
3757
+
username=username,
3759
+
display_name=self.author_name or self.title,
3760
+
email=self.author_email,
3761
+
homepage=self.author_uri or self.link,
3762
+
icon=self.logo or self.icon or self.image_url,
3766
+
class DuplicateMap(BaseModel):
3767
+
"""Maps duplicate entry IDs to canonical entry IDs."""
3769
+
duplicates: dict[str, str] = {} # duplicate_id -> canonical_id
3770
+
comment: str = "Entry IDs that map to the same canonical content"
3772
+
def add_duplicate(self, duplicate_id: str, canonical_id: str) -> None:
3773
+
"""Add a duplicate mapping."""
3774
+
self.duplicates[duplicate_id] = canonical_id
3776
+
def remove_duplicate(self, duplicate_id: str) -> bool:
3777
+
"""Remove a duplicate mapping. Returns True if existed."""
3778
+
return self.duplicates.pop(duplicate_id, None) is not None
3780
+
def get_canonical(self, entry_id: str) -> str:
3781
+
"""Get canonical ID for an entry (returns original if not duplicate)."""
3782
+
return self.duplicates.get(entry_id, entry_id)
3784
+
def is_duplicate(self, entry_id: str) -> bool:
3785
+
"""Check if entry ID is marked as duplicate."""
3786
+
return entry_id in self.duplicates
3788
+
def get_duplicates_for_canonical(self, canonical_id: str) -> list[str]:
3789
+
"""Get all duplicate IDs that map to a canonical ID."""
3792
+
for duplicate_id, canonical in self.duplicates.items()
3793
+
if canonical == canonical_id
3797
+
<file path="src/thicket/models/user.py">
3798
+
"""User metadata models for thicket."""
3800
+
from datetime import datetime
3801
+
from typing import Optional
3803
+
from pydantic import BaseModel, ConfigDict
3806
+
class UserMetadata(BaseModel):
3807
+
"""Metadata about a user stored in the Git repository."""
3809
+
model_config = ConfigDict(
3810
+
json_encoders={datetime: lambda v: v.isoformat()},
3811
+
str_strip_whitespace=True,
3815
+
display_name: Optional[str] = None
3816
+
email: Optional[str] = None
3817
+
homepage: Optional[str] = None
3818
+
icon: Optional[str] = None
3819
+
feeds: list[str] = []
3820
+
directory: str # Directory name in Git store
3822
+
last_updated: datetime
3823
+
entry_count: int = 0
3825
+
def update_timestamp(self) -> None:
3826
+
"""Update the last_updated timestamp to now."""
3827
+
self.last_updated = datetime.now()
3829
+
def increment_entry_count(self, count: int = 1) -> None:
3830
+
"""Increment the entry count by the given amount."""
3831
+
self.entry_count += count
3832
+
self.update_timestamp()
3835
+
class GitStoreIndex(BaseModel):
3836
+
"""Index of all users and their directories in the Git store."""
3838
+
model_config = ConfigDict(
3839
+
json_encoders={datetime: lambda v: v.isoformat()}
3842
+
users: dict[str, UserMetadata] = {} # username -> UserMetadata
3844
+
last_updated: datetime
3845
+
total_entries: int = 0
3847
+
def add_user(self, user_metadata: UserMetadata) -> None:
3848
+
"""Add or update a user in the index."""
3849
+
self.users[user_metadata.username] = user_metadata
3850
+
self.last_updated = datetime.now()
3852
+
def remove_user(self, username: str) -> bool:
3853
+
"""Remove a user from the index. Returns True if user existed."""
3854
+
if username in self.users:
3855
+
del self.users[username]
3856
+
self.last_updated = datetime.now()
3860
+
def get_user(self, username: str) -> Optional[UserMetadata]:
3861
+
"""Get user metadata by username."""
3862
+
return self.users.get(username)
3864
+
def update_entry_count(self, username: str, count: int) -> None:
3865
+
"""Update entry count for a user and total."""
3866
+
user = self.get_user(username)
3868
+
user.increment_entry_count(count)
3869
+
self.total_entries += count
3870
+
self.last_updated = datetime.now()
3872
+
def recalculate_totals(self) -> None:
3873
+
"""Recalculate total entries from all users."""
3874
+
self.total_entries = sum(user.entry_count for user in self.users.values())
3875
+
self.last_updated = datetime.now()
3878
+
<file path="src/thicket/utils/__init__.py">
3879
+
"""Utility modules for thicket."""
3881
+
# This module will contain shared utilities
3882
+
# For now, it's empty but can be expanded with common functions
3885
+
<file path="src/thicket/__init__.py">
3886
+
"""Thicket: A CLI tool for persisting Atom/RSS feeds in Git repositories."""
3888
+
__version__ = "0.1.0"
3889
+
__author__ = "thicket"
3890
+
__email__ = "thicket@example.com"
3893
+
<file path="src/thicket/__main__.py">
3894
+
"""Entry point for running thicket as a module."""
3896
+
from .cli.main import app
3898
+
if __name__ == "__main__":
3902
+
<file path=".gitignore">
3903
+
# Byte-compiled / optimized / DLL files
3911
+
# Distribution / packaging
3925
+
share/python-wheels/
3932
+
# Usually these files are written by a python script from a template
3933
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
3939
+
pip-delete-this-directory.txt
3941
+
# Unit test / coverage reports
3964
+
db.sqlite3-journal
3973
+
# Sphinx documentation
3980
+
# Jupyter Notebook
3981
+
.ipynb_checkpoints
3988
+
# For a library or package, you might want to ignore these files since the code is
3989
+
# intended to run in multiple environments; otherwise, check them in:
3993
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
3994
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
3995
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
3996
+
# install all needed dependencies.
4000
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
4001
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
4002
+
# commonly ignored for libraries.
4006
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
4007
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
4008
+
# commonly ignored for libraries.
4009
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
4014
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
4015
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
4016
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
4023
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
4025
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
4026
+
# in the .venv directory. It is recommended not to include this directory in version control.
4029
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
4033
+
celerybeat-schedule
4036
+
# SageMath parsed files
4049
+
# Spyder project settings
4053
+
# Rope project settings
4056
+
# mkdocs documentation
4064
+
# Pyre type checker
4067
+
# pytype static type analyzer
4070
+
# Cython debug symbols
4074
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
4075
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
4076
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
4077
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
4081
+
# Abstra is an AI-powered process automation framework.
4082
+
# Ignore directories containing user credentials, local state, and settings.
4083
+
# Learn more at https://abstra.io/docs
4086
+
# Visual Studio Code
4087
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
4088
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
4089
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
4090
+
# you could uncomment the following to ignore the entire vscode folder
4096
+
# PyPI configuration file
4105
+
.streamlit/secrets.toml
4110
+
<file path="CLAUDE.md">
4111
+
My goal is to build a CLI tool called thicket in Python that maintains a Git repository within which Atom feeds can be persisted, including their contents.
4113
+
# Python Environment and Package Management
4115
+
This project uses `uv` for Python package management and virtual environment handling.
4117
+
## Running Commands
4119
+
ALWAYS use `uv run` to execute Python commands:
4121
+
- Run the CLI: `uv run -m thicket`
4122
+
- Run tests: `uv run pytest`
4123
+
- Type checking: `uv run mypy src/`
4124
+
- Linting: `uv run ruff check src/`
4125
+
- Format code: `uv run ruff format src/`
4126
+
- Compile check: `uv run python -m py_compile <file>`
4128
+
## Package Management
4130
+
- Add dependencies: `uv add <package>`
4131
+
- Add dev dependencies: `uv add --dev <package>`
4132
+
- Install dependencies: `uv sync`
4133
+
- Update dependencies: `uv lock --upgrade`
4135
+
# Project Structure
4137
+
The configuration file specifies:
4138
+
- the location of a git store
4139
+
- a list of usernames and target Atom/RSS feed(s) and optional metadata about the username such as their email, homepage, icon and display name
4140
+
- a cache directory to store temporary results such as feed downloads and their last modification date that speed up operations across runs of the tool
4142
+
The Git data store should:
4143
+
- have a subdirectory per user
4144
+
- within that directory, an entry per Atom entry indexed by the Atom id for that entry. The id should be sanitised consistently to be a safe filename. RSS feed should be normalized to Atom before storing it.
4145
+
- within each entry file, the metadata of the Atom feed converted into a JSON format that preserves as much metadata as possible.
4146
+
- have a JSON file in the Git repository that indexes the users, their associated directories within the Git repository, and any other metadata about that user from the config file
4147
+
The CLI should be modern and use cool progress bars and any otfrom ecosystem libraries.
4149
+
The intention behind the Git repository is that it can be queried by other websites in order to build a webblog structure of comments that link to other blogs.
4152
+
<file path="pyproject.toml">
4154
+
requires = ["hatchling"]
4155
+
build-backend = "hatchling.build"
4159
+
dynamic = ["version"]
4160
+
description = "A CLI tool for persisting Atom/RSS feeds in Git repositories"
4161
+
readme = "README.md"
4163
+
requires-python = ">=3.9"
4165
+
{name = "thicket", email = "thicket@example.com"},
4168
+
"Development Status :: 3 - Alpha",
4169
+
"Intended Audience :: Developers",
4170
+
"License :: OSI Approved :: MIT License",
4171
+
"Operating System :: OS Independent",
4172
+
"Programming Language :: Python :: 3",
4173
+
"Programming Language :: Python :: 3.9",
4174
+
"Programming Language :: Python :: 3.10",
4175
+
"Programming Language :: Python :: 3.11",
4176
+
"Programming Language :: Python :: 3.12",
4177
+
"Programming Language :: Python :: 3.13",
4178
+
"Topic :: Internet :: WWW/HTTP :: Dynamic Content :: News/Diary",
4179
+
"Topic :: Software Development :: Version Control :: Git",
4180
+
"Topic :: Text Processing :: Markup :: XML",
4185
+
"GitPython>=3.1.40",
4186
+
"feedparser>=6.0.11",
4187
+
"pydantic>=2.11.0",
4188
+
"pydantic-settings>=2.10.0",
4190
+
"pendulum>=3.0.0",
4192
+
"platformdirs>=4.0.0",
4194
+
"email_validator",
4198
+
[project.optional-dependencies]
4201
+
"pytest-asyncio>=0.24.0",
4202
+
"pytest-cov>=6.0.0",
4206
+
"types-PyYAML>=6.0.0",
4210
+
Homepage = "https://github.com/example/thicket"
4211
+
Documentation = "https://github.com/example/thicket"
4212
+
Repository = "https://github.com/example/thicket"
4213
+
"Bug Tracker" = "https://github.com/example/thicket/issues"
4216
+
thicket = "thicket.cli.main:app"
4218
+
[tool.hatch.version]
4219
+
path = "src/thicket/__init__.py"
4221
+
[tool.hatch.build.targets.wheel]
4222
+
packages = ["src/thicket"]
4226
+
target-version = ['py39']
4227
+
include = '\.pyi?$'
4228
+
extend-exclude = '''
4243
+
target-version = "py39"
4248
+
"E", # pycodestyle errors
4249
+
"W", # pycodestyle warnings
4252
+
"B", # flake8-bugbear
4253
+
"C4", # flake8-comprehensions
4257
+
"E501", # line too long, handled by black
4258
+
"B008", # do not perform function calls in argument defaults
4259
+
"C901", # too complex
4262
+
[tool.ruff.lint.per-file-ignores]
4263
+
"__init__.py" = ["F401"]
4266
+
python_version = "3.9"
4267
+
check_untyped_defs = true
4268
+
disallow_any_generics = true
4269
+
disallow_incomplete_defs = true
4270
+
disallow_untyped_defs = true
4271
+
no_implicit_optional = true
4272
+
warn_redundant_casts = true
4273
+
warn_unused_ignores = true
4274
+
warn_return_any = true
4275
+
strict_optional = true
4277
+
[[tool.mypy.overrides]]
4283
+
ignore_missing_imports = true
4285
+
[tool.pytest.ini_options]
4286
+
testpaths = ["tests"]
4287
+
python_files = ["test_*.py"]
4288
+
python_classes = ["Test*"]
4289
+
python_functions = ["test_*"]
4292
+
"--strict-markers",
4293
+
"--strict-config",
4294
+
"--cov=src/thicket",
4295
+
"--cov-report=term-missing",
4296
+
"--cov-report=html",
4297
+
"--cov-report=xml",
4299
+
filterwarnings = [
4301
+
"ignore::UserWarning",
4302
+
"ignore::DeprecationWarning",
4305
+
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
4306
+
"integration: marks tests as integration tests",
4309
+
[tool.coverage.run]
4313
+
[tool.coverage.report]
4315
+
"pragma: no cover",
4318
+
"if settings.DEBUG",
4319
+
"raise AssertionError",
4320
+
"raise NotImplementedError",
4322
+
"if __name__ == .__main__.:",
4323
+
"class .*\\bProtocol\\):",
4324
+
"@(abc\\.)?abstractmethod",
4328
+
<file path="src/thicket/cli/commands/__init__.py">
4329
+
"""CLI commands for thicket."""
4331
+
# Import all commands to register them with the main app
4332
+
from . import add, duplicates, generate, index_cmd, info_cmd, init, links_cmd, list_cmd, sync
4334
+
__all__ = ["add", "duplicates", "generate", "index_cmd", "info_cmd", "init", "links_cmd", "list_cmd", "sync"]
4337
+
<file path="src/thicket/cli/commands/add.py">
4338
+
"""Add command for thicket."""
4341
+
from pathlib import Path
4342
+
from typing import Optional
4345
+
from pydantic import HttpUrl, ValidationError
4347
+
from ...core.feed_parser import FeedParser
4348
+
from ...core.git_store import GitStore
4349
+
from ..main import app
4350
+
from ..utils import (
4359
+
@app.command("add")
4361
+
subcommand: str = typer.Argument(..., help="Subcommand: 'user' or 'feed'"),
4362
+
username: str = typer.Argument(..., help="Username"),
4363
+
feed_url: Optional[str] = typer.Argument(None, help="Feed URL (required for 'user' command)"),
4364
+
email: Optional[str] = typer.Option(None, "--email", "-e", help="User email"),
4365
+
homepage: Optional[str] = typer.Option(None, "--homepage", "-h", help="User homepage"),
4366
+
icon: Optional[str] = typer.Option(None, "--icon", "-i", help="User icon URL"),
4367
+
display_name: Optional[str] = typer.Option(None, "--display-name", "-d", help="User display name"),
4368
+
config_file: Optional[Path] = typer.Option(
4369
+
Path("thicket.yaml"), "--config", help="Configuration file path"
4371
+
auto_discover: bool = typer.Option(
4372
+
True, "--auto-discover/--no-auto-discover", help="Auto-discover user metadata from feed"
4375
+
"""Add a user or feed to thicket."""
4377
+
if subcommand == "user":
4378
+
add_user(username, feed_url, email, homepage, icon, display_name, config_file, auto_discover)
4379
+
elif subcommand == "feed":
4380
+
add_feed(username, feed_url, config_file)
4382
+
print_error(f"Unknown subcommand: {subcommand}")
4383
+
print_error("Use 'user' or 'feed'")
4384
+
raise typer.Exit(1)
4389
+
feed_url: Optional[str],
4390
+
email: Optional[str],
4391
+
homepage: Optional[str],
4392
+
icon: Optional[str],
4393
+
display_name: Optional[str],
4394
+
config_file: Path,
4395
+
auto_discover: bool,
4397
+
"""Add a new user with feed."""
4400
+
print_error("Feed URL is required when adding a user")
4401
+
raise typer.Exit(1)
4403
+
# Validate feed URL
4405
+
validated_feed_url = HttpUrl(feed_url)
4406
+
except ValidationError:
4407
+
print_error(f"Invalid feed URL: {feed_url}")
4408
+
raise typer.Exit(1) from None
4410
+
# Load configuration
4411
+
config = load_config(config_file)
4413
+
# Initialize Git store
4414
+
git_store = GitStore(config.git_store)
4416
+
# Check if user already exists
4417
+
existing_user = git_store.get_user(username)
4419
+
print_error(f"User '{username}' already exists")
4420
+
print_error("Use 'thicket add feed' to add additional feeds")
4421
+
raise typer.Exit(1)
4423
+
# Auto-discover metadata if enabled
4424
+
discovered_metadata = None
4426
+
discovered_metadata = asyncio.run(discover_feed_metadata(validated_feed_url))
4428
+
# Prepare user data with manual overrides taking precedence
4429
+
user_display_name = display_name or (discovered_metadata.author_name or discovered_metadata.title if discovered_metadata else None)
4430
+
user_email = email or (discovered_metadata.author_email if discovered_metadata else None)
4431
+
user_homepage = homepage or (str(discovered_metadata.author_uri or discovered_metadata.link) if discovered_metadata else None)
4432
+
user_icon = icon or (str(discovered_metadata.logo or discovered_metadata.icon or discovered_metadata.image_url) if discovered_metadata else None)
4434
+
# Add user to Git store
4435
+
git_store.add_user(
4436
+
username=username,
4437
+
display_name=user_display_name,
4439
+
homepage=user_homepage,
4441
+
feeds=[str(validated_feed_url)],
4445
+
git_store.commit_changes(f"Add user: {username}")
4447
+
print_success(f"Added user '{username}' with feed: {feed_url}")
4449
+
if discovered_metadata and auto_discover:
4450
+
print_info("Auto-discovered metadata:")
4451
+
if user_display_name:
4452
+
print_info(f" Display name: {user_display_name}")
4454
+
print_info(f" Email: {user_email}")
4456
+
print_info(f" Homepage: {user_homepage}")
4458
+
print_info(f" Icon: {user_icon}")
4461
+
def add_feed(username: str, feed_url: Optional[str], config_file: Path) -> None:
4462
+
"""Add a feed to an existing user."""
4465
+
print_error("Feed URL is required")
4466
+
raise typer.Exit(1)
4468
+
# Validate feed URL
4470
+
validated_feed_url = HttpUrl(feed_url)
4471
+
except ValidationError:
4472
+
print_error(f"Invalid feed URL: {feed_url}")
4473
+
raise typer.Exit(1) from None
4475
+
# Load configuration
4476
+
config = load_config(config_file)
4478
+
# Initialize Git store
4479
+
git_store = GitStore(config.git_store)
4481
+
# Check if user exists
4482
+
user = git_store.get_user(username)
4484
+
print_error(f"User '{username}' not found")
4485
+
print_error("Use 'thicket add user' to add a new user")
4486
+
raise typer.Exit(1)
4488
+
# Check if feed already exists
4489
+
if str(validated_feed_url) in user.feeds:
4490
+
print_error(f"Feed already exists for user '{username}': {feed_url}")
4491
+
raise typer.Exit(1)
4493
+
# Add feed to user
4494
+
updated_feeds = user.feeds + [str(validated_feed_url)]
4495
+
if git_store.update_user(username, feeds=updated_feeds):
4496
+
git_store.commit_changes(f"Add feed to user {username}: {feed_url}")
4497
+
print_success(f"Added feed to user '{username}': {feed_url}")
4499
+
print_error(f"Failed to add feed to user '{username}'")
4500
+
raise typer.Exit(1)
4503
+
async def discover_feed_metadata(feed_url: HttpUrl):
4504
+
"""Discover metadata from a feed URL."""
4506
+
with create_progress() as progress:
4507
+
task = progress.add_task("Discovering feed metadata...", total=None)
4509
+
parser = FeedParser()
4510
+
content = await parser.fetch_feed(feed_url)
4511
+
metadata, _ = parser.parse_feed(content, feed_url)
4513
+
progress.update(task, completed=True)
4516
+
except Exception as e:
4517
+
print_error(f"Failed to discover feed metadata: {e}")
4521
+
<file path="src/thicket/cli/commands/duplicates.py">
4522
+
"""Duplicates command for thicket."""
4524
+
from pathlib import Path
4525
+
from typing import Optional
4528
+
from rich.table import Table
4530
+
from ...core.git_store import GitStore
4531
+
from ..main import app
4532
+
from ..utils import (
4542
+
@app.command("duplicates")
4543
+
def duplicates_command(
4544
+
action: str = typer.Argument(..., help="Action: 'list', 'add', 'remove'"),
4545
+
duplicate_id: Optional[str] = typer.Argument(None, help="Duplicate entry ID"),
4546
+
canonical_id: Optional[str] = typer.Argument(None, help="Canonical entry ID"),
4547
+
config_file: Optional[Path] = typer.Option(
4548
+
Path("thicket.yaml"), "--config", help="Configuration file path"
4551
+
"""Manage duplicate entry mappings."""
4553
+
# Load configuration
4554
+
config = load_config(config_file)
4556
+
# Initialize Git store
4557
+
git_store = GitStore(config.git_store)
4559
+
if action == "list":
4560
+
list_duplicates(git_store)
4561
+
elif action == "add":
4562
+
add_duplicate(git_store, duplicate_id, canonical_id)
4563
+
elif action == "remove":
4564
+
remove_duplicate(git_store, duplicate_id)
4566
+
print_error(f"Unknown action: {action}")
4567
+
print_error("Use 'list', 'add', or 'remove'")
4568
+
raise typer.Exit(1)
4571
+
def list_duplicates(git_store: GitStore) -> None:
4572
+
"""List all duplicate mappings."""
4573
+
duplicates = git_store.get_duplicates()
4575
+
if not duplicates.duplicates:
4576
+
if get_tsv_mode():
4577
+
print("No duplicate mappings found")
4579
+
print_info("No duplicate mappings found")
4582
+
if get_tsv_mode():
4583
+
print("Duplicate ID\tCanonical ID")
4584
+
for duplicate_id, canonical_id in duplicates.duplicates.items():
4585
+
print(f"{duplicate_id}\t{canonical_id}")
4586
+
print(f"Total duplicates: {len(duplicates.duplicates)}")
4588
+
table = Table(title="Duplicate Entry Mappings")
4589
+
table.add_column("Duplicate ID", style="red")
4590
+
table.add_column("Canonical ID", style="green")
4592
+
for duplicate_id, canonical_id in duplicates.duplicates.items():
4593
+
table.add_row(duplicate_id, canonical_id)
4595
+
console.print(table)
4596
+
print_info(f"Total duplicates: {len(duplicates.duplicates)}")
4599
+
def add_duplicate(git_store: GitStore, duplicate_id: Optional[str], canonical_id: Optional[str]) -> None:
4600
+
"""Add a duplicate mapping."""
4601
+
if not duplicate_id:
4602
+
print_error("Duplicate ID is required")
4603
+
raise typer.Exit(1)
4605
+
if not canonical_id:
4606
+
print_error("Canonical ID is required")
4607
+
raise typer.Exit(1)
4609
+
# Check if duplicate_id already exists
4610
+
duplicates = git_store.get_duplicates()
4611
+
if duplicates.is_duplicate(duplicate_id):
4612
+
existing_canonical = duplicates.get_canonical(duplicate_id)
4613
+
print_error(f"Duplicate ID already mapped to: {existing_canonical}")
4614
+
print_error("Use 'remove' first to change the mapping")
4615
+
raise typer.Exit(1)
4617
+
# Check if we're trying to make a canonical ID point to itself
4618
+
if duplicate_id == canonical_id:
4619
+
print_error("Duplicate ID cannot be the same as canonical ID")
4620
+
raise typer.Exit(1)
4623
+
git_store.add_duplicate(duplicate_id, canonical_id)
4626
+
git_store.commit_changes(f"Add duplicate mapping: {duplicate_id} -> {canonical_id}")
4628
+
print_success(f"Added duplicate mapping: {duplicate_id} -> {canonical_id}")
4631
+
def remove_duplicate(git_store: GitStore, duplicate_id: Optional[str]) -> None:
4632
+
"""Remove a duplicate mapping."""
4633
+
if not duplicate_id:
4634
+
print_error("Duplicate ID is required")
4635
+
raise typer.Exit(1)
4637
+
# Check if mapping exists
4638
+
duplicates = git_store.get_duplicates()
4639
+
if not duplicates.is_duplicate(duplicate_id):
4640
+
print_error(f"No duplicate mapping found for: {duplicate_id}")
4641
+
raise typer.Exit(1)
4643
+
canonical_id = duplicates.get_canonical(duplicate_id)
4645
+
# Remove the mapping
4646
+
if git_store.remove_duplicate(duplicate_id):
4648
+
git_store.commit_changes(f"Remove duplicate mapping: {duplicate_id} -> {canonical_id}")
4649
+
print_success(f"Removed duplicate mapping: {duplicate_id} -> {canonical_id}")
4651
+
print_error(f"Failed to remove duplicate mapping: {duplicate_id}")
4652
+
raise typer.Exit(1)
4655
+
<file path="src/thicket/cli/commands/sync.py">
4656
+
"""Sync command for thicket."""
4659
+
from pathlib import Path
4660
+
from typing import Optional
4663
+
from rich.progress import track
4665
+
from ...core.feed_parser import FeedParser
4666
+
from ...core.git_store import GitStore
4667
+
from ..main import app
4668
+
from ..utils import (
4678
+
all_users: bool = typer.Option(
4679
+
False, "--all", "-a", help="Sync all users and feeds"
4681
+
user: Optional[str] = typer.Option(
4682
+
None, "--user", "-u", help="Sync specific user only"
4684
+
config_file: Optional[Path] = typer.Option(
4685
+
Path("thicket.yaml"), "--config", help="Configuration file path"
4687
+
dry_run: bool = typer.Option(
4688
+
False, "--dry-run", help="Show what would be synced without making changes"
4691
+
"""Sync feeds and store entries in Git repository."""
4693
+
# Load configuration
4694
+
config = load_config(config_file)
4696
+
# Initialize Git store
4697
+
git_store = GitStore(config.git_store)
4699
+
# Determine which users to sync from git repository
4700
+
users_to_sync = []
4702
+
index = git_store._load_index()
4703
+
users_to_sync = list(index.users.values())
4705
+
user_metadata = git_store.get_user(user)
4706
+
if not user_metadata:
4707
+
print_error(f"User '{user}' not found in git repository")
4708
+
raise typer.Exit(1)
4709
+
users_to_sync = [user_metadata]
4711
+
print_error("Specify --all to sync all users or --user to sync a specific user")
4712
+
raise typer.Exit(1)
4714
+
if not users_to_sync:
4715
+
print_info("No users configured to sync")
4719
+
total_new_entries = 0
4720
+
total_updated_entries = 0
4722
+
for user_metadata in users_to_sync:
4723
+
print_info(f"Syncing user: {user_metadata.username}")
4725
+
user_new_entries = 0
4726
+
user_updated_entries = 0
4728
+
# Sync each feed for the user
4729
+
for feed_url in track(user_metadata.feeds, description=f"Syncing {user_metadata.username}'s feeds"):
4731
+
new_entries, updated_entries = asyncio.run(
4732
+
sync_feed(git_store, user_metadata.username, feed_url, dry_run)
4734
+
user_new_entries += new_entries
4735
+
user_updated_entries += updated_entries
4737
+
except Exception as e:
4738
+
print_error(f"Failed to sync feed {feed_url}: {e}")
4741
+
print_info(f"User {user_metadata.username}: {user_new_entries} new, {user_updated_entries} updated")
4742
+
total_new_entries += user_new_entries
4743
+
total_updated_entries += user_updated_entries
4745
+
# Commit changes if not dry run
4746
+
if not dry_run and (total_new_entries > 0 or total_updated_entries > 0):
4747
+
commit_message = f"Sync feeds: {total_new_entries} new entries, {total_updated_entries} updated"
4748
+
git_store.commit_changes(commit_message)
4749
+
print_success(f"Committed changes: {commit_message}")
4753
+
print_info(f"Dry run complete: would sync {total_new_entries} new entries, {total_updated_entries} updated")
4755
+
print_success(f"Sync complete: {total_new_entries} new entries, {total_updated_entries} updated")
4758
+
async def sync_feed(git_store: GitStore, username: str, feed_url, dry_run: bool) -> tuple[int, int]:
4759
+
"""Sync a single feed for a user."""
4761
+
parser = FeedParser()
4764
+
# Fetch and parse feed
4765
+
content = await parser.fetch_feed(feed_url)
4766
+
metadata, entries = parser.parse_feed(content, feed_url)
4769
+
updated_entries = 0
4771
+
# Process each entry
4772
+
for entry in entries:
4774
+
# Check if entry already exists
4775
+
existing_entry = git_store.get_entry(username, entry.id)
4777
+
if existing_entry:
4778
+
# Check if entry has been updated
4779
+
if existing_entry.updated != entry.updated:
4781
+
git_store.store_entry(username, entry)
4782
+
updated_entries += 1
4786
+
git_store.store_entry(username, entry)
4789
+
except Exception as e:
4790
+
print_error(f"Failed to process entry {entry.id}: {e}")
4793
+
return new_entries, updated_entries
4795
+
except Exception as e:
4796
+
print_error(f"Failed to sync feed {feed_url}: {e}")
4800
+
<file path="src/thicket/models/config.py">
4801
+
"""Configuration models for thicket."""
4803
+
from pathlib import Path
4804
+
from typing import Optional
4806
+
from pydantic import BaseModel, EmailStr, HttpUrl
4807
+
from pydantic_settings import BaseSettings, SettingsConfigDict
4810
+
class UserConfig(BaseModel):
4811
+
"""Configuration for a single user and their feeds."""
4814
+
feeds: list[HttpUrl]
4815
+
email: Optional[EmailStr] = None
4816
+
homepage: Optional[HttpUrl] = None
4817
+
icon: Optional[HttpUrl] = None
4818
+
display_name: Optional[str] = None
4821
+
class ThicketConfig(BaseSettings):
4822
+
"""Main configuration for thicket."""
4824
+
model_config = SettingsConfigDict(
4825
+
env_prefix="THICKET_",
4827
+
yaml_file="thicket.yaml",
4828
+
case_sensitive=False,
4833
+
users: list[UserConfig] = []
4836
+
<file path="src/thicket/cli/commands/links_cmd.py">
4837
+
"""CLI command for extracting and categorizing all outbound links from blog entries."""
4841
+
from pathlib import Path
4842
+
from typing import Dict, List, Optional, Set
4843
+
from urllib.parse import urljoin, urlparse
4846
+
from rich.console import Console
4847
+
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
4848
+
from rich.table import Table
4850
+
from ...core.git_store import GitStore
4851
+
from ..main import app
4852
+
from ..utils import load_config, get_tsv_mode
4854
+
console = Console()
4858
+
"""Represents a link found in a blog entry."""
4860
+
def __init__(self, url: str, entry_id: str, username: str):
4862
+
self.entry_id = entry_id
4863
+
self.username = username
4865
+
def to_dict(self) -> dict:
4866
+
"""Convert to dictionary for JSON serialization."""
4869
+
"entry_id": self.entry_id,
4870
+
"username": self.username
4874
+
def from_dict(cls, data: dict) -> "LinkData":
4875
+
"""Create from dictionary."""
4878
+
entry_id=data["entry_id"],
4879
+
username=data["username"]
4883
+
class LinkCategorizer:
4884
+
"""Categorizes links as internal, user, or unknown."""
4886
+
def __init__(self, user_domains: Dict[str, Set[str]]):
4887
+
self.user_domains = user_domains
4888
+
# Create reverse mapping of domain -> username
4889
+
self.domain_to_user = {}
4890
+
for username, domains in user_domains.items():
4891
+
for domain in domains:
4892
+
self.domain_to_user[domain] = username
4894
+
def categorize_url(self, url: str, source_username: str) -> tuple[str, Optional[str]]:
4896
+
Categorize a URL as 'internal', 'user', or 'unknown'.
4897
+
Returns (category, target_username).
4900
+
parsed = urlparse(url)
4901
+
domain = parsed.netloc.lower()
4903
+
# Check if it's a link to the same user's domain (internal)
4904
+
if domain in self.user_domains.get(source_username, set()):
4905
+
return "internal", source_username
4907
+
# Check if it's a link to another user's domain
4908
+
if domain in self.domain_to_user:
4909
+
return "user", self.domain_to_user[domain]
4911
+
# Everything else is unknown
4912
+
return "unknown", None
4915
+
return "unknown", None
4918
+
class LinkExtractor:
4919
+
"""Extracts and resolves links from blog entries."""
4921
+
def __init__(self):
4922
+
# Pattern for extracting links from HTML
4923
+
self.link_pattern = re.compile(r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', re.IGNORECASE | re.DOTALL)
4924
+
self.url_pattern = re.compile(r'https?://[^\s<>"]+')
4926
+
def extract_links_from_html(self, html_content: str, base_url: str) -> List[tuple[str, str]]:
4927
+
"""Extract all links from HTML content and resolve them against base URL."""
4930
+
# Extract links from <a> tags
4931
+
for match in self.link_pattern.finditer(html_content):
4932
+
url = match.group(1)
4933
+
text = re.sub(r'<[^>]+>', '', match.group(2)).strip() # Remove HTML tags from link text
4935
+
# Resolve relative URLs against base URL
4936
+
resolved_url = urljoin(base_url, url)
4937
+
links.append((resolved_url, text))
4942
+
def extract_links_from_entry(self, entry, username: str, base_url: str) -> List[LinkData]:
4943
+
"""Extract all links from a blog entry."""
4946
+
# Combine all text content for analysis
4947
+
content_to_search = []
4949
+
content_to_search.append(entry.content)
4951
+
content_to_search.append(entry.summary)
4953
+
for content in content_to_search:
4954
+
extracted_links = self.extract_links_from_html(content, base_url)
4956
+
for url, link_text in extracted_links:
4958
+
if not url or url.startswith('#'):
4961
+
link_data = LinkData(
4963
+
entry_id=entry.id,
4967
+
links.append(link_data)
4974
+
config_file: Optional[Path] = typer.Option(
4975
+
Path("thicket.yaml"),
4978
+
help="Path to configuration file",
4980
+
output_file: Optional[Path] = typer.Option(
4984
+
help="Path to output unified links file (default: links.json in git store)",
4986
+
verbose: bool = typer.Option(
4990
+
help="Show detailed progress information",
4993
+
"""Extract and categorize all outbound links from blog entries.
4995
+
This command analyzes all blog entries to extract outbound links,
4996
+
resolve them properly with respect to the feed's base URL, and
4997
+
categorize them as internal, user, or unknown links.
4999
+
Creates a unified links.json file containing all link data.
5002
+
# Load configuration
5003
+
config = load_config(config_file)
5005
+
# Initialize Git store
5006
+
git_store = GitStore(config.git_store)
5008
+
# Build user domain mapping
5010
+
console.print("Building user domain mapping...")
5012
+
index = git_store._load_index()
5015
+
for username, user_metadata in index.users.items():
5018
+
# Add domains from feeds
5019
+
for feed_url in user_metadata.feeds:
5020
+
domain = urlparse(feed_url).netloc.lower()
5022
+
domains.add(domain)
5024
+
# Add domain from homepage
5025
+
if user_metadata.homepage:
5026
+
domain = urlparse(str(user_metadata.homepage)).netloc.lower()
5028
+
domains.add(domain)
5030
+
user_domains[username] = domains
5033
+
console.print(f"Found {len(user_domains)} users with {sum(len(d) for d in user_domains.values())} total domains")
5035
+
# Initialize components
5036
+
link_extractor = LinkExtractor()
5037
+
categorizer = LinkCategorizer(user_domains)
5040
+
users = list(index.users.keys())
5043
+
console.print("[yellow]No users found in Git store[/yellow]")
5044
+
raise typer.Exit(0)
5046
+
# Process all entries
5048
+
link_categories = {"internal": [], "user": [], "unknown": []}
5049
+
link_dict = {} # Dictionary with link URL as key, maps to list of atom IDs
5050
+
reverse_dict = {} # Dictionary with atom ID as key, maps to list of URLs
5054
+
TextColumn("[progress.description]{task.description}"),
5056
+
TaskProgressColumn(),
5060
+
# Count total entries first
5061
+
counting_task = progress.add_task("Counting entries...", total=len(users))
5064
+
for username in users:
5065
+
entries = git_store.list_entries(username)
5066
+
total_entries += len(entries)
5067
+
progress.advance(counting_task)
5069
+
progress.remove_task(counting_task)
5072
+
processing_task = progress.add_task(
5073
+
f"Processing {total_entries} entries...",
5074
+
total=total_entries
5077
+
for username in users:
5078
+
entries = git_store.list_entries(username)
5079
+
user_metadata = index.users[username]
5081
+
# Get base URL for this user (use first feed URL)
5082
+
base_url = str(user_metadata.feeds[0]) if user_metadata.feeds else "https://example.com"
5084
+
for entry in entries:
5085
+
# Extract links from this entry
5086
+
entry_links = link_extractor.extract_links_from_entry(entry, username, base_url)
5088
+
# Track unique links per entry
5089
+
entry_urls_seen = set()
5091
+
# Categorize each link
5092
+
for link_data in entry_links:
5093
+
# Skip if we've already seen this URL in this entry
5094
+
if link_data.url in entry_urls_seen:
5096
+
entry_urls_seen.add(link_data.url)
5098
+
category, target_username = categorizer.categorize_url(link_data.url, username)
5100
+
# Add to link dictionary (URL as key, maps to list of atom IDs)
5101
+
if link_data.url not in link_dict:
5102
+
link_dict[link_data.url] = []
5103
+
if link_data.entry_id not in link_dict[link_data.url]:
5104
+
link_dict[link_data.url].append(link_data.entry_id)
5106
+
# Also add to reverse mapping (atom ID -> list of URLs)
5107
+
if link_data.entry_id not in reverse_dict:
5108
+
reverse_dict[link_data.entry_id] = []
5109
+
if link_data.url not in reverse_dict[link_data.entry_id]:
5110
+
reverse_dict[link_data.entry_id].append(link_data.url)
5112
+
# Add category info to link data for categories tracking
5113
+
link_info = link_data.to_dict()
5114
+
link_info["category"] = category
5115
+
link_info["target_username"] = target_username
5117
+
all_links.append(link_info)
5118
+
link_categories[category].append(link_info)
5120
+
progress.advance(processing_task)
5122
+
if verbose and entry_links:
5123
+
console.print(f" Found {len(entry_links)} links in {username}:{entry.title[:50]}...")
5125
+
# Determine output path
5127
+
output_path = output_file
5129
+
output_path = config.git_store / "links.json"
5131
+
# Save all extracted links (not just filtered ones)
5133
+
console.print("Preparing output data...")
5135
+
# Build a set of all URLs that correspond to posts in the git database
5136
+
registered_urls = set()
5138
+
# Get all entries from all users and build URL mappings
5139
+
for username in users:
5140
+
entries = git_store.list_entries(username)
5141
+
user_metadata = index.users[username]
5143
+
for entry in entries:
5144
+
# Try to match entry URLs with extracted links
5145
+
if hasattr(entry, 'link') and entry.link:
5146
+
registered_urls.add(str(entry.link))
5148
+
# Also check entry alternate links if they exist
5149
+
if hasattr(entry, 'links') and entry.links:
5150
+
for link in entry.links:
5151
+
if hasattr(link, 'href') and link.href:
5152
+
registered_urls.add(str(link.href))
5154
+
# Build unified structure with metadata
5155
+
unified_links = {}
5156
+
reverse_mapping = {}
5158
+
for url, entry_ids in link_dict.items():
5159
+
unified_links[url] = {
5160
+
"referencing_entries": entry_ids
5163
+
# Find target username if this is a tracked post
5164
+
if url in registered_urls:
5165
+
for username in users:
5166
+
user_domains_set = {domain for domain in user_domains.get(username, [])}
5167
+
if any(domain in url for domain in user_domains_set):
5168
+
unified_links[url]["target_username"] = username
5171
+
# Build reverse mapping
5172
+
for entry_id in entry_ids:
5173
+
if entry_id not in reverse_mapping:
5174
+
reverse_mapping[entry_id] = []
5175
+
if url not in reverse_mapping[entry_id]:
5176
+
reverse_mapping[entry_id].append(url)
5178
+
# Create unified output data
5180
+
"links": unified_links,
5181
+
"reverse_mapping": reverse_mapping,
5182
+
"user_domains": {k: list(v) for k, v in user_domains.items()}
5186
+
console.print(f"Found {len(registered_urls)} registered post URLs")
5187
+
console.print(f"Found {len(link_dict)} total links, {sum(1 for link in unified_links.values() if 'target_username' in link)} tracked posts")
5189
+
# Save unified data
5190
+
with open(output_path, "w") as f:
5191
+
json.dump(output_data, f, indent=2, default=str)
5194
+
if not get_tsv_mode():
5195
+
console.print("\n[green]✓ Links extraction completed successfully[/green]")
5197
+
# Create summary table or TSV output
5198
+
if get_tsv_mode():
5199
+
print("Category\tCount\tDescription")
5200
+
print(f"Internal\t{len(link_categories['internal'])}\tLinks to same user's domain")
5201
+
print(f"User\t{len(link_categories['user'])}\tLinks to other tracked users")
5202
+
print(f"Unknown\t{len(link_categories['unknown'])}\tLinks to external sites")
5203
+
print(f"Total Extracted\t{len(all_links)}\tAll extracted links")
5204
+
print(f"Saved to Output\t{len(output_data['links'])}\tLinks saved to output file")
5205
+
print(f"Cross-references\t{sum(1 for link in unified_links.values() if 'target_username' in link)}\tLinks to registered posts only")
5207
+
table = Table(title="Links Summary")
5208
+
table.add_column("Category", style="cyan")
5209
+
table.add_column("Count", style="green")
5210
+
table.add_column("Description", style="white")
5212
+
table.add_row("Internal", str(len(link_categories["internal"])), "Links to same user's domain")
5213
+
table.add_row("User", str(len(link_categories["user"])), "Links to other tracked users")
5214
+
table.add_row("Unknown", str(len(link_categories["unknown"])), "Links to external sites")
5215
+
table.add_row("Total Extracted", str(len(all_links)), "All extracted links")
5216
+
table.add_row("Saved to Output", str(len(output_data['links'])), "Links saved to output file")
5217
+
table.add_row("Cross-references", str(sum(1 for link in unified_links.values() if 'target_username' in link)), "Links to registered posts only")
5219
+
console.print(table)
5221
+
# Show user links if verbose
5222
+
if verbose and link_categories["user"]:
5223
+
if get_tsv_mode():
5224
+
print("User Link Source\tUser Link Target\tLink Count")
5225
+
user_link_counts = {}
5227
+
for link in link_categories["user"]:
5228
+
key = f"{link['username']} -> {link['target_username']}"
5229
+
user_link_counts[key] = user_link_counts.get(key, 0) + 1
5231
+
for link_pair, count in sorted(user_link_counts.items(), key=lambda x: x[1], reverse=True)[:10]:
5232
+
source, target = link_pair.split(" -> ")
5233
+
print(f"{source}\t{target}\t{count}")
5235
+
console.print("\n[bold]User-to-user links:[/bold]")
5236
+
user_link_counts = {}
5238
+
for link in link_categories["user"]:
5239
+
key = f"{link['username']} -> {link['target_username']}"
5240
+
user_link_counts[key] = user_link_counts.get(key, 0) + 1
5242
+
for link_pair, count in sorted(user_link_counts.items(), key=lambda x: x[1], reverse=True)[:10]:
5243
+
console.print(f" {link_pair}: {count} links")
5245
+
if not get_tsv_mode():
5246
+
console.print(f"\nUnified links data saved to: {output_path}")
5248
+
except Exception as e:
5249
+
console.print(f"[red]Error extracting links: {e}[/red]")
5251
+
console.print_exception()
5252
+
raise typer.Exit(1)
5255
+
<file path="src/thicket/cli/commands/list_cmd.py">
5256
+
"""List command for thicket."""
5259
+
from pathlib import Path
5260
+
from typing import Optional
5263
+
from rich.table import Table
5265
+
from ...core.git_store import GitStore
5266
+
from ..main import app
5267
+
from ..utils import (
5271
+
print_feeds_table,
5272
+
print_feeds_table_from_git,
5274
+
print_users_table,
5275
+
print_users_table_from_git,
5276
+
print_entries_tsv,
5281
+
@app.command("list")
5283
+
what: str = typer.Argument(..., help="What to list: 'users', 'feeds', 'entries'"),
5284
+
user: Optional[str] = typer.Option(
5285
+
None, "--user", "-u", help="Filter by specific user"
5287
+
limit: Optional[int] = typer.Option(
5288
+
None, "--limit", "-l", help="Limit number of results"
5290
+
config_file: Optional[Path] = typer.Option(
5291
+
Path("thicket.yaml"), "--config", help="Configuration file path"
5294
+
"""List users, feeds, or entries."""
5296
+
# Load configuration
5297
+
config = load_config(config_file)
5299
+
# Initialize Git store
5300
+
git_store = GitStore(config.git_store)
5302
+
if what == "users":
5303
+
list_users(git_store)
5304
+
elif what == "feeds":
5305
+
list_feeds(git_store, user)
5306
+
elif what == "entries":
5307
+
list_entries(git_store, user, limit)
5309
+
print_error(f"Unknown list type: {what}")
5310
+
print_error("Use 'users', 'feeds', or 'entries'")
5311
+
raise typer.Exit(1)
5314
+
def list_users(git_store: GitStore) -> None:
5315
+
"""List all users."""
5316
+
index = git_store._load_index()
5317
+
users = list(index.users.values())
5320
+
print_info("No users configured")
5323
+
print_users_table_from_git(users)
5326
+
def list_feeds(git_store: GitStore, username: Optional[str] = None) -> None:
5327
+
"""List feeds, optionally filtered by user."""
5329
+
user = git_store.get_user(username)
5331
+
print_error(f"User '{username}' not found")
5332
+
raise typer.Exit(1)
5334
+
if not user.feeds:
5335
+
print_info(f"No feeds configured for user '{username}'")
5338
+
print_feeds_table_from_git(git_store, username)
5341
+
def list_entries(git_store: GitStore, username: Optional[str] = None, limit: Optional[int] = None) -> None:
5342
+
"""List entries, optionally filtered by user."""
5345
+
# List entries for specific user
5346
+
user = git_store.get_user(username)
5348
+
print_error(f"User '{username}' not found")
5349
+
raise typer.Exit(1)
5351
+
entries = git_store.list_entries(username, limit)
5353
+
print_info(f"No entries found for user '{username}'")
5356
+
print_entries_table([entries], [username])
5359
+
# List entries for all users
5361
+
all_usernames = []
5363
+
index = git_store._load_index()
5364
+
for user in index.users.values():
5365
+
entries = git_store.list_entries(user.username, limit)
5367
+
all_entries.append(entries)
5368
+
all_usernames.append(user.username)
5370
+
if not all_entries:
5371
+
print_info("No entries found")
5374
+
print_entries_table(all_entries, all_usernames)
5377
+
def _clean_html_content(content: Optional[str]) -> str:
5378
+
"""Clean HTML content for display in table."""
5382
+
# Remove HTML tags
5383
+
clean_text = re.sub(r'<[^>]+>', ' ', content)
5384
+
# Replace multiple whitespace with single space
5385
+
clean_text = re.sub(r'\s+', ' ', clean_text)
5386
+
# Strip and limit length
5387
+
clean_text = clean_text.strip()
5388
+
if len(clean_text) > 100:
5389
+
clean_text = clean_text[:97] + "..."
5394
+
def print_entries_table(entries_by_user: list[list], usernames: list[str]) -> None:
5395
+
"""Print a table of entries."""
5396
+
if get_tsv_mode():
5397
+
print_entries_tsv(entries_by_user, usernames)
5400
+
table = Table(title="Feed Entries")
5401
+
table.add_column("User", style="cyan", no_wrap=True)
5402
+
table.add_column("Title", style="bold")
5403
+
table.add_column("Updated", style="blue")
5404
+
table.add_column("URL", style="green")
5406
+
# Combine all entries with usernames
5408
+
for entries, username in zip(entries_by_user, usernames):
5409
+
for entry in entries:
5410
+
all_entries.append((username, entry))
5412
+
# Sort by updated time (newest first)
5413
+
all_entries.sort(key=lambda x: x[1].updated, reverse=True)
5415
+
for username, entry in all_entries:
5416
+
# Format updated time
5417
+
updated_str = entry.updated.strftime("%Y-%m-%d %H:%M")
5419
+
# Truncate title if too long
5420
+
title = entry.title
5421
+
if len(title) > 50:
5422
+
title = title[:47] + "..."
5431
+
console.print(table)
5434
+
<file path="src/thicket/cli/main.py">
5435
+
"""Main CLI application using Typer."""
5438
+
from rich.console import Console
5440
+
from .. import __version__
5442
+
app = typer.Typer(
5444
+
help="A CLI tool for persisting Atom/RSS feeds in Git repositories",
5445
+
no_args_is_help=True,
5446
+
rich_markup_mode="rich",
5449
+
console = Console()
5451
+
# Global state for TSV output mode
5455
+
def version_callback(value: bool) -> None:
5456
+
"""Show version and exit."""
5458
+
console.print(f"thicket version {__version__}")
5459
+
raise typer.Exit()
5464
+
version: bool = typer.Option(
5468
+
help="Show the version and exit",
5469
+
callback=version_callback,
5472
+
tsv: bool = typer.Option(
5475
+
help="Output in tab-separated values format without truncation",
5478
+
"""Thicket: A CLI tool for persisting Atom/RSS feeds in Git repositories."""
5483
+
# Import commands to register them
5484
+
from .commands import add, duplicates, generate, index_cmd, info_cmd, init, links_cmd, list_cmd, sync
5486
+
if __name__ == "__main__":
5490
+
<file path="src/thicket/core/git_store.py">
5491
+
"""Git repository operations for thicket."""
5494
+
from datetime import datetime
5495
+
from pathlib import Path
5496
+
from typing import Optional
5499
+
from git import Repo
5501
+
from ..models import AtomEntry, DuplicateMap, GitStoreIndex, UserMetadata
5505
+
"""Manages the Git repository for storing feed entries."""
5507
+
def __init__(self, repo_path: Path):
5508
+
"""Initialize the Git store."""
5509
+
self.repo_path = repo_path
5510
+
self.repo: Optional[Repo] = None
5511
+
self._ensure_repo()
5513
+
def _ensure_repo(self) -> None:
5514
+
"""Ensure the Git repository exists and is initialized."""
5515
+
if not self.repo_path.exists():
5516
+
self.repo_path.mkdir(parents=True, exist_ok=True)
5519
+
self.repo = Repo(self.repo_path)
5520
+
except git.InvalidGitRepositoryError:
5521
+
# Initialize new repository
5522
+
self.repo = Repo.init(self.repo_path)
5523
+
self._create_initial_structure()
5525
+
def _create_initial_structure(self) -> None:
5526
+
"""Create initial Git store structure."""
5527
+
# Create index.json
5528
+
index = GitStoreIndex(
5529
+
created=datetime.now(),
5530
+
last_updated=datetime.now(),
5532
+
self._save_index(index)
5534
+
# Create duplicates.json
5535
+
duplicates = DuplicateMap()
5536
+
self._save_duplicates(duplicates)
5538
+
# Create initial commit
5539
+
self.repo.index.add(["index.json", "duplicates.json"])
5540
+
self.repo.index.commit("Initial thicket repository structure")
5542
+
def _save_index(self, index: GitStoreIndex) -> None:
5543
+
"""Save the index to index.json."""
5544
+
index_path = self.repo_path / "index.json"
5545
+
with open(index_path, "w") as f:
5546
+
json.dump(index.model_dump(mode="json", exclude_none=True), f, indent=2, default=str)
5548
+
def _load_index(self) -> GitStoreIndex:
5549
+
"""Load the index from index.json."""
5550
+
index_path = self.repo_path / "index.json"
5551
+
if not index_path.exists():
5552
+
return GitStoreIndex(
5553
+
created=datetime.now(),
5554
+
last_updated=datetime.now(),
5557
+
with open(index_path) as f:
5558
+
data = json.load(f)
5560
+
return GitStoreIndex(**data)
5562
+
def _save_duplicates(self, duplicates: DuplicateMap) -> None:
5563
+
"""Save duplicates map to duplicates.json."""
5564
+
duplicates_path = self.repo_path / "duplicates.json"
5565
+
with open(duplicates_path, "w") as f:
5566
+
json.dump(duplicates.model_dump(exclude_none=True), f, indent=2)
5568
+
def _load_duplicates(self) -> DuplicateMap:
5569
+
"""Load duplicates map from duplicates.json."""
5570
+
duplicates_path = self.repo_path / "duplicates.json"
5571
+
if not duplicates_path.exists():
5572
+
return DuplicateMap()
5574
+
with open(duplicates_path) as f:
5575
+
data = json.load(f)
5577
+
return DuplicateMap(**data)
5579
+
def add_user(self, username: str, display_name: Optional[str] = None,
5580
+
email: Optional[str] = None, homepage: Optional[str] = None,
5581
+
icon: Optional[str] = None, feeds: Optional[list[str]] = None) -> UserMetadata:
5582
+
"""Add a new user to the Git store."""
5583
+
index = self._load_index()
5585
+
# Create user directory
5586
+
user_dir = self.repo_path / username
5587
+
user_dir.mkdir(exist_ok=True)
5589
+
# Create user metadata
5590
+
user_metadata = UserMetadata(
5591
+
username=username,
5592
+
display_name=display_name,
5594
+
homepage=homepage,
5596
+
feeds=feeds or [],
5597
+
directory=username,
5598
+
created=datetime.now(),
5599
+
last_updated=datetime.now(),
5604
+
index.add_user(user_metadata)
5605
+
self._save_index(index)
5607
+
return user_metadata
5609
+
def get_user(self, username: str) -> Optional[UserMetadata]:
5610
+
"""Get user metadata by username."""
5611
+
index = self._load_index()
5612
+
return index.get_user(username)
5614
+
def update_user(self, username: str, **kwargs) -> bool:
5615
+
"""Update user metadata."""
5616
+
index = self._load_index()
5617
+
user = index.get_user(username)
5622
+
# Update user metadata
5623
+
for key, value in kwargs.items():
5624
+
if hasattr(user, key) and value is not None:
5625
+
setattr(user, key, value)
5627
+
user.update_timestamp()
5631
+
index.add_user(user)
5632
+
self._save_index(index)
5636
+
def store_entry(self, username: str, entry: AtomEntry) -> bool:
5637
+
"""Store an entry in the user's directory."""
5638
+
user = self.get_user(username)
5642
+
# Sanitize entry ID for filename
5643
+
from .feed_parser import FeedParser
5644
+
parser = FeedParser()
5645
+
safe_id = parser.sanitize_entry_id(entry.id)
5647
+
# Create entry file
5648
+
user_dir = self.repo_path / user.directory
5649
+
entry_path = user_dir / f"{safe_id}.json"
5651
+
# Check if entry already exists
5652
+
entry_exists = entry_path.exists()
5655
+
with open(entry_path, "w") as f:
5656
+
json.dump(entry.model_dump(mode="json", exclude_none=True), f, indent=2, default=str)
5658
+
# Update user metadata if new entry
5659
+
if not entry_exists:
5660
+
index = self._load_index()
5661
+
index.update_entry_count(username, 1)
5662
+
self._save_index(index)
5666
+
def get_entry(self, username: str, entry_id: str) -> Optional[AtomEntry]:
5667
+
"""Get an entry by username and entry ID."""
5668
+
user = self.get_user(username)
5672
+
# Sanitize entry ID
5673
+
from .feed_parser import FeedParser
5674
+
parser = FeedParser()
5675
+
safe_id = parser.sanitize_entry_id(entry_id)
5677
+
entry_path = self.repo_path / user.directory / f"{safe_id}.json"
5678
+
if not entry_path.exists():
5681
+
with open(entry_path) as f:
5682
+
data = json.load(f)
5684
+
return AtomEntry(**data)
5686
+
def list_entries(self, username: str, limit: Optional[int] = None) -> list[AtomEntry]:
5687
+
"""List entries for a user."""
5688
+
user = self.get_user(username)
5692
+
user_dir = self.repo_path / user.directory
5693
+
if not user_dir.exists():
5697
+
entry_files = sorted(user_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
5701
+
entry_files = entry_files[:limit]
5703
+
for entry_file in entry_files:
5705
+
with open(entry_file) as f:
5706
+
data = json.load(f)
5707
+
entries.append(AtomEntry(**data))
5709
+
# Skip invalid entries
5714
+
def get_duplicates(self) -> DuplicateMap:
5715
+
"""Get the duplicates map."""
5716
+
return self._load_duplicates()
5718
+
def add_duplicate(self, duplicate_id: str, canonical_id: str) -> None:
5719
+
"""Add a duplicate mapping."""
5720
+
duplicates = self._load_duplicates()
5721
+
duplicates.add_duplicate(duplicate_id, canonical_id)
5722
+
self._save_duplicates(duplicates)
5724
+
def remove_duplicate(self, duplicate_id: str) -> bool:
5725
+
"""Remove a duplicate mapping."""
5726
+
duplicates = self._load_duplicates()
5727
+
result = duplicates.remove_duplicate(duplicate_id)
5728
+
self._save_duplicates(duplicates)
5731
+
def commit_changes(self, message: str) -> None:
5732
+
"""Commit all changes to the Git repository."""
5737
+
self.repo.git.add(A=True)
5739
+
# Check if there are changes to commit
5740
+
if self.repo.index.diff("HEAD"):
5741
+
self.repo.index.commit(message)
5743
+
def get_stats(self) -> dict:
5744
+
"""Get statistics about the Git store."""
5745
+
index = self._load_index()
5746
+
duplicates = self._load_duplicates()
5749
+
"total_users": len(index.users),
5750
+
"total_entries": index.total_entries,
5751
+
"total_duplicates": len(duplicates.duplicates),
5752
+
"last_updated": index.last_updated,
5753
+
"repository_size": sum(f.stat().st_size for f in self.repo_path.rglob("*") if f.is_file()),
5756
+
def search_entries(self, query: str, username: Optional[str] = None,
5757
+
limit: Optional[int] = None) -> list[tuple[str, AtomEntry]]:
5758
+
"""Search entries by content."""
5761
+
# Get users to search
5762
+
index = self._load_index()
5763
+
users = [index.get_user(username)] if username else list(index.users.values())
5764
+
users = [u for u in users if u is not None]
5766
+
for user in users:
5767
+
user_dir = self.repo_path / user.directory
5768
+
if not user_dir.exists():
5771
+
entry_files = user_dir.glob("*.json")
5773
+
for entry_file in entry_files:
5775
+
with open(entry_file) as f:
5776
+
data = json.load(f)
5778
+
entry = AtomEntry(**data)
5780
+
# Simple text search in title, summary, and content
5781
+
searchable_text = " ".join(filter(None, [
5783
+
entry.summary or "",
5784
+
entry.content or "",
5787
+
if query.lower() in searchable_text:
5788
+
results.append((user.username, entry))
5790
+
if limit and len(results) >= limit:
5794
+
# Skip invalid entries
5797
+
# Sort by updated time (newest first)
5798
+
results.sort(key=lambda x: x[1].updated, reverse=True)
5800
+
return results[:limit] if limit else results
5803
+
<file path="ARCH.md">
5804
+
# Thicket Architecture Design
5807
+
Thicket is a modern CLI tool for persisting Atom/RSS feeds in a Git repository, designed to enable distributed webblog comment structures.
5809
+
## Technology Stack
5811
+
### Core Libraries
5813
+
#### CLI Framework
5814
+
- **Typer** (0.15.x) - Modern CLI framework with type hints
5815
+
- **Rich** (13.x) - Beautiful terminal output, progress bars, and tables
5816
+
- **prompt-toolkit** - Interactive prompts when needed
5818
+
#### Feed Processing
5819
+
- **feedparser** (6.0.11) - Universal feed parser supporting RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0
5820
+
- Alternative: **atoma** for stricter Atom/RSS parsing with JSON feed support
5821
+
- Alternative: **fastfeedparser** for high-performance parsing (10x faster)
5823
+
#### Git Integration
5824
+
- **GitPython** (3.1.44) - High-level git operations, requires git CLI
5825
+
- Alternative: **pygit2** (1.18.0) - Direct libgit2 bindings, better for authentication
5828
+
- **httpx** (0.28.x) - Modern async/sync HTTP client with connection pooling
5829
+
- **aiohttp** (3.11.x) - For async-only operations if needed
5831
+
#### Configuration & Data Models
5832
+
- **pydantic** (2.11.x) - Data validation and settings management
5833
+
- **pydantic-settings** (2.10.x) - Configuration file handling with env var support
5836
+
- **pendulum** (3.x) - Better datetime handling
5837
+
- **bleach** (6.x) - HTML sanitization for feed content
5838
+
- **platformdirs** (4.x) - Cross-platform directory paths
5840
+
## Project Structure
5844
+
├── pyproject.toml # Modern Python packaging
5845
+
├── README.md # Project documentation
5846
+
├── ARCH.md # This file
5847
+
├── CLAUDE.md # Project instructions
5852
+
│ ├── __main__.py # Entry point for `python -m thicket`
5853
+
│ ├── cli/ # CLI commands and interface
5854
+
│ │ ├── __init__.py
5855
+
│ │ ├── main.py # Main CLI app with Typer
5856
+
│ │ ├── commands/ # Subcommands
5857
+
│ │ │ ├── __init__.py
5858
+
│ │ │ ├── init.py # Initialize git store
5859
+
│ │ │ ├── add.py # Add users and feeds
5860
+
│ │ │ ├── sync.py # Sync feeds
5861
+
│ │ │ ├── list_cmd.py # List users/feeds
5862
+
│ │ │ ├── duplicates.py # Manage duplicate entries
5863
+
│ │ │ ├── links_cmd.py # Extract and categorize links
5864
+
│ │ │ └── index_cmd.py # Build reference index and show threads
5865
+
│ │ └── utils.py # CLI utilities (progress, formatting)
5866
+
│ ├── core/ # Core business logic
5867
+
│ │ ├── __init__.py
5868
+
│ │ ├── feed_parser.py # Feed parsing and normalization
5869
+
│ │ ├── git_store.py # Git repository operations
5870
+
│ │ └── reference_parser.py # Link extraction and threading
5871
+
│ ├── models/ # Pydantic data models
5872
+
│ │ ├── __init__.py
5873
+
│ │ ├── config.py # Configuration models
5874
+
│ │ ├── feed.py # Feed/Entry models
5875
+
│ │ └── user.py # User metadata models
5876
+
│ └── utils/ # Shared utilities
5880
+
│ ├── conftest.py # pytest configuration
5881
+
│ ├── test_feed_parser.py
5882
+
│ ├── test_git_store.py
5883
+
│ └── fixtures/ # Test data
5886
+
└── examples/ # Example configurations
5891
+
### Configuration File (YAML/TOML)
5893
+
class ThicketConfig(BaseSettings):
5894
+
git_store: Path # Git repository location
5895
+
cache_dir: Path # Cache directory
5896
+
users: list[UserConfig]
5898
+
model_config = SettingsConfigDict(
5899
+
env_prefix="THICKET_",
5901
+
yaml_file="thicket.yaml"
5904
+
class UserConfig(BaseModel):
5906
+
feeds: list[HttpUrl]
5907
+
email: Optional[EmailStr] = None
5908
+
homepage: Optional[HttpUrl] = None
5909
+
icon: Optional[HttpUrl] = None
5910
+
display_name: Optional[str] = None
5913
+
### Feed Storage Format
5915
+
class AtomEntry(BaseModel):
5916
+
id: str # Original Atom ID
5920
+
published: Optional[datetime]
5921
+
summary: Optional[str]
5922
+
content: Optional[str] # Full body content from Atom entry
5923
+
content_type: Optional[str] = "html" # text, html, xhtml
5924
+
author: Optional[dict]
5925
+
categories: list[str] = []
5926
+
rights: Optional[str] = None # Copyright info
5927
+
source: Optional[str] = None # Source feed URL
5928
+
# Additional Atom fields preserved during RSS->Atom conversion
5930
+
model_config = ConfigDict(
5932
+
datetime: lambda v: v.isoformat()
5936
+
class DuplicateMap(BaseModel):
5937
+
"""Maps duplicate entry IDs to canonical entry IDs"""
5938
+
duplicates: dict[str, str] = {} # duplicate_id -> canonical_id
5939
+
comment: str = "Entry IDs that map to the same canonical content"
5941
+
def add_duplicate(self, duplicate_id: str, canonical_id: str) -> None:
5942
+
"""Add a duplicate mapping"""
5943
+
self.duplicates[duplicate_id] = canonical_id
5945
+
def remove_duplicate(self, duplicate_id: str) -> bool:
5946
+
"""Remove a duplicate mapping. Returns True if existed."""
5947
+
return self.duplicates.pop(duplicate_id, None) is not None
5949
+
def get_canonical(self, entry_id: str) -> str:
5950
+
"""Get canonical ID for an entry (returns original if not duplicate)"""
5951
+
return self.duplicates.get(entry_id, entry_id)
5953
+
def is_duplicate(self, entry_id: str) -> bool:
5954
+
"""Check if entry ID is marked as duplicate"""
5955
+
return entry_id in self.duplicates
5958
+
## Git Repository Structure
5961
+
├── index.json # User directory index
5962
+
├── duplicates.json # Manual curation of duplicate entries
5963
+
├── links.json # Unified links, references, and mapping data
5965
+
│ ├── entry_id_1.json # Sanitized entry files
5966
+
│ ├── entry_id_2.json
5972
+
## Key Design Decisions
5974
+
### 1. Feed Normalization & Auto-Discovery
5975
+
- All RSS feeds converted to Atom format before storage
5976
+
- Preserves maximum metadata during conversion
5977
+
- Sanitizes HTML content to prevent XSS
5978
+
- **Auto-discovery**: Extracts user metadata from feed during `add user` command
5980
+
### 2. ID Sanitization
5981
+
- Consistent algorithm to convert Atom IDs to safe filenames
5982
+
- Handles edge cases (very long IDs, special characters)
5983
+
- Maintains reversibility where possible
5985
+
### 3. Git Operations
5986
+
- Uses GitPython for simplicity (no authentication required)
5987
+
- Single main branch for all users and entries
5988
+
- Atomic commits per sync operation
5989
+
- Meaningful commit messages with feed update summaries
5990
+
- Preserves complete history - never delete entries even if they disappear from feeds
5992
+
### 4. Caching Strategy
5993
+
- HTTP caching with Last-Modified/ETag support
5994
+
- Local cache of parsed feeds with TTL
5995
+
- Cache invalidation on configuration changes
5996
+
- Git store serves as permanent historical archive beyond feed depth limits
5998
+
### 5. Error Handling
5999
+
- Graceful handling of feed parsing errors
6000
+
- Retry logic for network failures
6001
+
- Clear error messages with recovery suggestions
6003
+
## CLI Command Structure
6006
+
# Initialize a new git store
6007
+
thicket init /path/to/store
6009
+
# Add a user with feeds (auto-discovers metadata from feed)
6010
+
thicket add user "alyssa" \
6011
+
--feed "https://example.com/feed.atom"
6012
+
# Auto-populates: email, homepage, icon, display_name from feed metadata
6014
+
# Add a user with manual overrides
6015
+
thicket add user "alyssa" \
6016
+
--feed "https://example.com/feed.atom" \
6017
+
--email "alyssa@example.com" \
6018
+
--homepage "https://alyssa.example.com" \
6019
+
--icon "https://example.com/avatar.png" \
6020
+
--display-name "Alyssa P. Hacker"
6022
+
# Add additional feed to existing user
6023
+
thicket add feed "alyssa" "https://example.com/other-feed.rss"
6025
+
# Sync all feeds (designed for cron usage)
6026
+
thicket sync --all
6028
+
# Sync specific user
6029
+
thicket sync --user alyssa
6031
+
# List users and their feeds
6032
+
thicket list users
6033
+
thicket list feeds --user alyssa
6035
+
# Manage duplicate entries
6036
+
thicket duplicates list
6037
+
thicket duplicates add <entry_id_1> <entry_id_2> # Mark as duplicates
6038
+
thicket duplicates remove <entry_id_1> <entry_id_2> # Unmark duplicates
6040
+
# Link processing and threading
6041
+
thicket links --verbose # Extract and categorize all links
6042
+
thicket index --verbose # Build reference index for threading
6043
+
thicket threads # Show conversation threads
6044
+
thicket threads --username user1 # Show threads for specific user
6045
+
thicket threads --min-size 3 # Show threads with minimum size
6048
+
## Performance Considerations
6050
+
1. **Concurrent Feed Fetching**: Use httpx with asyncio for parallel downloads
6051
+
2. **Incremental Updates**: Only fetch/parse feeds that have changed
6052
+
3. **Efficient Git Operations**: Batch commits, use shallow clones where appropriate
6053
+
4. **Progress Feedback**: Rich progress bars for long operations
6055
+
## Security Considerations
6057
+
1. **HTML Sanitization**: Use bleach to clean feed content
6058
+
2. **URL Validation**: Strict validation of feed URLs
6059
+
3. **Git Security**: No credentials stored in repository
6060
+
4. **Path Traversal**: Careful sanitization of filenames
6062
+
## Future Enhancements
6064
+
1. **Web Interface**: Optional web UI for browsing the git store
6065
+
2. **Webhooks**: Notify external services on feed updates
6066
+
3. **Feed Discovery**: Auto-discover feeds from HTML pages
6067
+
4. **Export Formats**: Generate static sites, OPML exports
6068
+
5. **Federation**: P2P sync between thicket instances
6070
+
## Requirements Clarification
6072
+
**✓ Resolved Requirements:**
6073
+
1. **Feed Update Frequency**: Designed for cron usage - no built-in scheduling needed
6074
+
2. **Duplicate Handling**: Manual curation via `duplicates.json` file with CLI commands
6075
+
3. **Git Branching**: Single main branch for all users and entries
6076
+
4. **Authentication**: No feeds require authentication currently
6077
+
5. **Content Storage**: Store complete Atom entry body content as provided
6078
+
6. **Deleted Entries**: Preserve all entries in Git store permanently (historical archive)
6079
+
7. **History Depth**: Git store maintains full history beyond feed depth limits
6080
+
8. **Feed Auto-Discovery**: Extract user metadata from feed during `add user` command
6082
+
## Duplicate Entry Management
6084
+
### Duplicate Detection Strategy
6085
+
- **Manual Curation**: Duplicates identified and managed manually via CLI
6086
+
- **Storage**: `duplicates.json` file in Git root maps entry IDs to canonical entries
6087
+
- **Structure**: `{"duplicate_id": "canonical_id", ...}`
6088
+
- **CLI Commands**: Add/remove duplicate mappings with validation
6089
+
- **Query Resolution**: Search/list commands resolve duplicates to canonical entries
6091
+
### Duplicate File Format
6094
+
"https://example.com/feed/entry/123": "https://canonical.com/posts/same-post",
6095
+
"https://mirror.com/articles/456": "https://canonical.com/posts/same-post",
6096
+
"comment": "Entry IDs that map to the same canonical content"
6100
+
## Feed Metadata Auto-Discovery
6102
+
### Extraction Strategy
6103
+
When adding a new user with `thicket add user`, the system fetches and parses the feed to extract:
6105
+
- **Display Name**: From `feed.title` or `feed.author.name`
6106
+
- **Email**: From `feed.author.email` or `feed.managingEditor`
6107
+
- **Homepage**: From `feed.link` or `feed.author.uri`
6108
+
- **Icon**: From `feed.logo`, `feed.icon`, or `feed.image.url`
6110
+
### Discovery Priority Order
6111
+
1. **Author Information**: Prefer `feed.author.*` fields (more specific to person)
6112
+
2. **Feed-Level**: Fall back to feed-level metadata
6113
+
3. **Manual Override**: CLI flags always take precedence over discovered values
6114
+
4. **Update Behavior**: Auto-discovery only runs during initial `add user`, not on sync
6116
+
### Extracted Metadata Format
6118
+
class FeedMetadata(BaseModel):
6119
+
title: Optional[str] = None
6120
+
author_name: Optional[str] = None
6121
+
author_email: Optional[EmailStr] = None
6122
+
author_uri: Optional[HttpUrl] = None
6123
+
link: Optional[HttpUrl] = None
6124
+
logo: Optional[HttpUrl] = None
6125
+
icon: Optional[HttpUrl] = None
6126
+
image_url: Optional[HttpUrl] = None
6128
+
def to_user_config(self, username: str, feed_url: HttpUrl) -> UserConfig:
6129
+
"""Convert discovered metadata to UserConfig with fallbacks"""
6130
+
return UserConfig(
6131
+
username=username,
6133
+
display_name=self.author_name or self.title,
6134
+
email=self.author_email,
6135
+
homepage=self.author_uri or self.link,
6136
+
icon=self.logo or self.icon or self.image_url
6140
+
## Link Processing and Threading Architecture
6143
+
The thicket system implements a sophisticated link processing and threading system to create email-style threaded views of blog entries by tracking cross-references between different blogs.
6145
+
### Link Processing Pipeline
6147
+
#### 1. Link Extraction (`thicket links`)
6148
+
The `links` command systematically extracts all outbound links from blog entries and categorizes them:
6151
+
class LinkData(BaseModel):
6152
+
url: str # Fully resolved URL
6153
+
entry_id: str # Source entry ID
6154
+
username: str # Source username
6155
+
context: str # Surrounding text context
6156
+
category: str # "internal", "user", or "unknown"
6157
+
target_username: Optional[str] # Target user if applicable
6160
+
**Link Categories:**
6161
+
- **Internal**: Links to the same user's domain (self-references)
6162
+
- **User**: Links to other tracked users' domains
6163
+
- **Unknown**: Links to external sites not tracked by thicket
6165
+
#### 2. URL Resolution
6166
+
All links are properly resolved using the Atom feed's base URL to handle:
6167
+
- Relative URLs (converted to absolute)
6168
+
- Protocol-relative URLs
6169
+
- Fragment identifiers
6170
+
- Redirects and canonical URLs
6172
+
#### 3. Domain Mapping
6173
+
The system builds a comprehensive domain mapping from user configuration:
6174
+
- Feed URLs → domain extraction
6175
+
- Homepage URLs → domain extraction
6176
+
- Reverse mapping: domain → username
6178
+
### Threading System
6180
+
#### 1. Reference Index Generation (`thicket index`)
6181
+
Creates a bidirectional reference index from the categorized links:
6184
+
class BlogReference(BaseModel):
6185
+
source_entry_id: str
6186
+
source_username: str
6188
+
target_username: Optional[str]
6189
+
target_entry_id: Optional[str]
6193
+
#### 2. Thread Detection Algorithm
6194
+
Uses graph traversal to find connected blog entries:
6195
+
- **Outbound references**: Links from an entry to other entries
6196
+
- **Inbound references**: Links to an entry from other entries
6197
+
- **Thread members**: All entries connected through references
6199
+
#### 3. Threading Display (`thicket threads`)
6200
+
Creates email-style threaded views:
6201
+
- Chronological ordering within threads
6202
+
- Reference counts (outbound/inbound)
6203
+
- Context preservation
6204
+
- Filtering options (user, entry, minimum size)
6206
+
### Data Structures
6208
+
#### links.json Format (Unified Structure)
6212
+
"https://example.com/post/123": {
6213
+
"referencing_entries": ["https://blog.user.com/entry/456"],
6214
+
"target_username": "user2"
6216
+
"https://external-site.com/article": {
6217
+
"referencing_entries": ["https://blog.user.com/entry/789"]
6220
+
"reverse_mapping": {
6221
+
"https://blog.user.com/entry/456": ["https://example.com/post/123"],
6222
+
"https://blog.user.com/entry/789": ["https://external-site.com/article"]
6226
+
"source_entry_id": "https://blog.user.com/entry/456",
6227
+
"source_username": "user1",
6228
+
"target_url": "https://example.com/post/123",
6229
+
"target_username": "user2",
6230
+
"target_entry_id": "https://example.com/post/123",
6231
+
"context": "As mentioned in this post..."
6235
+
"user1": ["blog.user.com"],
6236
+
"user2": ["example.com"]
6241
+
This unified structure eliminates duplication by:
6242
+
- Storing each URL only once with minimal metadata
6243
+
- Including all link data, reference data, and mappings in one file
6244
+
- Using presence of `target_username` to identify tracked vs external links
6245
+
- Providing bidirectional mappings for efficient queries
6247
+
### Unified Structure Benefits
6249
+
- **Eliminates Duplication**: Each URL appears only once with metadata
6250
+
- **Single Source of Truth**: All link-related data in one file
6251
+
- **Efficient Queries**: Fast lookups for both directions (URL→entries, entry→URLs)
6252
+
- **Atomic Updates**: All link data changes together
6253
+
- **Reduced I/O**: Fewer file operations
6255
+
### Implementation Benefits
6257
+
1. **Systematic Link Processing**: All links are extracted and categorized consistently
6258
+
2. **Proper URL Resolution**: Handles relative URLs and base URL resolution correctly
6259
+
3. **Domain-based Categorization**: Automatically identifies user-to-user references
6260
+
4. **Bidirectional Indexing**: Supports both "who links to whom" and "who is linked by whom"
6261
+
5. **Thread Discovery**: Finds conversation threads automatically
6262
+
6. **Rich Context**: Preserves surrounding text for each link
6263
+
7. **Performance**: Pre-computed indexes for fast threading queries
6268
+
# Extract and categorize all links
6269
+
thicket links --verbose
6271
+
# Build reference index for threading
6272
+
thicket index --verbose
6274
+
# Show all conversation threads
6277
+
# Show threads for specific user
6278
+
thicket threads --username user1
6280
+
# Show threads with minimum size
6281
+
thicket threads --min-size 3
6284
+
### Integration with Existing Commands
6286
+
The link processing system integrates seamlessly with existing thicket commands:
6287
+
- `thicket sync` updates entries, requiring `thicket links` to be run afterward
6288
+
- `thicket index` uses the output from `thicket links` for improved accuracy
6289
+
- `thicket threads` provides the user-facing threading interface
6291
+
## Current Implementation Status
6293
+
### ✅ Completed Features
6294
+
1. **Core Infrastructure**
6295
+
- Modern CLI with Typer and Rich
6296
+
- Pydantic data models for type safety
6297
+
- Git repository operations with GitPython
6298
+
- Feed parsing and normalization with feedparser
6300
+
2. **User and Feed Management**
6301
+
- `thicket init` - Initialize git store
6302
+
- `thicket add` - Add users and feeds with auto-discovery
6303
+
- `thicket sync` - Sync feeds with progress tracking
6304
+
- `thicket list` - List users, feeds, and entries
6305
+
- `thicket duplicates` - Manage duplicate entries
6307
+
3. **Link Processing and Threading**
6308
+
- `thicket links` - Extract and categorize all outbound links
6309
+
- `thicket index` - Build reference index from links
6310
+
- `thicket threads` - Display threaded conversation views
6311
+
- Proper URL resolution with base URL handling
6312
+
- Domain-based link categorization
6313
+
- Context preservation for links
6315
+
### 📊 System Performance
6316
+
- **Link Extraction**: Successfully processes thousands of blog entries
6317
+
- **Categorization**: Identifies internal, user, and unknown links
6318
+
- **Threading**: Creates email-style threaded views of conversations
6319
+
- **Storage**: Efficient JSON-based data structures for links and references
6321
+
### 🔧 Current Architecture Highlights
6322
+
- **Modular Design**: Clear separation between CLI, core logic, and models
6323
+
- **Type Safety**: Comprehensive Pydantic models for data validation
6324
+
- **Rich CLI**: Beautiful progress bars, tables, and error handling
6325
+
- **Extensible**: Easy to add new commands and features
6326
+
- **Git Integration**: All data stored in version-controlled JSON files
6328
+
### 🎯 Proven Functionality
6329
+
The system has been tested with real blog data and successfully:
6330
+
- Extracted 14,396 total links from blog entries
6331
+
- Categorized 3,994 internal links, 363 user-to-user links, and 10,039 unknown links
6332
+
- Built comprehensive domain mappings for 16 users across 20 domains
6333
+
- Generated threaded views showing blog conversation patterns
6335
+
### 🚀 Ready for Use
6336
+
The thicket system is now fully functional for:
6337
+
- Maintaining Git repositories of blog feeds
6338
+
- Tracking cross-references between blogs
6339
+
- Creating threaded views of blog conversations
6340
+
- Discovering blog interaction patterns
6341
+
- Building distributed comment systems
6344
+
<file path="src/thicket/cli/utils.py">
6345
+
"""CLI utilities and helpers."""
6347
+
from pathlib import Path
6348
+
from typing import Optional
6351
+
from rich.console import Console
6352
+
from rich.progress import Progress, SpinnerColumn, TextColumn
6353
+
from rich.table import Table
6355
+
from ..models import ThicketConfig, UserMetadata
6356
+
from ..core.git_store import GitStore
6358
+
console = Console()
6361
+
def get_tsv_mode() -> bool:
6362
+
"""Get the global TSV mode setting."""
6363
+
from .main import tsv_mode
6367
+
def load_config(config_path: Optional[Path] = None) -> ThicketConfig:
6368
+
"""Load thicket configuration from file or environment."""
6369
+
if config_path and config_path.exists():
6372
+
with open(config_path) as f:
6373
+
config_data = yaml.safe_load(f)
6375
+
# Convert to ThicketConfig
6376
+
return ThicketConfig(**config_data)
6378
+
# Try to load from default locations or environment
6380
+
# First try to find thicket.yaml in current directory
6381
+
default_config = Path("thicket.yaml")
6382
+
if default_config.exists():
6384
+
with open(default_config) as f:
6385
+
config_data = yaml.safe_load(f)
6386
+
return ThicketConfig(**config_data)
6388
+
# Fall back to environment variables
6389
+
return ThicketConfig()
6390
+
except Exception as e:
6391
+
console.print(f"[red]Error loading configuration: {e}[/red]")
6392
+
console.print("[yellow]Run 'thicket init' to create a new configuration.[/yellow]")
6393
+
raise typer.Exit(1) from e
6396
+
def save_config(config: ThicketConfig, config_path: Path) -> None:
6397
+
"""Save thicket configuration to file."""
6400
+
config_data = config.model_dump(mode="json", exclude_none=True)
6402
+
# Convert Path objects to strings for YAML serialization
6403
+
config_data["git_store"] = str(config_data["git_store"])
6404
+
config_data["cache_dir"] = str(config_data["cache_dir"])
6406
+
with open(config_path, "w") as f:
6407
+
yaml.dump(config_data, f, default_flow_style=False, sort_keys=False)
6410
+
def create_progress() -> Progress:
6411
+
"""Create a Rich progress display."""
6414
+
TextColumn("[progress.description]{task.description}"),
6420
+
def print_users_table(config: ThicketConfig) -> None:
6421
+
"""Print a table of users and their feeds."""
6422
+
if get_tsv_mode():
6423
+
print_users_tsv(config)
6426
+
table = Table(title="Users and Feeds")
6427
+
table.add_column("Username", style="cyan", no_wrap=True)
6428
+
table.add_column("Display Name", style="magenta")
6429
+
table.add_column("Email", style="blue")
6430
+
table.add_column("Homepage", style="green")
6431
+
table.add_column("Feeds", style="yellow")
6433
+
for user in config.users:
6434
+
feeds_str = "\n".join(str(feed) for feed in user.feeds)
6437
+
user.display_name or "",
6439
+
str(user.homepage) if user.homepage else "",
6443
+
console.print(table)
6446
+
def print_feeds_table(config: ThicketConfig, username: Optional[str] = None) -> None:
6447
+
"""Print a table of feeds, optionally filtered by username."""
6448
+
if get_tsv_mode():
6449
+
print_feeds_tsv(config, username)
6452
+
table = Table(title=f"Feeds{f' for {username}' if username else ''}")
6453
+
table.add_column("Username", style="cyan", no_wrap=True)
6454
+
table.add_column("Feed URL", style="blue")
6455
+
table.add_column("Status", style="green")
6457
+
users = [config.find_user(username)] if username else config.users
6458
+
users = [u for u in users if u is not None]
6460
+
for user in users:
6461
+
for feed in user.feeds:
6465
+
"Active", # TODO: Add actual status checking
6468
+
console.print(table)
6471
+
def confirm_action(message: str, default: bool = False) -> bool:
6472
+
"""Prompt for confirmation."""
6473
+
return typer.confirm(message, default=default)
6476
+
def print_success(message: str) -> None:
6477
+
"""Print a success message."""
6478
+
console.print(f"[green]✓[/green] {message}")
6481
+
def print_error(message: str) -> None:
6482
+
"""Print an error message."""
6483
+
console.print(f"[red]✗[/red] {message}")
6486
+
def print_warning(message: str) -> None:
6487
+
"""Print a warning message."""
6488
+
console.print(f"[yellow]⚠[/yellow] {message}")
6491
+
def print_info(message: str) -> None:
6492
+
"""Print an info message."""
6493
+
console.print(f"[blue]ℹ[/blue] {message}")
6496
+
def print_users_table_from_git(users: list[UserMetadata]) -> None:
6497
+
"""Print a table of users from git repository."""
6498
+
if get_tsv_mode():
6499
+
print_users_tsv_from_git(users)
6502
+
table = Table(title="Users and Feeds")
6503
+
table.add_column("Username", style="cyan", no_wrap=True)
6504
+
table.add_column("Display Name", style="magenta")
6505
+
table.add_column("Email", style="blue")
6506
+
table.add_column("Homepage", style="green")
6507
+
table.add_column("Feeds", style="yellow")
6509
+
for user in users:
6510
+
feeds_str = "\n".join(user.feeds)
6513
+
user.display_name or "",
6515
+
user.homepage or "",
6519
+
console.print(table)
6522
+
def print_feeds_table_from_git(git_store: GitStore, username: Optional[str] = None) -> None:
6523
+
"""Print a table of feeds from git repository."""
6524
+
if get_tsv_mode():
6525
+
print_feeds_tsv_from_git(git_store, username)
6528
+
table = Table(title=f"Feeds{f' for {username}' if username else ''}")
6529
+
table.add_column("Username", style="cyan", no_wrap=True)
6530
+
table.add_column("Feed URL", style="blue")
6531
+
table.add_column("Status", style="green")
6534
+
user = git_store.get_user(username)
6535
+
users = [user] if user else []
6537
+
index = git_store._load_index()
6538
+
users = list(index.users.values())
6540
+
for user in users:
6541
+
for feed in user.feeds:
6545
+
"Active", # TODO: Add actual status checking
6548
+
console.print(table)
6551
+
def print_users_tsv(config: ThicketConfig) -> None:
6552
+
"""Print users in TSV format."""
6553
+
print("Username\tDisplay Name\tEmail\tHomepage\tFeeds")
6554
+
for user in config.users:
6555
+
feeds_str = ",".join(str(feed) for feed in user.feeds)
6556
+
print(f"{user.username}\t{user.display_name or ''}\t{user.email or ''}\t{user.homepage or ''}\t{feeds_str}")
6559
+
def print_users_tsv_from_git(users: list[UserMetadata]) -> None:
6560
+
"""Print users from git repository in TSV format."""
6561
+
print("Username\tDisplay Name\tEmail\tHomepage\tFeeds")
6562
+
for user in users:
6563
+
feeds_str = ",".join(user.feeds)
6564
+
print(f"{user.username}\t{user.display_name or ''}\t{user.email or ''}\t{user.homepage or ''}\t{feeds_str}")
6567
+
def print_feeds_tsv(config: ThicketConfig, username: Optional[str] = None) -> None:
6568
+
"""Print feeds in TSV format."""
6569
+
print("Username\tFeed URL\tStatus")
6570
+
users = [config.find_user(username)] if username else config.users
6571
+
users = [u for u in users if u is not None]
6573
+
for user in users:
6574
+
for feed in user.feeds:
6575
+
print(f"{user.username}\t{feed}\tActive")
6578
+
def print_feeds_tsv_from_git(git_store: GitStore, username: Optional[str] = None) -> None:
6579
+
"""Print feeds from git repository in TSV format."""
6580
+
print("Username\tFeed URL\tStatus")
6583
+
user = git_store.get_user(username)
6584
+
users = [user] if user else []
6586
+
index = git_store._load_index()
6587
+
users = list(index.users.values())
6589
+
for user in users:
6590
+
for feed in user.feeds:
6591
+
print(f"{user.username}\t{feed}\tActive")
6594
+
def print_entries_tsv(entries_by_user: list[list], usernames: list[str]) -> None:
6595
+
"""Print entries in TSV format."""
6596
+
print("User\tAtom ID\tTitle\tUpdated\tURL")
6598
+
# Combine all entries with usernames
6600
+
for entries, username in zip(entries_by_user, usernames):
6601
+
for entry in entries:
6602
+
all_entries.append((username, entry))
6604
+
# Sort by updated time (newest first)
6605
+
all_entries.sort(key=lambda x: x[1].updated, reverse=True)
6607
+
for username, entry in all_entries:
6608
+
# Format updated time
6609
+
updated_str = entry.updated.strftime("%Y-%m-%d %H:%M")
6611
+
# Escape tabs and newlines in title to preserve TSV format
6612
+
title = entry.title.replace('\t', ' ').replace('\n', ' ').replace('\r', ' ')
6614
+
print(f"{username}\t{entry.id}\t{title}\t{updated_str}\t{entry.link}")
anil.recoil.org