Atom feed for our EEG site
anil.recoil.org
1# /// script
2# requires-python = ">=3.11"
3# dependencies = [
4# "feedparser",
5# "beautifulsoup4",
6# "urllib3",
7# ]
8# ///
9# Do not delete the above as its needed for `uv run`
10#!/usr/bin/env python3
11
12import json
13import feedparser
14import sys
15import os
16from bs4 import BeautifulSoup
17import re
18from urllib.parse import urlparse, urljoin
19
20def extract_links_from_html(html_content, base_url=None):
21 """Extract and normalize links from HTML content"""
22 soup = BeautifulSoup(html_content, 'html.parser')
23 links = []
24
25 for a_tag in soup.find_all('a', href=True):
26 href = a_tag['href'].strip()
27
28 # Skip empty links, anchors, javascript, and mailto
29 if not href or href.startswith(('#', 'javascript:', 'mailto:')):
30 continue
31
32 # Convert relative URLs to absolute if we have a base URL
33 if base_url and not href.startswith(('http://', 'https://')):
34 href = urljoin(base_url, href)
35
36 links.append(href)
37
38 return links
39
40def normalize_url(url):
41 """Normalize URLs to consistently match them"""
42 if not url:
43 return ""
44
45 # Handle common URL shorteners or redirects (not implemented)
46
47 # Parse the URL
48 parsed = urlparse(url)
49
50 # Ensure scheme is consistent
51 scheme = parsed.scheme.lower() or 'http'
52
53 # Normalize netloc (lowercase, remove 'www.' prefix optionally)
54 netloc = parsed.netloc.lower()
55 if netloc.startswith('www.'):
56 netloc = netloc[4:]
57
58 # Remove trailing slashes and index.html/index.php
59 path = parsed.path.rstrip('/')
60 for index_file in ['/index.html', '/index.php', '/index.htm']:
61 if path.endswith(index_file):
62 path = path[:-len(index_file)]
63
64 # Remove common fragments and query parameters that don't affect content
65 # (like tracking params, utm_*, etc.)
66 query_parts = []
67 if parsed.query:
68 for param in parsed.query.split('&'):
69 if '=' in param:
70 key, value = param.split('=', 1)
71 if not key.startswith(('utm_', 'ref', 'source')):
72 query_parts.append(f"{key}={value}")
73
74 query = '&'.join(query_parts)
75
76 # Remove common hash fragments
77 fragment = ''
78
79 # Special case for common blogging platforms
80 # Medium, WordPress, Ghost, etc. may have specific URL patterns
81
82 # Reconstruct the URL
83 normalized = f"{scheme}://{netloc}{path}"
84 if query:
85 normalized += f"?{query}"
86 if fragment:
87 normalized += f"#{fragment}"
88
89 return normalized
90
91def analyze_feed():
92 # Parse the aggregated feed
93 print(f"Parsing eeg.xml...", file=sys.stderr)
94 feed_data = feedparser.parse("eeg.xml")
95
96 # Add debug info about the feed
97 print(f"Feed title: {feed_data.feed.get('title', 'Unknown')}", file=sys.stderr)
98 print(f"Feed version: {feed_data.get('version', 'Unknown')}", file=sys.stderr)
99
100 if not feed_data or not hasattr(feed_data, 'entries'):
101 print("Error: Could not parse feed or no entries found", file=sys.stderr)
102 return
103
104 print(f"Found {len(feed_data.entries)} entries in the aggregated feed", file=sys.stderr)
105
106 all_entries = []
107 entry_urls = {} # Maps normalized URLs to entry data
108
109 # First pass: collect all entries and their URLs
110 for entry in feed_data.entries:
111 # Get link
112 link = entry.get('link', '')
113 if not link:
114 continue
115
116 # Normalize the entry URL to help with matching
117 normalized_link = normalize_url(link)
118
119 # Get feed title (stored as category in the aggregated feed)
120 feed_title = "Unknown"
121 if hasattr(entry, 'tags') and entry.tags:
122 feed_title = entry.tags[0].term
123
124 # Get description/content
125 if hasattr(entry, 'content') and entry.content:
126 content = entry.content[0].value
127 else:
128 content = entry.get('summary', '')
129
130 # Extract all links from content, using the entry link as base URL for resolving relative URLs
131 content_links = extract_links_from_html(content, base_url=link)
132
133 # Get unique ID
134 entry_id = entry.get('id', link)
135
136 entry_data = {
137 'title': entry.get('title', 'No title'),
138 'link': link,
139 'normalized_link': normalized_link,
140 'feed_title': feed_title,
141 'id': entry_id,
142 'content_links': content_links,
143 'references': [], # Will be filled in the second pass
144 'referenced_by': [], # Will be filled in the second pass
145 'external_links': [] # Links to content outside the feed
146 }
147
148 all_entries.append(entry_data)
149 entry_urls[normalized_link] = entry_data
150
151 print(f"Total entries processed: {len(all_entries)}", file=sys.stderr)
152
153 # Second pass: analyze links between entries
154 for entry in all_entries:
155 # Keep track of references to avoid duplicates
156 reference_ids = set()
157 normalized_content_links = [normalize_url(link) for link in entry['content_links']]
158
159 for i, normalized_link in enumerate(normalized_content_links):
160 original_link = entry['content_links'][i] if i < len(entry['content_links']) else normalized_link
161
162 if normalized_link in entry_urls and normalized_link != entry['normalized_link']:
163 # This entry links to another entry in the feed
164 referenced_entry = entry_urls[normalized_link]
165
166 # Avoid duplicate references
167 if referenced_entry['id'] in reference_ids:
168 continue
169
170 reference_ids.add(referenced_entry['id'])
171 elif normalized_link not in entry_urls and normalized_link != entry['normalized_link']:
172 # This is a link to something outside the feed
173 # Track as an external link
174 if not any(ext_link['url'] == original_link for ext_link in entry['external_links']):
175 external_link = {
176 'url': original_link,
177 'normalized_url': normalized_link,
178 'in_feed': False # Mark as external to the feed
179 }
180 entry['external_links'].append(external_link)
181 continue
182
183 if normalized_link in entry_urls and normalized_link != entry['normalized_link']:
184 # Add to the references of the current entry
185 entry['references'].append({
186 'id': referenced_entry['id'],
187 'link': referenced_entry['link'],
188 'title': referenced_entry['title'],
189 'feed_title': referenced_entry['feed_title'],
190 'in_feed': True # Mark as a reference to a post in the feed
191 })
192
193 # Add to the referenced_by of the referenced entry
194 # Check if this entry is already in referenced_by
195 already_referenced = any(ref['id'] == entry['id'] for ref in referenced_entry['referenced_by'])
196 if not already_referenced:
197 referenced_entry['referenced_by'].append({
198 'id': entry['id'],
199 'link': entry['link'],
200 'title': entry['title'],
201 'feed_title': entry['feed_title'],
202 'in_feed': True # Mark as a reference from a post in the feed
203 })
204
205 # Create the thread data structure
206 thread_data = {}
207 for entry in all_entries:
208 thread_data[entry['id']] = {
209 'id': entry['id'],
210 'title': entry['title'],
211 'link': entry['link'],
212 'feed_title': entry['feed_title'],
213 'references': entry['references'],
214 'referenced_by': entry['referenced_by'],
215 'external_links': entry['external_links']
216 }
217
218 # Write the thread data to a JSON file
219 with open('threads.json', 'w') as f:
220 json.dump(thread_data, f, indent=2)
221
222 print(f"Thread data successfully written to threads.json", file=sys.stderr)
223
224 # Generate some statistics
225 entries_with_references = sum(1 for entry in all_entries if entry['references'])
226 entries_with_referenced_by = sum(1 for entry in all_entries if entry['referenced_by'])
227 entries_with_external_links = sum(1 for entry in all_entries if entry['external_links'])
228 total_internal_references = sum(len(entry['references']) for entry in all_entries)
229 total_external_links = sum(len(entry['external_links']) for entry in all_entries)
230
231 print(f"\nThread Analysis:", file=sys.stderr)
232 print(f"Total entries: {len(all_entries)}", file=sys.stderr)
233 print(f"Entries that reference other entries in the feed: {entries_with_references}", file=sys.stderr)
234 print(f"Entries referenced by other entries in the feed: {entries_with_referenced_by}", file=sys.stderr)
235 print(f"Entries with external links: {entries_with_external_links}", file=sys.stderr)
236 print(f"Total internal references: {total_internal_references}", file=sys.stderr)
237 print(f"Total external links: {total_external_links}", file=sys.stderr)
238
239if __name__ == "__main__":
240 analyze_feed()