···
8
-
from datetime import datetime
10
-
from typing import Any, Dict, List, Optional, Set, Tuple
9
+
from typing import Any, Optional
from zulip_bots.lib import BotHandler
# Handle imports for both direct execution and package import
15
+
from ..cli.commands.sync import sync_feed
from ..core.git_store import GitStore
from ..models import AtomEntry, ThicketConfig
18
-
from ..cli.commands.sync import sync_feed
# When run directly by zulip-bots, add the package to path
src_dir = Path(__file__).parent.parent.parent
if str(src_dir) not in sys.path:
sys.path.insert(0, str(src_dir))
26
+
from thicket.cli.commands.sync import sync_feed
from thicket.core.git_store import GitStore
from thicket.models import AtomEntry, ThicketConfig
28
-
from thicket.cli.commands.sync import sync_feed
···
self.logger = logging.getLogger(__name__)
self.git_store: Optional[GitStore] = None
self.config: Optional[ThicketConfig] = None
39
-
self.posted_entries: Set[str] = set()
39
+
self.posted_entries: set[str] = set()
# Bot configuration from storage
self.stream_name: Optional[str] = None
self.topic_name: Optional[str] = None
self.sync_interval: int = 300 # 5 minutes default
self.max_entries_per_sync: int = 10
self.config_path: Optional[Path] = None
48
+
# Bot behavior settings (loaded from botrc)
49
+
self.rate_limit_delay: int = 5
50
+
self.posts_per_batch: int = 5
51
+
self.catchup_entries: int = 5
52
+
self.config_change_notifications: bool = True
53
+
self.username_claim_notifications: bool = True
55
+
# Track last sync time for schedule queries
56
+
self.last_sync_time: Optional[float] = None
# Debug mode configuration
self.debug_user: Optional[str] = None
self.debug_zulip_user_id: Optional[str] = None
"""Return bot usage instructions."""
This bot automatically monitors thicket feeds and posts new articles.
60
-
- `@mention status` - Show current bot status and configuration
70
+
- `@mention status` - Show current bot status and configuration
- `@mention sync now` - Force an immediate sync
- `@mention reset` - Clear posting history (will repost recent entries)
- `@mention config stream <stream_name>` - Set target stream
64
-
- `@mention config topic <topic_name>` - Set target topic
74
+
- `@mention config topic <topic_name>` - Set target topic
- `@mention config interval <seconds>` - Set sync interval
76
+
- `@mention schedule` - Show sync schedule and next run time
77
+
- `@mention claim <username>` - Claim a thicket username for your Zulip account
- `@mention help` - Show this help message
def initialize(self, bot_handler: BotHandler) -> None:
"""Initialize the bot with persistent storage."""
self.logger.info("Initializing ThicketBot")
# Get configuration from environment (set by CLI)
self.debug_user = os.getenv("THICKET_DEBUG_USER")
config_path_env = os.getenv("THICKET_CONFIG_PATH")
self.config_path = Path(config_path_env)
self.logger.info(f"Using thicket config: {self.config_path}")
92
+
# Load default configuration from botrc file
93
+
self._load_botrc_defaults()
# Load bot configuration from persistent storage
self._load_bot_config(bot_handler)
# Initialize thicket components
self._initialize_thicket()
self._load_posted_entries(bot_handler)
# Validate debug mode if enabled
self._validate_debug_mode(bot_handler)
self.logger.error(f"Failed to initialize thicket: {e}")
# Start background sync loop
self._schedule_sync(bot_handler)
99
-
def handle_message(self, message: Dict[str, Any], bot_handler: BotHandler) -> None:
114
+
def handle_message(self, message: dict[str, Any], bot_handler: BotHandler) -> None:
"""Handle incoming Zulip messages."""
content = message["content"].strip()
sender = message["sender_full_name"]
# Only respond to mentions
if not self._is_mentioned(content, bot_handler):
cleaned_content = self._clean_mention(content, bot_handler)
command_parts = cleaned_content.split()
self._send_help(message, bot_handler)
command = command_parts[0].lower()
self._send_help(message, bot_handler)
elif command == "status":
self._send_status(message, bot_handler, sender)
123
-
elif command == "sync" and len(command_parts) > 1 and command_parts[1] == "now":
140
+
and len(command_parts) > 1
141
+
and command_parts[1] == "now"
self._handle_force_sync(message, bot_handler, sender)
self._handle_reset_command(message, bot_handler, sender)
elif command == "config":
128
-
self._handle_config_command(message, bot_handler, command_parts[1:], sender)
147
+
self._handle_config_command(
148
+
message, bot_handler, command_parts[1:], sender
150
+
elif command == "schedule":
151
+
self._handle_schedule_command(message, bot_handler, sender)
152
+
elif command == "claim":
153
+
self._handle_claim_command(
154
+
message, bot_handler, command_parts[1:], sender
130
-
bot_handler.send_reply(message, f"Unknown command: {command}. Type `@mention help` for usage.")
157
+
bot_handler.send_reply(
159
+
f"Unknown command: {command}. Type `@mention help` for usage.",
self.logger.error(f"Error handling command '{command}': {e}")
bot_handler.send_reply(message, f"Error processing command: {str(e)}")
···
# Get bot's actual name from Zulip
bot_info = bot_handler._client.get_profile()
140
-
if bot_info.get('result') == 'success':
141
-
bot_name = bot_info.get('full_name', '').lower()
170
+
if bot_info.get("result") == "success":
171
+
bot_name = bot_info.get("full_name", "").lower()
143
-
return f"@{bot_name}" in content.lower() or f"@**{bot_name}**" in content.lower()
174
+
f"@{bot_name}" in content.lower()
175
+
or f"@**{bot_name}**" in content.lower()
self.logger.debug(f"Could not get bot profile: {e}")
# Fallback to generic check
return "@thicket" in content.lower()
def _clean_mention(self, content: str, bot_handler: BotHandler) -> str:
"""Remove bot mention from message content."""
# Get bot's actual name from Zulip
bot_info = bot_handler._client.get_profile()
157
-
if bot_info.get('result') == 'success':
158
-
bot_name = bot_info.get('full_name', '')
190
+
if bot_info.get("result") == "success":
191
+
bot_name = bot_info.get("full_name", "")
# Remove @bot_name or @**bot_name**
escaped_name = re.escape(bot_name)
162
-
content = re.sub(rf'@(?:\*\*)?{escaped_name}(?:\*\*)?', '', content, flags=re.IGNORECASE).strip()
196
+
rf"@(?:\*\*)?{escaped_name}(?:\*\*)?",
199
+
flags=re.IGNORECASE,
self.logger.debug(f"Could not get bot profile for mention cleaning: {e}")
# Fallback to removing @thicket
168
-
content = re.sub(r'@(?:\*\*)?thicket(?:\*\*)?', '', content, flags=re.IGNORECASE).strip()
207
+
r"@(?:\*\*)?thicket(?:\*\*)?", "", content, flags=re.IGNORECASE
171
-
def _send_help(self, message: Dict[str, Any], bot_handler: BotHandler) -> None:
211
+
def _send_help(self, message: dict[str, Any], bot_handler: BotHandler) -> None:
bot_handler.send_reply(message, self.usage())
175
-
def _send_status(self, message: Dict[str, Any], bot_handler: BotHandler, sender: str) -> None:
216
+
self, message: dict[str, Any], bot_handler: BotHandler, sender: str
"""Send bot status information."""
f"**Thicket Bot Status** (requested by {sender})",
184
-
status_lines.extend([
185
-
f"🐛 **Debug Mode:** ENABLED",
186
-
f"🎯 **Debug User:** {self.debug_user}",
226
+
status_lines.extend(
228
+
"🐛 **Debug Mode:** ENABLED",
229
+
f"🎯 **Debug User:** {self.debug_user}",
190
-
status_lines.extend([
191
-
f"📍 **Stream:** {self.stream_name or 'Not configured'}",
192
-
f"📝 **Topic:** {self.topic_name or 'Not configured'}",
234
+
status_lines.extend(
236
+
f"📍 **Stream:** {self.stream_name or 'Not configured'}",
237
+
f"📝 **Topic:** {self.topic_name or 'Not configured'}",
242
+
status_lines.extend(
244
+
f"⏱️ **Sync Interval:** {self.sync_interval}s ({self.sync_interval // 60}m {self.sync_interval % 60}s)",
245
+
f"📊 **Max Entries/Sync:** {self.max_entries_per_sync}",
246
+
f"📁 **Config Path:** {self.config_path or 'Not configured'}",
248
+
f"📄 **Tracked Entries:** {len(self.posted_entries)}",
249
+
f"🔄 **Catchup Mode:** {'Active (first run)' if len(self.posted_entries) == 0 else 'Inactive'}",
250
+
f"✅ **Thicket Initialized:** {'Yes' if self.git_store else 'No'}",
196
-
status_lines.extend([
197
-
f"⏱️ **Sync Interval:** {self.sync_interval}s ({self.sync_interval // 60}m {self.sync_interval % 60}s)",
198
-
f"📊 **Max Entries/Sync:** {self.max_entries_per_sync}",
199
-
f"📁 **Config Path:** {self.config_path or 'Not configured'}",
201
-
f"📄 **Tracked Entries:** {len(self.posted_entries)}",
202
-
f"🔄 **Catchup Mode:** {'Active (first run)' if len(self.posted_entries) == 0 else 'Inactive'}",
203
-
f"✅ **Thicket Initialized:** {'Yes' if self.git_store else 'No'}",
252
+
self._get_schedule_info(),
bot_handler.send_reply(message, "\n".join(status_lines))
208
-
def _handle_force_sync(self, message: Dict[str, Any], bot_handler: BotHandler, sender: str) -> None:
258
+
def _handle_force_sync(
259
+
self, message: dict[str, Any], bot_handler: BotHandler, sender: str
"""Handle immediate sync request."""
if not self._check_initialization(message, bot_handler):
213
-
bot_handler.send_reply(message, f"🔄 Starting immediate sync... (requested by {sender})")
265
+
bot_handler.send_reply(
266
+
message, f"🔄 Starting immediate sync... (requested by {sender})"
new_entries = self._perform_sync(bot_handler)
219
-
f"✅ Sync completed! Found {len(new_entries)} new entries."
272
+
message, f"✅ Sync completed! Found {len(new_entries)} new entries."
self.logger.error(f"Force sync failed: {e}")
bot_handler.send_reply(message, f"❌ Sync failed: {str(e)}")
225
-
def _handle_reset_command(self, message: Dict[str, Any], bot_handler: BotHandler, sender: str) -> None:
278
+
def _handle_reset_command(
279
+
self, message: dict[str, Any], bot_handler: BotHandler, sender: str
"""Handle reset command to clear posted entries tracking."""
self.posted_entries.clear()
self._save_posted_entries(bot_handler)
232
-
f"✅ Posting history reset! Recent entries will be posted on next sync. (requested by {sender})"
287
+
f"✅ Posting history reset! Recent entries will be posted on next sync. (requested by {sender})",
self.logger.info(f"Posted entries tracking reset by {sender}")
self.logger.error(f"Reset failed: {e}")
bot_handler.send_reply(message, f"❌ Reset failed: {str(e)}")
294
+
def _handle_schedule_command(
295
+
self, message: dict[str, Any], bot_handler: BotHandler, sender: str
297
+
"""Handle schedule query command."""
298
+
schedule_info = self._get_schedule_info()
299
+
bot_handler.send_reply(
301
+
f"**Thicket Bot Schedule** (requested by {sender})\n\n{schedule_info}",
304
+
def _handle_claim_command(
306
+
message: dict[str, Any],
307
+
bot_handler: BotHandler,
311
+
"""Handle username claiming command."""
313
+
bot_handler.send_reply(message, "Usage: `@mention claim <username>`")
316
+
if not self._check_initialization(message, bot_handler):
319
+
username = args[0].strip()
321
+
# Get sender's Zulip user info
322
+
sender_user_id = message.get("sender_id")
323
+
sender_email = message.get("sender_email")
325
+
if not sender_user_id or not sender_email:
326
+
bot_handler.send_reply(
327
+
message, "❌ Could not determine your Zulip user information."
332
+
# Get current Zulip server from environment
333
+
zulip_site_url = os.getenv("THICKET_ZULIP_SITE_URL", "")
334
+
server_url = zulip_site_url.replace("https://", "").replace("http://", "")
337
+
bot_handler.send_reply(
338
+
message, "❌ Could not determine Zulip server URL."
342
+
# Check if username exists in thicket
343
+
user = self.git_store.get_user(username)
345
+
bot_handler.send_reply(
347
+
f"❌ Username `{username}` not found in thicket. Available users: {', '.join(self.git_store.list_users())}",
351
+
# Check if username is already claimed for this server
352
+
existing_zulip_id = user.get_zulip_mention(server_url)
353
+
if existing_zulip_id:
354
+
# Check if it's claimed by the same user
355
+
if existing_zulip_id == sender_email or str(existing_zulip_id) == str(
358
+
bot_handler.send_reply(
360
+
f"✅ Username `{username}` is already claimed by you on {server_url}!",
363
+
bot_handler.send_reply(
365
+
f"❌ Username `{username}` is already claimed by another user on {server_url}.",
369
+
# Claim the username - prefer email for consistency
370
+
success = self.git_store.add_zulip_association(
371
+
username, server_url, sender_email
376
+
f"🎉 Successfully claimed username `{username}` for **{sender}** on {server_url}!\n"
377
+
+ "You will now be mentioned when new articles are posted from this user's feeds."
379
+
bot_handler.send_reply(message, reply_msg)
381
+
# Send notification to configured stream if enabled and not in debug mode
383
+
self.username_claim_notifications
384
+
and not self.debug_user
385
+
and self.stream_name
386
+
and self.topic_name
389
+
notification_msg = f"👋 **{sender}** claimed thicket username `{username}` on {server_url}"
390
+
bot_handler.send_message(
393
+
"to": self.stream_name,
394
+
"subject": self.topic_name,
395
+
"content": notification_msg,
398
+
except Exception as e:
400
+
f"Failed to send username claim notification: {e}"
404
+
f"User {sender} ({sender_email}) claimed username {username} on {server_url}"
407
+
bot_handler.send_reply(
409
+
f"❌ Failed to claim username `{username}`. This shouldn't happen - please contact an administrator.",
412
+
except Exception as e:
413
+
self.logger.error(f"Error processing claim for {username} by {sender}: {e}")
414
+
bot_handler.send_reply(message, f"❌ Error processing claim: {str(e)}")
def _handle_config_command(
241
-
message: Dict[str, Any],
242
-
bot_handler: BotHandler,
418
+
message: dict[str, Any],
419
+
bot_handler: BotHandler,
"""Handle configuration commands."""
248
-
bot_handler.send_reply(message, "Usage: `@mention config <setting> <value>`")
425
+
bot_handler.send_reply(
426
+
message, "Usage: `@mention config <setting> <value>`"
setting = args[0].lower()
value = " ".join(args[1:])
434
+
old_value = self.stream_name
self._save_bot_config(bot_handler)
257
-
bot_handler.send_reply(message, f"✅ Stream set to: **{value}** (by {sender})")
437
+
bot_handler.send_reply(
438
+
message, f"✅ Stream set to: **{value}** (by {sender})"
440
+
self._send_config_change_notification(
441
+
bot_handler, sender, "stream", old_value, value
445
+
old_value = self.topic_name
self._save_bot_config(bot_handler)
262
-
bot_handler.send_reply(message, f"✅ Topic set to: **{value}** (by {sender})")
448
+
bot_handler.send_reply(
449
+
message, f"✅ Topic set to: **{value}** (by {sender})"
451
+
self._send_config_change_notification(
452
+
bot_handler, sender, "topic", old_value, value
elif setting == "interval":
268
-
bot_handler.send_reply(message, "❌ Interval must be at least 60 seconds")
459
+
bot_handler.send_reply(
460
+
message, "❌ Interval must be at least 60 seconds"
463
+
old_value = self.sync_interval
self.sync_interval = interval
self._save_bot_config(bot_handler)
272
-
bot_handler.send_reply(message, f"✅ Sync interval set to: **{interval}s** (by {sender})")
466
+
bot_handler.send_reply(
467
+
message, f"✅ Sync interval set to: **{interval}s** (by {sender})"
469
+
self._send_config_change_notification(
274
-
bot_handler.send_reply(message, "❌ Invalid interval value. Must be a number of seconds.")
477
+
bot_handler.send_reply(
478
+
message, "❌ Invalid interval value. Must be a number of seconds."
481
+
elif setting == "max_entries":
483
+
max_entries = int(value)
484
+
if max_entries < 1 or max_entries > 50:
485
+
bot_handler.send_reply(
486
+
message, "❌ Max entries must be between 1 and 50"
489
+
old_value = self.max_entries_per_sync
490
+
self.max_entries_per_sync = max_entries
491
+
self._save_bot_config(bot_handler)
492
+
bot_handler.send_reply(
494
+
f"✅ Max entries per sync set to: **{max_entries}** (by {sender})",
496
+
self._send_config_change_notification(
499
+
"max entries per sync",
504
+
bot_handler.send_reply(
505
+
message, "❌ Invalid max entries value. Must be a number."
279
-
f"❌ Unknown setting: {setting}. Available: stream, topic, interval"
511
+
f"❌ Unknown setting: {setting}. Available: stream, topic, interval, max_entries",
def _load_bot_config(self, bot_handler: BotHandler) -> None:
···
config = json.loads(config_data)
self.stream_name = config.get("stream_name")
289
-
self.topic_name = config.get("topic_name")
521
+
self.topic_name = config.get("topic_name")
self.sync_interval = config.get("sync_interval", 300)
self.max_entries_per_sync = config.get("max_entries_per_sync", 10)
292
-
except Exception as e:
524
+
self.last_sync_time = config.get("last_sync_time")
# Bot config not found on first run is expected
···
"topic_name": self.topic_name,
"sync_interval": self.sync_interval,
"max_entries_per_sync": self.max_entries_per_sync,
537
+
"last_sync_time": self.last_sync_time,
bot_handler.storage.put("bot_config", json.dumps(config_data))
self.logger.error(f"Error saving bot config: {e}")
543
+
def _load_botrc_defaults(self) -> None:
544
+
"""Load default configuration from botrc file."""
546
+
import configparser
547
+
from pathlib import Path
549
+
botrc_path = Path("bot-config/botrc")
550
+
if not botrc_path.exists():
551
+
self.logger.info("No botrc file found, using hardcoded defaults")
554
+
config = configparser.ConfigParser()
555
+
config.read(botrc_path)
557
+
if "bot" in config:
558
+
bot_section = config["bot"]
559
+
self.sync_interval = bot_section.getint("sync_interval", 300)
560
+
self.max_entries_per_sync = bot_section.getint(
561
+
"max_entries_per_sync", 10
563
+
self.rate_limit_delay = bot_section.getint("rate_limit_delay", 5)
564
+
self.posts_per_batch = bot_section.getint("posts_per_batch", 5)
566
+
# Set defaults only if not already configured
567
+
default_stream = bot_section.get("default_stream", "").strip()
568
+
default_topic = bot_section.get("default_topic", "").strip()
570
+
self.stream_name = default_stream
572
+
self.topic_name = default_topic
574
+
if "catchup" in config:
575
+
catchup_section = config["catchup"]
576
+
self.catchup_entries = catchup_section.getint("catchup_entries", 5)
578
+
if "notifications" in config:
579
+
notifications_section = config["notifications"]
580
+
self.config_change_notifications = notifications_section.getboolean(
581
+
"config_change_notifications", True
583
+
self.username_claim_notifications = notifications_section.getboolean(
584
+
"username_claim_notifications", True
587
+
self.logger.info(f"Loaded configuration from {botrc_path}")
589
+
except Exception as e:
590
+
self.logger.error(f"Error loading botrc defaults: {e}")
591
+
self.logger.info("Using hardcoded defaults")
def _initialize_thicket(self) -> None:
"""Initialize thicket components."""
if not self.config_path or not self.config_path.exists():
raise ValueError("Thicket config file not found")
# Load thicket configuration
with open(self.config_path) as f:
config_data = yaml.safe_load(f)
self.config = ThicketConfig(**config_data)
self.git_store = GitStore(self.config.git_store)
self.logger.info("Thicket components initialized successfully")
def _validate_debug_mode(self, bot_handler: BotHandler) -> None:
"""Validate debug mode configuration."""
if not self.debug_user or not self.git_store:
# Get current Zulip server from environment
zulip_site_url = os.getenv("THICKET_ZULIP_SITE_URL", "")
server_url = zulip_site_url.replace("https://", "").replace("http://", "")
# Check if debug user exists in thicket
user = self.git_store.get_user(self.debug_user)
raise ValueError(f"Debug user '{self.debug_user}' not found in thicket")
# Check if user has Zulip association for this server
raise ValueError("Could not determine Zulip server URL")
zulip_user_id = user.get_zulip_mention(server_url)
345
-
raise ValueError(f"User '{self.debug_user}' has no Zulip association for server '{server_url}'")
631
+
f"User '{self.debug_user}' has no Zulip association for server '{server_url}'"
# Try to look up the actual Zulip user ID from the email address
# But don't fail if we can't - we'll try again when sending messages
actual_user_id = self._lookup_zulip_user_id(bot_handler, zulip_user_id)
if actual_user_id and actual_user_id != zulip_user_id:
# Successfully resolved to numeric ID
self.debug_zulip_user_id = actual_user_id
353
-
self.logger.info(f"Debug mode enabled: Will send DMs to {self.debug_user} (email: {zulip_user_id}, user_id: {actual_user_id}) on {server_url}")
641
+
f"Debug mode enabled: Will send DMs to {self.debug_user} (email: {zulip_user_id}, user_id: {actual_user_id}) on {server_url}"
# Keep the email address, will resolve later when sending
self.debug_zulip_user_id = zulip_user_id
357
-
self.logger.info(f"Debug mode enabled: Will send DMs to {self.debug_user} ({zulip_user_id}) on {server_url} (will resolve user ID when sending)")
647
+
f"Debug mode enabled: Will send DMs to {self.debug_user} ({zulip_user_id}) on {server_url} (will resolve user ID when sending)"
359
-
def _lookup_zulip_user_id(self, bot_handler: BotHandler, email_or_id: str) -> Optional[str]:
650
+
def _lookup_zulip_user_id(
651
+
self, bot_handler: BotHandler, email_or_id: str
652
+
) -> Optional[str]:
"""Look up Zulip user ID from email address or return the ID if it's already numeric."""
# If it's already a numeric user ID, return it
if email_or_id.isdigit():
client = bot_handler._client
self.logger.error("No Zulip client available for user lookup")
# First try the get_user_by_email API if available
user_result = client.get_user_by_email(email_or_id)
374
-
if user_result.get('result') == 'success':
375
-
user_data = user_result.get('user', {})
376
-
user_id = user_data.get('user_id')
667
+
if user_result.get("result") == "success":
668
+
user_data = user_result.get("user", {})
669
+
user_id = user_data.get("user_id")
378
-
self.logger.info(f"Found user ID {user_id} for '{email_or_id}' via get_user_by_email API")
672
+
f"Found user ID {user_id} for '{email_or_id}' via get_user_by_email API"
except (AttributeError, Exception):
# Fallback: Get all users and search through them
users_result = client.get_users()
385
-
if users_result.get('result') == 'success':
386
-
for user in users_result['members']:
387
-
user_email = user.get('email', '')
388
-
delivery_email = user.get('delivery_email', '')
390
-
if (user_email == email_or_id or
391
-
delivery_email == email_or_id or
392
-
str(user.get('user_id')) == email_or_id):
393
-
user_id = user.get('user_id')
680
+
if users_result.get("result") == "success":
681
+
for user in users_result["members"]:
682
+
user_email = user.get("email", "")
683
+
delivery_email = user.get("delivery_email", "")
686
+
user_email == email_or_id
687
+
or delivery_email == email_or_id
688
+
or str(user.get("user_id")) == email_or_id
690
+
user_id = user.get("user_id")
396
-
self.logger.error(f"No user found with identifier '{email_or_id}'. Searched {len(users_result['members'])} users.")
694
+
f"No user found with identifier '{email_or_id}'. Searched {len(users_result['members'])} users."
399
-
self.logger.error(f"Failed to get users: {users_result.get('msg', 'Unknown error')}")
699
+
f"Failed to get users: {users_result.get('msg', 'Unknown error')}"
self.logger.error(f"Error looking up user ID for '{email_or_id}': {e}")
406
-
def _lookup_zulip_user_info(self, bot_handler: BotHandler, email_or_id: str) -> Tuple[Optional[str], Optional[str]]:
707
+
def _lookup_zulip_user_info(
708
+
self, bot_handler: BotHandler, email_or_id: str
709
+
) -> tuple[Optional[str], Optional[str]]:
"""Look up both Zulip user ID and full name from email address."""
if email_or_id.isdigit():
client = bot_handler._client
# Try get_user_by_email API first
user_result = client.get_user_by_email(email_or_id)
419
-
if user_result.get('result') == 'success':
420
-
user_data = user_result.get('user', {})
421
-
user_id = user_data.get('user_id')
422
-
full_name = user_data.get('full_name', '')
722
+
if user_result.get("result") == "success":
723
+
user_data = user_result.get("user", {})
724
+
user_id = user_data.get("user_id")
725
+
full_name = user_data.get("full_name", "")
return str(user_id), full_name
# Fallback: search all users
users_result = client.get_users()
430
-
if users_result.get('result') == 'success':
431
-
for user in users_result['members']:
432
-
if (user.get('email') == email_or_id or
433
-
user.get('delivery_email') == email_or_id):
434
-
return str(user.get('user_id')), user.get('full_name', '')
733
+
if users_result.get("result") == "success":
734
+
for user in users_result["members"]:
736
+
user.get("email") == email_or_id
737
+
or user.get("delivery_email") == email_or_id
739
+
return str(user.get("user_id")), user.get("full_name", "")
self.logger.error(f"Error looking up user info for '{email_or_id}': {e}")
···
def _save_posted_entries(self, bot_handler: BotHandler) -> None:
"""Save the set of posted entries."""
455
-
bot_handler.storage.put("posted_entries", json.dumps(list(self.posted_entries)))
760
+
bot_handler.storage.put(
761
+
"posted_entries", json.dumps(list(self.posted_entries))
self.logger.error(f"Error saving posted entries: {e}")
459
-
def _check_initialization(self, message: Dict[str, Any], bot_handler: BotHandler) -> bool:
766
+
def _check_initialization(
767
+
self, message: dict[str, Any], bot_handler: BotHandler
"""Check if thicket is properly initialized."""
if not self.git_store or not self.config:
464
-
"❌ Thicket not initialized. Please check configuration."
772
+
message, "❌ Thicket not initialized. Please check configuration."
# In debug mode, we don't need stream/topic configuration
if not self.stream_name or not self.topic_name:
475
-
"❌ Stream and topic must be configured first. Use `@mention config stream <name>` and `@mention config topic <name>`"
783
+
"❌ Stream and topic must be configured first. Use `@mention config stream <name>` and `@mention config topic <name>`",
def _schedule_sync(self, bot_handler: BotHandler) -> None:
"""Schedule periodic sync operations."""
487
-
can_sync = (self.git_store and
488
-
((self.stream_name and self.topic_name) or
796
+
can_sync = self.git_store and (
797
+
(self.stream_name and self.topic_name) or self.debug_user
self._perform_sync(bot_handler)
time.sleep(self.sync_interval)
self.logger.error(f"Error in sync loop: {e}")
time.sleep(60) # Wait before retrying
# Start background thread
sync_thread = threading.Thread(target=sync_loop, daemon=True)
504
-
def _perform_sync(self, bot_handler: BotHandler) -> List[AtomEntry]:
814
+
def _perform_sync(self, bot_handler: BotHandler) -> list[AtomEntry]:
"""Perform thicket sync and return new entries."""
if not self.config or not self.git_store:
509
-
new_entries: List[Tuple[AtomEntry, str]] = [] # (entry, username) pairs
819
+
new_entries: list[tuple[AtomEntry, str]] = [] # (entry, username) pairs
is_first_run = len(self.posted_entries) == 0
# Get all users and their feeds from git store
users_with_feeds = self.git_store.list_all_users_with_feeds()
for username, feed_urls in users_with_feeds:
for feed_url in feed_urls:
···
asyncio.set_event_loop(loop)
new_count, _ = loop.run_until_complete(
524
-
sync_feed(self.git_store, username, str(feed_url), dry_run=False)
835
+
self.git_store, username, str(feed_url), dry_run=False
# Get the newly added entries
531
-
entries_to_check = self.git_store.list_entries(username, limit=new_count)
843
+
entries_to_check = self.git_store.list_entries(
844
+
username, limit=new_count
# Always check for catchup mode on first run
535
-
# Catchup mode: get last 5 entries on first run
536
-
catchup_entries = self.git_store.list_entries(username, limit=5)
537
-
entries_to_check = catchup_entries if not entries_to_check else entries_to_check
849
+
# Catchup mode: get configured number of entries on first run
850
+
catchup_entries = self.git_store.list_entries(
851
+
username, limit=self.catchup_entries
853
+
entries_to_check = (
855
+
if not entries_to_check
856
+
else entries_to_check
for entry in entries_to_check:
entry_key = f"{username}:{entry.id}"
if entry_key not in self.posted_entries:
new_entries.append((entry, username))
if len(new_entries) >= self.max_entries_per_sync:
550
-
self.logger.error(f"Error syncing feed {feed_url} for user {username}: {e}")
871
+
f"Error syncing feed {feed_url} for user {username}: {e}"
if len(new_entries) >= self.max_entries_per_sync:
# Post new entries to Zulip with rate limiting
for i, (entry, username) in enumerate(new_entries):
self._post_entry_to_zulip(entry, bot_handler, username)
self.posted_entries.add(f"{username}:{entry.id}")
564
-
# Rate limiting: pause after every 5 messages
565
-
if posted_count % 5 == 0 and i < len(new_entries) - 1:
886
+
# Rate limiting: pause after configured number of messages
888
+
posted_count % self.posts_per_batch == 0
889
+
and i < len(new_entries) - 1
891
+
time.sleep(self.rate_limit_delay)
self._save_posted_entries(bot_handler)
895
+
# Update last sync time
896
+
self.last_sync_time = time.time()
return [entry for entry, _ in new_entries]
572
-
def _post_entry_to_zulip(self, entry: AtomEntry, bot_handler: BotHandler, username: str) -> None:
900
+
def _post_entry_to_zulip(
901
+
self, entry: AtomEntry, bot_handler: BotHandler, username: str
"""Post a single entry to the configured Zulip stream/topic or debug user DM."""
# Get current Zulip server from environment
zulip_site_url = os.getenv("THICKET_ZULIP_SITE_URL", "")
server_url = zulip_site_url.replace("https://", "").replace("http://", "")
# Build author/date info consistently
if server_url and self.git_store:
···
zulip_user_id = user.get_zulip_mention(server_url)
# Look up the actual Zulip full name for proper @mention
587
-
_, zulip_full_name = self._lookup_zulip_user_info(bot_handler, zulip_user_id)
917
+
_, zulip_full_name = self._lookup_zulip_user_info(
918
+
bot_handler, zulip_user_id
display_name = zulip_full_name or user.display_name or username
# Check if author is different from the user - avoid redundancy
author_name = entry.author and entry.author.get("name")
if author_name and author_name.lower() != display_name.lower():
author_info = f" (by {author_name})"
599
-
published_info = f" • {entry.published.strftime('%Y-%m-%d')}"
932
+
f" • {entry.published.strftime('%Y-%m-%d')}"
mention_info = f"@**{display_name}** posted{author_info}{published_info}:\n\n"
# If no Zulip user found, use consistent format without @mention
user = self.git_store.get_user(username) if self.git_store else None
display_name = user.display_name if user else username
author_name = entry.author and entry.author.get("name")
if author_name and author_name.lower() != display_name.lower():
author_info = f" (by {author_name})"
published_info = f" • {entry.published.strftime('%Y-%m-%d')}"
618
-
mention_info = f"**{display_name}** posted{author_info}{published_info}:\n\n"
953
+
f"**{display_name}** posted{author_info}{published_info}:\n\n"
# Format the message with HTML processing
# Process HTML in summary and truncate if needed
processed_summary = self._process_html_content(entry.summary)
if len(processed_summary) > 400:
processed_summary = processed_summary[:397] + "..."
message_lines.append(f"\n{processed_summary}")
message_content = mention_info + "\n".join(message_lines)
# Choose destination based on mode
if self.debug_user and self.debug_zulip_user_id:
debug_message = f"🐛 **DEBUG:** New article from thicket user `{username}`:\n\n{message_content}"
# Ensure we have the numeric user ID
user_id_to_use = self.debug_zulip_user_id
if not user_id_to_use.isdigit():
# Need to look up the numeric ID
644
-
resolved_id = self._lookup_zulip_user_id(bot_handler, user_id_to_use)
980
+
resolved_id = self._lookup_zulip_user_id(
981
+
bot_handler, user_id_to_use
user_id_to_use = resolved_id
647
-
self.logger.debug(f"Resolved {self.debug_zulip_user_id} to user ID {user_id_to_use}")
986
+
f"Resolved {self.debug_zulip_user_id} to user ID {user_id_to_use}"
649
-
self.logger.error(f"Could not resolve user ID for {self.debug_zulip_user_id}")
990
+
f"Could not resolve user ID for {self.debug_zulip_user_id}"
# For private messages, user_id needs to be an integer, not string
user_id_int = int(user_id_to_use)
655
-
bot_handler.send_message({
657
-
"to": [user_id_int], # Use integer user ID
658
-
"content": debug_message
997
+
bot_handler.send_message(
1000
+
"to": [user_id_int], # Use integer user ID
1001
+
"content": debug_message,
# If conversion to int fails, user_id_to_use might be an email
663
-
bot_handler.send_message({
665
-
"to": [user_id_to_use], # Try as string (email)
666
-
"content": debug_message
1007
+
bot_handler.send_message(
1009
+
"type": "private",
1010
+
"to": [user_id_to_use], # Try as string (email)
1011
+
"content": debug_message,
669
-
self.logger.error(f"Failed to send DM to {self.debug_user} (tried both int and string): {e2}")
1015
+
self.logger.error(
1016
+
f"Failed to send DM to {self.debug_user} (tried both int and string): {e2}"
672
-
self.logger.error(f"Failed to send DM to {self.debug_user} ({user_id_to_use}): {e}")
1020
+
self.logger.error(
1021
+
f"Failed to send DM to {self.debug_user} ({user_id_to_use}): {e}"
674
-
self.logger.info(f"Posted entry to debug user {self.debug_user}: {entry.title}")
1025
+
f"Posted entry to debug user {self.debug_user}: {entry.title}"
# Normal mode: send to stream/topic
677
-
bot_handler.send_message({
679
-
"to": self.stream_name,
680
-
"subject": self.topic_name,
681
-
"content": message_content
683
-
self.logger.info(f"Posted entry to stream: {entry.title} (user: {username})")
1029
+
bot_handler.send_message(
1032
+
"to": self.stream_name,
1033
+
"subject": self.topic_name,
1034
+
"content": message_content,
1038
+
f"Posted entry to stream: {entry.title} (user: {username})"
self.logger.error(f"Error posting entry to Zulip: {e}")
···
"""Process HTML content from feeds to clean Zulip-compatible markdown."""
# Try to use markdownify for proper HTML to Markdown conversion
from markdownify import markdownify as md
# Convert HTML to Markdown with compact settings for summaries
heading_style="ATX", # Use # for headings (but we'll post-process these)
bullets="-", # Use - for bullets
702
-
convert=['a', 'b', 'strong', 'i', 'em', 'code', 'pre', 'p', 'br', 'ul', 'ol', 'li', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']
# Post-process to convert headings to bold for compact summaries
# Convert markdown headers to bold with period
708
-
markdown = re.sub(r'^#{1,6}\s*(.+)$', r'**\1.**', markdown, flags=re.MULTILINE)
1084
+
markdown = re.sub(
1085
+
r"^#{1,6}\s*(.+)$", r"**\1.**", markdown, flags=re.MULTILINE
# Clean up excessive newlines and make more compact
711
-
markdown = re.sub(r'\n\s*\n\s*\n+', ' ', markdown) # Multiple newlines become space
712
-
markdown = re.sub(r'\n\s*\n', '. ', markdown) # Double newlines become sentence breaks
713
-
markdown = re.sub(r'\n', ' ', markdown) # Single newlines become spaces
1089
+
markdown = re.sub(
1090
+
r"\n\s*\n\s*\n+", " ", markdown
1091
+
) # Multiple newlines become space
1092
+
markdown = re.sub(
1093
+
r"\n\s*\n", ". ", markdown
1094
+
) # Double newlines become sentence breaks
1095
+
markdown = re.sub(r"\n", " ", markdown) # Single newlines become spaces
# Clean up double periods and excessive whitespace
716
-
markdown = re.sub(r'\.\.+', '.', markdown)
717
-
markdown = re.sub(r'\s+', ' ', markdown)
1098
+
markdown = re.sub(r"\.\.+", ".", markdown)
1099
+
markdown = re.sub(r"\s+", " ", markdown)
# Fallback: manual HTML processing
# Convert headings to bold with periods for compact summaries
726
-
content = re.sub(r'<h[1-6](?:\s[^>]*)?>([^<]*)</h[1-6]>', r'**\1.** ', content, flags=re.IGNORECASE)
1110
+
r"<h[1-6](?:\s[^>]*)?>([^<]*)</h[1-6]>",
1113
+
flags=re.IGNORECASE,
# Convert common HTML elements to Markdown
729
-
content = re.sub(r'<(?:strong|b)(?:\s[^>]*)?>([^<]*)</(?:strong|b)>', r'**\1**', content, flags=re.IGNORECASE)
730
-
content = re.sub(r'<(?:em|i)(?:\s[^>]*)?>([^<]*)</(?:em|i)>', r'*\1*', content, flags=re.IGNORECASE)
731
-
content = re.sub(r'<code(?:\s[^>]*)?>([^<]*)</code>', r'`\1`', content, flags=re.IGNORECASE)
732
-
content = re.sub(r'<a(?:\s[^>]*?)?\s*href=["\']([^"\']*)["\'](?:\s[^>]*)?>([^<]*)</a>', r'[\2](\1)', content, flags=re.IGNORECASE)
1118
+
r"<(?:strong|b)(?:\s[^>]*)?>([^<]*)</(?:strong|b)>",
1121
+
flags=re.IGNORECASE,
1124
+
r"<(?:em|i)(?:\s[^>]*)?>([^<]*)</(?:em|i)>",
1127
+
flags=re.IGNORECASE,
1130
+
r"<code(?:\s[^>]*)?>([^<]*)</code>",
1133
+
flags=re.IGNORECASE,
1136
+
r'<a(?:\s[^>]*?)?\s*href=["\']([^"\']*)["\'](?:\s[^>]*)?>([^<]*)</a>',
1139
+
flags=re.IGNORECASE,
# Convert block elements to spaces instead of newlines for compactness
735
-
content = re.sub(r'<br\s*/?>', ' ', content, flags=re.IGNORECASE)
736
-
content = re.sub(r'</p>\s*<p>', '. ', content, flags=re.IGNORECASE)
737
-
content = re.sub(r'</?(?:p|div)(?:\s[^>]*)?>', ' ', content, flags=re.IGNORECASE)
1143
+
content = re.sub(r"<br\s*/?>", " ", content, flags=re.IGNORECASE)
1144
+
content = re.sub(r"</p>\s*<p>", ". ", content, flags=re.IGNORECASE)
1146
+
r"</?(?:p|div)(?:\s[^>]*)?>", " ", content, flags=re.IGNORECASE
# Remove remaining HTML tags
740
-
content = re.sub(r'<[^>]+>', '', content)
1150
+
content = re.sub(r"<[^>]+>", "", content)
# Clean up whitespace and make compact
743
-
content = re.sub(r'\s+', ' ', content) # Multiple whitespace becomes single space
744
-
content = re.sub(r'\.\.+', '.', content) # Multiple periods become single period
1154
+
r"\s+", " ", content
1155
+
) # Multiple whitespace becomes single space
1157
+
r"\.\.+", ".", content
1158
+
) # Multiple periods become single period
self.logger.error(f"Error processing HTML content: {e}")
# Last resort: just strip HTML tags
751
-
return re.sub(r'<[^>]+>', '', html_content).strip()
1166
+
return re.sub(r"<[^>]+>", "", html_content).strip()
1168
+
def _get_schedule_info(self) -> str:
1169
+
"""Get schedule information string."""
1172
+
if self.last_sync_time:
1175
+
last_sync = datetime.datetime.fromtimestamp(self.last_sync_time)
1176
+
next_sync = last_sync + datetime.timedelta(seconds=self.sync_interval)
1177
+
now = datetime.datetime.now()
1179
+
# Calculate time until next sync
1180
+
time_until_next = next_sync - now
1182
+
if time_until_next.total_seconds() > 0:
1183
+
minutes, seconds = divmod(int(time_until_next.total_seconds()), 60)
1184
+
hours, minutes = divmod(minutes, 60)
1187
+
time_str = f"{hours}h {minutes}m {seconds}s"
1189
+
time_str = f"{minutes}m {seconds}s"
1191
+
time_str = f"{seconds}s"
1195
+
f"🕐 **Last Sync:** {last_sync.strftime('%H:%M:%S')}",
1196
+
f"⏰ **Next Sync:** {next_sync.strftime('%H:%M:%S')} (in {time_str})",
1202
+
f"🕐 **Last Sync:** {last_sync.strftime('%H:%M:%S')}",
1203
+
f"⏰ **Next Sync:** Due now (running every {self.sync_interval}s)",
1207
+
lines.append("🕐 **Last Sync:** Never (bot starting up)")
1209
+
# Add sync frequency info
1210
+
if self.sync_interval >= 3600:
1212
+
f"{self.sync_interval // 3600}h {(self.sync_interval % 3600) // 60}m"
1214
+
elif self.sync_interval >= 60:
1215
+
frequency_str = f"{self.sync_interval // 60}m {self.sync_interval % 60}s"
1217
+
frequency_str = f"{self.sync_interval}s"
1219
+
lines.append(f"🔄 **Sync Frequency:** Every {frequency_str}")
1221
+
return "\n".join(lines)
754
-
handler_class = ThicketBotHandler
1223
+
def _send_config_change_notification(
1225
+
bot_handler: BotHandler,
1228
+
old_value: Optional[str],
1231
+
"""Send configuration change notification if enabled."""
1232
+
if not self.config_change_notifications or self.debug_user:
1235
+
# Don't send notification if stream/topic aren't configured yet
1236
+
if not self.stream_name or not self.topic_name:
1240
+
old_display = old_value if old_value else "(not set)"
1241
+
notification_msg = (
1242
+
f"⚙️ **{changer}** changed {setting}: `{old_display}` → `{new_value}`"
1245
+
bot_handler.send_message(
1248
+
"to": self.stream_name,
1249
+
"subject": self.topic_name,
1250
+
"content": notification_msg,
1253
+
except Exception as e:
1254
+
self.logger.error(f"Failed to send config change notification: {e}")
1257
+
handler_class = ThicketBotHandler
anil.recoil.org