commit 2fde1bc6d1a5960a2b038e0a0ed3fcb2020e8e59 · bretton.dev/coves

.env.dev

···

       128
       128
        
       PDS_INSTANCE_HANDLE=testuser123.local.coves.dev

     

       129
       129
        
       PDS_INSTANCE_PASSWORD=test-password-123

     

       130
       130
        
       

     

       131
       131
       +
       # Kagi News Aggregator DID (for trusted thumbnail URLs)

     

       132
       132
       +
       KAGI_AGGREGATOR_DID=did:plc:yyf34padpfjknejyutxtionr

     

       133
       133
       +
       

     

       131
       134
        
       # =============================================================================

     

       132
       135
        
       # Development Settings

     

       133
       136
        
       # =============================================================================

+1 -1

aggregators/kagi-news/README.md

···

       8
       8
        
       - Fetches RSS feeds from Kagi News daily via CRON

     

       9
       9
        
       - Parses HTML descriptions to extract structured content (highlights, perspectives, sources)

     

       10
       10
        
       - Formats posts using Coves rich text with facets (bold, italic, links)

     

       11
       11
       -
       - Hot-links images from Kagi's proxy (no blob upload)

     

       11
       11
       +
       - Thumbnails are automatically extracted by the server's unfurl service

     

       12
       12
        
       - Posts to configured communities via XRPC

     

       13
       13
        
       

     

       14
       14
        
       ## Project Structure

+13 -14

aggregators/kagi-news/src/coves_client.py

···

       72
       72
        
               content: str,

     

       73
       73
        
               facets: List[Dict],

     

       74
       74
        
               title: Optional[str] = None,

     

       75
       75
       -
               embed: Optional[Dict] = None

     

       75
       75
       +
               embed: Optional[Dict] = None,

     

       76
       76
       +
               thumbnail_url: Optional[str] = None

     

       76
       77
        
           ) -> str:

     

       77
       78
        
               """

     

       78
       79
        
               Create a post in a community.

     
···

       83
       84
        
                   facets: Rich text facets (formatting, links)

     

       84
       85
        
                   title: Optional post title

     

       85
       86
        
                   embed: Optional external embed

     

       87
       87
       +
                   thumbnail_url: Optional thumbnail URL (for trusted aggregators only)

     

       86
       88
        
       

     

       87
       89
        
               Returns:

     

       88
       90
        
                   AT Proto URI of created post (e.g., "at://did:plc:.../social.coves.post/...")

     
···

       109
       111
        
                   if embed:

     

       110
       112
        
                       post_data["embed"] = embed

     

       111
       113
        
       

     

       114
       114
       +
                   # Add thumbnail URL at top level if provided (for trusted aggregators)

     

       115
       115
       +
                   if thumbnail_url:

     

       116
       116
       +
                       post_data["thumbnailUrl"] = thumbnail_url

     

       117
       117
       +
       

     

       112
       118
        
                   # Use Coves-specific endpoint (not direct PDS write)

     

       113
       119
        
                   # This provides validation, authorization, and business logic

     

       114
       120
        
                   logger.info(f"Creating post in community: {community_handle}")

     
···

       141
       147
        
               self,

     

       142
       148
        
               uri: str,

     

       143
       149
        
               title: str,

     

       144
       144
       -
               description: str,

     

       145
       145
       -
               thumb: Optional[str] = None

     

       150
       150
       +
               description: str

     

       146
       151
        
           ) -> Dict:

     

       147
       152
        
               """

     

       148
       153
        
               Create external embed object for hot-linked content.

     

       149
       154
        
       

     

       150
       155
        
               Args:

     

       151
       151
       -
                   uri: External URL (story link)

     

       152
       152
       -
                   title: Story title

     

       153
       153
       -
                   description: Story description/summary

     

       154
       154
       -
                   thumb: Optional thumbnail image URL

     

       156
       156
       +
                   uri: URL of the external content

     

       157
       157
       +
                   title: Title of the content

     

       158
       158
       +
                   description: Description/summary

     

       155
       159
        
       

     

       156
       160
        
               Returns:

     

       157
       157
       -
                   External embed dictionary

     

       161
       161
       +
                   Embed dictionary ready for post creation

     

       158
       162
        
               """

     

       159
       159
       -
               embed = {

     

       163
       163
       +
               return {

     

       160
       164
        
                   "$type": "social.coves.embed.external",

     

       161
       165
        
                   "external": {

     

       162
       166
        
                       "uri": uri,

     
···

       164
       168
        
                       "description": description

     

       165
       169
        
                   }

     

       166
       170
        
               }

     

       167
       167
       -
       

     

       168
       168
       -
               if thumb:

     

       169
       169
       -
                   embed["external"]["thumb"] = thumb

     

       170
       170
       -
       

     

       171
       171
       -
               return embed

     

       172
       171
        
       

     

       173
       172
        
           def _get_timestamp(self) -> str:

     

       174
       173
        
               """

+4 -3

aggregators/kagi-news/src/main.py

···

       173
       173
        
                       embed = self.coves_client.create_external_embed(

     

       174
       174
        
                           uri=story.link,

     

       175
       175
        
                           title=story.title,

     

       176
       176
       -
                           description=story.summary[:200] if len(story.summary) > 200 else story.summary,

     

       177
       177
       -
                           thumb=story.image_url

     

       176
       176
       +
                           description=story.summary[:200] if len(story.summary) > 200 else story.summary

     

       178
       177
        
                       )

     

       179
       178
        
       

     

       180
       179
        
                       # Post to community

     

       180
       180
       +
                       # Pass thumbnail URL from RSS feed at top level for trusted aggregator upload

     

       181
       181
        
                       try:

     

       182
       182
        
                           post_uri = self.coves_client.create_post(

     

       183
       183
        
                               community_handle=feed_config.community_handle,

     

       184
       184
        
                               title=story.title,

     

       185
       185
        
                               content=rich_text["content"],

     

       186
       186
        
                               facets=rich_text["facets"],

     

       187
       187
       -
                               embed=embed

     

       187
       187
       +
                               embed=embed,

     

       188
       188
       +
                               thumbnail_url=story.image_url  # From RSS feed - server will validate and upload

     

       188
       189
        
                           )

     

       189
       190
        
       

     

       190
       191
        
                           # Mark as posted (only if successful)

+5 -14

aggregators/kagi-news/tests/test_e2e.py

···

       398
       398
        
           Verifies:

     

       399
       399
        
           - Embed structure matches social.coves.embed.external

     

       400
       400
        
           - All required fields are present

     

       401
       401
       -
           - Optional thumbnail is included when provided

     

       401
       401
       +
           - Thumbnails are handled by server's unfurl service (not included in client)

     

       402
       402
        
           """

     

       403
       403
        
           handle, password = aggregator_credentials

     

       404
       404
        
       

     
···

       408
       408
        
               password=password

     

       409
       409
        
           )

     

       410
       410
        
       

     

       411
       411
       -
           # Test with thumbnail

     

       411
       411
       +
           # Create external embed (server will handle thumbnail extraction)

     

       412
       412
        
           embed = client.create_external_embed(

     

       413
       413
        
               uri="https://example.com/story",

     

       414
       414
        
               title="Test Story",

     

       415
       415
       -
               description="Test description",

     

       416
       416
       -
               thumb="https://example.com/image.jpg"

     

       415
       415
       +
               description="Test description"

     

       417
       416
        
           )

     

       418
       417
        
       

     

       419
       418
        
           assert embed["$type"] == "social.coves.embed.external"

     

       420
       419
        
           assert embed["external"]["uri"] == "https://example.com/story"

     

       421
       420
        
           assert embed["external"]["title"] == "Test Story"

     

       422
       421
        
           assert embed["external"]["description"] == "Test description"

     

       423
       423
       -
           assert embed["external"]["thumb"] == "https://example.com/image.jpg"

     

       424
       424
       -
       

     

       425
       425
       -
           # Test without thumbnail

     

       426
       426
       -
           embed_no_thumb = client.create_external_embed(

     

       427
       427
       -
               uri="https://example.com/story2",

     

       428
       428
       -
               title="Test Story 2",

     

       429
       429
       -
               description="Test description 2"

     

       430
       430
       -
           )

     

       431
       431
       -
       

     

       432
       432
       -
           assert "thumb" not in embed_no_thumb["external"]

     

       422
       422
       +
           # Thumbnail is not included - server's unfurl service handles it

     

       423
       423
       +
           assert "thumb" not in embed["external"]

     

       433
       424
        
           print("\n✅ External embed format correct")

+4 -3

aggregators/kagi-news/tests/test_main.py

···

       404
       404
        
               mock_client.create_post.return_value = "at://did:plc:test/social.coves.post/abc123"

     

       405
       405
        
       

     

       406
       406
        
               # Mock create_external_embed to return proper embed structure

     

       407
       407
       +
               # Note: Thumbnails are handled by server's unfurl service, not client

     

       407
       408
        
               mock_client.create_external_embed.return_value = {

     

       408
       409
        
                   "$type": "social.coves.embed.external",

     

       409
       410
        
                   "external": {

     

       410
       411
        
                       "uri": sample_story.link,

     

       411
       412
        
                       "title": sample_story.title,

     

       412
       412
       -
                       "description": sample_story.summary,

     

       413
       413
       -
                       "thumb": sample_story.image_url

     

       413
       413
       +
                       "description": sample_story.summary

     

       414
       414
        
                   }

     

       415
       415
        
               }

     

       416
       416
        
       

     
···

       457
       457
        
                   assert call_kwargs["embed"]["$type"] == "social.coves.embed.external"

     

       458
       458
        
                   assert call_kwargs["embed"]["external"]["uri"] == sample_story.link

     

       459
       459
        
                   assert call_kwargs["embed"]["external"]["title"] == sample_story.title

     

       460
       460
       -
                   assert call_kwargs["embed"]["external"]["thumb"] == sample_story.image_url

     

       460
       460
       +
                   # Thumbnail is not included - server's unfurl service handles it

     

       461
       461
       +
                   assert "thumb" not in call_kwargs["embed"]["external"]

+60

docs/PRD_BACKLOG.md

···

       601
       601
        
       

     

       602
       602
        
       ---

     

       603
       603
        
       

     

       604
       604
       +
       ### Unfurl Cache Cleanup Background Job

     

       605
       605
       +
       **Added:** 2025-11-07 | **Effort:** 2-3 hours | **Priority:** Performance/Maintenance

     

       606
       606
       +
       

     

       607
       607
       +
       **Problem:** The `unfurl_cache` table will grow indefinitely as expired entries are not deleted. While the cache uses lazy expiration (checking `expires_at` on read), old records remain in the database consuming disk space.

     

       608
       608
       +
       

     

       609
       609
       +
       **Impact:**

     

       610
       610
       +
       - 📊 ~1KB per cached URL

     

       611
       611
       +
       - 📈 At 10K cached URLs = ~10MB (negligible for alpha)

     

       612
       612
       +
       - ⚠️ At 1M cached URLs = ~1GB (potential issue at scale)

     

       613
       613
       +
       - 🐌 Table bloat can slow down queries over time

     

       614
       614
       +
       

     

       615
       615
       +
       **Current Mitigation:**

     

       616
       616
       +
       - ✅ Lazy expiration: Cache hits check `expires_at` and refetch if expired

     

       617
       617
       +
       - ✅ Indexed on `expires_at` for efficient expiration queries

     

       618
       618
       +
       - ✅ Not critical for alpha (growth is gradual)

     

       619
       619
       +
       

     

       620
       620
       +
       **Solution (Beta/Production):**

     

       621
       621
       +
       Implement background cleanup job to delete expired entries:

     

       622
       622
       +
       

     

       623
       623
       +
       ```go

     

       624
       624
       +
       // Periodic cleanup (run daily or weekly)

     

       625
       625
       +
       func (r *unfurlRepository) CleanupExpired(ctx context.Context) (int64, error) {

     

       626
       626
       +
           query := `DELETE FROM unfurl_cache WHERE expires_at < NOW()`

     

       627
       627
       +
           result, err := r.db.ExecContext(ctx, query)

     

       628
       628
       +
           if err != nil {

     

       629
       629
       +
               return 0, err

     

       630
       630
       +
           }

     

       631
       631
       +
           return result.RowsAffected()

     

       632
       632
       +
       }

     

       633
       633
       +
       ```

     

       634
       634
       +
       

     

       635
       635
       +
       **Implementation Options:**

     

       636
       636
       +
       1. **Cron job**: Separate process runs cleanup on schedule

     

       637
       637
       +
       2. **Background goroutine**: Service-level background task with configurable interval

     

       638
       638
       +
       3. **PostgreSQL pg_cron extension**: Database-level scheduled cleanup

     

       639
       639
       +
       

     

       640
       640
       +
       **Recommended Approach:**

     

       641
       641
       +
       - Phase 1 (Beta): Background goroutine running weekly cleanup

     

       642
       642
       +
       - Phase 2 (Production): Migrate to pg_cron or external cron for reliability

     

       643
       643
       +
       

     

       644
       644
       +
       **Configuration:**

     

       645
       645
       +
       ```bash

     

       646
       646
       +
       UNFURL_CACHE_CLEANUP_ENABLED=true

     

       647
       647
       +
       UNFURL_CACHE_CLEANUP_INTERVAL=168h  # 7 days

     

       648
       648
       +
       ```

     

       649
       649
       +
       

     

       650
       650
       +
       **Monitoring:**

     

       651
       651
       +
       - Log cleanup operations: `[UNFURL-CACHE-CLEANUP] Deleted 1234 expired entries`

     

       652
       652
       +
       - Track table size growth over time

     

       653
       653
       +
       - Alert if table exceeds threshold (e.g., 100MB)

     

       654
       654
       +
       

     

       655
       655
       +
       **Files to Create:**

     

       656
       656
       +
       - `internal/core/unfurl/cleanup.go` - Background cleanup service

     

       657
       657
       +
       

     

       658
       658
       +
       **Related:**

     

       659
       659
       +
       - Implemented in oEmbed unfurling feature (2025-11-07)

     

       660
       660
       +
       - Cache table: [migration XXX_create_unfurl_cache.sql](../internal/db/migrations/)

     

       661
       661
       +
       

     

       662
       662
       +
       ---

     

       663
       663
       +
       

     

       604
       664
        
       ## 🔵 P3: Technical Debt

     

       605
       665
        
       

     

       606
       666
        
       ### Consolidate Environment Variable Validation

+348

scripts/generate_test_comments.go

···

       1
       1
       +
       package main

     

       2
       2
       +
       

     

       3
       3
       +
       import (

     

       4
       4
       +
       	"database/sql"

     

       5
       5
       +
       	"fmt"

     

       6
       6
       +
       	"log"

     

       7
       7
       +
       	"math/rand"

     

       8
       8
       +
       	"time"

     

       9
       9
       +
       

     

       10
       10
       +
       	_ "github.com/lib/pq"

     

       11
       11
       +
       )

     

       12
       12
       +
       

     

       13
       13
       +
       // Post URI: at://did:plc:hcuo3qx2lr7h7dquusbeobht/social.coves.community.post/3m4yohkzbkc2b

     

       14
       14
       +
       // Community DID: did:plc:hcuo3qx2lr7h7dquusbeobht

     

       15
       15
       +
       // Community Handle: test-usnews.community.coves.social

     

       16
       16
       +
       

     

       17
       17
       +
       const (

     

       18
       18
       +
       	postURI      = "at://did:plc:hcuo3qx2lr7h7dquusbeobht/social.coves.community.post/3m4yohkzbkc2b"

     

       19
       19
       +
       	postCID      = "bafyzohran123"

     

       20
       20
       +
       	communityDID = "did:plc:hcuo3qx2lr7h7dquusbeobht"

     

       21
       21
       +
       )

     

       22
       22
       +
       

     

       23
       23
       +
       type User struct {

     

       24
       24
       +
       	DID    string

     

       25
       25
       +
       	Handle string

     

       26
       26
       +
       	Name   string

     

       27
       27
       +
       }

     

       28
       28
       +
       

     

       29
       29
       +
       type Comment struct {

     

       30
       30
       +
       	URI       string

     

       31
       31
       +
       	CID       string

     

       32
       32
       +
       	RKey      string

     

       33
       33
       +
       	DID       string

     

       34
       34
       +
       	RootURI   string

     

       35
       35
       +
       	RootCID   string

     

       36
       36
       +
       	ParentURI string

     

       37
       37
       +
       	ParentCID string

     

       38
       38
       +
       	Content   string

     

       39
       39
       +
       	CreatedAt time.Time

     

       40
       40
       +
       }

     

       41
       41
       +
       

     

       42
       42
       +
       var userNames = []string{

     

       43
       43
       +
       	"sarah_jenkins", "michael_chen", "jessica_rodriguez", "david_nguyen",

     

       44
       44
       +
       	"emily_williams", "james_patel", "ashley_garcia", "robert_kim",

     

       45
       45
       +
       	"jennifer_lee", "william_martinez", "amanda_johnson", "daniel_brown",

     

       46
       46
       +
       	"melissa_davis", "christopher_wilson", "rebecca_anderson", "matthew_taylor",

     

       47
       47
       +
       	"laura_thomas", "anthony_moore", "stephanie_jackson", "joshua_white",

     

       48
       48
       +
       	"nicole_harris", "ryan_martin", "rachel_thompson", "kevin_garcia",

     

       49
       49
       +
       	"michelle_robinson", "brandon_clark", "samantha_lewis", "justin_walker",

     

       50
       50
       +
       	"kimberly_hall", "tyler_allen", "brittany_young", "andrew_king",

     

       51
       51
       +
       }

     

       52
       52
       +
       

     

       53
       53
       +
       var positiveComments = []string{

     

       54
       54
       +
       	"This is such fantastic news! Zohran represents real progressive values and I couldn't be happier with this outcome!",

     

       55
       55
       +
       	"Finally! A mayor who actually understands the needs of working families. This is a historic moment for NYC!",

     

       56
       56
       +
       	"What an incredible victory! Zohran's grassroots campaign shows that people power still matters in politics.",

     

       57
       57
       +
       	"I'm so proud of our city today. This win gives me hope for the future of progressive politics!",

     

       58
       58
       +
       	"This is exactly what NYC needed. Zohran's policies on housing and healthcare are going to transform our city!",

     

       59
       59
       +
       	"Congratulations to Zohran! His commitment to affordable housing is going to make such a difference.",

     

       60
       60
       +
       	"I've been following his campaign since day one and I'm thrilled to see him win. He truly deserves this!",

     

       61
       61
       +
       	"This victory is proof that authentic progressive candidates can win. So excited for what's ahead!",

     

       62
       62
       +
       	"Zohran's dedication to public transit and climate action is exactly what we need. Great day for NYC!",

     

       63
       63
       +
       	"What a momentous occasion! His policies on education are going to help so many families.",

     

       64
       64
       +
       	"I'm emotional reading this! Zohran gives me so much hope for the direction of our city.",

     

       65
       65
       +
       	"This is the change we've been waiting for! Can't wait to see his vision become reality.",

     

       66
       66
       +
       	"His campaign was inspiring from start to finish. This win is well-deserved!",

     

       67
       67
       +
       	"Finally, a mayor who will prioritize working people over corporate interests!",

     

       68
       68
       +
       	"The grassroots organizing that made this happen was incredible to witness. Democracy in action!",

     

       69
       69
       +
       	"Zohran's focus on social justice is refreshing. This is a win for all New Yorkers!",

     

       70
       70
       +
       	"I volunteered for his campaign and this victory means everything. So proud!",

     

       71
       71
       +
       	"This gives me faith in our democratic process. People-powered campaigns can still win!",

     

       72
       72
       +
       	"His policies on criminal justice reform are exactly what NYC needs right now.",

     

       73
       73
       +
       	"What an amazing day for progressive politics! Zohran is going to do great things.",

     

       74
       74
       +
       }

     

       75
       75
       +
       

     

       76
       76
       +
       var replyComments = []string{

     

       77
       77
       +
       	"Absolutely agree! This is going to be transformative.",

     

       78
       78
       +
       	"Couldn't have said it better myself!",

     

       79
       79
       +
       	"Yes! This is exactly right.",

     

       80
       80
       +
       	"100% this! So well said.",

     

       81
       81
       +
       	"This perfectly captures how I feel too!",

     

       82
       82
       +
       	"Exactly my thoughts! Great perspective.",

     

       83
       83
       +
       	"So true! I'm equally excited.",

     

       84
       84
       +
       	"Well put! I share your optimism.",

     

       85
       85
       +
       	"This! Absolutely this!",

     

       86
       86
       +
       	"I feel the same way! Great comment.",

     

       87
       87
       +
       	"You took the words right out of my mouth!",

     

       88
       88
       +
       	"Perfectly stated! I agree completely.",

     

       89
       89
       +
       	"Yes yes yes! This is it exactly.",

     

       90
       90
       +
       	"This is spot on! Thank you for sharing.",

     

       91
       91
       +
       	"I couldn't agree more with this take!",

     

       92
       92
       +
       	"Exactly what I was thinking! Well said.",

     

       93
       93
       +
       	"This captures it perfectly!",

     

       94
       94
       +
       	"So much this! Great comment.",

     

       95
       95
       +
       	"You nailed it! I feel exactly the same.",

     

       96
       96
       +
       	"This is the best take I've seen! Agreed!",

     

       97
       97
       +
       }

     

       98
       98
       +
       

     

       99
       99
       +
       var deepReplyComments = []string{

     

       100
       100
       +
       	"And it's not just about the policies, it's about the movement he's building!",

     

       101
       101
       +
       	"This thread is giving me life! So glad to see so many people excited.",

     

       102
       102
       +
       	"I love seeing all this positive energy! We're going to change NYC together.",

     

       103
       103
       +
       	"Reading these comments makes me even more hopeful. We did this!",

     

       104
       104
       +
       	"The solidarity in this thread is beautiful. This is what democracy looks like!",

     

       105
       105
       +
       	"I'm so grateful to be part of this community right now. Historic moment!",

     

       106
       106
       +
       	"This conversation shows how ready people are for real change.",

     

       107
       107
       +
       	"Seeing this support gives me so much hope for what's possible.",

     

       108
       108
       +
       	"This is the kind of energy we need to keep up! Let's go!",

     

       109
       109
       +
       	"I'm saving this thread to look back on this incredible moment.",

     

       110
       110
       +
       }

     

       111
       111
       +
       

     

       112
       112
       +
       func generateTID() string {

     

       113
       113
       +
       	// Simple TID generator for testing (timestamp in microseconds + random)

     

       114
       114
       +
       	now := time.Now().UnixMicro()

     

       115
       115
       +
       	return fmt.Sprintf("%d%04d", now, rand.Intn(10000))

     

       116
       116
       +
       }

     

       117
       117
       +
       

     

       118
       118
       +
       func createUser(db *sql.DB, handle, name string, idx int) (*User, error) {

     

       119
       119
       +
       	did := fmt.Sprintf("did:plc:testuser%d%d", time.Now().Unix(), idx)

     

       120
       120
       +
       	user := &User{

     

       121
       121
       +
       		DID:    did,

     

       122
       122
       +
       		Handle: handle,

     

       123
       123
       +
       		Name:   name,

     

       124
       124
       +
       	}

     

       125
       125
       +
       

     

       126
       126
       +
       	query := `

     

       127
       127
       +
       		INSERT INTO users (did, handle, pds_url, created_at, updated_at)

     

       128
       128
       +
       		VALUES ($1, $2, $3, NOW(), NOW())

     

       129
       129
       +
       		ON CONFLICT (did) DO NOTHING

     

       130
       130
       +
       	`

     

       131
       131
       +
       

     

       132
       132
       +
       	_, err := db.Exec(query, user.DID, user.Handle, "http://localhost:3001")

     

       133
       133
       +
       	if err != nil {

     

       134
       134
       +
       		return nil, fmt.Errorf("failed to create user: %w", err)

     

       135
       135
       +
       	}

     

       136
       136
       +
       

     

       137
       137
       +
       	log.Printf("Created user: %s (%s)", user.Handle, user.DID)

     

       138
       138
       +
       	return user, nil

     

       139
       139
       +
       }

     

       140
       140
       +
       

     

       141
       141
       +
       func createComment(db *sql.DB, user *User, content, parentURI, parentCID string, createdAt time.Time) (*Comment, error) {

     

       142
       142
       +
       	rkey := generateTID()

     

       143
       143
       +
       	uri := fmt.Sprintf("at://%s/social.coves.feed.comment/%s", user.DID, rkey)

     

       144
       144
       +
       	cid := fmt.Sprintf("bafy%s", rkey)

     

       145
       145
       +
       

     

       146
       146
       +
       	comment := &Comment{

     

       147
       147
       +
       		URI:       uri,

     

       148
       148
       +
       		CID:       cid,

     

       149
       149
       +
       		RKey:      rkey,

     

       150
       150
       +
       		DID:       user.DID,

     

       151
       151
       +
       		RootURI:   postURI,

     

       152
       152
       +
       		RootCID:   postCID,

     

       153
       153
       +
       		ParentURI: parentURI,

     

       154
       154
       +
       		ParentCID: parentCID,

     

       155
       155
       +
       		Content:   content,

     

       156
       156
       +
       		CreatedAt: createdAt,

     

       157
       157
       +
       	}

     

       158
       158
       +
       

     

       159
       159
       +
       	query := `

     

       160
       160
       +
       		INSERT INTO comments (

     

       161
       161
       +
       			uri, cid, rkey, commenter_did, root_uri, root_cid,

     

       162
       162
       +
       			parent_uri, parent_cid, content, created_at, indexed_at

     

       163
       163
       +
       		) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW())

     

       164
       164
       +
       		ON CONFLICT (uri) DO NOTHING

     

       165
       165
       +
       		RETURNING id

     

       166
       166
       +
       	`

     

       167
       167
       +
       

     

       168
       168
       +
       	var id int64

     

       169
       169
       +
       	err := db.QueryRow(query,

     

       170
       170
       +
       		comment.URI, comment.CID, comment.RKey, comment.DID,

     

       171
       171
       +
       		comment.RootURI, comment.RootCID, comment.ParentURI, comment.ParentCID,

     

       172
       172
       +
       		comment.Content, comment.CreatedAt,

     

       173
       173
       +
       	).Scan(&id)

     

       174
       174
       +
       	if err != nil {

     

       175
       175
       +
       		return nil, fmt.Errorf("failed to create comment: %w", err)

     

       176
       176
       +
       	}

     

       177
       177
       +
       

     

       178
       178
       +
       	log.Printf("Created comment by %s: %.50s...", user.Handle, content)

     

       179
       179
       +
       	return comment, nil

     

       180
       180
       +
       }

     

       181
       181
       +
       

     

       182
       182
       +
       func updateCommentCount(db *sql.DB, parentURI string, isPost bool) error {

     

       183
       183
       +
       	if isPost {

     

       184
       184
       +
       		_, err := db.Exec(`

     

       185
       185
       +
       			UPDATE posts

     

       186
       186
       +
       			SET comment_count = comment_count + 1

     

       187
       187
       +
       			WHERE uri = $1

     

       188
       188
       +
       		`, parentURI)

     

       189
       189
       +
       		return err

     

       190
       190
       +
       	}

     

       191
       191
       +
       

     

       192
       192
       +
       	_, err := db.Exec(`

     

       193
       193
       +
       		UPDATE comments

     

       194
       194
       +
       		SET reply_count = reply_count + 1

     

       195
       195
       +
       		WHERE uri = $1

     

       196
       196
       +
       	`, parentURI)

     

       197
       197
       +
       	return err

     

       198
       198
       +
       }

     

       199
       199
       +
       

     

       200
       200
       +
       func main() {

     

       201
       201
       +
       	// Connect to dev database

     

       202
       202
       +
       	dbURL := "postgres://dev_user:dev_password@localhost:5435/coves_dev?sslmode=disable"

     

       203
       203
       +
       	db, err := sql.Open("postgres", dbURL)

     

       204
       204
       +
       	if err != nil {

     

       205
       205
       +
       		log.Fatalf("Failed to connect to database: %v", err)

     

       206
       206
       +
       	}

     

       207
       207
       +
       	defer db.Close()

     

       208
       208
       +
       

     

       209
       209
       +
       	if err := db.Ping(); err != nil {

     

       210
       210
       +
       		log.Fatalf("Failed to ping database: %v", err)

     

       211
       211
       +
       	}

     

       212
       212
       +
       

     

       213
       213
       +
       	log.Println("Connected to database successfully!")

     

       214
       214
       +
       	log.Printf("Post URI: %s", postURI)

     

       215
       215
       +
       	log.Println("Starting to generate test data...")

     

       216
       216
       +
       

     

       217
       217
       +
       	rand.Seed(time.Now().UnixNano())

     

       218
       218
       +
       

     

       219
       219
       +
       	// Create users

     

       220
       220
       +
       	log.Println("\n=== Creating Users ===")

     

       221
       221
       +
       	users := make([]*User, 0, len(userNames))

     

       222
       222
       +
       	for i, name := range userNames {

     

       223
       223
       +
       		handle := fmt.Sprintf("%s.bsky.social", name)

     

       224
       224
       +
       		user, err := createUser(db, handle, name, i)

     

       225
       225
       +
       		if err != nil {

     

       226
       226
       +
       			log.Printf("Warning: Failed to create user %s: %v", name, err)

     

       227
       227
       +
       			continue

     

       228
       228
       +
       		}

     

       229
       229
       +
       		users = append(users, user)

     

       230
       230
       +
       	}

     

       231
       231
       +
       

     

       232
       232
       +
       	log.Printf("\nCreated %d users", len(users))

     

       233
       233
       +
       

     

       234
       234
       +
       	// Generate comments with varied timing

     

       235
       235
       +
       	log.Println("\n=== Creating Top-Level Comments ===")

     

       236
       236
       +
       	baseTime := time.Now().Add(-2 * time.Hour) // Comments from 2 hours ago

     

       237
       237
       +
       	topLevelComments := make([]*Comment, 0)

     

       238
       238
       +
       

     

       239
       239
       +
       	// Create 15-20 top-level comments

     

       240
       240
       +
       	numTopLevel := 15 + rand.Intn(6)

     

       241
       241
       +
       	for i := 0; i < numTopLevel && i < len(users); i++ {

     

       242
       242
       +
       		user := users[i]

     

       243
       243
       +
       		content := positiveComments[i%len(positiveComments)]

     

       244
       244
       +
       		createdAt := baseTime.Add(time.Duration(i*5+rand.Intn(3)) * time.Minute)

     

       245
       245
       +
       

     

       246
       246
       +
       		comment, err := createComment(db, user, content, postURI, postCID, createdAt)

     

       247
       247
       +
       		if err != nil {

     

       248
       248
       +
       			log.Printf("Warning: Failed to create top-level comment: %v", err)

     

       249
       249
       +
       			continue

     

       250
       250
       +
       		}

     

       251
       251
       +
       

     

       252
       252
       +
       		topLevelComments = append(topLevelComments, comment)

     

       253
       253
       +
       

     

       254
       254
       +
       		// Update post comment count

     

       255
       255
       +
       		if err := updateCommentCount(db, postURI, true); err != nil {

     

       256
       256
       +
       			log.Printf("Warning: Failed to update post comment count: %v", err)

     

       257
       257
       +
       		}

     

       258
       258
       +
       

     

       259
       259
       +
       		// Small delay to avoid timestamp collisions

     

       260
       260
       +
       		time.Sleep(10 * time.Millisecond)

     

       261
       261
       +
       	}

     

       262
       262
       +
       

     

       263
       263
       +
       	log.Printf("Created %d top-level comments", len(topLevelComments))

     

       264
       264
       +
       

     

       265
       265
       +
       	// Create first-level replies (replies to top-level comments)

     

       266
       266
       +
       	log.Println("\n=== Creating First-Level Replies ===")

     

       267
       267
       +
       	firstLevelReplies := make([]*Comment, 0)

     

       268
       268
       +
       

     

       269
       269
       +
       	for i, parentComment := range topLevelComments {

     

       270
       270
       +
       		// 60% chance of having replies

     

       271
       271
       +
       		if rand.Float64() > 0.6 {

     

       272
       272
       +
       			continue

     

       273
       273
       +
       		}

     

       274
       274
       +
       

     

       275
       275
       +
       		// 1-3 replies per comment

     

       276
       276
       +
       		numReplies := 1 + rand.Intn(3)

     

       277
       277
       +
       		for j := 0; j < numReplies; j++ {

     

       278
       278
       +
       			userIdx := (i*3 + j + len(topLevelComments)) % len(users)

     

       279
       279
       +
       			user := users[userIdx]

     

       280
       280
       +
       			content := replyComments[rand.Intn(len(replyComments))]

     

       281
       281
       +
       			createdAt := parentComment.CreatedAt.Add(time.Duration(5+rand.Intn(10)) * time.Minute)

     

       282
       282
       +
       

     

       283
       283
       +
       			comment, err := createComment(db, user, content, parentComment.URI, parentComment.CID, createdAt)

     

       284
       284
       +
       			if err != nil {

     

       285
       285
       +
       				log.Printf("Warning: Failed to create first-level reply: %v", err)

     

       286
       286
       +
       				continue

     

       287
       287
       +
       			}

     

       288
       288
       +
       

     

       289
       289
       +
       			firstLevelReplies = append(firstLevelReplies, comment)

     

       290
       290
       +
       

     

       291
       291
       +
       			// Update parent comment reply count

     

       292
       292
       +
       			if err := updateCommentCount(db, parentComment.URI, false); err != nil {

     

       293
       293
       +
       				log.Printf("Warning: Failed to update comment reply count: %v", err)

     

       294
       294
       +
       			}

     

       295
       295
       +
       

     

       296
       296
       +
       			time.Sleep(10 * time.Millisecond)

     

       297
       297
       +
       		}

     

       298
       298
       +
       	}

     

       299
       299
       +
       

     

       300
       300
       +
       	log.Printf("Created %d first-level replies", len(firstLevelReplies))

     

       301
       301
       +
       

     

       302
       302
       +
       	// Create second-level replies (replies to replies) - testing nested threading

     

       303
       303
       +
       	log.Println("\n=== Creating Second-Level Replies ===")

     

       304
       304
       +
       	secondLevelCount := 0

     

       305
       305
       +
       

     

       306
       306
       +
       	for i, parentComment := range firstLevelReplies {

     

       307
       307
       +
       		// 40% chance of having deep replies

     

       308
       308
       +
       		if rand.Float64() > 0.4 {

     

       309
       309
       +
       			continue

     

       310
       310
       +
       		}

     

       311
       311
       +
       

     

       312
       312
       +
       		// 1-2 deep replies

     

       313
       313
       +
       		numReplies := 1 + rand.Intn(2)

     

       314
       314
       +
       		for j := 0; j < numReplies; j++ {

     

       315
       315
       +
       			userIdx := (i*2 + j + len(topLevelComments) + len(firstLevelReplies)) % len(users)

     

       316
       316
       +
       			user := users[userIdx]

     

       317
       317
       +
       			content := deepReplyComments[rand.Intn(len(deepReplyComments))]

     

       318
       318
       +
       			createdAt := parentComment.CreatedAt.Add(time.Duration(3+rand.Intn(7)) * time.Minute)

     

       319
       319
       +
       

     

       320
       320
       +
       			_, err := createComment(db, user, content, parentComment.URI, parentComment.CID, createdAt)

     

       321
       321
       +
       			if err != nil {

     

       322
       322
       +
       				log.Printf("Warning: Failed to create second-level reply: %v", err)

     

       323
       323
       +
       				continue

     

       324
       324
       +
       			}

     

       325
       325
       +
       

     

       326
       326
       +
       			secondLevelCount++

     

       327
       327
       +
       

     

       328
       328
       +
       			// Update parent comment reply count

     

       329
       329
       +
       			if err := updateCommentCount(db, parentComment.URI, false); err != nil {

     

       330
       330
       +
       				log.Printf("Warning: Failed to update comment reply count: %v", err)

     

       331
       331
       +
       			}

     

       332
       332
       +
       

     

       333
       333
       +
       			time.Sleep(10 * time.Millisecond)

     

       334
       334
       +
       		}

     

       335
       335
       +
       	}

     

       336
       336
       +
       

     

       337
       337
       +
       	log.Printf("Created %d second-level replies", secondLevelCount)

     

       338
       338
       +
       

     

       339
       339
       +
       	// Print summary

     

       340
       340
       +
       	totalComments := len(topLevelComments) + len(firstLevelReplies) + secondLevelCount

     

       341
       341
       +
       	log.Println("\n=== Summary ===")

     

       342
       342
       +
       	log.Printf("Total users created: %d", len(users))

     

       343
       343
       +
       	log.Printf("Total comments created: %d", totalComments)

     

       344
       344
       +
       	log.Printf("  - Top-level comments: %d", len(topLevelComments))

     

       345
       345
       +
       	log.Printf("  - First-level replies: %d", len(firstLevelReplies))

     

       346
       346
       +
       	log.Printf("  - Second-level replies: %d", secondLevelCount)

     

       347
       347
       +
       	log.Println("\nDone! Check the post at !test-usnews for the comments.")

     

       348
       348
       +
       }

+134

scripts/post_streamable.py

···

       1
       1
       +
       #!/usr/bin/env python3

     

       2
       2
       +
       """

     

       3
       3
       +
       Quick script to post a Streamable video to test-usnews community.

     

       4
       4
       +
       Uses the kagi-news CovesClient infrastructure.

     

       5
       5
       +
       """

     

       6
       6
       +
       

     

       7
       7
       +
       import sys

     

       8
       8
       +
       import os

     

       9
       9
       +
       

     

       10
       10
       +
       # Add kagi-news src to path to use CovesClient

     

       11
       11
       +
       sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../aggregators/kagi-news'))

     

       12
       12
       +
       

     

       13
       13
       +
       from src.coves_client import CovesClient

     

       14
       14
       +
       

     

       15
       15
       +
       def main():

     

       16
       16
       +
           # Configuration

     

       17
       17
       +
           COVES_API_URL = "http://localhost:8081"

     

       18
       18
       +
           PDS_URL = "http://localhost:3001"

     

       19
       19
       +
       

     

       20
       20
       +
           # Use PDS instance credentials (from .env.dev)

     

       21
       21
       +
           HANDLE = "testuser123.local.coves.dev"

     

       22
       22
       +
           PASSWORD = "test-password-123"

     

       23
       23
       +
       

     

       24
       24
       +
           # Post details

     

       25
       25
       +
           COMMUNITY_HANDLE = "test-usnews.community.coves.social"

     

       26
       26
       +
       

     

       27
       27
       +
           # Post 1: Streamable video

     

       28
       28
       +
           STREAMABLE_URL = "https://streamable.com/7kpdft"

     

       29
       29
       +
           STREAMABLE_TITLE = "NBACentral - \"Your son don't wanna be here, we know it's your last weekend. Enjoy ..."

     

       30
       30
       +
       

     

       31
       31
       +
           # Post 2: Reddit highlight

     

       32
       32
       +
           REDDIT_URL = "https://www.reddit.com/r/nba/comments/1orfsgm/highlight_giannis_antetokounmpo_41_pts_15_reb_9/"

     

       33
       33
       +
           REDDIT_TITLE = "[Highlight] Giannis Antetokounmpo (41 PTS, 15 REB, 9 AST) tallies his 56th career regular season game of 40+ points, passing Kareem Abdul-Jabbar for the most such games in franchise history. Milwaukee defeats Chicago 126-110 to win their NBA Cup opener."

     

       34
       34
       +
       

     

       35
       35
       +
           # Initialize client

     

       36
       36
       +
           print(f"Initializing Coves client...")

     

       37
       37
       +
           print(f"  API URL: {COVES_API_URL}")

     

       38
       38
       +
           print(f"  PDS URL: {PDS_URL}")

     

       39
       39
       +
           print(f"  Handle: {HANDLE}")

     

       40
       40
       +
       

     

       41
       41
       +
           client = CovesClient(

     

       42
       42
       +
               api_url=COVES_API_URL,

     

       43
       43
       +
               handle=HANDLE,

     

       44
       44
       +
               password=PASSWORD,

     

       45
       45
       +
               pds_url=PDS_URL

     

       46
       46
       +
           )

     

       47
       47
       +
       

     

       48
       48
       +
           # Authenticate

     

       49
       49
       +
           print("\nAuthenticating...")

     

       50
       50
       +
           try:

     

       51
       51
       +
               client.authenticate()

     

       52
       52
       +
               print(f"✓ Authenticated as {client.did}")

     

       53
       53
       +
           except Exception as e:

     

       54
       54
       +
               print(f"✗ Authentication failed: {e}")

     

       55
       55
       +
               return 1

     

       56
       56
       +
       

     

       57
       57
       +
           # Post 1: Streamable video

     

       58
       58
       +
           print("\n" + "="*60)

     

       59
       59
       +
           print("POST 1: STREAMABLE VIDEO")

     

       60
       60
       +
           print("="*60)

     

       61
       61
       +
       

     

       62
       62
       +
           print("\nCreating minimal external embed (URI only)...")

     

       63
       63
       +
           streamable_embed = {

     

       64
       64
       +
               "$type": "social.coves.embed.external",

     

       65
       65
       +
               "external": {

     

       66
       66
       +
                   "uri": STREAMABLE_URL

     

       67
       67
       +
               }

     

       68
       68
       +
           }

     

       69
       69
       +
           print(f"✓ Embed created with URI only (unfurl service should enrich)")

     

       70
       70
       +
       

     

       71
       71
       +
           print(f"\nPosting to {COMMUNITY_HANDLE}...")

     

       72
       72
       +
           print(f"  Title: {STREAMABLE_TITLE}")

     

       73
       73
       +
           print(f"  Video: {STREAMABLE_URL}")

     

       74
       74
       +
       

     

       75
       75
       +
           try:

     

       76
       76
       +
               post_uri = client.create_post(

     

       77
       77
       +
                   community_handle=COMMUNITY_HANDLE,

     

       78
       78
       +
                   title=STREAMABLE_TITLE,

     

       79
       79
       +
                   content="",

     

       80
       80
       +
                   facets=[],

     

       81
       81
       +
                   embed=streamable_embed

     

       82
       82
       +
               )

     

       83
       83
       +
       

     

       84
       84
       +
               print(f"\n✓ Streamable post created successfully!")

     

       85
       85
       +
               print(f"  URI: {post_uri}")

     

       86
       86
       +
       

     

       87
       87
       +
           except Exception as e:

     

       88
       88
       +
               print(f"\n✗ Streamable post creation failed: {e}")

     

       89
       89
       +
               import traceback

     

       90
       90
       +
               traceback.print_exc()

     

       91
       91
       +
               return 1

     

       92
       92
       +
       

     

       93
       93
       +
           # Post 2: Reddit highlight

     

       94
       94
       +
           print("\n" + "="*60)

     

       95
       95
       +
           print("POST 2: REDDIT HIGHLIGHT")

     

       96
       96
       +
           print("="*60)

     

       97
       97
       +
       

     

       98
       98
       +
           print("\nCreating minimal external embed (URI only)...")

     

       99
       99
       +
           reddit_embed = {

     

       100
       100
       +
               "$type": "social.coves.embed.external",

     

       101
       101
       +
               "external": {

     

       102
       102
       +
                   "uri": REDDIT_URL

     

       103
       103
       +
               }

     

       104
       104
       +
           }

     

       105
       105
       +
           print(f"✓ Embed created with URI only (unfurl service should enrich)")

     

       106
       106
       +
       

     

       107
       107
       +
           print(f"\nPosting to {COMMUNITY_HANDLE}...")

     

       108
       108
       +
           print(f"  Title: {REDDIT_TITLE}")

     

       109
       109
       +
           print(f"  URL: {REDDIT_URL}")

     

       110
       110
       +
       

     

       111
       111
       +
           try:

     

       112
       112
       +
               post_uri = client.create_post(

     

       113
       113
       +
                   community_handle=COMMUNITY_HANDLE,

     

       114
       114
       +
                   title=REDDIT_TITLE,

     

       115
       115
       +
                   content="",

     

       116
       116
       +
                   facets=[],

     

       117
       117
       +
                   embed=reddit_embed

     

       118
       118
       +
               )

     

       119
       119
       +
       

     

       120
       120
       +
               print(f"\n✓ Reddit post created successfully!")

     

       121
       121
       +
               print(f"  URI: {post_uri}")

     

       122
       122
       +
               print(f"\n" + "="*60)

     

       123
       123
       +
               print("Both posts created! Check them out at !test-usnews")

     

       124
       124
       +
               print("="*60)

     

       125
       125
       +
               return 0

     

       126
       126
       +
       

     

       127
       127
       +
           except Exception as e:

     

       128
       128
       +
               print(f"\n✗ Reddit post creation failed: {e}")

     

       129
       129
       +
               import traceback

     

       130
       130
       +
               traceback.print_exc()

     

       131
       131
       +
               return 1

     

       132
       132
       +
       

     

       133
       133
       +
       if __name__ == "__main__":

     

       134
       134
       +
           sys.exit(main())