commit f708c525413c58689fb435742788626a30130f0a · anil.recoil.org/thicket

+28 -25

ARCH.md

···

       157
       157
        
       git-store/

     

       158
       158
        
       ├── index.json              # User directory index

     

       159
       159
        
       ├── duplicates.json         # Manual curation of duplicate entries

     

       160
       160
       -
       ├── links.json              # All outbound links categorized by type

     

       161
       161
       -
       ├── references.json         # Cross-reference index for threading

     

       160
       160
       +
       ├── links.json              # Unified links, references, and mapping data

     

       162
       161
        
       ├── user1/

     

       163
       162
        
       │   ├── entry_id_1.json     # Sanitized entry files

     

       164
       163
        
       │   ├── entry_id_2.json

     
···

       403
       402
        
       

     

       404
       403
        
       ### Data Structures

     

       405
       404
        
       

     

       406
       406
       -
       #### links.json Format

     

       405
       405
       +
       #### links.json Format (Unified Structure)

     

       407
       406
        
       ```json

     

       408
       407
        
       {

     

       409
       409
       -
         "links": [

     

       410
       410
       -
           {

     

       411
       411
       -
             "url": "https://example.com/post/123",

     

       412
       412
       -
             "entry_id": "https://blog.user.com/entry/456",

     

       413
       413
       -
             "username": "user1",

     

       414
       414
       -
             "context": "As mentioned in this post...",

     

       415
       415
       -
             "category": "user",

     

       408
       408
       +
         "links": {

     

       409
       409
       +
           "https://example.com/post/123": {

     

       410
       410
       +
             "referencing_entries": ["https://blog.user.com/entry/456"],

     

       411
       411
       +
             "is_tracked_post": true,

     

       416
       412
        
             "target_username": "user2"

     

       413
       413
       +
           },

     

       414
       414
       +
           "https://external-site.com/article": {

     

       415
       415
       +
             "referencing_entries": ["https://blog.user.com/entry/789"],

     

       416
       416
       +
             "is_tracked_post": false

     

       417
       417
        
           }

     

       418
       418
       -
         ],

     

       419
       419
       -
         "categories": {

     

       420
       420
       -
           "internal": 1234,

     

       421
       421
       -
           "user": 456,

     

       422
       422
       -
           "unknown": 7890

     

       423
       418
        
         },

     

       424
       424
       -
         "user_domains": {

     

       425
       425
       -
           "user1": ["blog.user.com", "user.com"],

     

       426
       426
       -
           "user2": ["example.com"]

     

       427
       427
       -
         }

     

       428
       428
       -
       }

     

       429
       429
       -
       ```

     

       430
       430
       -
       

     

       431
       431
       -
       #### references.json Format

     

       432
       432
       -
       ```json

     

       433
       433
       -
       {

     

       419
       419
       +
         "reverse_mapping": {

     

       420
       420
       +
           "https://blog.user.com/entry/456": ["https://example.com/post/123"],

     

       421
       421
       +
           "https://blog.user.com/entry/789": ["https://external-site.com/article"]

     

       422
       422
       +
         },

     

       434
       423
        
         "references": [

     

       435
       424
        
           {

     

       436
       425
        
             "source_entry_id": "https://blog.user.com/entry/456",

     
···

       447
       436
        
         }

     

       448
       437
        
       }

     

       449
       438
        
       ```

     

       439
       439
       +
       

     

       440
       440
       +
       This unified structure eliminates duplication by:

     

       441
       441
       +
       - Storing each URL only once with metadata flags

     

       442
       442
       +
       - Including all link data, reference data, and mappings in one file

     

       443
       443
       +
       - Using `is_tracked_post` to identify internal vs external links

     

       444
       444
       +
       - Providing bidirectional mappings for efficient queries

     

       445
       445
       +
       

     

       446
       446
       +
       ### Unified Structure Benefits

     

       447
       447
       +
       

     

       448
       448
       +
       - **Eliminates Duplication**: Each URL appears only once with metadata

     

       449
       449
       +
       - **Single Source of Truth**: All link-related data in one file

     

       450
       450
       +
       - **Efficient Queries**: Fast lookups for both directions (URL→entries, entry→URLs)

     

       451
       451
       +
       - **Atomic Updates**: All link data changes together

     

       452
       452
       +
       - **Reduced I/O**: Fewer file operations

     

       450
       453
        
       

     

       451
       454
        
       ### Implementation Benefits

     

       452
       455

+42 -11

src/thicket/cli/commands/index_cmd.py

···

       35
       35
        
               None,

     

       36
       36
        
               "--output",

     

       37
       37
        
               "-o",

     

       38
       38
       -
               help="Path to output index file (default: references.json in git store)",

     

       38
       38
       +
               help="Path to output index file (default: updates links.json in git store)",

     

       39
       39
        
           ),

     

       40
       40
        
           verbose: bool = typer.Option(

     

       41
       41
        
               False,

     
···

       49
       49
        
           This command analyzes all blog entries to detect cross-references between

     

       50
       50
        
           different blogs, creating an index that can be used to build threaded

     

       51
       51
        
           views of related content.

     

       52
       52
       +
           

     

       53
       53
       +
           Updates the unified links.json file with reference data.

     

       52
       54
        
           """

     

       53
       55
        
           try:

     

       54
       56
        
               # Load configuration

     
···

       154
       156
        
               if output_file:

     

       155
       157
        
                   output_path = output_file

     

       156
       158
        
               else:

     

       157
       157
       -
                   output_path = config.git_store / "references.json"

     

       159
       159
       +
                   output_path = config.git_store / "links.json"

     

       160
       160
       +
       

     

       161
       161
       +
               # Load existing links data or create new structure

     

       162
       162
       +
               if output_path.exists() and not output_file:

     

       163
       163
       +
                   # Load existing unified structure

     

       164
       164
       +
                   with open(output_path) as f:

     

       165
       165
       +
                       existing_data = json.load(f)

     

       166
       166
       +
               else:

     

       167
       167
       +
                   # Create new structure

     

       168
       168
       +
                   existing_data = {

     

       169
       169
       +
                       "links": {},

     

       170
       170
       +
                       "reverse_mapping": {},

     

       171
       171
       +
                       "user_domains": {}

     

       172
       172
       +
                   }

     

       173
       173
       +
               

     

       174
       174
       +
               # Update with reference data

     

       175
       175
       +
               existing_data["references"] = ref_index.to_dict()["references"]

     

       176
       176
       +
               existing_data["user_domains"] = {k: list(v) for k, v in user_domains.items()}

     

       158
       177
        
       

     

       159
       159
       -
               # Save reference index

     

       178
       178
       +
               # Save updated structure

     

       160
       179
        
               with open(output_path, "w") as f:

     

       161
       161
       -
                   json.dump(ref_index.to_dict(), f, indent=2, default=str)

     

       180
       180
       +
                   json.dump(existing_data, f, indent=2, default=str)

     

       162
       181
        
       

     

       163
       182
        
               # Show summary

     

       164
       183
        
               if not get_tsv_mode():

     
···

       248
       267
        
               None,

     

       249
       268
        
               "--index",

     

       250
       269
        
               "-i",

     

       251
       251
       -
               help="Path to reference index file (default: references.json in git store)",

     

       270
       270
       +
               help="Path to reference index file (default: links.json in git store)",

     

       252
       271
        
           ),

     

       253
       272
        
           username: Optional[str] = typer.Option(

     

       254
       273
        
               None,

     
···

       274
       293
        
           This command uses the reference index to show which blog entries

     

       275
       294
        
           are connected through cross-references, creating an email-style

     

       276
       295
        
           threaded view of the conversation.

     

       296
       296
       +
           

     

       297
       297
       +
           Reads reference data from the unified links.json file.

     

       277
       298
        
           """

     

       278
       299
        
           try:

     

       279
       300
        
               # Load configuration

     
···

       283
       304
        
               if index_file:

     

       284
       305
        
                   index_path = index_file

     

       285
       306
        
               else:

     

       286
       286
       -
                   index_path = config.git_store / "references.json"

     

       307
       307
       +
                   index_path = config.git_store / "links.json"

     

       287
       308
        
       

     

       288
       309
        
               if not index_path.exists():

     

       289
       289
       -
                   console.print(f"[red]Reference index not found: {index_path}[/red]")

     

       290
       290
       -
                   console.print("Run 'thicket index' first to build the reference index")

     

       310
       310
       +
                   console.print(f"[red]Links file not found: {index_path}[/red]")

     

       311
       311
       +
                   console.print("Run 'thicket links' and 'thicket index' first to build the reference index")

     

       291
       312
        
                   raise typer.Exit(1)

     

       292
       313
        
       

     

       293
       293
       -
               # Load reference index

     

       314
       314
       +
               # Load unified data

     

       294
       315
        
               with open(index_path) as f:

     

       295
       295
       -
                   index_data = json.load(f)

     

       316
       316
       +
                   unified_data = json.load(f)

     

       317
       317
       +
       

     

       318
       318
       +
               # Check if references exist in the unified structure

     

       319
       319
       +
               if "references" not in unified_data:

     

       320
       320
       +
                   console.print(f"[red]No references found in {index_path}[/red]")

     

       321
       321
       +
                   console.print("Run 'thicket index' first to build the reference index")

     

       322
       322
       +
                   raise typer.Exit(1)

     

       296
       323
        
       

     

       297
       297
       -
               ref_index = ReferenceIndex.from_dict(index_data)

     

       324
       324
       +
               # Extract reference data and reconstruct ReferenceIndex

     

       325
       325
       +
               ref_index = ReferenceIndex.from_dict({

     

       326
       326
       +
                   "references": unified_data["references"],

     

       327
       327
       +
                   "user_domains": unified_data.get("user_domains", {})

     

       328
       328
       +
               })

     

       298
       329
        
       

     

       299
       330
        
               # Initialize Git store to get entry details

     

       300
       331
        
               git_store = GitStore(config.git_store)

+12 -6

src/thicket/cli/commands/info_cmd.py

···

       106
       106
        
                   raise typer.Exit(1)

     

       107
       107
        
               

     

       108
       108
        
               # Load reference index if available

     

       109
       109
       -
               references_path = config.git_store / "references.json"

     

       109
       109
       +
               links_path = config.git_store / "links.json"

     

       110
       110
        
               ref_index = None

     

       111
       111
       -
               if references_path.exists():

     

       112
       112
       -
                   with open(references_path) as f:

     

       113
       113
       -
                       index_data = json.load(f)

     

       114
       114
       -
                   ref_index = ReferenceIndex.from_dict(index_data)

     

       111
       111
       +
               if links_path.exists():

     

       112
       112
       +
                   with open(links_path) as f:

     

       113
       113
       +
                       unified_data = json.load(f)

     

       114
       114
       +
                   

     

       115
       115
       +
                   # Check if references exist in the unified structure

     

       116
       116
       +
                   if "references" in unified_data:

     

       117
       117
       +
                       ref_index = ReferenceIndex.from_dict({

     

       118
       118
       +
                           "references": unified_data["references"],

     

       119
       119
       +
                           "user_domains": unified_data.get("user_domains", {})

     

       120
       120
       +
                       })

     

       115
       121
        
               

     

       116
       122
        
               # Display information

     

       117
       123
        
               if get_tsv_mode():

     
···

       122
       128
        
                   if ref_index:

     

       123
       129
        
                       _display_link_info(entry, found_username, ref_index)

     

       124
       130
        
                   else:

     

       125
       125
       -
                       console.print("\n[yellow]No reference index found. Run 'thicket index' to build cross-reference data.[/yellow]")

     

       131
       131
       +
                       console.print("\n[yellow]No reference index found. Run 'thicket links' and 'thicket index' to build cross-reference data.[/yellow]")

     

       126
       132
        
                   

     

       127
       133
        
                   # Optionally display content

     

       128
       134
        
                   if show_content and entry.content:

+45 -44

src/thicket/cli/commands/links_cmd.py

···

       145
       145
        
               None,

     

       146
       146
        
               "--output",

     

       147
       147
        
               "-o",

     

       148
       148
       -
               help="Path to output links file (default: links.json in git store)",

     

       149
       149
       -
           ),

     

       150
       150
       -
           mapping_file: Optional[Path] = typer.Option(

     

       151
       151
       -
               None,

     

       152
       152
       -
               "--mapping",

     

       153
       153
       -
               "-m",

     

       154
       154
       -
               help="Path to output URL <-> atom ID mapping file (default: url_mapping.json in git store)",

     

       148
       148
       +
               help="Path to output unified links file (default: links.json in git store)",

     

       155
       149
        
           ),

     

       156
       150
        
           verbose: bool = typer.Option(

     

       157
       151
        
               False,

     
···

       165
       159
        
           This command analyzes all blog entries to extract outbound links,

     

       166
       160
        
           resolve them properly with respect to the feed's base URL, and

     

       167
       161
        
           categorize them as internal, user, or unknown links.

     

       162
       162
       +
           

     

       163
       163
       +
           Creates a unified links.json file containing all link data.

     

       168
       164
        
           """

     

       169
       165
        
           try:

     

       170
       166
        
               # Load configuration

     
···

       290
       286
        
                           if verbose and entry_links:

     

       291
       287
        
                               console.print(f"  Found {len(entry_links)} links in {username}:{entry.title[:50]}...")

     

       292
       288
        
               

     

       293
       293
       -
               # Determine output paths

     

       289
       289
       +
               # Determine output path

     

       294
       290
        
               if output_file:

     

       295
       291
        
                   output_path = output_file

     

       296
       292
        
               else:

     

       297
       293
        
                   output_path = config.git_store / "links.json"

     

       298
       298
       -
               

     

       299
       299
       -
               if mapping_file:

     

       300
       300
       -
                   mapping_path = mapping_file

     

       301
       301
       -
               else:

     

       302
       302
       -
                   mapping_path = config.git_store / "url_mapping.json"

     

       303
       294
        
               

     

       304
       295
        
               # Save all extracted links (not just filtered ones)

     

       305
       296
        
               if verbose:

     
···

       324
       315
        
                               if hasattr(link, 'href') and link.href:

     

       325
       316
        
                                   registered_urls.add(str(link.href))

     

       326
       317
        
               

     

       327
       327
       -
               # Create filtered version for URL mapping (only links to registered posts)

     

       328
       328
       -
               filtered_link_dict = {}

     

       329
       329
       -
               filtered_reverse_dict = {}

     

       318
       318
       +
               # Build unified structure with metadata

     

       319
       319
       +
               unified_links = {}

     

       320
       320
       +
               reverse_mapping = {}

     

       330
       321
        
               

     

       331
       322
        
               for url, entry_ids in link_dict.items():

     

       332
       332
       -
                   if url in registered_urls:

     

       333
       333
       -
                       filtered_link_dict[url] = entry_ids

     

       334
       334
       -
                       

     

       335
       335
       -
                       # Also update reverse mapping

     

       336
       336
       -
                       for entry_id in entry_ids:

     

       337
       337
       -
                           if entry_id not in filtered_reverse_dict:

     

       338
       338
       -
                               filtered_reverse_dict[entry_id] = []

     

       339
       339
       -
                           if url not in filtered_reverse_dict[entry_id]:

     

       340
       340
       -
                               filtered_reverse_dict[entry_id].append(url)

     

       323
       323
       +
                   is_tracked = url in registered_urls

     

       324
       324
       +
                   target_username = None

     

       325
       325
       +
                   

     

       326
       326
       +
                   # Find target username if this is a tracked post

     

       327
       327
       +
                   if is_tracked:

     

       328
       328
       +
                       for username in users:

     

       329
       329
       +
                           user_domains_set = {domain for domain in user_domains.get(username, [])}

     

       330
       330
       +
                           if any(domain in url for domain in user_domains_set):

     

       331
       331
       +
                               target_username = username

     

       332
       332
       +
                               break

     

       333
       333
       +
                   

     

       334
       334
       +
                   unified_links[url] = {

     

       335
       335
       +
                       "referencing_entries": entry_ids,

     

       336
       336
       +
                       "is_tracked_post": is_tracked

     

       337
       337
       +
                   }

     

       338
       338
       +
                   

     

       339
       339
       +
                   if target_username:

     

       340
       340
       +
                       unified_links[url]["target_username"] = target_username

     

       341
       341
       +
                   

     

       342
       342
       +
                   # Build reverse mapping

     

       343
       343
       +
                   for entry_id in entry_ids:

     

       344
       344
       +
                       if entry_id not in reverse_mapping:

     

       345
       345
       +
                           reverse_mapping[entry_id] = []

     

       346
       346
       +
                       if url not in reverse_mapping[entry_id]:

     

       347
       347
       +
                           reverse_mapping[entry_id].append(url)

     

       341
       348
        
               

     

       342
       342
       -
               # Use all links for main output, not filtered ones

     

       343
       343
       -
               output_data = link_dict

     

       349
       349
       +
               # Create unified output data

     

       350
       350
       +
               output_data = {

     

       351
       351
       +
                   "links": unified_links,

     

       352
       352
       +
                   "reverse_mapping": reverse_mapping,

     

       353
       353
       +
                   "user_domains": {k: list(v) for k, v in user_domains.items()}

     

       354
       354
       +
               }

     

       344
       355
        
               

     

       345
       356
        
               if verbose:

     

       346
       357
        
                   console.print(f"Found {len(registered_urls)} registered post URLs")

     

       347
       347
       -
                   console.print(f"Found {len(link_dict)} total links, {len(filtered_link_dict)} links to registered posts")

     

       358
       358
       +
                   console.print(f"Found {len(link_dict)} total links, {sum(1 for link in unified_links.values() if link['is_tracked_post'])} tracked posts")

     

       348
       359
        
               

     

       349
       349
       -
               # Save links data (URL -> atom ID mapping, all links)

     

       360
       360
       +
               # Save unified data

     

       350
       361
        
               with open(output_path, "w") as f:

     

       351
       362
        
                   json.dump(output_data, f, indent=2, default=str)

     

       352
       352
       -
               

     

       353
       353
       -
               # Save bidirectional mapping file (filtered)

     

       354
       354
       -
               mapping_data = {

     

       355
       355
       -
                   "url_to_atom": filtered_link_dict,

     

       356
       356
       -
                   "atom_to_urls": filtered_reverse_dict

     

       357
       357
       -
               }

     

       358
       358
       -
               

     

       359
       359
       -
               with open(mapping_path, "w") as f:

     

       360
       360
       -
                   json.dump(mapping_data, f, indent=2, default=str)

     

       361
       363
        
               

     

       362
       364
        
               # Show summary

     

       363
       365
        
               if not get_tsv_mode():

     
···

       370
       372
        
                   print(f"User\t{len(link_categories['user'])}\tLinks to other tracked users")

     

       371
       373
        
                   print(f"Unknown\t{len(link_categories['unknown'])}\tLinks to external sites")

     

       372
       374
        
                   print(f"Total Extracted\t{len(all_links)}\tAll extracted links")

     

       373
       373
       -
                   print(f"Saved to Output\t{len(output_data)}\tLinks saved to output file")

     

       374
       374
       -
                   print(f"Cross-references\t{len(filtered_link_dict)}\tLinks to registered posts only")

     

       375
       375
       +
                   print(f"Saved to Output\t{len(output_data['links'])}\tLinks saved to output file")

     

       376
       376
       +
                   print(f"Cross-references\t{sum(1 for link in unified_links.values() if link['is_tracked_post'])}\tLinks to registered posts only")

     

       375
       377
        
               else:

     

       376
       378
        
                   table = Table(title="Links Summary")

     

       377
       379
        
                   table.add_column("Category", style="cyan")

     
···

       382
       384
        
                   table.add_row("User", str(len(link_categories["user"])), "Links to other tracked users")

     

       383
       385
        
                   table.add_row("Unknown", str(len(link_categories["unknown"])), "Links to external sites")

     

       384
       386
        
                   table.add_row("Total Extracted", str(len(all_links)), "All extracted links")

     

       385
       385
       -
                   table.add_row("Saved to Output", str(len(output_data)), "Links saved to output file")

     

       386
       386
       -
                   table.add_row("Cross-references", str(len(filtered_link_dict)), "Links to registered posts only")

     

       387
       387
       +
                   table.add_row("Saved to Output", str(len(output_data['links'])), "Links saved to output file")

     

       388
       388
       +
                   table.add_row("Cross-references", str(sum(1 for link in unified_links.values() if link['is_tracked_post'])), "Links to registered posts only")

     

       387
       389
        
                   

     

       388
       390
        
                   console.print(table)

     

       389
       391
        
               

     
···

       412
       414
        
                           console.print(f"  {link_pair}: {count} links")

     

       413
       415
        
               

     

       414
       416
        
               if not get_tsv_mode():

     

       415
       415
       -
                   console.print(f"\nLinks output saved to: {output_path}")

     

       416
       416
       -
                   console.print(f"URL mapping saved to: {mapping_path}")

     

       417
       417
       +
                   console.print(f"\nUnified links data saved to: {output_path}")

     

       417
       418
        
       

     

       418
       419
        
           except Exception as e:

     

       419
       420
        
               console.print(f"[red]Error extracting links: {e}[/red]")