Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
nekomimi.pet
1import { getAllSites } from './db';
2import { fetchSiteRecord, getPdsForDid, downloadAndCacheSite, isCached } from './utils';
3import { logger } from './observability';
4import { markSiteAsBeingCached, unmarkSiteAsBeingCached } from './cache';
5
6export interface BackfillOptions {
7 skipExisting?: boolean; // Skip sites already in cache
8 concurrency?: number; // Number of sites to cache concurrently
9 maxSites?: number; // Maximum number of sites to backfill (for testing)
10}
11
12export interface BackfillStats {
13 total: number;
14 cached: number;
15 skipped: number;
16 failed: number;
17 duration: number;
18}
19
20/**
21 * Backfill all sites from the database into the local cache
22 */
23export async function backfillCache(options: BackfillOptions = {}): Promise<BackfillStats> {
24 const {
25 skipExisting = true,
26 concurrency = 3,
27 maxSites,
28 } = options;
29
30 const startTime = Date.now();
31 const stats: BackfillStats = {
32 total: 0,
33 cached: 0,
34 skipped: 0,
35 failed: 0,
36 duration: 0,
37 };
38
39 logger.info('Starting cache backfill', { skipExisting, concurrency, maxSites });
40 console.log(`
41╔══════════════════════════════════════════╗
42║ CACHE BACKFILL STARTING ║
43╚══════════════════════════════════════════╝
44 `);
45
46 try {
47 // Get all sites from database
48 let sites = await getAllSites();
49 stats.total = sites.length;
50
51 logger.info(`Found ${sites.length} sites in database`);
52 console.log(`📊 Found ${sites.length} sites in database`);
53
54 // Limit if specified
55 if (maxSites && maxSites > 0) {
56 sites = sites.slice(0, maxSites);
57 console.log(`⚙️ Limited to ${maxSites} sites for backfill`);
58 }
59
60 // Process sites in batches
61 const batches: typeof sites[] = [];
62 for (let i = 0; i < sites.length; i += concurrency) {
63 batches.push(sites.slice(i, i + concurrency));
64 }
65
66 let processed = 0;
67 for (const batch of batches) {
68 await Promise.all(
69 batch.map(async (site) => {
70 try {
71 // Check if already cached
72 if (skipExisting && isCached(site.did, site.rkey)) {
73 stats.skipped++;
74 processed++;
75 logger.debug(`Skipping already cached site`, { did: site.did, rkey: site.rkey });
76 console.log(`⏭️ [${processed}/${sites.length}] Skipped (cached): ${site.display_name || site.rkey}`);
77 return;
78 }
79
80 // Fetch site record
81 const siteData = await fetchSiteRecord(site.did, site.rkey);
82 if (!siteData) {
83 stats.failed++;
84 processed++;
85 logger.error('Site record not found during backfill', null, { did: site.did, rkey: site.rkey });
86 console.log(`❌ [${processed}/${sites.length}] Failed (not found): ${site.display_name || site.rkey}`);
87 return;
88 }
89
90 // Get PDS endpoint
91 const pdsEndpoint = await getPdsForDid(site.did);
92 if (!pdsEndpoint) {
93 stats.failed++;
94 processed++;
95 logger.error('PDS not found during backfill', null, { did: site.did });
96 console.log(`❌ [${processed}/${sites.length}] Failed (no PDS): ${site.display_name || site.rkey}`);
97 return;
98 }
99
100 // Mark site as being cached to prevent serving stale content during update
101 markSiteAsBeingCached(site.did, site.rkey);
102
103 try {
104 // Download and cache site
105 await downloadAndCacheSite(site.did, site.rkey, siteData.record, pdsEndpoint, siteData.cid);
106 stats.cached++;
107 processed++;
108 logger.info('Successfully cached site during backfill', { did: site.did, rkey: site.rkey });
109 console.log(`✅ [${processed}/${sites.length}] Cached: ${site.display_name || site.rkey}`);
110 } finally {
111 // Always unmark, even if caching fails
112 unmarkSiteAsBeingCached(site.did, site.rkey);
113 }
114 } catch (err) {
115 stats.failed++;
116 processed++;
117 logger.error('Failed to cache site during backfill', err, { did: site.did, rkey: site.rkey });
118 console.log(`❌ [${processed}/${sites.length}] Failed: ${site.display_name || site.rkey}`);
119 }
120 })
121 );
122 }
123
124 stats.duration = Date.now() - startTime;
125
126 console.log(`
127╔══════════════════════════════════════════╗
128║ CACHE BACKFILL COMPLETED ║
129╚══════════════════════════════════════════╝
130
131📊 Total Sites: ${stats.total}
132✅ Cached: ${stats.cached}
133⏭️ Skipped: ${stats.skipped}
134❌ Failed: ${stats.failed}
135⏱️ Duration: ${(stats.duration / 1000).toFixed(2)}s
136 `);
137
138 logger.info('Cache backfill completed', stats);
139 } catch (err) {
140 logger.error('Cache backfill failed', err);
141 console.error('❌ Cache backfill failed:', err);
142 }
143
144 return stats;
145}