Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
nekomimi.pet
1import { getAllSites } from './db';
2import { fetchSiteRecord, getPdsForDid, downloadAndCacheSite, isCached } from './utils';
3import { createLogger } from '@wisp/observability';
4import { markSiteAsBeingCached, unmarkSiteAsBeingCached } from './cache';
5import { clearRedirectRulesCache } from './site-cache';
6
7const logger = createLogger('hosting-service');
8
9export interface BackfillOptions {
10 skipExisting?: boolean; // Skip sites already in cache
11 concurrency?: number; // Number of sites to cache concurrently
12 maxSites?: number; // Maximum number of sites to backfill (for testing)
13}
14
15export interface BackfillStats {
16 total: number;
17 cached: number;
18 skipped: number;
19 failed: number;
20 duration: number;
21}
22
23/**
24 * Backfill all sites from the database into the local cache
25 */
26export async function backfillCache(options: BackfillOptions = {}): Promise<BackfillStats> {
27 const {
28 skipExisting = true,
29 concurrency = 10, // Increased from 3 to 10 for better parallelization
30 maxSites,
31 } = options;
32
33 const startTime = Date.now();
34 const stats: BackfillStats = {
35 total: 0,
36 cached: 0,
37 skipped: 0,
38 failed: 0,
39 duration: 0,
40 };
41
42 logger.info('Starting cache backfill', { skipExisting, concurrency, maxSites });
43 console.log(`
44╔══════════════════════════════════════════╗
45║ CACHE BACKFILL STARTING ║
46╚══════════════════════════════════════════╝
47 `);
48
49 try {
50 // Get all sites from database
51 let sites = await getAllSites();
52 stats.total = sites.length;
53
54 logger.info(`Found ${sites.length} sites in database`);
55 console.log(`📊 Found ${sites.length} sites in database`);
56
57 // Limit if specified
58 if (maxSites && maxSites > 0) {
59 sites = sites.slice(0, maxSites);
60 console.log(`⚙️ Limited to ${maxSites} sites for backfill`);
61 }
62
63 // Process sites in batches
64 const batches: typeof sites[] = [];
65 for (let i = 0; i < sites.length; i += concurrency) {
66 batches.push(sites.slice(i, i + concurrency));
67 }
68
69 let processed = 0;
70 for (const batch of batches) {
71 await Promise.all(
72 batch.map(async (site) => {
73 try {
74 // Check if already cached
75 if (skipExisting && isCached(site.did, site.rkey)) {
76 stats.skipped++;
77 processed++;
78 logger.debug(`Skipping already cached site`, { did: site.did, rkey: site.rkey });
79 console.log(`⏭️ [${processed}/${sites.length}] Skipped (cached): ${site.display_name || site.rkey}`);
80 return;
81 }
82
83 // Fetch site record
84 const siteData = await fetchSiteRecord(site.did, site.rkey);
85 if (!siteData) {
86 stats.failed++;
87 processed++;
88 logger.error('Site record not found during backfill', null, { did: site.did, rkey: site.rkey });
89 console.log(`❌ [${processed}/${sites.length}] Failed (not found): ${site.display_name || site.rkey}`);
90 return;
91 }
92
93 // Get PDS endpoint
94 const pdsEndpoint = await getPdsForDid(site.did);
95 if (!pdsEndpoint) {
96 stats.failed++;
97 processed++;
98 logger.error('PDS not found during backfill', null, { did: site.did });
99 console.log(`❌ [${processed}/${sites.length}] Failed (no PDS): ${site.display_name || site.rkey}`);
100 return;
101 }
102
103 // Mark site as being cached to prevent serving stale content during update
104 markSiteAsBeingCached(site.did, site.rkey);
105
106 try {
107 // Download and cache site
108 await downloadAndCacheSite(site.did, site.rkey, siteData.record, pdsEndpoint, siteData.cid);
109 // Clear redirect rules cache since the site was updated
110 clearRedirectRulesCache(site.did, site.rkey);
111 stats.cached++;
112 processed++;
113 logger.info('Successfully cached site during backfill', { did: site.did, rkey: site.rkey });
114 console.log(`✅ [${processed}/${sites.length}] Cached: ${site.display_name || site.rkey}`);
115 } finally {
116 // Always unmark, even if caching fails
117 unmarkSiteAsBeingCached(site.did, site.rkey);
118 }
119 } catch (err) {
120 stats.failed++;
121 processed++;
122 logger.error('Failed to cache site during backfill', err, { did: site.did, rkey: site.rkey });
123 console.log(`❌ [${processed}/${sites.length}] Failed: ${site.display_name || site.rkey}`);
124 }
125 })
126 );
127 }
128
129 stats.duration = Date.now() - startTime;
130
131 console.log(`
132╔══════════════════════════════════════════╗
133║ CACHE BACKFILL COMPLETED ║
134╚══════════════════════════════════════════╝
135
136📊 Total Sites: ${stats.total}
137✅ Cached: ${stats.cached}
138⏭️ Skipped: ${stats.skipped}
139❌ Failed: ${stats.failed}
140⏱️ Duration: ${(stats.duration / 1000).toFixed(2)}s
141 `);
142
143 logger.info('Cache backfill completed', stats);
144 } catch (err) {
145 logger.error('Cache backfill failed', err);
146 console.error('❌ Cache backfill failed:', err);
147 }
148
149 return stats;
150}