Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
nekomimi.pet
1#!/usr/bin/env bun
2/**
3 * Screenshot Sites Script
4 *
5 * Takes screenshots of all sites in the database.
6 * Usage: bun run scripts/screenshot-sites.ts
7 */
8
9import { chromium } from 'playwright'
10import { db } from '../src/lib/db'
11import { mkdir } from 'fs/promises'
12import { join } from 'path'
13
14const SCREENSHOTS_DIR = join(process.cwd(), 'screenshots')
15const VIEWPORT_WIDTH = 1920
16const VIEWPORT_HEIGHT = 1080
17const TIMEOUT = 10000 // 10 seconds
18const MAX_RETRIES = 1
19const CONCURRENCY = 10 // Number of parallel screenshots
20
21interface Site {
22 did: string
23 rkey: string
24}
25
26/**
27 * Get all sites from the database
28 */
29async function getAllSites(): Promise<Site[]> {
30 const rows = await db`
31 SELECT did, rkey
32 FROM sites
33 ORDER BY created_at DESC
34 `
35
36 return rows as Site[]
37}
38
39/**
40 * Determine the URL to screenshot for a site
41 * Priority: custom domain (verified) → wisp domain → fallback to sites.wisp.place
42 */
43async function getSiteUrl(site: Site): Promise<string> {
44 // Check for custom domain mapped to this site
45 const customDomains = await db`
46 SELECT domain FROM custom_domains
47 WHERE did = ${site.did} AND rkey = ${site.rkey} AND verified = true
48 LIMIT 1
49 `
50 if (customDomains.length > 0) {
51 return `https://${customDomains[0].domain}`
52 }
53
54 // Check for wisp domain mapped to this site
55 const wispDomains = await db`
56 SELECT domain FROM domains
57 WHERE did = ${site.did} AND rkey = ${site.rkey}
58 LIMIT 1
59 `
60 if (wispDomains.length > 0) {
61 return `https://${wispDomains[0].domain}`
62 }
63
64 // Fallback to direct serving URL
65 return `https://sites.wisp.place/${site.did}/${site.rkey}`
66}
67
68/**
69 * Sanitize filename to remove invalid characters
70 */
71function sanitizeFilename(str: string): string {
72 return str.replace(/[^a-z0-9_-]/gi, '_').toLowerCase()
73}
74
75/**
76 * Take a screenshot of a site with retry logic
77 */
78async function screenshotSite(
79 page: any,
80 site: Site,
81 retries: number = MAX_RETRIES
82): Promise<{ success: boolean; error?: string }> {
83 const url = await getSiteUrl(site)
84 // Use the URL as filename (remove https:// and sanitize)
85 const urlForFilename = url.replace(/^https?:\/\//, '')
86 const filename = `${sanitizeFilename(urlForFilename)}.png`
87 const filepath = join(SCREENSHOTS_DIR, filename)
88
89 for (let attempt = 0; attempt <= retries; attempt++) {
90 try {
91 // Navigate to the site
92 await page.goto(url, {
93 waitUntil: 'networkidle',
94 timeout: TIMEOUT
95 })
96
97 // Wait a bit for any dynamic content
98 await page.waitForTimeout(1000)
99
100 // Take screenshot
101 await page.screenshot({
102 path: filepath,
103 fullPage: false, // Just viewport, not full scrollable page
104 type: 'png'
105 })
106
107 return { success: true }
108
109 } catch (error) {
110 const errorMsg = error instanceof Error ? error.message : String(error)
111
112 if (attempt < retries) {
113 continue
114 }
115
116 return { success: false, error: errorMsg }
117 }
118 }
119
120 return { success: false, error: 'Unknown error' }
121}
122
123/**
124 * Main function
125 */
126async function main() {
127 console.log('🚀 Starting site screenshot process...\n')
128
129 // Create screenshots directory if it doesn't exist
130 await mkdir(SCREENSHOTS_DIR, { recursive: true })
131 console.log(`📁 Screenshots will be saved to: ${SCREENSHOTS_DIR}\n`)
132
133 // Get all sites
134 console.log('📊 Fetching sites from database...')
135 const sites = await getAllSites()
136 console.log(` Found ${sites.length} sites\n`)
137
138 if (sites.length === 0) {
139 console.log('No sites to screenshot. Exiting.')
140 return
141 }
142
143 // Launch browser
144 console.log('🌐 Launching browser...\n')
145 const browser = await chromium.launch({
146 headless: true
147 })
148
149 const context = await browser.newContext({
150 viewport: {
151 width: VIEWPORT_WIDTH,
152 height: VIEWPORT_HEIGHT
153 },
154 userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 WispScreenshotBot/1.0'
155 })
156
157 // Track results
158 const results = {
159 success: 0,
160 failed: 0,
161 errors: [] as { site: string; error: string }[]
162 }
163
164 // Process sites in parallel batches
165 console.log(`📸 Screenshotting ${sites.length} sites with concurrency ${CONCURRENCY}...\n`)
166
167 for (let i = 0; i < sites.length; i += CONCURRENCY) {
168 const batch = sites.slice(i, i + CONCURRENCY)
169 const batchNum = Math.floor(i / CONCURRENCY) + 1
170 const totalBatches = Math.ceil(sites.length / CONCURRENCY)
171
172 console.log(`[Batch ${batchNum}/${totalBatches}] Processing ${batch.length} sites...`)
173
174 // Create a page for each site in the batch
175 const batchResults = await Promise.all(
176 batch.map(async (site, idx) => {
177 const page = await context.newPage()
178 const globalIdx = i + idx + 1
179 console.log(` [${globalIdx}/${sites.length}] ${site.did}/${site.rkey}`)
180
181 const result = await screenshotSite(page, site)
182 await page.close()
183
184 return { site, result }
185 })
186 )
187
188 // Aggregate results
189 for (const { site, result } of batchResults) {
190 if (result.success) {
191 results.success++
192 } else {
193 results.failed++
194 results.errors.push({
195 site: `${site.did}/${site.rkey}`,
196 error: result.error || 'Unknown error'
197 })
198 }
199 }
200
201 console.log(` Batch complete: ${batchResults.filter(r => r.result.success).length}/${batch.length} successful\n`)
202 }
203
204 // Cleanup
205 await browser.close()
206
207 // Print summary
208 console.log('╔════════════════════════════════════════════════════════════════╗')
209 console.log('║ SCREENSHOT SUMMARY ║')
210 console.log('╚════════════════════════════════════════════════════════════════╝\n')
211 console.log(`Total sites: ${sites.length}`)
212 console.log(`✅ Successful: ${results.success}`)
213 console.log(`❌ Failed: ${results.failed}`)
214
215 if (results.errors.length > 0) {
216 console.log('\nFailed sites:')
217 for (const err of results.errors) {
218 console.log(` - ${err.site}: ${err.error}`)
219 }
220 }
221
222 console.log(`\n📁 Screenshots saved to: ${SCREENSHOTS_DIR}\n`)
223}
224
225// Run the script
226main().catch((error) => {
227 console.error('Fatal error:', error)
228 process.exit(1)
229})