A fast, local-first "redirection engine" for !bang users with a few extra features ^-^
dunkirk.sh
1import { bangs } from "./hashbang";
2import { writeFileSync } from "fs";
3import { Worker, isMainThread, parentPort, workerData } from "worker_threads";
4import { cpus } from "os";
5import dns from "dns";
6import util from "util";
7
8const TEST_QUERY = "test query";
9const TIMEOUT = 5000;
10const BATCH_SIZE = 10;
11
12const dnsCache = new Map();
13const lookup = util.promisify(dns.lookup);
14const urlCache = new Map<string, boolean>();
15
16const resolveHost = async (hostname: string) => {
17 if (dnsCache.has(hostname)) {
18 return dnsCache.get(hostname);
19 }
20 const address = await lookup(hostname);
21 dnsCache.set(hostname, address);
22 return address;
23};
24
25const testUrl = async (url: string, retries = 3): Promise<boolean> => {
26 if (urlCache.has(url)) {
27 return urlCache.get(url)!;
28 }
29
30 for (let i = 0; i < retries; i++) {
31 try {
32 const hostname = new URL(url).hostname;
33 await resolveHost(hostname);
34
35 const res = await fetch(url, {
36 signal: AbortSignal.timeout(TIMEOUT),
37 headers: { "User-Agent": "Mozilla/5.0" },
38 });
39 const result = res.status === 200;
40 urlCache.set(url, result);
41 return result;
42 } catch (err) {
43 if (i === retries - 1) {
44 urlCache.set(url, false);
45 return false;
46 }
47 await new Promise((resolve) => setTimeout(resolve, 1000 * i));
48 }
49 }
50 return false;
51};
52
53if (isMainThread) {
54 const brokenBangs: { bang: string; url: string }[] = [];
55 const bangEntries = Object.entries(bangs);
56 const numThreads = cpus().length;
57 const chunkSize = Math.ceil(bangEntries.length / numThreads);
58 const chunks = Array.from({ length: numThreads }, (_, i) =>
59 bangEntries.slice(i * chunkSize, (i + 1) * chunkSize),
60 );
61
62 let completedBangs = 0;
63 const startTime = Date.now();
64 const spinChars = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
65 let spinIdx = 0;
66
67 const updateProgress = setInterval(() => {
68 const elapsedSeconds = (Date.now() - startTime) / 1000;
69 const bangsPerSecond = (completedBangs / elapsedSeconds).toFixed(2);
70 try {
71 process.stdout.clearLine(0);
72 process.stdout.cursorTo(0);
73 process.stdout.write(
74 `${spinChars[spinIdx]} Completed ${completedBangs}/${bangEntries.length} bangs (${bangsPerSecond}/s)`,
75 );
76 spinIdx = (spinIdx + 1) % spinChars.length;
77 } catch (err) {
78 process.stdout.write("\r");
79 }
80 }, 500);
81
82 const workers = chunks.map((chunk) => {
83 return new Promise((resolve) => {
84 const worker = new Worker(__filename, {
85 workerData: chunk,
86 });
87
88 worker.on("message", (msg) => {
89 if (msg.type === "progress") {
90 completedBangs++;
91 } else {
92 brokenBangs.push(...msg);
93 }
94 });
95
96 worker.on("exit", resolve);
97 });
98 });
99
100 Promise.all(workers).then(() => {
101 clearInterval(updateProgress);
102 process.stdout.write("\n");
103 const brokenOutput = JSON.stringify(brokenBangs, null, 2);
104 writeFileSync("src/bangs/broken-bangs.json", brokenOutput);
105 console.log("Broken bangs written to broken-bangs.json");
106 });
107} else {
108 const brokenBangsChunk: { bang: string; url: string }[] = [];
109 const chunk = workerData as [string, { u: string }][];
110
111 (async () => {
112 for (let i = 0; i < chunk.length; i += BATCH_SIZE) {
113 const batch = chunk.slice(i, i + BATCH_SIZE);
114 const results = await Promise.all(
115 batch.map(async ([key, bang]) => {
116 const url = bang.u.replace("{{{s}}}", encodeURIComponent(TEST_QUERY));
117 const isWorking = await testUrl(url);
118 return { key, url, isWorking };
119 }),
120 );
121
122 for (const { key, url, isWorking } of results) {
123 if (!isWorking) {
124 brokenBangsChunk.push({ bang: key, url });
125 try {
126 process.stdout.clearLine(0);
127 process.stdout.cursorTo(0);
128 } catch (err) {
129 process.stdout.write("\r");
130 }
131 console.log(`Bang ${key} is broken (${url})`);
132 }
133 parentPort?.postMessage({ type: "progress" });
134 }
135
136 if (brokenBangsChunk.length > 1000) {
137 parentPort?.postMessage(brokenBangsChunk);
138 brokenBangsChunk.length = 0;
139 }
140 }
141
142 if (brokenBangsChunk.length > 0) {
143 parentPort?.postMessage(brokenBangsChunk);
144 }
145 })();
146}