···

       
76
       
 
       
    }

     

       
77
       
 
       
  }

     

       
78
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
79
       
 
       
  getPage(url: URL) {

     

       
80
       
 
       
    let page = this.#pages.get(url.toString());

     

       
81
       
 
       
    if (!page) {

     
···

       
143
       
 
       
  }

     

       
144
       
 
       
}

     

       
145
       
 
       


     

       
146
       
-
       
async function crawlPages(page: Page, visited = new Set<Page>([page])): Promise<Page[]> {

     

       
147
       
 
       
  const links = await page.getLinks();

     

       
148
       
 
       
  if (links) {

     

       
149
       
-
       
    log(`crawling (${links.length} links)`, page.url.pathname);

     

       
150
       
 
       
    for (const link of links) {

     

       
151
       
 
       
      if (visited.has(link)) {

     

       
152
       
 
       
        continue;

     

       
153
       
 
       
      } else {

     

       
154
       
 
       
        visited.add(page);

     

       
155
       
-
       
        await crawlPages(link, visited);

     

       
156
       
 
       
      }

     

       
157
       
 
       
    }

     

       
158
       
 
       
  }

     
···

       
183
       
 
       
    }

     

       
184
       
 
       
  }

     

       
185
       
 
       
  const kruskal = new KruskalMST(graph);

     

       
186
       
-
       
  const output = new Set<Page>();

     

       
187
       
 
       
  for (const edge of kruskal.mst) {

     

       
188
       
 
       
    const page = pages[edge.from()];

     

       
189
       
 
       
    if (page) output.add(page);

     

···

       
76
       
 
       
    }

     

       
77
       
 
       
  }

     

       
78
       
 
       


     

       
79
       
+
       
  get origin() {

     

       
80
       
+
       
    return this.#origin;

     

       
81
       
+
       
  }

     

       
82
       
+
       


     

       
83
       
 
       
  getPage(url: URL) {

     

       
84
       
 
       
    let page = this.#pages.get(url.toString());

     

       
85
       
 
       
    if (!page) {

     
···

       
147
       
 
       
  }

     

       
148
       
 
       
}

     

       
149
       
 
       


     

       
150
       
+
       
async function crawlPages(page: Page, visited = new Set<Page>([page]), depth = 1): Promise<Page[]> {

     

       
151
       
 
       
  const links = await page.getLinks();

     

       
152
       
 
       
  if (links) {

     

       
153
       
+
       
    log(`crawling (${links.length} links, depth = ${depth})`, page.url.pathname);

     

       
154
       
 
       
    for (const link of links) {

     

       
155
       
 
       
      if (visited.has(link)) {

     

       
156
       
 
       
        continue;

     

       
157
       
 
       
      } else {

     

       
158
       
 
       
        visited.add(page);

     

       
159
       
+
       
        await crawlPages(link, visited, depth + 1);

     

       
160
       
 
       
      }

     

       
161
       
 
       
    }

     

       
162
       
 
       
  }

     
···

       
187
       
 
       
    }

     

       
188
       
 
       
  }

     

       
189
       
 
       
  const kruskal = new KruskalMST(graph);

     

       
190
       
+
       
  const output = new Set<Page>([root.origin]);

     

       
191
       
 
       
  for (const edge of kruskal.mst) {

     

       
192
       
 
       
    const page = pages[edge.from()];

     

       
193
       
 
       
    if (page) output.add(page);

     

···

       
19
       
 
       


     

       
20
       
 
       
const SYSTEM_PROMPT = `

     

       
21
       
 
       
Reformat markdown content you're given into an llms-full.txt file, also in markdown format

     

       
22
       
-
       
- Where the format isn't easily understandable by AI, reformat it faithfully to make it processable

     

       
23
       
-
       
- Reformat for an AI and paraphrase where necessary, but don't add interpretations

     

       
24
       
-
       
- Preserve code snippets and keep them in TypeScript or TypeScript typings format

     

       
25
       
-
       
- Avoid using emphasis or excessive markdown syntax, but keep code snippets where they are

     

       
26
       
-
       
- Don't mention other content, pages, or external content (Remove sentences such as "Refer to", "Read more")

     

       
27
       
-
       
- When encountering a markdown table, ensure that you don't output a separate legend, and keep all relevant information in the table

     

       
28
       
-
       
- Don't use any knowledge you may have on the subject. Only output what you're given.

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
29
       
 
       
`;

     

       
30
       
 
       


     

       
31
       
 
       
const ai = createOpenAI({

     
···

       
51
       
 
       
  const { text } = await generateText({

     

       
52
       
 
       
    model: createFallback({

     

       
53
       
 
       
      models: [

     

       
54
       
-
       
        ollama('gemma:7b'),

     

       
55
       
-
       
        ai('@hf/google/gemma-7b-it'),

     

       
56
       
 
       
      ],

     

       
57
       
 
       
      onError(error, modelId) {

     

       
58
       
 
       
        log(`error using model ${modelId}`, error);

     

       
59
       
 
       
      },

     

       
60
       
 
       
    }),

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
61
       
 
       
    system: SYSTEM_PROMPT.trim(),

     

       
62
       
 
       
    prompt: input,

     

       
63
       
 
       
  });

     

···

       
19
       
 
       


     

       
20
       
 
       
const SYSTEM_PROMPT = `

     

       
21
       
 
       
Reformat markdown content you're given into an llms-full.txt file, also in markdown format

     

       
22
       
+
       
- Reformat for an AI and paraphrase where necessary, but be faithful to the original

     

       
23
       
+
       
- Avoid using emphasis and use Github Flavored markdown syntax

     

       
24
       
+
       
- Keep code snippets and keep them in TypeScript or TypeScript typings format

     

       
25
       
+
       
- Don't mention other content, pages, or external content (Remove sentences such as "Refer to", "Read more", "Learn how to")

     

       
26
       
+
       
- For markdown tables, keep all relevant information in the table and remove table legends and emoji

     

       
27
       
+
       
- Remove icon legends or irrelevant text

     

       
28
       
+
       
- Don't add to the content or use any knowledge you may have on the subject

     

       
29
       
+
       
- Format the output in AI-friendly markdown and preserve inline code

     

       
30
       
+
       
- Remove sub-headings if they don't add crucial information or context

     

       
31
       
+
       
- Don't wrap your output in a code block

     

       
32
       
 
       
`;

     

       
33
       
 
       


     

       
34
       
 
       
const ai = createOpenAI({

     
···

       
54
       
 
       
  const { text } = await generateText({

     

       
55
       
 
       
    model: createFallback({

     

       
56
       
 
       
      models: [

     

       
57
       
+
       
        ollama('mistral-small3.1:24b'),

     

       
58
       
+
       
        ai('@cf/mistralai/mistral-small-3.1-24b-instruct'),

     

       
59
       
 
       
      ],

     

       
60
       
 
       
      onError(error, modelId) {

     

       
61
       
 
       
        log(`error using model ${modelId}`, error);

     

       
62
       
 
       
      },

     

       
63
       
 
       
    }),

     

       
64
       
+
       
    maxSteps: 5,

     

       
65
       
+
       
    experimental_continueSteps: true,

     

       
66
       
+
       
    temperature: 0.05,

     

       
67
       
 
       
    system: SYSTEM_PROMPT.trim(),

     

       
68
       
 
       
    prompt: input,

     

       
69
       
 
       
  });