···

       
8
       
 
       
        "@mozilla/readability": "^0.6.0",

     

       
9
       
 
       
        "@tsconfig/bun": "^1.0.8",

     

       
10
       
 
       
        "ai": "^4.3.16",

     

       
11
       
-
       
        "ai-fallback": "^0.1.5",

     

       
12
       
 
       
        "debug": "^4.4.1",

     

       
13
       
 
       
        "happy-dom": "^18.0.1",

     

       
14
       
 
       
        "hast-util-sanitize": "^5.0.2",

     

       
15
       
 
       
        "js-graph-algorithms": "^1.0.18",

     

       
16
       
 
       
        "mdast": "^3.0.0",

     

       
17
       
-
       
        "ollama-ai-provider": "^1.2.0",

     

       
18
       
 
       
        "prettier": "^3.5.3",

     

       
19
       
 
       
        "rehype-parse": "^9.0.1",

     

       
20
       
 
       
        "rehype-remark": "^10.0.1",

     
···

       
69
       
 
       
    "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="],

     

       
70
       
 
       


     

       
71
       
 
       
    "ai": ["ai@4.3.16", "", { "dependencies": { "@ai-sdk/provider": "1.1.3", "@ai-sdk/provider-utils": "2.2.8", "@ai-sdk/react": "1.2.12", "@ai-sdk/ui-utils": "1.2.11", "@opentelemetry/api": "1.9.0", "jsondiffpatch": "0.6.0" }, "peerDependencies": { "react": "^18 || ^19 || ^19.0.0-rc", "zod": "^3.23.8" }, "optionalPeers": ["react"] }, "sha512-KUDwlThJ5tr2Vw0A1ZkbDKNME3wzWhuVfAOwIvFUzl1TPVDFAXDFTXio3p+jaKneB+dKNCvFFlolYmmgHttG1g=="],

     

       
72
       
-
       


     

       
73
       
-
       
    "ai-fallback": ["ai-fallback@0.1.5", "", { "dependencies": { "@ai-sdk/provider": "^1", "@ai-sdk/provider-utils": "^2" } }, "sha512-/FhTd9SGMEUDYBKbO3ZyfS0CBGglJByMbMRQOGjjDYlxZinFZtn99w1SPh4NZYJWIP5jjoewytfZjp+30QPT1A=="],

     

       
74
       
 
       


     

       
75
       
 
       
    "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="],

     

       
76
       
 
       


     
···

       
238
       
 
       


     

       
239
       
 
       
    "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],

     

       
240
       
 
       


     

       
241
       
-
       
    "ollama-ai-provider": ["ollama-ai-provider@1.2.0", "", { "dependencies": { "@ai-sdk/provider": "^1.0.0", "@ai-sdk/provider-utils": "^2.0.0", "partial-json": "0.1.7" }, "peerDependencies": { "zod": "^3.0.0" }, "optionalPeers": ["zod"] }, "sha512-jTNFruwe3O/ruJeppI/quoOUxG7NA6blG3ZyQj3lei4+NnJo7bi3eIRWqlVpRlu/mbzbFXeJSBuYQWF6pzGKww=="],

     

       
242
       
-
       


     

       
243
       
 
       
    "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],

     

       
244
       
-
       


     

       
245
       
-
       
    "partial-json": ["partial-json@0.1.7", "", {}, "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA=="],

     

       
246
       
 
       


     

       
247
       
 
       
    "prettier": ["prettier@3.5.3", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw=="],

     

       
248
       
 
       


     

···

       
8
       
 
       
        "@mozilla/readability": "^0.6.0",

     

       
9
       
 
       
        "@tsconfig/bun": "^1.0.8",

     

       
10
       
 
       
        "ai": "^4.3.16",

     

       
0
       
 
       

     

       
11
       
 
       
        "debug": "^4.4.1",

     

       
12
       
 
       
        "happy-dom": "^18.0.1",

     

       
13
       
 
       
        "hast-util-sanitize": "^5.0.2",

     

       
14
       
 
       
        "js-graph-algorithms": "^1.0.18",

     

       
15
       
 
       
        "mdast": "^3.0.0",

     

       
0
       
 
       

     

       
16
       
 
       
        "prettier": "^3.5.3",

     

       
17
       
 
       
        "rehype-parse": "^9.0.1",

     

       
18
       
 
       
        "rehype-remark": "^10.0.1",

     
···

       
67
       
 
       
    "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="],

     

       
68
       
 
       


     

       
69
       
 
       
    "ai": ["ai@4.3.16", "", { "dependencies": { "@ai-sdk/provider": "1.1.3", "@ai-sdk/provider-utils": "2.2.8", "@ai-sdk/react": "1.2.12", "@ai-sdk/ui-utils": "1.2.11", "@opentelemetry/api": "1.9.0", "jsondiffpatch": "0.6.0" }, "peerDependencies": { "react": "^18 || ^19 || ^19.0.0-rc", "zod": "^3.23.8" }, "optionalPeers": ["react"] }, "sha512-KUDwlThJ5tr2Vw0A1ZkbDKNME3wzWhuVfAOwIvFUzl1TPVDFAXDFTXio3p+jaKneB+dKNCvFFlolYmmgHttG1g=="],

     

       
0
       
 
       

     

       
0
       
 
       

     

       
70
       
 
       


     

       
71
       
 
       
    "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="],

     

       
72
       
 
       


     
···

       
234
       
 
       


     

       
235
       
 
       
    "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],

     

       
236
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
237
       
 
       
    "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],

     

       
0
       
 
       

     

       
0
       
 
       

     

       
238
       
 
       


     

       
239
       
 
       
    "prettier": ["prettier@3.5.3", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw=="],

     

       
240
       
 
       


     

···

       
11
       
 
       
    "@mozilla/readability": "^0.6.0",

     

       
12
       
 
       
    "@tsconfig/bun": "^1.0.8",

     

       
13
       
 
       
    "ai": "^4.3.16",

     

       
14
       
-
       
    "ai-fallback": "^0.1.5",

     

       
15
       
 
       
    "debug": "^4.4.1",

     

       
16
       
 
       
    "happy-dom": "^18.0.1",

     

       
17
       
 
       
    "hast-util-sanitize": "^5.0.2",

     

       
18
       
 
       
    "js-graph-algorithms": "^1.0.18",

     

       
19
       
 
       
    "mdast": "^3.0.0",

     

       
20
       
-
       
    "ollama-ai-provider": "^1.2.0",

     

       
21
       
 
       
    "prettier": "^3.5.3",

     

       
22
       
 
       
    "rehype-parse": "^9.0.1",

     

       
23
       
 
       
    "rehype-remark": "^10.0.1",

     

···

       
11
       
 
       
    "@mozilla/readability": "^0.6.0",

     

       
12
       
 
       
    "@tsconfig/bun": "^1.0.8",

     

       
13
       
 
       
    "ai": "^4.3.16",

     

       
0
       
 
       

     

       
14
       
 
       
    "debug": "^4.4.1",

     

       
15
       
 
       
    "happy-dom": "^18.0.1",

     

       
16
       
 
       
    "hast-util-sanitize": "^5.0.2",

     

       
17
       
 
       
    "js-graph-algorithms": "^1.0.18",

     

       
18
       
 
       
    "mdast": "^3.0.0",

     

       
0
       
 
       

     

       
19
       
 
       
    "prettier": "^3.5.3",

     

       
20
       
 
       
    "rehype-parse": "^9.0.1",

     

       
21
       
 
       
    "rehype-remark": "^10.0.1",

     

···

       
19
       
 
       
  log('crawl', site.name);

     

       
20
       
 
       
  const pages = await crawl(site);

     

       
21
       
 
       
  const contents: string[] = [];

     

       
22
       
-
       
  for (const page of pages) {

     

       
0
       
 
       

     

       
23
       
 
       
    const content = await page.getContent();

     

       
24
       
 
       
    if (content) contents.push(content);

     

       
0
       
 
       

     

       
25
       
 
       
  }

     

       
26
       
 
       
  const output = await concatMarkdown(contents);

     

       
27
       
 
       
  const formatted = await formatMarkdown(output);

     

···

       
19
       
 
       
  log('crawl', site.name);

     

       
20
       
 
       
  const pages = await crawl(site);

     

       
21
       
 
       
  const contents: string[] = [];

     

       
22
       
+
       
  for (let idx = 0; idx < pages.length; idx++) {

     

       
23
       
+
       
    const page = pages[idx]!;

     

       
24
       
 
       
    const content = await page.getContent();

     

       
25
       
 
       
    if (content) contents.push(content);

     

       
26
       
+
       
    log(`completed page ${idx + 1} of ${pages.length}`);

     

       
27
       
 
       
  }

     

       
28
       
 
       
  const output = await concatMarkdown(contents);

     

       
29
       
 
       
  const formatted = await formatMarkdown(output);

     

···

       
2
       
 
       
import { createFallback } from 'ai-fallback';

     

       
3
       
 
       
import { streamText } from 'ai';

     

       
4
       
 
       
import { createOpenAI } from '@ai-sdk/openai';

     

       
5
       
-
       
import { createOllama } from 'ollama-ai-provider';

     

       
6
       
 
       
import * as fs from 'node:fs/promises';

     

       
7
       
 
       
import * as path from 'node:path';

     

       
8
       
 
       


     
···

       
14
       
 
       
await fs.mkdir(cacheDir, { recursive: true });

     

       
15
       
 
       
const getCacheFile = makeCacheFileHelper(cacheDir, '.txt');

     

       
16
       
 
       


     

       
17
       
-
       
if (!process.env.OPENAI_API_KEY) throw new Error('Missing OPENAI_API_KEY env var');

     

       
18
       
-
       
if (!process.env.OPENAI_API_URL) throw new Error('Missing OPENAI_API_URL env var');

     

       
19
       
-
       


     

       
20
       
 
       
const SYSTEM_PROMPT = `

     

       
21
       
 
       
Reformat markdown content you're given into an llms-full.txt file, also in markdown format

     

       
22
       
 
       
- Reformat for an AI and paraphrase where necessary, but be faithful to the original

     

       
23
       
-
       
- Avoid using emphasis and use Github Flavored markdown syntax

     

       
24
       
 
       
- Keep code snippets and keep them in TypeScript or TypeScript typings format

     

       
25
       
-
       
- Don't mention other content, pages, or external content (Remove sentences such as "Refer to", "Read more", "Learn how to")

     

       
26
       
-
       
- For markdown tables, keep all relevant information in the table and remove table legends and emoji

     

       
27
       
-
       
- Remove icon legends or irrelevant text

     

       
28
       
-
       
- Don't add to the content or use any knowledge you may have on the subject

     

       
29
       
-
       
- Format the output in AI-friendly markdown and preserve inline code

     

       
30
       
-
       
- Remove sub-headings if they don't add crucial information or context

     

       
31
       
-
       
- Don't wrap your output in a code block

     

       
32
       
 
       
`;

     

       
33
       
 
       


     

       
34
       
 
       
const ai = createOpenAI({

     
···

       
36
       
 
       
  baseURL: process.env.OPENAI_API_URL,

     

       
37
       
 
       
});

     

       
38
       
 
       


     

       
39
       
-
       
const ollama = createOllama({

     

       
40
       
-
       
  baseURL: 'http://localhost:11434/api',

     

       
41
       
 
       
});

     

       
42
       
 
       


     

       
43
       
 
       
export async function rewriteMarkdown(url: URL, input: string) {

     
···

       
52
       
 
       
  } catch {}

     

       
53
       
 
       
  log('prompting to rewrite', url.pathname);

     

       
54
       
 
       
  const { textStream } = streamText({

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
55
       
 
       
    model: createFallback({

     

       
56
       
 
       
      models: [

     

       
57
       
-
       
        ollama('mistral-small3.1:24b'),

     

       
58
       
 
       
        ai('@cf/mistralai/mistral-small-3.1-24b-instruct'),

     

       
59
       
 
       
      ],

     

       
60
       
 
       
      onError(error, modelId) {

     

       
61
       
 
       
        log(`error using model ${modelId}`, error);

     

       
62
       
 
       
      },

     

       
63
       
 
       
    }),

     

       
64
       
-
       
    maxSteps: 5,

     

       
65
       
-
       
    experimental_continueSteps: true,

     

       
66
       
-
       
    temperature: 0.05,

     

       
67
       
-
       
    system: SYSTEM_PROMPT.trim(),

     

       
68
       
-
       
    prompt: input,

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
69
       
 
       
  });

     

       
70
       
-
       
  const output = [];

     

       
71
       
 
       
  for await (const chunk of textStream)

     

       
72
       
 
       
    output.push(chunk);

     

       
73
       
 
       
  const text = output.join('');

     

···

       
2
       
 
       
import { createFallback } from 'ai-fallback';

     

       
3
       
 
       
import { streamText } from 'ai';

     

       
4
       
 
       
import { createOpenAI } from '@ai-sdk/openai';

     

       
0
       
 
       

     

       
5
       
 
       
import * as fs from 'node:fs/promises';

     

       
6
       
 
       
import * as path from 'node:path';

     

       
7
       
 
       


     
···

       
13
       
 
       
await fs.mkdir(cacheDir, { recursive: true });

     

       
14
       
 
       
const getCacheFile = makeCacheFileHelper(cacheDir, '.txt');

     

       
15
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
16
       
 
       
const SYSTEM_PROMPT = `

     

       
17
       
 
       
Reformat markdown content you're given into an llms-full.txt file, also in markdown format

     

       
18
       
 
       
- Reformat for an AI and paraphrase where necessary, but be faithful to the original

     

       
0
       
 
       

     

       
19
       
 
       
- Keep code snippets and keep them in TypeScript or TypeScript typings format

     

       
20
       
+
       
- For markdown tables, keep all relevant information in the table

     

       
21
       
+
       
- Don't mention other content, pages, or external content

     

       
22
       
+
       
- Don't write your own content

     

       
23
       
+
       
- Don't add or use any knowledge you may have on the subject

     

       
24
       
+
       
- Don't add your own interpretation or notes and only reinterpret the input content

     

       
25
       
+
       
- Don't wrap the output in a markdown code block

     

       
26
       
+
       
Only return the reformatted markdown content and stop when you've processed all input markdown content

     

       
27
       
 
       
`;

     

       
28
       
 
       


     

       
29
       
 
       
const ai = createOpenAI({

     
···

       
31
       
 
       
  baseURL: process.env.OPENAI_API_URL,

     

       
32
       
 
       
});

     

       
33
       
 
       


     

       
34
       
+
       
const ollama = createOpenAI({

     

       
35
       
+
       
  baseURL: 'http://localhost:11434/v1',

     

       
36
       
 
       
});

     

       
37
       
 
       


     

       
38
       
 
       
export async function rewriteMarkdown(url: URL, input: string) {

     
···

       
47
       
 
       
  } catch {}

     

       
48
       
 
       
  log('prompting to rewrite', url.pathname);

     

       
49
       
 
       
  const { textStream } = streamText({

     

       
50
       
+
       
    temperature: 0.05,

     

       
51
       
+
       
    maxSteps: 5,

     

       
52
       
+
       
    experimental_continueSteps: true,

     

       
53
       
 
       
    model: createFallback({

     

       
54
       
 
       
      models: [

     

       
55
       
+
       
        ollama('phi4:14b'),

     

       
56
       
 
       
        ai('@cf/mistralai/mistral-small-3.1-24b-instruct'),

     

       
57
       
 
       
      ],

     

       
58
       
 
       
      onError(error, modelId) {

     

       
59
       
 
       
        log(`error using model ${modelId}`, error);

     

       
60
       
 
       
      },

     

       
61
       
 
       
    }),

     

       
62
       
+
       
    onStepFinish({ finishReason, text }) {

     

       
63
       
+
       
      if (finishReason !== 'stop')

     

       
64
       
+
       
        log(`inference step (length: ${text.length})`, finishReason); 

     

       
65
       
+
       
    },

     

       
66
       
+
       
    messages: [

     

       
67
       
+
       
      {

     

       
68
       
+
       
        role: 'system',

     

       
69
       
+
       
        content: SYSTEM_PROMPT.trim(),

     

       
70
       
+
       
      },

     

       
71
       
+
       
      {

     

       
72
       
+
       
        role: 'user',

     

       
73
       
+
       
        content: input,

     

       
74
       
+
       
      },

     

       
75
       
+
       
    ],

     

       
76
       
 
       
  });

     

       
77
       
+
       
  const output: string[] = [];

     

       
78
       
 
       
  for await (const chunk of textStream)

     

       
79
       
 
       
    output.push(chunk);

     

       
80
       
 
       
  const text = output.join('');

     

···

       
18
       
 
       
    .use(rehypeParse)

     

       
19
       
 
       
    .use(rehypeSanitize, {

     

       
20
       
 
       
      tagNames: [

     

       
21
       
-
       
        ...defaultSanitizeSchema.tagNames!,

     

       
22
       
 
       
        'content-region',

     

       
23
       
 
       
        'footer',

     

       
24
       
 
       
        'header',

     
···

       
27
       
 
       
        'section',

     

       
28
       
 
       
        'nav',

     

       
29
       
 
       
      ],

     

       
30
       
-
       
      strip: ['script', 'style'],

     

       
31
       
 
       
    })

     

       
32
       
 
       
    .use(rehypeStringify)

     

       
33
       
 
       
    .process(html)

     

···

       
18
       
 
       
    .use(rehypeParse)

     

       
19
       
 
       
    .use(rehypeSanitize, {

     

       
20
       
 
       
      tagNames: [

     

       
21
       
+
       
        ...defaultSanitizeSchema.tagNames!.filter((tag) => tag !== 'details'),

     

       
22
       
 
       
        'content-region',

     

       
23
       
 
       
        'footer',

     

       
24
       
 
       
        'header',

     
···

       
27
       
 
       
        'section',

     

       
28
       
 
       
        'nav',

     

       
29
       
 
       
      ],

     

       
30
       
+
       
      strip: ['script', 'style', 'details'],

     

       
31
       
 
       
    })

     

       
32
       
 
       
    .use(rehypeStringify)

     

       
33
       
 
       
    .process(html)