commit 9b238cdb6c37026f2757107ab082f465679325a5 · kitten.sh/rn-llms-txt-gen

-8

bun.lock

···

       8
       8
        
               "@mozilla/readability": "^0.6.0",

     

       9
       9
        
               "@tsconfig/bun": "^1.0.8",

     

       10
       10
        
               "ai": "^4.3.16",

     

       11
       11
       -
               "ai-fallback": "^0.1.5",

     

       12
       11
        
               "debug": "^4.4.1",

     

       13
       12
        
               "happy-dom": "^18.0.1",

     

       14
       13
        
               "hast-util-sanitize": "^5.0.2",

     

       15
       14
        
               "js-graph-algorithms": "^1.0.18",

     

       16
       15
        
               "mdast": "^3.0.0",

     

       17
       17
       -
               "ollama-ai-provider": "^1.2.0",

     

       18
       16
        
               "prettier": "^3.5.3",

     

       19
       17
        
               "rehype-parse": "^9.0.1",

     

       20
       18
        
               "rehype-remark": "^10.0.1",

     
···

       69
       67
        
           "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="],

     

       70
       68
        
       

     

       71
       69
        
           "ai": ["ai@4.3.16", "", { "dependencies": { "@ai-sdk/provider": "1.1.3", "@ai-sdk/provider-utils": "2.2.8", "@ai-sdk/react": "1.2.12", "@ai-sdk/ui-utils": "1.2.11", "@opentelemetry/api": "1.9.0", "jsondiffpatch": "0.6.0" }, "peerDependencies": { "react": "^18 || ^19 || ^19.0.0-rc", "zod": "^3.23.8" }, "optionalPeers": ["react"] }, "sha512-KUDwlThJ5tr2Vw0A1ZkbDKNME3wzWhuVfAOwIvFUzl1TPVDFAXDFTXio3p+jaKneB+dKNCvFFlolYmmgHttG1g=="],

     

       72
       72
       -
       

     

       73
       73
       -
           "ai-fallback": ["ai-fallback@0.1.5", "", { "dependencies": { "@ai-sdk/provider": "^1", "@ai-sdk/provider-utils": "^2" } }, "sha512-/FhTd9SGMEUDYBKbO3ZyfS0CBGglJByMbMRQOGjjDYlxZinFZtn99w1SPh4NZYJWIP5jjoewytfZjp+30QPT1A=="],

     

       74
       70
        
       

     

       75
       71
        
           "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="],

     

       76
       72
        
       

     
···

       238
       234
        
       

     

       239
       235
        
           "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],

     

       240
       236
        
       

     

       241
       241
       -
           "ollama-ai-provider": ["ollama-ai-provider@1.2.0", "", { "dependencies": { "@ai-sdk/provider": "^1.0.0", "@ai-sdk/provider-utils": "^2.0.0", "partial-json": "0.1.7" }, "peerDependencies": { "zod": "^3.0.0" }, "optionalPeers": ["zod"] }, "sha512-jTNFruwe3O/ruJeppI/quoOUxG7NA6blG3ZyQj3lei4+NnJo7bi3eIRWqlVpRlu/mbzbFXeJSBuYQWF6pzGKww=="],

     

       242
       242
       -
       

     

       243
       237
        
           "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],

     

       244
       244
       -
       

     

       245
       245
       -
           "partial-json": ["partial-json@0.1.7", "", {}, "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA=="],

     

       246
       238
        
       

     

       247
       239
        
           "prettier": ["prettier@3.5.3", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw=="],

     

       248
       240

-2

package.json

···

       11
       11
        
           "@mozilla/readability": "^0.6.0",

     

       12
       12
        
           "@tsconfig/bun": "^1.0.8",

     

       13
       13
        
           "ai": "^4.3.16",

     

       14
       14
       -
           "ai-fallback": "^0.1.5",

     

       15
       14
        
           "debug": "^4.4.1",

     

       16
       15
        
           "happy-dom": "^18.0.1",

     

       17
       16
        
           "hast-util-sanitize": "^5.0.2",

     

       18
       17
        
           "js-graph-algorithms": "^1.0.18",

     

       19
       18
        
           "mdast": "^3.0.0",

     

       20
       20
       -
           "ollama-ai-provider": "^1.2.0",

     

       21
       19
        
           "prettier": "^3.5.3",

     

       22
       20
        
           "rehype-parse": "^9.0.1",

     

       23
       21
        
           "rehype-remark": "^10.0.1",

+3 -1

src/index.ts

···

       19
       19
        
         log('crawl', site.name);

     

       20
       20
        
         const pages = await crawl(site);

     

       21
       21
        
         const contents: string[] = [];

     

       22
       22
       -
         for (const page of pages) {

     

       22
       22
       +
         for (let idx = 0; idx < pages.length; idx++) {

     

       23
       23
       +
           const page = pages[idx]!;

     

       23
       24
        
           const content = await page.getContent();

     

       24
       25
        
           if (content) contents.push(content);

     

       26
       26
       +
           log(`completed page ${idx + 1} of ${pages.length}`);

     

       25
       27
        
         }

     

       26
       28
        
         const output = await concatMarkdown(contents);

     

       27
       29
        
         const formatted = await formatMarkdown(output);

+28 -21

src/rewrite.ts

···

       2
       2
        
       import { createFallback } from 'ai-fallback';

     

       3
       3
        
       import { streamText } from 'ai';

     

       4
       4
        
       import { createOpenAI } from '@ai-sdk/openai';

     

       5
       5
       -
       import { createOllama } from 'ollama-ai-provider';

     

       6
       5
        
       import * as fs from 'node:fs/promises';

     

       7
       6
        
       import * as path from 'node:path';

     

       8
       7
        
       

     
···

       14
       13
        
       await fs.mkdir(cacheDir, { recursive: true });

     

       15
       14
        
       const getCacheFile = makeCacheFileHelper(cacheDir, '.txt');

     

       16
       15
        
       

     

       17
       17
       -
       if (!process.env.OPENAI_API_KEY) throw new Error('Missing OPENAI_API_KEY env var');

     

       18
       18
       -
       if (!process.env.OPENAI_API_URL) throw new Error('Missing OPENAI_API_URL env var');

     

       19
       19
       -
       

     

       20
       16
        
       const SYSTEM_PROMPT = `

     

       21
       17
        
       Reformat markdown content you're given into an llms-full.txt file, also in markdown format

     

       22
       18
        
       - Reformat for an AI and paraphrase where necessary, but be faithful to the original

     

       23
       23
       -
       - Avoid using emphasis and use Github Flavored markdown syntax

     

       24
       19
        
       - Keep code snippets and keep them in TypeScript or TypeScript typings format

     

       25
       25
       -
       - Don't mention other content, pages, or external content (Remove sentences such as "Refer to", "Read more", "Learn how to")

     

       26
       26
       -
       - For markdown tables, keep all relevant information in the table and remove table legends and emoji

     

       27
       27
       -
       - Remove icon legends or irrelevant text

     

       28
       28
       -
       - Don't add to the content or use any knowledge you may have on the subject

     

       29
       29
       -
       - Format the output in AI-friendly markdown and preserve inline code

     

       30
       30
       -
       - Remove sub-headings if they don't add crucial information or context

     

       31
       31
       -
       - Don't wrap your output in a code block

     

       20
       20
       +
       - For markdown tables, keep all relevant information in the table

     

       21
       21
       +
       - Don't mention other content, pages, or external content

     

       22
       22
       +
       - Don't write your own content

     

       23
       23
       +
       - Don't add or use any knowledge you may have on the subject

     

       24
       24
       +
       - Don't add your own interpretation or notes and only reinterpret the input content

     

       25
       25
       +
       - Don't wrap the output in a markdown code block

     

       26
       26
       +
       Only return the reformatted markdown content and stop when you've processed all input markdown content

     

       32
       27
        
       `;

     

       33
       28
        
       

     

       34
       29
        
       const ai = createOpenAI({

     
···

       36
       31
        
         baseURL: process.env.OPENAI_API_URL,

     

       37
       32
        
       });

     

       38
       33
        
       

     

       39
       39
       -
       const ollama = createOllama({

     

       40
       40
       -
         baseURL: 'http://localhost:11434/api',

     

       34
       34
       +
       const ollama = createOpenAI({

     

       35
       35
       +
         baseURL: 'http://localhost:11434/v1',

     

       41
       36
        
       });

     

       42
       37
        
       

     

       43
       38
        
       export async function rewriteMarkdown(url: URL, input: string) {

     
···

       52
       47
        
         } catch {}

     

       53
       48
        
         log('prompting to rewrite', url.pathname);

     

       54
       49
        
         const { textStream } = streamText({

     

       50
       50
       +
           temperature: 0.05,

     

       51
       51
       +
           maxSteps: 5,

     

       52
       52
       +
           experimental_continueSteps: true,

     

       55
       53
        
           model: createFallback({

     

       56
       54
        
             models: [

     

       57
       57
       -
               ollama('mistral-small3.1:24b'),

     

       55
       55
       +
               ollama('phi4:14b'),

     

       58
       56
        
               ai('@cf/mistralai/mistral-small-3.1-24b-instruct'),

     

       59
       57
        
             ],

     

       60
       58
        
             onError(error, modelId) {

     

       61
       59
        
               log(`error using model ${modelId}`, error);

     

       62
       60
        
             },

     

       63
       61
        
           }),

     

       64
       64
       -
           maxSteps: 5,

     

       65
       65
       -
           experimental_continueSteps: true,

     

       66
       66
       -
           temperature: 0.05,

     

       67
       67
       -
           system: SYSTEM_PROMPT.trim(),

     

       68
       68
       -
           prompt: input,

     

       62
       62
       +
           onStepFinish({ finishReason, text }) {

     

       63
       63
       +
             if (finishReason !== 'stop')

     

       64
       64
       +
               log(`inference step (length: ${text.length})`, finishReason); 

     

       65
       65
       +
           },

     

       66
       66
       +
           messages: [

     

       67
       67
       +
             {

     

       68
       68
       +
               role: 'system',

     

       69
       69
       +
               content: SYSTEM_PROMPT.trim(),

     

       70
       70
       +
             },

     

       71
       71
       +
             {

     

       72
       72
       +
               role: 'user',

     

       73
       73
       +
               content: input,

     

       74
       74
       +
             },

     

       75
       75
       +
           ],

     

       69
       76
        
         });

     

       70
       70
       -
         const output = [];

     

       77
       77
       +
         const output: string[] = [];

     

       71
       78
        
         for await (const chunk of textStream)

     

       72
       79
        
           output.push(chunk);

     

       73
       80
        
         const text = output.join('');

+2 -2

src/unified.ts

···

       18
       18
        
           .use(rehypeParse)

     

       19
       19
        
           .use(rehypeSanitize, {

     

       20
       20
        
             tagNames: [

     

       21
       21
       -
               ...defaultSanitizeSchema.tagNames!,

     

       21
       21
       +
               ...defaultSanitizeSchema.tagNames!.filter((tag) => tag !== 'details'),

     

       22
       22
        
               'content-region',

     

       23
       23
        
               'footer',

     

       24
       24
        
               'header',

     
···

       27
       27
        
               'section',

     

       28
       28
        
               'nav',

     

       29
       29
        
             ],

     

       30
       30
       -
             strip: ['script', 'style'],

     

       30
       30
       +
             strip: ['script', 'style', 'details'],

     

       31
       31
        
           })

     

       32
       32
        
           .use(rehypeStringify)

     

       33
       33
        
           .process(html)