commit 860166ccbe0983d35f151eee70d355439923a61f · kitten.sh/rn-llms-txt-gen

+8 -4

src/page.ts

···

       76
       76
        
           }

     

       77
       77
        
         }

     

       78
       78
        
       

     

       79
       79
       +
         get origin() {

     

       80
       80
       +
           return this.#origin;

     

       81
       81
       +
         }

     

       82
       82
       +
       

     

       79
       83
        
         getPage(url: URL) {

     

       80
       84
        
           let page = this.#pages.get(url.toString());

     

       81
       85
        
           if (!page) {

     
···

       143
       147
        
         }

     

       144
       148
        
       }

     

       145
       149
        
       

     

       146
       146
       -
       async function crawlPages(page: Page, visited = new Set<Page>([page])): Promise<Page[]> {

     

       150
       150
       +
       async function crawlPages(page: Page, visited = new Set<Page>([page]), depth = 1): Promise<Page[]> {

     

       147
       151
        
         const links = await page.getLinks();

     

       148
       152
        
         if (links) {

     

       149
       149
       -
           log(`crawling (${links.length} links)`, page.url.pathname);

     

       153
       153
       +
           log(`crawling (${links.length} links, depth = ${depth})`, page.url.pathname);

     

       150
       154
        
           for (const link of links) {

     

       151
       155
        
             if (visited.has(link)) {

     

       152
       156
        
               continue;

     

       153
       157
        
             } else {

     

       154
       158
        
               visited.add(page);

     

       155
       155
       -
               await crawlPages(link, visited);

     

       159
       159
       +
               await crawlPages(link, visited, depth + 1);

     

       156
       160
        
             }

     

       157
       161
        
           }

     

       158
       162
        
         }

     
···

       183
       187
        
           }

     

       184
       188
        
         }

     

       185
       189
        
         const kruskal = new KruskalMST(graph);

     

       186
       186
       -
         const output = new Set<Page>();

     

       190
       190
       +
         const output = new Set<Page>([root.origin]);

     

       187
       191
        
         for (const edge of kruskal.mst) {

     

       188
       192
        
           const page = pages[edge.from()];

     

       189
       193
        
           if (page) output.add(page);

+15 -9

src/rewrite.ts

···

       19
       19
        
       

     

       20
       20
        
       const SYSTEM_PROMPT = `

     

       21
       21
        
       Reformat markdown content you're given into an llms-full.txt file, also in markdown format

     

       22
       22
       -
       - Where the format isn't easily understandable by AI, reformat it faithfully to make it processable

     

       23
       23
       -
       - Reformat for an AI and paraphrase where necessary, but don't add interpretations

     

       24
       24
       -
       - Preserve code snippets and keep them in TypeScript or TypeScript typings format

     

       25
       25
       -
       - Avoid using emphasis or excessive markdown syntax, but keep code snippets where they are

     

       26
       26
       -
       - Don't mention other content, pages, or external content (Remove sentences such as "Refer to", "Read more")

     

       27
       27
       -
       - When encountering a markdown table, ensure that you don't output a separate legend, and keep all relevant information in the table

     

       28
       28
       -
       - Don't use any knowledge you may have on the subject. Only output what you're given.

     

       22
       22
       +
       - Reformat for an AI and paraphrase where necessary, but be faithful to the original

     

       23
       23
       +
       - Avoid using emphasis and use Github Flavored markdown syntax

     

       24
       24
       +
       - Keep code snippets and keep them in TypeScript or TypeScript typings format

     

       25
       25
       +
       - Don't mention other content, pages, or external content (Remove sentences such as "Refer to", "Read more", "Learn how to")

     

       26
       26
       +
       - For markdown tables, keep all relevant information in the table and remove table legends and emoji

     

       27
       27
       +
       - Remove icon legends or irrelevant text

     

       28
       28
       +
       - Don't add to the content or use any knowledge you may have on the subject

     

       29
       29
       +
       - Format the output in AI-friendly markdown and preserve inline code

     

       30
       30
       +
       - Remove sub-headings if they don't add crucial information or context

     

       31
       31
       +
       - Don't wrap your output in a code block

     

       29
       32
        
       `;

     

       30
       33
        
       

     

       31
       34
        
       const ai = createOpenAI({

     
···

       51
       54
        
         const { text } = await generateText({

     

       52
       55
        
           model: createFallback({

     

       53
       56
        
             models: [

     

       54
       54
       -
               ollama('gemma:7b'),

     

       55
       55
       -
               ai('@hf/google/gemma-7b-it'),

     

       57
       57
       +
               ollama('mistral-small3.1:24b'),

     

       58
       58
       +
               ai('@cf/mistralai/mistral-small-3.1-24b-instruct'),

     

       56
       59
        
             ],

     

       57
       60
        
             onError(error, modelId) {

     

       58
       61
        
               log(`error using model ${modelId}`, error);

     

       59
       62
        
             },

     

       60
       63
        
           }),

     

       64
       64
       +
           maxSteps: 5,

     

       65
       65
       +
           experimental_continueSteps: true,

     

       66
       66
       +
           temperature: 0.05,

     

       61
       67
        
           system: SYSTEM_PROMPT.trim(),

     

       62
       68
        
           prompt: input,

     

       63
       69
        
         });