commit b95450e9b9a4b69a850df429e7d38b3f3e10f7f3 · kitten.sh/rn-llms-txt-gen

+2 -2

src/page.ts

···

       5
       5
        
       import { WeightedDiGraph, KruskalMST, Edge } from 'js-graph-algorithms';

     

       6
       6
        
       import { extractContent, extractLinks, parseBody } from "./dom";

     

       7
       7
        
       import { fetchHtml } from "./fetch";

     

       8
       8
       -
       import { htmlToMarkdown, sanitizeHtml } from "./unified";

     

       8
       8
       +
       import { htmlToMarkdown, sanitizeHtml, transferTitle } from "./unified";

     

       9
       9
        
       import { rewriteMarkdown } from './rewrite';

     

       10
       10
        
       import { makeCacheFileHelper } from './path';

     

       11
       11
        
       

     
···

       143
       143
        
           const markdown = await extractContentToMarkdown(this.url, html);

     

       144
       144
        
           if (!markdown) return (this.#content = null);

     

       145
       145
        
           const rewritten = await rewriteMarkdown(this.url, markdown);

     

       146
       146
       -
           return (this.#content = rewritten);

     

       146
       146
       +
           return (this.#content = await transferTitle(markdown, rewritten));

     

       147
       147
        
         }

     

       148
       148
        
       }

     

       149
       149

src/types.d.ts

···

       1
       1
       +
       declare module 'remark-title' {

     

       2
       2
       +
         interface TitleOptions {

     

       3
       3
       +
           title: string;

     

       4
       4
       +
         }

     

       5
       5
       +
         export default function remarkTitle(opts: TitleOptions): (tree: Root) => undefined;

     

       6
       6
       +
         export type Root = import('mdast').Root;

     

       7
       7
       +
       }

+66 -1

src/unified.ts

···

       1
       1
       -
       import type { List, ListItem, Root } from 'mdast';

     

       1
       1
       +
       import type { List, ListItem, PhrasingContent, Root } from 'mdast';

     

       2
       2
        
       import { unified } from 'unified';

     

       3
       3
        
       import { visit } from 'unist-util-visit';

     

       4
       4
        
       import { defaultSchema as defaultSanitizeSchema } from 'hast-util-sanitize';

     
···

       12
       12
        
       import remarkGfm from 'remark-gfm';

     

       13
       13
        
       import remarkParse from 'remark-parse';

     

       14
       14
        
       import remarkSqueezeParagraphs from 'remark-squeeze-paragraphs';

     

       15
       15
       +
       import remarkTitle from 'remark-title';

     

       15
       16
        
       

     

       16
       17
        
       export async function sanitizeHtml(html: string): Promise<string> {

     

       17
       18
        
         const vfile = await unified()

     
···

       158
       159
        
           );

     

       159
       160
        
         return md.toString();

     

       160
       161
        
       }

     

       162
       162
       +
       

     

       163
       163
       +
       function extractTitle(markdown: string): string | null {

     

       164
       164
       +
         let depth: number | null = null;

     

       165
       165
       +
         let title: string | null = null;

     

       166
       166
       +
         const toString = (nodes: PhrasingContent[]): string =>

     

       167
       167
       +
           nodes.map((node) => {

     

       168
       168
       +
             switch (node.type) {

     

       169
       169
       +
               case 'break':

     

       170
       170
       +
                 return '\n';

     

       171
       171
       +
               case 'delete':

     

       172
       172
       +
               case 'emphasis':

     

       173
       173
       +
               case 'link':

     

       174
       174
       +
               case 'strong':

     

       175
       175
       +
                 return toString(node.children);

     

       176
       176
       +
               case 'inlineCode':

     

       177
       177
       +
                 return `\`${node.value}\``;

     

       178
       178
       +
               case 'text':

     

       179
       179
       +
                 return node.value;

     

       180
       180
       +
               case 'footnoteReference':

     

       181
       181
       +
               case 'html':

     

       182
       182
       +
               case 'image':

     

       183
       183
       +
               case 'imageReference':

     

       184
       184
       +
               case 'linkReference':

     

       185
       185
       +
               default:

     

       186
       186
       +
                 return '';

     

       187
       187
       +
             }

     

       188
       188
       +
           }).join('');

     

       189
       189
       +
         const tree = unified()

     

       190
       190
       +
           .use(remarkParse, { fragment: true })

     

       191
       191
       +
           .use(remarkGfm, {

     

       192
       192
       +
             tablePipeAlign: false,

     

       193
       193
       +
             tableCellPadding: false,

     

       194
       194
       +
           })

     

       195
       195
       +
           .parse(markdown);

     

       196
       196
       +
         visit(tree, function (node) {

     

       197
       197
       +
           if (node.type !== 'heading')

     

       198
       198
       +
             return;

     

       199
       199
       +
           if (!depth || node.depth < depth)

     

       200
       200
       +
             title = toString(node.children);

     

       201
       201
       +
         });

     

       202
       202
       +
         return title;

     

       203
       203
       +
       }

     

       204
       204
       +
       

     

       205
       205
       +
       export async function transferTitle(from: string, to: string): Promise<string> {

     

       206
       206
       +
         const title = extractTitle(from);

     

       207
       207
       +
         if (!title) return to;

     

       208
       208
       +
         const md = await unified()

     

       209
       209
       +
           .use(remarkParse, { fragment: true })

     

       210
       210
       +
           .use(remarkGfm, {

     

       211
       211
       +
             tablePipeAlign: false,

     

       212
       212
       +
             tableCellPadding: false,

     

       213
       213
       +
           })

     

       214
       214
       +
           .use(remarkTitle, {

     

       215
       215
       +
             title,

     

       216
       216
       +
           })

     

       217
       217
       +
           .use(remarkStringify, {

     

       218
       218
       +
             bullet: '-',

     

       219
       219
       +
             incrementListMarker: false,

     

       220
       220
       +
             ruleSpaces: false,

     

       221
       221
       +
             tightDefinitions: true,

     

       222
       222
       +
           })

     

       223
       223
       +
           .process(to);

     

       224
       224
       +
         return md.toString();

     

       225
       225
       +
       }