···

       
26
       
 
       
    log(`completed page ${idx + 1} of ${pages.length}`);

     

       
27
       
 
       
  }

     

       
28
       
 
       
  const output = await concatMarkdown(contents);

     

       
29
       
-
       
  const formatted = await formatMarkdown(output);

     

       
30
       
 
       
  const file = path.join(outputPath, `llms-full-${site.name}.txt`);

     

       
31
       
 
       
  await fs.writeFile(file, formatted, 'utf-8');

     

       
32
       
 
       
}

     

···

       
26
       
 
       
    log(`completed page ${idx + 1} of ${pages.length}`);

     

       
27
       
 
       
  }

     

       
28
       
 
       
  const output = await concatMarkdown(contents);

     

       
29
       
+
       
  const formatted = await formatMarkdown(site.name, output);

     

       
30
       
 
       
  const file = path.join(outputPath, `llms-full-${site.name}.txt`);

     

       
31
       
 
       
  await fs.writeFile(file, formatted, 'utf-8');

     

       
32
       
 
       
}

     

···

       
4
       
 
       
import remarkStringify from 'remark-stringify';

     

       
5
       
 
       
import remarkGfm from 'remark-gfm';

     

       
6
       
 
       


     

       
7
       
-
       
async function reformat(content: string): Promise<string> {

     

       
0
       
 
       

     

       
0
       
 
       

     

       
8
       
 
       
  // NOTE: We reformat with remark again to get rid of prettier's

     

       
9
       
 
       
  // table formatting mostly. This doesn't work well as LLM input

     

       
10
       
 
       
  const md = await unified()

     
···

       
13
       
 
       
      tablePipeAlign: false,

     

       
14
       
 
       
      tableCellPadding: false,

     

       
15
       
 
       
    })

     

       
0
       
 
       

     

       
16
       
 
       
    .use(remarkStringify, {

     

       
17
       
 
       
      bullet: '-',

     

       
18
       
 
       
      incrementListMarker: false,

     
···

       
23
       
 
       
  return md.toString();

     

       
24
       
 
       
}

     

       
25
       
 
       


     

       
26
       
-
       
export async function formatMarkdown(input: string) {

     

       
27
       
-
       
  return reformat(await format(input, {

     

       
28
       
-
       
    semi: false,

     

       
29
       
-
       
    singleQuote: false,

     

       
30
       
-
       
    trailingComma: 'es5',

     

       
31
       
-
       
    proseWrap: 'never',

     

       
32
       
-
       
    parser: 'markdown',

     

       
33
       
-
       
  }));

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
34
       
 
       
}

     

···

       
4
       
 
       
import remarkStringify from 'remark-stringify';

     

       
5
       
 
       
import remarkGfm from 'remark-gfm';

     

       
6
       
 
       


     

       
7
       
+
       
import { remarkTitle } from './unified';

     

       
8
       
+
       


     

       
9
       
+
       
async function reformat(content: string, title: string): Promise<string> {

     

       
10
       
 
       
  // NOTE: We reformat with remark again to get rid of prettier's

     

       
11
       
 
       
  // table formatting mostly. This doesn't work well as LLM input

     

       
12
       
 
       
  const md = await unified()

     
···

       
15
       
 
       
      tablePipeAlign: false,

     

       
16
       
 
       
      tableCellPadding: false,

     

       
17
       
 
       
    })

     

       
18
       
+
       
    .use(remarkTitle, { title })

     

       
19
       
 
       
    .use(remarkStringify, {

     

       
20
       
 
       
      bullet: '-',

     

       
21
       
 
       
      incrementListMarker: false,

     
···

       
26
       
 
       
  return md.toString();

     

       
27
       
 
       
}

     

       
28
       
 
       


     

       
29
       
+
       
export async function formatMarkdown(title: string, input: string) {

     

       
30
       
+
       
  return reformat(

     

       
31
       
+
       
    await format(input, {

     

       
32
       
+
       
      semi: false,

     

       
33
       
+
       
      singleQuote: false,

     

       
34
       
+
       
      trailingComma: 'es5',

     

       
35
       
+
       
      proseWrap: 'never',

     

       
36
       
+
       
      parser: 'markdown',

     

       
37
       
+
       
    }),

     

       
38
       
+
       
    title

     

       
39
       
+
       
  );

     

       
40
       
 
       
}

     

···

       
199
       
 
       
  }

     

       
200
       
 
       
}

     

       
201
       
 
       


     

       
202
       
-
       
function remarkTitle(opts: { title: string }) {

     

       
203
       
 
       
  return function checkTitleTransformer(root: Root) {

     

       
204
       
 
       
    const node = root.children[0]!;

     

       
205
       
 
       
    const replacement: Heading = {

     

···

       
199
       
 
       
  }

     

       
200
       
 
       
}

     

       
201
       
 
       


     

       
202
       
+
       
export function remarkTitle(opts: { title: string }) {

     

       
203
       
 
       
  return function checkTitleTransformer(root: Root) {

     

       
204
       
 
       
    const node = root.children[0]!;

     

       
205
       
 
       
    const replacement: Heading = {