pyrox.dev
1import collections
2import json
3import sys
4from typing import Any, Dict, List
5
6# for MD conversion
7import mistune
8import re
9from xml.sax.saxutils import escape, quoteattr
10
11JSON = Dict[str, Any]
12
13class Key:
14 def __init__(self, path: List[str]):
15 self.path = path
16 def __hash__(self):
17 result = 0
18 for id in self.path:
19 result ^= hash(id)
20 return result
21 def __eq__(self, other):
22 return type(self) is type(other) and self.path == other.path
23
24Option = collections.namedtuple('Option', ['name', 'value'])
25
26# pivot a dict of options keyed by their display name to a dict keyed by their path
27def pivot(options: Dict[str, JSON]) -> Dict[Key, Option]:
28 result: Dict[Key, Option] = dict()
29 for (name, opt) in options.items():
30 result[Key(opt['loc'])] = Option(name, opt)
31 return result
32
33# pivot back to indexed-by-full-name
34# like the docbook build we'll just fail if multiple options with differing locs
35# render to the same option name.
36def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]:
37 result: Dict[str, Dict] = dict()
38 for (key, opt) in options.items():
39 if opt.name in result:
40 raise RuntimeError(
41 'multiple options with colliding ids found',
42 opt.name,
43 result[opt.name]['loc'],
44 opt.value['loc'],
45 )
46 result[opt.name] = opt.value
47 return result
48
49admonitions = {
50 '.warning': 'warning',
51 '.important': 'important',
52 '.note': 'note'
53}
54class Renderer(mistune.renderers.BaseRenderer):
55 def _get_method(self, name):
56 try:
57 return super(Renderer, self)._get_method(name)
58 except AttributeError:
59 def not_supported(*args, **kwargs):
60 raise NotImplementedError("md node not supported yet", name, args, **kwargs)
61 return not_supported
62
63 def text(self, text):
64 return escape(text)
65 def paragraph(self, text):
66 return text + "\n\n"
67 def newline(self):
68 return "<literallayout>\n</literallayout>"
69 def codespan(self, text):
70 return f"<literal>{escape(text)}</literal>"
71 def block_code(self, text, info=None):
72 info = f" language={quoteattr(info)}" if info is not None else ""
73 return f"<programlisting{info}>\n{escape(text)}</programlisting>"
74 def link(self, link, text=None, title=None):
75 tag = "link"
76 if link[0:1] == '#':
77 if text == "":
78 tag = "xref"
79 attr = "linkend"
80 link = quoteattr(link[1:])
81 else:
82 # try to faithfully reproduce links that were of the form <link href="..."/>
83 # in docbook format
84 if text == link:
85 text = ""
86 attr = "xlink:href"
87 link = quoteattr(link)
88 return f"<{tag} {attr}={link}>{text}</{tag}>"
89 def list(self, text, ordered, level, start=None):
90 if ordered:
91 raise NotImplementedError("ordered lists not supported yet")
92 return f"<itemizedlist>\n{text}\n</itemizedlist>"
93 def list_item(self, text, level):
94 return f"<listitem><para>{text}</para></listitem>\n"
95 def block_text(self, text):
96 return text
97 def emphasis(self, text):
98 return f"<emphasis>{text}</emphasis>"
99 def strong(self, text):
100 return f"<emphasis role=\"strong\">{text}</emphasis>"
101 def admonition(self, text, kind):
102 if kind not in admonitions:
103 raise NotImplementedError(f"admonition {kind} not supported yet")
104 tag = admonitions[kind]
105 # we don't keep whitespace here because usually we'll contain only
106 # a single paragraph and the original docbook string is no longer
107 # available to restore the trailer.
108 return f"<{tag}><para>{text.rstrip()}</para></{tag}>"
109 def block_quote(self, text):
110 return f"<blockquote><para>{text}</para></blockquote>"
111 def command(self, text):
112 return f"<command>{escape(text)}</command>"
113 def option(self, text):
114 return f"<option>{escape(text)}</option>"
115 def file(self, text):
116 return f"<filename>{escape(text)}</filename>"
117 def var(self, text):
118 return f"<varname>{escape(text)}</varname>"
119 def env(self, text):
120 return f"<envar>{escape(text)}</envar>"
121 def manpage(self, page, section):
122 title = f"<refentrytitle>{escape(page)}</refentrytitle>"
123 vol = f"<manvolnum>{escape(section)}</manvolnum>"
124 return f"<citerefentry>{title}{vol}</citerefentry>"
125
126 def finalize(self, data):
127 return "".join(data)
128
129def p_command(md):
130 COMMAND_PATTERN = r'\{command\}`(.*?)`'
131 def parse(self, m, state):
132 return ('command', m.group(1))
133 md.inline.register_rule('command', COMMAND_PATTERN, parse)
134 md.inline.rules.append('command')
135
136def p_file(md):
137 FILE_PATTERN = r'\{file\}`(.*?)`'
138 def parse(self, m, state):
139 return ('file', m.group(1))
140 md.inline.register_rule('file', FILE_PATTERN, parse)
141 md.inline.rules.append('file')
142
143def p_var(md):
144 VAR_PATTERN = r'\{var\}`(.*?)`'
145 def parse(self, m, state):
146 return ('var', m.group(1))
147 md.inline.register_rule('var', VAR_PATTERN, parse)
148 md.inline.rules.append('var')
149
150def p_env(md):
151 ENV_PATTERN = r'\{env\}`(.*?)`'
152 def parse(self, m, state):
153 return ('env', m.group(1))
154 md.inline.register_rule('env', ENV_PATTERN, parse)
155 md.inline.rules.append('env')
156
157def p_option(md):
158 OPTION_PATTERN = r'\{option\}`(.*?)`'
159 def parse(self, m, state):
160 return ('option', m.group(1))
161 md.inline.register_rule('option', OPTION_PATTERN, parse)
162 md.inline.rules.append('option')
163
164def p_manpage(md):
165 MANPAGE_PATTERN = r'\{manpage\}`(.*?)\((.+?)\)`'
166 def parse(self, m, state):
167 return ('manpage', m.group(1), m.group(2))
168 md.inline.register_rule('manpage', MANPAGE_PATTERN, parse)
169 md.inline.rules.append('manpage')
170
171def p_admonition(md):
172 ADMONITION_PATTERN = re.compile(r'^::: \{([^\n]*?)\}\n(.*?)^:::$\n*', flags=re.MULTILINE|re.DOTALL)
173 def parse(self, m, state):
174 return {
175 'type': 'admonition',
176 'children': self.parse(m.group(2), state),
177 'params': [ m.group(1) ],
178 }
179 md.block.register_rule('admonition', ADMONITION_PATTERN, parse)
180 md.block.rules.append('admonition')
181
182md = mistune.create_markdown(renderer=Renderer(), plugins=[
183 p_command, p_file, p_var, p_env, p_option, p_manpage, p_admonition
184])
185
186# converts in-place!
187def convertMD(options: Dict[str, Any]) -> str:
188 def convertString(path: str, text: str) -> str:
189 try:
190 rendered = md(text)
191 # keep trailing spaces so we can diff the generated XML to check for conversion bugs.
192 return rendered.rstrip() + text[len(text.rstrip()):]
193 except:
194 print(f"error in {path}")
195 raise
196
197 def optionIs(option: Dict[str, Any], key: str, typ: str) -> bool:
198 if key not in option: return False
199 if type(option[key]) != dict: return False
200 if '_type' not in option[key]: return False
201 return option[key]['_type'] == typ
202
203 for (name, option) in options.items():
204 try:
205 if optionIs(option, 'description', 'mdDoc'):
206 option['description'] = convertString(name, option['description']['text'])
207 elif markdownByDefault:
208 option['description'] = convertString(name, option['description'])
209
210 if optionIs(option, 'example', 'literalMD'):
211 docbook = convertString(name, option['example']['text'])
212 option['example'] = { '_type': 'literalDocBook', 'text': docbook }
213 if optionIs(option, 'default', 'literalMD'):
214 docbook = convertString(name, option['default']['text'])
215 option['default'] = { '_type': 'literalDocBook', 'text': docbook }
216 except Exception as e:
217 raise Exception(f"Failed to render option {name}: {str(e)}")
218
219
220 return options
221
222warningsAreErrors = False
223errorOnDocbook = False
224markdownByDefault = False
225optOffset = 0
226for arg in sys.argv[1:]:
227 if arg == "--warnings-are-errors":
228 optOffset += 1
229 warningsAreErrors = True
230 if arg == "--error-on-docbook":
231 optOffset += 1
232 errorOnDocbook = True
233 if arg == "--markdown-by-default":
234 optOffset += 1
235 markdownByDefault = True
236
237options = pivot(json.load(open(sys.argv[1 + optOffset], 'r')))
238overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r')))
239
240# fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir
241for (k, v) in options.items():
242 # The _module options are not declared in nixos/modules
243 if v.value['loc'][0] != "_module":
244 v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}' if isinstance(s, str) else s, v.value['declarations']))
245
246# merge both descriptions
247for (k, v) in overrides.items():
248 cur = options.setdefault(k, v).value
249 for (ok, ov) in v.value.items():
250 if ok == 'declarations':
251 decls = cur[ok]
252 for d in ov:
253 if d not in decls:
254 decls += [d]
255 elif ok == "type":
256 # ignore types of placeholder options
257 if ov != "_unspecified" or cur[ok] == "_unspecified":
258 cur[ok] = ov
259 elif ov is not None or cur.get(ok, None) is None:
260 cur[ok] = ov
261
262severity = "error" if warningsAreErrors else "warning"
263
264def is_docbook(o, key):
265 val = o.get(key, {})
266 if not isinstance(val, dict):
267 return False
268 return val.get('_type', '') == 'literalDocBook'
269
270# check that every option has a description
271hasWarnings = False
272hasErrors = False
273hasDocBookErrors = False
274for (k, v) in options.items():
275 if errorOnDocbook:
276 if isinstance(v.value.get('description', {}), str):
277 hasErrors = True
278 hasDocBookErrors = True
279 print(
280 f"\x1b[1;31merror: option {v.name} description uses DocBook\x1b[0m",
281 file=sys.stderr)
282 elif is_docbook(v.value, 'defaultText'):
283 hasErrors = True
284 hasDocBookErrors = True
285 print(
286 f"\x1b[1;31merror: option {v.name} default uses DocBook\x1b[0m",
287 file=sys.stderr)
288 elif is_docbook(v.value, 'example'):
289 hasErrors = True
290 hasDocBookErrors = True
291 print(
292 f"\x1b[1;31merror: option {v.name} example uses DocBook\x1b[0m",
293 file=sys.stderr)
294
295 if v.value.get('description', None) is None:
296 hasWarnings = True
297 print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr)
298 v.value['description'] = "This option has no description."
299 if v.value.get('type', "unspecified") == "unspecified":
300 hasWarnings = True
301 print(
302 f"\x1b[1;31m{severity}: option {v.name} has no type. Please specify a valid type, see " +
303 "https://nixos.org/manual/nixos/stable/index.html#sec-option-types\x1b[0m", file=sys.stderr)
304
305if hasDocBookErrors:
306 print("Explanation: The documentation contains descriptions, examples, or defaults written in DocBook. " +
307 "NixOS is in the process of migrating from DocBook to Markdown, and " +
308 "DocBook is disallowed for in-tree modules. To change your contribution to "+
309 "use Markdown, apply mdDoc and literalMD. For example:\n" +
310 "\n" +
311 " example.foo = mkOption {\n" +
312 " description = lib.mdDoc ''your description'';\n" +
313 " defaultText = lib.literalMD ''your description of default'';\n" +
314 " }\n" +
315 "\n" +
316 " example.enable = mkEnableOption (lib.mdDoc ''your thing'');",
317 file = sys.stderr)
318
319if hasErrors:
320 sys.exit(1)
321if hasWarnings and warningsAreErrors:
322 print(
323 "\x1b[1;31m" +
324 "Treating warnings as errors. Set documentation.nixos.options.warningsAreErrors " +
325 "to false to ignore these warnings." +
326 "\x1b[0m",
327 file=sys.stderr)
328 sys.exit(1)
329
330json.dump(convertMD(unpivot(options)), fp=sys.stdout)