this repo has no description
dekker.one
1from minizinc_testing import yaml
2from minizinc_testing.spec import CachedResult
3
4import pytest
5
6# pylint: disable=import-error,no-name-in-module
7from py.xml import html
8from html import escape
9import pytest_html
10import re
11import minizinc as mzn
12from difflib import HtmlDiff
13import sys
14
15
16def pytest_configure(config):
17 pytest.solver_cache = {}
18 search = config.getoption("--driver")
19 if search is not None:
20 driver = mzn.find_driver([search])
21 if driver is None:
22 raise Exception("Failed to find MiniZinc driver in {}".format(search))
23 driver.make_default()
24
25
26def pytest_addoption(parser):
27 parser.addoption(
28 "--solvers",
29 action="store",
30 metavar="SOLVERS",
31 help="only run tests with the comma separated SOLVERS.",
32 )
33 parser.addoption(
34 "--suite",
35 action="append",
36 default=[],
37 metavar="SUITE_NAME",
38 help="Use the given YAML configuration from suites.yml",
39 )
40 parser.addoption(
41 "--all-suites", action="store_true", dest="feature", help="Run all test suites"
42 )
43 parser.addoption(
44 "--driver",
45 action="store",
46 metavar="MINIZINC",
47 help="Directory containing MiniZinc executable",
48 )
49
50
51def pytest_collect_file(parent, path):
52 if path.ext == ".mzn":
53 return MznFile.from_parent(parent, fspath=path)
54
55
56def pytest_html_results_table_header(cells):
57 cells.insert(2, html.th("Solver", class_="sortable", col="solver"))
58 cells.insert(3, html.th("Checker", class_="sortable", col="checker"))
59 cells.pop()
60
61
62def pytest_html_results_table_row(report, cells):
63 if hasattr(report, "user_properties"):
64 props = {k: v for k, v in report.user_properties}
65 cells.insert(2, html.td(props["solver"]))
66 cells.insert(3, html.td(props["checker"] if "checker" in props else "-"))
67 cells.pop()
68
69
70@pytest.hookimpl(hookwrapper=True)
71def pytest_runtest_makereport(item, call):
72 outcome = yield
73 report = outcome.get_result()
74 extra = getattr(report, "extra", [])
75 if report.when == "call" and report.outcome != "skipped":
76 props = {k: v for k, v in report.user_properties}
77 if "compare" in props:
78 required, obtained = props["compare"]
79 html_content = """
80 <button class="copy-button" onclick="this.nextElementSibling.select();document.execCommand('copy');this.textContent = 'Copied!';">Copy obtained output to clipboard</button>
81 <textarea class="hidden-textarea" readonly>{}</textarea>
82 """.format(
83 escape(obtained)
84 )
85 actual = obtained.split("\n")
86 htmldiff = HtmlDiff(2)
87 html_content += '<h4>Diffs</h4><div class="diffs">'
88 html_content += "<hr>".join(
89 htmldiff.make_table(
90 expected.split("\n"),
91 actual,
92 fromdesc="expected",
93 todesc="actual",
94 context=True,
95 )
96 for expected in required
97 )
98 html_content += "</div>"
99 extra.append(pytest_html.extras.html(html_content))
100 report.extra = extra
101
102
103def pytest_metadata(metadata):
104 # Ensure that secrets don't get shown
105 # Can likely be removed after pytest-metadata is updated
106 metadata.pop("CI_JOB_TOKEN", None)
107 metadata.pop("CI_REPOSITORY_URL", None)
108 metadata.pop("CI_REGISTRY_PASSWORD", None)
109
110
111class MznFile(pytest.File):
112 def collect(self):
113 with open("./spec/suites.yml", encoding="utf-8") as suites_file:
114 suites = yaml.load(suites_file)
115
116 if not self.config.getoption("--all-suites"):
117 enabled_suites = self.config.getoption("--suite")
118 if len(enabled_suites) == 0:
119 suites = {"default": suites["default"]}
120 else:
121 suites = {k: v for k, v in suites.items() if k in enabled_suites}
122
123 with self.fspath.open(encoding="utf-8") as file:
124 contents = file.read()
125 yaml_comment = re.match(r"\/\*\*\*\n(.*?)\n\*\*\*\/", contents, flags=re.S)
126 if yaml_comment is None:
127 pytest.skip(
128 "skipping {} as no tests specified".format(str(self.fspath))
129 )
130 else:
131 tests = [doc for doc in yaml.load_all(yaml_comment.group(1))]
132
133 for suite_name, suite in suites.items():
134 if any(self.fspath.fnmatch(glob) for glob in suite.includes):
135 for i, spec in enumerate(tests):
136 for solver in spec.solvers:
137 base = (
138 str(i) if spec.name is yaml.Undefined else spec.name
139 )
140 name = "{}.{}.{}".format(suite_name, base, solver)
141 cache = CachedResult()
142 yield SolveItem.from_parent(
143 self,
144 name=name,
145 spec=spec,
146 solver=solver,
147 cache=cache,
148 markers=spec.markers,
149 suite=suite,
150 )
151 for checker in spec.check_against:
152 yield CheckItem.from_parent(
153 self,
154 name="{}:{}".format(name, checker),
155 cache=cache,
156 solver=solver,
157 checker=checker,
158 markers=spec.markers,
159 suite=suite,
160 )
161
162
163class MznItem(pytest.Item):
164 def __init__(self, name, parent, solver, markers, suite):
165 super().__init__(name, parent)
166 self.user_properties.append(("solver", solver))
167 for marker in markers:
168 self.add_marker(marker)
169
170 if self.config.getoption("--solvers") is not None:
171 self.allowed = [
172 x.strip() for x in self.config.getoption("--solvers").split(",")
173 ]
174 self.allowed = [x for x in self.allowed if x in suite.solvers]
175 else:
176 self.allowed = suite.solvers
177
178 if not self.solver_allowed(solver):
179 self.add_marker(
180 pytest.mark.skip("skipping {} not in {}".format(solver, self.allowed))
181 )
182
183 if not self.solver_exists(solver):
184 self.add_marker(pytest.mark.skip("Solver {} not available".format(solver)))
185
186 def solver_allowed(self, solver):
187 return self.allowed is None or solver in self.allowed
188
189 def solver_exists(self, solver):
190 solver_exists = pytest.solver_cache.get(solver, None)
191 if solver_exists is None:
192 try:
193 s = mzn.Solver.lookup(solver)
194 empty_model = mzn.Model()
195 empty_model.add_string("solve satisfy;")
196 instance = mzn.Instance(s, empty_model)
197 instance.solve()
198 solver_exists = True
199 except (mzn.MiniZincError, LookupError) as error:
200 solver_exists = False
201 finally:
202 pytest.solver_cache[solver] = solver_exists
203 return solver_exists
204
205
206class SolveItem(MznItem):
207 def __init__(self, name, parent, spec, solver, cache, markers, suite):
208 super().__init__(name, parent, solver, markers, suite)
209 self.spec = spec
210 self.solver = solver
211 self.cache = cache
212 self.default_options = suite.options
213 self.strict = suite.strict
214
215 def runtest(self):
216 model, result, required, obtained = self.spec.run(
217 str(self.fspath), self.solver, default_options=self.default_options
218 )
219
220 # To pass model and result to checker test item
221 self.cache.model = model
222 self.cache.result = result
223 self.cache.obtained = obtained
224
225 passed = self.spec.passed(result)
226
227 if not passed:
228 # Test fails if we still haven't passed
229 expected = [yaml.dump(exp) for exp in required]
230 actual = yaml.dump(obtained)
231 self.user_properties.append(("compare", (expected, actual)))
232 message = "expected one of\n\n{}\n\nbut got\n\n{}".format(
233 "\n---\n".join(expected), actual
234 )
235
236 # Doesn't match, so backup by checking against another solver
237 if isinstance(result, mzn.Result) and result.status.has_solution():
238 checkers = [s for s in self.spec.solvers if s is not self.solver]
239 if len(checkers) > 0:
240 checker = checkers[0]
241 non_strict_pass = self.cache.test(checker)
242 status = "but passed" if non_strict_pass else "and failed"
243 message += "\n\n{} check against {}.".format(status, checker)
244
245 if not self.strict and non_strict_pass:
246 print(message, file=sys.stderr)
247 return
248
249 assert False, message
250
251 def reportinfo(self):
252 return self.fspath, 0, "{}::{}".format(str(self.fspath), self.name)
253
254
255class CheckItem(MznItem):
256 def __init__(self, name, parent, cache, solver, checker, markers, suite):
257 super().__init__(name, parent, solver, markers, suite)
258 if not self.solver_allowed(checker):
259 self.add_marker(
260 pytest.mark.skip(
261 "skipping checker {} not in {}".format(checker, self.allowed)
262 )
263 )
264 if not self.solver_exists(checker):
265 self.add_marker(
266 pytest.mark.skip("skipping checker {} not available".format(checker))
267 )
268
269 self.cache = cache
270 self.solver = solver
271 self.checker = checker
272 self.user_properties.append(("checker", checker))
273 self.add_marker(pytest.mark.check)
274
275 def runtest(self):
276 if (
277 not isinstance(self.cache.result, mzn.Result)
278 or not self.cache.result.status.has_solution()
279 ):
280 pytest.skip("skipping check for no result/solution")
281 else:
282 passed = self.cache.test(self.checker)
283 assert passed, "failed when checking against {}. Got {}".format(
284 self.checker, yaml.dump(self.cache.obtained)
285 )
286
287 def reportinfo(self):
288 return self.fspath, 0, "{}::{}".format(str(self.fspath), self.name)