this repo has no description
dekker.one
1#!/usr/bin/env python3
2
3## Author: Gleb Belov@monash.edu 2017
4## This program runs MiniZinc solvers over a set of instances,
5## checks solutions and compares to given solution logs.
6
7## TODO type errors etc. when checking?
8## TODO -a for optimization only
9## TODO continuous output dumping as option
10## TODO CPU/user time limit, proper memory limit (setrlimit not working)
11
12import sys, io, re as regex, traceback
13import os.path, platform
14##import numpy
15import math
16import json, argparse
17import datetime
18from collections import OrderedDict
19
20import utils, json_config, json_log, mzn_exec, cmp_result_logs
21from json_config import s_CommentKey, s_AddKey
22
23s_ProgramDescr = 'MiniZinc testing automation. (c) 2018 Monash University, gleb.belov@monash.edu'
24s_ProgramDescrLong = ( "Allows checking of MiniZinc solutions by configurable checker profiles. The solutions can be input from previous logs or produced by a chosen solver profile, for 1 or several instances, with result comparison and some ranking. New solutions' summary logs, detailed stdout/err outputs, and statistics are saved in subfolder mzn-test/LOGS, /OUTPUTS, and /STATS, resp.")
25
26########################### GENERAL CONFIG. Could be in the JSON config actually #########################
27sDirResults = "mzn-test"
28sFlnSolUnchk = sDirResults + "/sol__unchk.json" ### Logfile to append immediate results
29sFlnSolCheckBase = sDirResults + "/LOGS/sol__{}.json" ### Logfile to append checked results. {} replaced by datetime
30sFlnSolFailBase = sDirResults + "/LOGS/sol__{}__FAIL.json" ### To save failed solutions
31sFlnSolLastDzn = sDirResults + "/sol_last.dzn" ### File to save the DZN solution for checking
32sFlnStdoutBase = sDirResults + "/OUTPUTS/{}.stdout.txt" ### File name base to dump stdout for any backend call
33sFlnStderrBase = sDirResults + "/OUTPUTS/{}.stderr.txt"
34sFlnStatLog = sDirResults + "/STATS/stat_log__{}.txt" ### The instance list name(s) will be inserted, if any
35
36sDZNOutputAgrs = "--output-mode dzn --output-objective" ## The flattener arguments to produce DZN-compatible output facilitating solution checking
37 ## Remove --output-objective in MZN Release <=2.1.7
38sFlatOptChecker = "--allow-multiple-assignments"
39
40s_UsageExamples = (
41 "\nUSAGE EXAMPLES:"
42 "\n(1) \"mzn-test.py model.mzn data.dzn [--checkDZN stdout.txt [--checkStderr stderr.txt]] [--chkPrf MINIZINC-CHK --chkPrf FZN-GECODE-CHK] [--tCheck 15] [--addSolverOption \"--fzn-flags '-D fPureCircuit=true'\"]\" ::: check the instance's solutions, optionally reading them from a DZN-formatted file (otherwise solving first), optionally overriding default checker list etc."
43 "\n(2) \"mzn-test.py --solver CPLEX -t 300 -l instList1.txt -l instList2.txt --name CPLEXTest_003 --result newLog00.json prevLog1.json prevLog2.json --failed failLog.json\""
44 " ::: solve instances using the specified solver profile and wall time limit 300 seconds. The instances are taken from the list files. The test is aliased CPLEXTest_003. Results are saved to newLog00.json and compared/ranked to those in prevLog's. (Probably) incorrect solutions are saved to failLog.json."
45 "\n(3) \"mzn-test.py [-l instList1.txt] -c prevLog1.json -c prevLog2.json [--runAndCmp]\" ::: compare existing logs, optionally limited to the given instances, optionally running new tests. USE SINGLE QUOTES ONLY INSIDE ARGUMENTS PASSED TO THE BACKENDS when running backends through shell."
46 )
47##############################################################################################
48################ Parameters of MZN-Test, including config and command-line
49##############################################################################################
50class MZT_Param:
51 def parseCmdLine(self):
52 parser = argparse.ArgumentParser(
53 description=s_ProgramDescr + '\n' + s_ProgramDescrLong,
54 epilog=s_UsageExamples)
55 parser.add_argument('instanceFiles', nargs='*', metavar='<instanceFile>',
56 help='model instance files, if no instance lists supplied, otherwise existing solution logs to compare with')
57 parser.add_argument('-l', '--instanceList', dest='l_InstLists', action='append', metavar='<instanceListFile>',
58 help='file with a list of instance input files, one instance per line,'
59 ' instance file types specified in config. Used for running tests or for instance selection in comparison mode')
60 parser.add_argument('--name', '--testName', metavar='<string>', help='alias of this test run, defaults to result log file name')
61 parser.add_argument('-c', '--compare', action="append", metavar='<logfile>',
62 help='summarize and compare results to existing <logfile>. Only compares the logs and does not run tests, unless --runAndCmp. The flags -c can be omitted if -l is used')
63 parser.add_argument('--runAndCmp', '--runAndCompare', '--run', action='store_true',
64 help='even if other logs are provided by -c, do run the tests and compare')
65 parser.add_argument('--solver', '--solverPrf', '--solverProfile', metavar='<prf_name>',
66 help='solver profile from those defined in config section \"SOLVER_PROFILES\"')
67 parser.add_argument('--call', '--solverCall', metavar='"<exe+flags or shell command(s) if --shellSolve 1>"',
68 help='solver backend call, should be quoted. Insert %%s where instance files need to be. Flatten with \''
69 + sDZNOutputAgrs + '\' to enable solution checking, unless the model has a suitable output definition.'
70 " Pass '--output-time' to the output filter (e.g., solns2out) to enable ranking by time")
71 parser.add_argument('-t', '--tSolve',
72 type=float,
73 metavar='<sec>', help='solver backend wall-time limit, default: '+
74 str(self.cfgDefault["BACKEND_DEFS"]["__BE_SOLVER"]["EXE"]["n_TimeoutRealHard"][0]))
75 parser.add_argument('--result', default=sFlnSolCheckBase, metavar='<file>',
76 help='save result log to <file>, default: \''+sFlnSolCheckBase+'\'')
77 parser.add_argument('--resultUnchk', default=sFlnSolUnchk, metavar='<file>', help='save unchecked result log to <file>')
78 parser.add_argument('--chkPrf', '--checkerPrf', '--checkerProfile', metavar='<prf_name>', action='append',
79 help='checker profile from those defined in config section \"CHECKER_PROFILES\", can be a few')
80 parser.add_argument('--tCheck',
81 type=float,
82 metavar='<sec>', help='checker backend wall-time limit, default: '+
83 str(self.cfgDefault["BACKEND_DEFS"]["__BE_CHECKER"]["EXE"]["n_TimeoutRealHard"][0]))
84 parser.add_argument('--nCheckMax', '--nCheck', '--nCheckedMax', ## default=self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0],
85 type=int,
86 metavar='<N>', help='max number of solutions checked per instance.'
87 ' Negative means checking starts from the last obtained solution. Default: '+
88 str(self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0]))
89 parser.add_argument('--nFailedSaveMax', ## default=self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0],
90 type=int,
91 metavar='<N>', help='max number of failed solution reports saved per instance, default: '+
92 str(self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0]))
93 parser.add_argument('--failed', ## default=self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0],
94 metavar='<file>', help='save failed check reports to <file>, default: \''+
95 self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0]+'\'')
96 parser.add_argument('--checkDZN', '--checkStdout', metavar='<stdout_file>',
97 help='for a single instance, check DZN-formatted solutions from a solver\'s std output dumped to <stdout_file>. The DZN format is produced, e.g., if the model is flattened with \'' + sDZNOutputAgrs + '\'')
98 parser.add_argument('--checkStderr', metavar='<stderr_file>',
99 help='with --checkDZN, read a solver\'s stderr log from <stderr_file> (not essential)')
100 parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='tee backend\'s stderr to screen, in addition to the instance\'s output dumpfile in mzn-test/OUTPUTS. Only works for --shellSolve 1')
101 parser.add_argument('--vc', '--verbose-check', dest='vc', action='store_true', help='same for checking')
102 ## parser.add_argument('--no-feature', dest='feature', action='store_false')
103 ## parser.set_defaults(feature=True)
104 parser.add_argument('--debug', '--printcall', type=int, metavar='<bitfield>', help='bit 1: print full solver call commands, bit 2: same for checker')
105 parser.add_argument('--shellSolve', type=int, metavar='0/1', help='backend call through shell when using psutils')
106 parser.add_argument('--psutils', type=int, metavar='0/1', help='backend call through psutils (seems buggy in 3.4.2)')
107 ## parser.add_argument('--fullPaths', action='store_true',
108 ## help='use full paths in instance identifiers. By default, it\'s the pure base filenames')
109
110 parser.add_argument('--mergeCfg', action="append", metavar='<file>', help='merge config from <file>')
111 parser.add_argument('--saveCfg', metavar='<file>', help='save internal config to <file>. Can be useful to modify some parameters and run with --mergeCfg')
112 parser.add_argument('--saveSolverCfg', metavar='<file>', help='save the final solver backend config to <file>')
113 parser.add_argument('--saveCheckerCfg', metavar='<file>', help='save the final checker backend config to <file>')
114 parser.add_argument('--addOption', '--addOptions', action="append", metavar='<text>', type=str, help='add <text> to any solver / checker call')
115 parser.add_argument('--addSolverOption', '--addSolverOptions', action="append", metavar='<text>', type=str, help='add <text> to the solver call')
116 parser.add_argument('--addCheckerOption', '--addCheckerOptions', action="append", metavar='<text>', type=str, help='add <text> to a checker call')
117 parser.add_argument('--useJoinedName', action="append", metavar='<...%s...>', type=str, help='add this to the call, with %%s being replaced by'
118 ' a joined filename from all the input filenames, e.g., "--writeModel MODELS/%%s.mps"')
119 self.args = parser.parse_args()
120 # print( "ARGS:\n", self.args )
121 ## Solver backend and checker backend list
122 self.slvBE = None
123 self.chkBEs = None
124
125 ## Get parameters from config and command line, merge values
126 def obtainParams(self):
127 self.parseCmdLine()
128 self.mergeValues()
129
130 def initCfgDefault(self):
131 ddd = {
132 s_CommentKey: [
133 "The default config structure for mzn-test.py. ", s_ProgramDescr,
134 "You can export this by --saveCfg and modify -> --mergeCfg,",
135 "even having only partial JSON subtree in a merged file(s).",
136 "Structure: COMMON_OPTIONS, SOLVER_/CHECKER_PROFILES, BACKEND_DEFS.",
137 "Solver and checkers are selected from pre-defined profiles,",
138 "which are in turn built from sequences of 'basic backend definitions'.",
139 "Comments are either separate keys or added as list elements ('/// ...')",
140 "in pre-selected positions; then, overriding items should keep that order."
141 ],
142 "COMMON_OPTIONS": {
143 s_CommentKey: [ "'Solvers' and 'Checkers' select the profiles to use for solving and checking.",
144 "At the moment only the 1st solver is used from Solvers."
145 "The selected profiles must be known in SOLVER_PROFILES / CHECKER_PROFILES, resp."
146 ],
147 "Solvers": [ "MINIZINC",
148 #"MZN-CPLEX",
149 "/// At the moment only the 1st element is used for solving" ],
150 "SOLUTION_CHECKING": {
151 "Checkers": ["GECODE-CHK", "GUROBI-CHK", "CPLEX-CHK" ],
152 "n_CheckedMax": [ -10, "/// Negative value means it's that many last solutions" ],
153 "n_FailedSaveMax": [ 3, "/// After that many failed solutions, stop checking the instance" ],
154 "s_FailedSaveFile": [ sFlnSolFailBase, "/// Filename to save failed solutions" ],
155 },
156 "Instance_List": {
157 s_CommentKey: [ "Params for instance lists.",
158 "Instance list is a file containing an instance's model files,",
159 "at most 1 instance per line.",
160 "InstanceFileExt: only files with this extensions from a list file",
161 "will be taken on each line" ],
162 "InstanceFileExt": [".mzn", ".dzn"] ## Add json? TODO
163 },
164 "runCommand": {
165 "windows": {
166 "runSilent": "echo \"WARNING. No timeout on Windows.\" & {2} 1>{3} 2>{4}",
167 "runVerbose": "echo \"WARNING. No timeout on Windows.\" & {2} 3>&1 1>{3} 2>&3 | tee {4} & echo >>{4}"
168 }
169 ,"non-windows": {
170 "runSilent": "ulimit -v {0}; timeout -k 1 {1} bash -c \"{2}\" 1>{3} 2>{4}",
171 "runVerbose": "ulimit -v {0}; timeout -k 1 {1} bash -c \"{2}\" 3>&1 1>{3} 2>&3 | tee {4}; echo >>{4}"
172 }
173 }
174 },
175 "SOLVER_PROFILES": {
176 s_CommentKey: [ "Similar to CHECKER_PROFILES." ],
177 "MINIZINC": [ "__BE_COMMON", "__BE_SOLVER", "BE_MINIZINC" ],
178 "FZN-GUROBI": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-GUROBI" ],
179 "FZN-CPLEX": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-CPLEX" ],
180 "FZN-CBC": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-CBC" ],
181 "GUROBI": [ "__BE_COMMON", "__BE_SOLVER", "BE_GUROBI" ],
182 "CPLEX": [ "__BE_COMMON", "__BE_SOLVER", "BE_CPLEX" ],
183 "XPRESS": [ "__BE_COMMON", "__BE_SOLVER", "BE_XPRESS" ],
184 "SCIP": [ "__BE_COMMON", "__BE_SOLVER", "BE_SCIP" ],
185 "CBC": [ "__BE_COMMON", "__BE_SOLVER", "BE_CBC" ],
186 "GECODE": [ "__BE_COMMON", "__BE_SOLVER", "BE_GECODE" ],
187 "CHUFFED": [ "__BE_COMMON", "__BE_SOLVER", "BE_CHUFFED" ],
188 "ORTOOLS": [ "__BE_COMMON", "__BE_SOLVER", "BE_ORTOOLS" ] #,
189 # "MZN-GUROBI": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-GUROBI" ],
190 # "MZN-CPLEX": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-CPLEX" ],
191 # "MZN-CBC": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-CBC" ],
192 # "MZN-GECODE": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-GECODE" ],
193 # "FZN-CHUFFED": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-CHUFFED" ]
194 },
195 "CHECKER_PROFILES": {
196 s_CommentKey: [ "Each profile gives a list of backend defs to use.",
197 "Later backends in the list can override/add options, ",
198 "for example for values to be read from the outputs.",
199 "Adding is only possible if the key is prefixed by '"+s_AddKey+"'"
200 ],
201 "MINIZINC-CHK": [ "__BE_COMMON", "__BE_CHECKER_OLDMINIZINC", "BE_MINIZINC" ],
202 "GECODE-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_GECODE" ],
203 "GUROBI-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_GUROBI" ],
204 "CPLEX-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_CPLEX" ],
205 "CHUFFED-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_CHUFFED" ],
206 "MZN-GECODE-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_MZN-GECODE" ],
207 "MZN-GUROBI-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_MZN-GUROBI" ],
208 "MZN-CPLEX-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_MZN-CPLEX" ],
209 "FZN-GECODE-CHK": [ "__BE_COMMON", "__BE_CHECKER_OLDMINIZINC", "BE_FZN-GECODE" ],
210 "FZN-GECODE-SHELL-CHK": [ "__BE_COMMON", "__BE_CHECKER_OLDMINIZINC", "BE_FZN-GECODE_SHELL" ],
211 "FZN-CHUFFED-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_FZN-CHUFFED" ]
212 },
213 "BACKEND_DEFS": {
214 s_CommentKey: [ "__BE_COMMON initializes a basic backend structure.",
215 "Each further backend in a profile list overrides or adds options."
216 ],
217 "__BE_COMMON": {
218 s_CommentKey: [ "THE INITIALIZING BACKEND." ],
219 "EXE": {
220 s_CommentKey: [ "Solver call parameters" ],
221 "s_SolverCall" : ["minizinc -v -s -a " + sDZNOutputAgrs + " %s",
222 "/// The 1st element defines the call line. %s is replaced by the instance filename(s)."],
223 "s_ExtraCmdline" : ["", "/// Only for __BE_SOLVER/__BE_CHECKER... subprofiles."
224 " The 1st element gives extra cmdline arguments to the call"],
225 "b_ThruShell" : [True, "/// Set True to call solver thru shell."
226 " Then you can do shell tricks but Ctrl+C may not kill all subprocesses etc."],
227 "n_TimeoutRealHard": [300, "/// Real-time timeout per instance, seconds,"
228 " for all solution steps together. Use mzn/backend options for CPU time limit."],
229 "n_VMEMLIMIT_SoftHard": [8000000, 8000000, "/// 2 limits, soft/hard, in KB. Platform-dependent in Python 3.6. Default 8 GB"],
230 },
231 "Stderr_Keylines": {
232 s_CommentKey: [ "A complete line in stderr will be interpreted accordingly.",
233 " Format: <outvar> : { <line>: <value>, ... }"
234 " You can add own things here (use '"+s_AddKey+"' before new var name)",
235 " which will be transferred into results" ]
236 },
237 "Stderr_Keyvalues": {
238 s_CommentKey: [ "Numerical values to be extracted from a line in stderr.",
239 " { <outvar>: [ <regex search pattern>, <regex to replace by spaces>, <value's pos in the line>] }."
240 ],
241 ### The %%%mzn-stat values appear in stdout (as of May 2019) but leave them here just in case
242 "Time_Flt": [ "%%%mzn-stat: flatTime", "[:=]", 3, "/// E.g., 'Flattening done, 3s' produces 3."
243 " !!! This is interpreted as successful flattening by the checker" ],
244 "ObjVal_Solver": [ "%%%mzn-stat objective=", "[,:/=]", 3, ## Need = to avoid mixup witht the bound
245 "/// The objval as reported by solver."],
246 "DualBnd_Solver": [ "%%%mzn-stat objectiveBound", "[,:/=]", 3 ],
247 "CPUTime_Solver": [ "%%%mzn-stat solveTime", "[,:/=]", 3 ],
248 "NNodes_Solver": [ "%%%mzn-stat nodes", "[,:/=]", 3 ],
249 },
250 "Stdout_Keylines": {
251 s_CommentKey: [ "Similar to Stderr_Keylines"],
252 "Sol_Status": {
253 "----------": 1,
254 "==========": 2,
255 "=====UNSATISFIABLE=====": -1,
256 "=====UNBOUNDED=====": -2,
257 "=====UNKNOWN=====": 0,
258 "=====UNSATorUNBOUNDED=====": -3,
259 "=====ERROR=====": -4
260 },
261 "Problem_Sense": {
262 "%%%mzn-stat: method=\"maximize\"": 1,
263 "%%%mzn-stat: method=\"minimize\"": -1,
264 "%%%mzn-stat: method=\"satisfy\"": 0,
265 }
266 },
267 "Stdout_Keyvalues": {
268 s_CommentKey: ["Similar to Stderr_Keyvalues." ],
269 "Time_Flt": [ "%%%mzn-stat: flatTime", "[:=]", 3, "/// E.g., 'Flattening done, 3s' produces 3."
270 " !!! This is interpreted as successful flattening by the checker" ],
271 "ObjVal_Solver": [ "%%%mzn-stat objective=", "[,:/=]", 3, ## Need = to avoid mixup witht the bound
272 "/// The objval as reported by solver."],
273 "DualBnd_Solver": [ "%%%mzn-stat objectiveBound", "[,:/=]", 3 ],
274 "CPUTime_Solver": [ "%%%mzn-stat solveTime", "[,:/=]", 3 ],
275 "NNodes_Solver": [ "%%%mzn-stat nodes", "[,:/=]", 3 ],
276 "ObjVal_MZN": [ "_objective", "[():=;%]", 2,
277 "/// The objective value as evaluated by MZN." ],
278 "RealTime_Solns2Out": [ "% time elapsed:", " ", 4 ],
279 }
280 },
281 "__BE_SOLVER": {
282 s_CommentKey: ["Specializations for a general solver" ],
283 "EXE": {
284 "s_ExtraCmdline" : ["-a"],
285 "b_ThruShell" : [True],
286 "n_TimeoutRealHard": [300],
287 # "n_VMEMLIMIT_SoftHard": [8000100000, 8100000000]
288 }
289 },
290 "__BE_CHECKER": {
291 s_CommentKey: ["Specializations for a general checker" ],
292 "EXE": {
293 "s_ExtraCmdline" : [sFlatOptChecker],
294 "b_ThruShell" : [True],
295 "n_TimeoutRealHard": [15],
296 # "n_VMEMLIMIT_SoftHard": [8100000000, 8100000000]
297 }
298 },
299 "__BE_CHECKER_OLDMINIZINC": {
300 s_CommentKey: ["Specializations for a general checker using the 1.6 MiniZinc driver" ],
301 "EXE": {
302 "s_ExtraCmdline" : ["--mzn2fzn-cmd 'mzn2fzn -v -s --output-mode dzn " + sFlatOptChecker + "'"],
303 "b_ThruShell" : [True],
304 "n_TimeoutRealHard": [15],
305 # "n_VMEMLIMIT_SoftHard": [8100000000, 8100000000]
306 }
307 },
308 "BE_MINIZINC": {
309 s_CommentKey: [ "------------------- Specializations for pure minizinc driver" ],
310 "EXE":{
311 "s_SolverCall": [ "minizinc --mzn2fzn-cmd 'mzn2fzn -v -s " + sDZNOutputAgrs + "' -s %s"], # _objective fails for checking
312 "b_ThruShell" : [True],
313 },
314 },
315 "BE_FZN-GUROBI": {
316 s_CommentKey: [ "------------------- Specializations for Gurobi solver instance" ],
317 "EXE":{
318 "s_SolverCall" : ["mzn2fzn -v -s -G linear " + sDZNOutputAgrs + " %s --fzn tmp.fzn --ozn tmp.ozn && mzn-gurobi -v -s tmp.fzn"], ## works without solns2out for now. Need thus when using shell call with system() TODO?
319 #"opt_writeModel": ["--writeModel"]
320 },
321 "Stderr_Keyvalues": {
322 s_AddKey+"Preslv_Rows": [ "Presolved:", " ", 2 ],
323 s_AddKey+"Preslv_Cols": [ "Presolved:", " ", 4 ],
324 s_AddKey+"Preslv_Non0": [ "Presolved:", " ", 6 ]
325 },
326 },
327 "BE_GUROBI": {
328 s_CommentKey: [ "------------------- Specializations for Gurobi solver instance" ],
329 "EXE":{
330 "s_SolverCall" : ["minizinc -v -s --solver gurobi --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
331 #"opt_writeModel": ["--writeModel"]
332 },
333 "Stderr_Keyvalues": {
334 s_AddKey+"Preslv_Rows": [ "Presolved:", " ", 2 ],
335 s_AddKey+"Preslv_Cols": [ "Presolved:", " ", 4 ],
336 s_AddKey+"Preslv_Non0": [ "Presolved:", " ", 6 ]
337 },
338 },
339 "BE_MZN-GUROBI": {
340 s_CommentKey: [ "------------------- Specializations for Gurobi solver instance" ],
341 "EXE":{
342 "s_SolverCall" : ["mzn-gurobi -v -s -G linear --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
343 #"opt_writeModel": ["--writeModel"]
344 },
345 "Stderr_Keyvalues": {
346 s_AddKey+"Preslv_Rows": [ "Presolved:", " ", 2 ],
347 s_AddKey+"Preslv_Cols": [ "Presolved:", " ", 4 ],
348 s_AddKey+"Preslv_Non0": [ "Presolved:", " ", 6 ]
349 },
350 },
351 "BE_FZN-CPLEX": {
352 s_CommentKey: [ "------------------- Specializations for IBM ILOG CPLEX solver instance" ],
353 "EXE": {
354 "s_SolverCall" : ["mzn2fzn -v -s -G linear " + sDZNOutputAgrs + " %s --fzn tmp.fzn --ozn tmp.ozn && mzn-cplex -v -s tmp.fzn"]
355 #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
356 #"b_ThruShell" : [True],
357 #"opt_writeModel": ["--writeModel"]
358 },
359 "Stderr_Keyvalues": {
360 s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ],
361 s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ],
362 s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ]
363 },
364 },
365 "BE_CPLEX": {
366 s_CommentKey: [ "------------------- Specializations for IBM ILOG CPLEX solver instance" ],
367 "EXE": {
368 "s_SolverCall" : ["minizinc -v -s --solver cplex --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
369 #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
370 #"b_ThruShell" : [True],
371 #"opt_writeModel": ["--writeModel"]
372 },
373 "Stderr_Keyvalues": {
374 s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ],
375 s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ],
376 s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ]
377 },
378 },
379 "BE_XPRESS": {
380 s_CommentKey: [ "------------------- Specializations for FICO XPRESS solver instance" ],
381 "EXE": {
382 "s_SolverCall" : ["minizinc -v -s --solver xpress --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
383 #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
384 #"b_ThruShell" : [True],
385 #"opt_writeModel": ["--writeModel"]
386 },
387 "Stderr_Keyvalues": { ## Is different for XPRESS and -v fails at the moment anyway
388 s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ],
389 s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ],
390 s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ]
391 },
392 },
393 "BE_MZN-CPLEX": {
394 s_CommentKey: [ "------------------- Specializations for IBM ILOG CPLEX solver instance" ],
395 "EXE": {
396 "s_SolverCall" : ["mzn-cplex -v -s -G linear --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
397 #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
398 #"b_ThruShell" : [True],
399 #"opt_writeModel": ["--writeModel"]
400 },
401 "Stderr_Keyvalues": {
402 s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ],
403 s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ],
404 s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ]
405 },
406 },
407 "BE_FZN-CBC": {
408 s_CommentKey: [ "------------------- Specializations for COIN-OR Branch&Cut solver instance" ],
409 "EXE": {
410 "s_SolverCall" : ["mzn-cbc -v -s -G linear --output-time " + sDZNOutputAgrs + " %s --fzn tmp.fzn --ozn tmp.ozn && mzn-cplex -v -s tmp.fzn"],
411 #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
412 #"b_ThruShell" : [True],
413 },
414 },
415 "BE_SCIP": {
416 s_CommentKey: [ "------------------- Specializations for SCIP solver instance" ],
417 "EXE": {
418 "s_SolverCall" : ["minizinc -v -s --solver scip --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
419 #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
420 #"b_ThruShell" : [True],
421 },
422 },
423 "BE_CBC": {
424 s_CommentKey: [ "------------------- Specializations for COIN-OR Branch&Cut solver instance" ],
425 "EXE": {
426 "s_SolverCall" : ["minizinc -v -s --solver osicbc --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
427 #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
428 #"b_ThruShell" : [True],
429 },
430 },
431 "BE_MZN-CBC": {
432 s_CommentKey: [ "------------------- Specializations for COIN-OR Branch&Cut solver instance" ],
433 "EXE": {
434 "s_SolverCall" : ["mzn-cbc -v -s -G linear --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
435 #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
436 #"b_ThruShell" : [True],
437 },
438 },
439 "BE_GECODE": {
440 s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ],
441 "EXE": {
442# "s_SolverCall" : ["minizinc -s --solver gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
443 "s_SolverCall" : ["minizinc -v -s --solver gecode " + sDZNOutputAgrs
444 + " %s"], # --time 300000
445 "b_ThruShell" : [True],
446 }
447 },
448 "BE_MZN-GECODE": {
449 s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ],
450 "EXE": {
451# "s_SolverCall" : ["mzn-fzn -s -G gecode --solver fzn-gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
452 "s_SolverCall" : ["mzn-gecode -v -s -G gecode " + sDZNOutputAgrs
453 + " %s"], # --time 300000
454 "b_ThruShell" : [True],
455 }
456 },
457 "BE_FZN-GECODE": {
458 s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ],
459 "EXE": {
460# "s_SolverCall" : ["mzn-fzn -s -G gecode --solver fzn-gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
461 "s_SolverCall" : ["minizinc -s -G gecode -f fzn-gecode --mzn2fzn-cmd 'mzn2fzn -v -s " + sDZNOutputAgrs + "' %s"],
462 "b_ThruShell" : [True],
463 }
464 },
465 "BE_FZN-GECODE_SHELL": {
466 s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ],
467 "EXE": {
468# "s_SolverCall" : ["mzn-fzn -s -G gecode --solver fzn-gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
469 "s_SolverCall" : ["mzn2fzn -v -s -G gecode " + sDZNOutputAgrs
470 + sFlatOptChecker + " %s --fzn tmp.fzn --ozn tmp.ozn && fzn-gecode tmp.fzn | solns2out tmp.ozn"],
471 "b_ThruShell" : [True],
472 "s_ExtraCmdline" : [""],
473 }
474 },
475 "BE_CHUFFED": {
476 s_CommentKey: [ "------------------- Specializations for Chuffed FlatZinc interpreter" ],
477 "EXE": {
478 "s_SolverCall" : ["minizinc -v -s --solver chuffed -f --output-time "
479 + sDZNOutputAgrs + " %s"], # _objective fails for checking
480 } ## --fzn-flags --time-out --fzn-flags 300
481 }
482 , "BE_ORTOOLS": {
483 s_CommentKey: [ "------------------- Specializations for OR-Tools FlatZinc interpreter" ],
484 "EXE": {
485 "s_SolverCall" : ["minizinc -v -s --solver ortools -f --output-time "
486 + sDZNOutputAgrs + " %s"], # _objective fails for checking
487 } ## --fzn-flags --time-out --fzn-flags 300
488 }
489 , "BE_FZN-CHUFFED": {
490 s_CommentKey: [ "------------------- Specializations for Chuffed FlatZinc interpreter" ],
491 "EXE": {
492 "s_SolverCall" : ["mzn-fzn -v -s -G chuffed --solver fzn-chuffed --fzn-flags -f --output-time "
493 + sDZNOutputAgrs + " %s"], # _objective fails for checking
494 } ## --fzn-flags --time-out --fzn-flags 300
495 }
496 }
497 }
498 return ddd
499 ## self.nNoOptAndAtLeast2Feas = 0
500
501 ## Read a cfg file, instead of/in addition to the default cfg
502 def mergeCfg(self, fln):
503 ddd1 = None
504 with open( fln, 'r' ) as rf:
505 ddd1 = json.load( rf )
506 self.cfg = mergeJSON( self.cfg, ddd1 )
507
508 ## Merge cmdline values with cfg, performing some immediate actions
509 ## And compile the backends constituting solver and checker(s)
510 def mergeValues(self):
511 if None!=self.args.mergeCfg: ### MERGE CFG FROM EXTRA FILES
512 for eCfg in self.args.mergeCfg:
513 self.mergeCfg( eCfg )
514 ################ Update some explicit cmdline params -- AFTER MERGING CFG FILES.
515 if None!=self.args.failed:
516 self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0] = self.args.failed
517 if None!=self.args.nCheckMax:
518 self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0] = self.args.nCheckMax
519 if None!=self.args.nFailedSaveMax:
520 self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0] = self.args.nFailedSaveMax
521 ################ SAVE FINAL CFG
522 if None!=self.args.saveCfg:
523 with utils.openFile_autoDir( self.args.saveCfg, 'w' ) as wf:
524 print( "Saving final config to", self.args.saveCfg )
525 json.dump( self.cfg, wf, sort_keys=True, indent=json_config.n_JSON_Indent )
526 ### COMPILE THE SOLVER BACKEND
527 if None!=self.args.solver:
528 self.cfg["COMMON_OPTIONS"]["Solvers"][0] = self.args.solver
529 slvPrfName = self.cfg["COMMON_OPTIONS"]["Solvers"][0]
530 slvPrf = self.cfg["SOLVER_PROFILES"][slvPrfName]
531 assert len(slvPrf)>0, "Solver profile '%s' should use at least a basic backend" % slvPrfName
532 self.slvBE = self.cfg["BACKEND_DEFS"][slvPrf[0]]
533 for i in range( 1, len( slvPrf ) ):
534 self.slvBE = json_config.mergeJSON( self.slvBE, self.cfg["BACKEND_DEFS"][slvPrf[i]] )
535 if None!=self.args.tSolve:
536 self.slvBE["EXE"]["n_TimeoutRealHard"][0] = self.args.tSolve
537 assert None==self.args.solver or None==self.args.call, "ERROR: both solver call and a solver profile specified."
538 if None!=self.args.call: ## After the compilation
539 self.slvBE["EXE"]["s_SolverCall"][0] = self.args.call
540 if None!=self.args.shellSolve:
541 self.slvBE["EXE"]["b_ThruShell"][0] = self.args.shellSolve!=0
542 print ( "\nSolver/checker configurations:\n SLV_CFG: ", json.dumps( self.slvBE["EXE"] ) )
543 ### COMPILE THE CHECKER BACKENDS
544 if None!=self.args.chkPrf and 0<len( self.args.chkPrf ):
545 self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["Checkers"] = self.args.chkPrf
546 chkPrfList = self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["Checkers"]
547 self.chkBEs = []
548 for chkPrfName in chkPrfList:
549 chkPrf = self.cfg["CHECKER_PROFILES"][chkPrfName]
550 assert len(chkPrf)>0, "Checker profile '%s' should use at least a basic backend" % chkPrfName
551 self.chkBEs.append( self.cfg["BACKEND_DEFS"][chkPrf[0]] )
552 for i in range( 1, len( chkPrf ) ):
553 self.chkBEs[-1] = json_config.mergeJSON( self.chkBEs[-1], self.cfg["BACKEND_DEFS"][chkPrf[i]] )
554 if None!=self.args.tCheck:
555 self.chkBEs[-1]["EXE"]["n_TimeoutRealHard"][0] = self.args.tCheck
556 print ( " CHK_CFG: ", json.dumps( self.chkBEs[-1]["EXE"] ) )
557 print( "" )
558 ### SAVE THE SOLVER BACKEND
559 if None!=self.args.saveSolverCfg:
560 with utils.openFile_autoDir( self.args.saveSolverCfg, 'w' ) as wf:
561 print( "Saving solver config to", self.args.saveSolverCfg )
562 json.dump( self.slvBE, wf, sort_keys=True, indent=json_config.n_JSON_Indent )
563 ### SAVE THE CHECKER BACKENDS
564 if None!=self.args.saveCheckerCfg:
565 with utils.openFile_autoDir( self.args.saveCheckerCfg, 'w' ) as wf:
566 print( "Saving checker config to", self.args.saveCheckerCfg )
567 json.dump( self.chkBE, wf, sort_keys=True, indent=json_config.n_JSON_Indent )
568 self.sThisName = self.args.result
569 if None!=self.args.name:
570 self.sThisName = self.args.name
571
572 def __init__(self):
573 self.cfgDefault = self.initCfgDefault()
574 self.cfg = self.cfgDefault
575 ### Further parameters
576 self.args = {}
577
578 def __str__(self):
579 s_Out = json.dumps( self.cfg, sort_keys=True, indent=json_config.n_JSON_Indent ) + '\n'
580 s_Out += str(self.args) + '\n'
581 s_Out += "\n >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> SOLVER BACKEND:\n"
582 s_Out += json.dumps( self.slvBE, sort_keys=True, indent=json_config.n_JSON_Indent ) + '\n'
583 s_Out += "\n >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> CHECKER BACKENDS:\n"
584 for chkBE in self.chkBEs:
585 s_Out += json.dumps( chkBE, sort_keys=True, indent=json_config.n_JSON_Indent ) + '\n'
586 return s_Out
587
588
589##############################################################################################
590################### The MZNTest class
591##############################################################################################
592class MznTest:
593
594 ## Produce a pair: first, a string identifying the instance which is given as a string of the instance files
595 ## Second, same without paths and file extensions for short printing
596 ## the elements are sorted according to the extensions list, then alphabetically
597 def getIName( self, s_Inst ):
598 lId = s_Inst.split() ## list of instance files
599 lExt = self.params.cfg["COMMON_OPTIONS"]["Instance_List"]["InstanceFileExt"]
600 lId = sorted( lId, key = lambda nm:
601 ( lExt.index( os.path.splitext(nm)[1] ) \
602 if os.path.splitext(nm)[1] in lExt else len(lExt), os.path.basename( nm ) ) )
603 lId2 = [ os.path.splitext( os.path.basename( fln ) )[0] for fln in lId ];
604 return ' '.join(lId), ' '.join(lId2);
605
606 def obtainParams( self ):
607 self.params.obtainParams()
608
609 ## If an instance was specified in cmdline, or model list(s) supplied
610 def compileExplicitModelLists(self):
611 ## Get cmdline filenames or from the --instList arguments
612 ## Can compile the list from log files, see below
613 self.params.instList = []
614 ## Only if -l not used, take the pos args
615 if (self.params.args.l_InstLists is None or 0==len( self.params.args.l_InstLists )) \
616 and self.params.args.instanceFiles is not None and 0<len( self.params.args.instanceFiles ):
617 self.params.instList.append( " ".join( self.params.args.instanceFiles ) )
618 ## Mode "compare only" if (comparison lists and not run option) or no instances
619 self.bCmpOnly = True if (not self.params.args.runAndCmp and (( \
620 self.params.args.compare is not None and 0<len(self.params.args.compare)) or \
621 ( self.params.args.l_InstLists is not None and 0<len( self.params.args.l_InstLists ) \
622 and self.params.args.instanceFiles is not None and 0<len( self.params.args.instanceFiles ) ))) \
623 else False
624 ## If -l used, compile the inst list files
625 if None!=self.params.args.l_InstLists and 0<len( self.params.args.l_InstLists ):
626 for sFln in self.params.args.l_InstLists:
627 with open( sFln, 'r' ) as rf:
628 for line in rf:
629 s_ModelFiles = ""
630 for wrd in line.split():
631 fIsMF = False
632 for ext in self.params.cfg["COMMON_OPTIONS"]["Instance_List"]["InstanceFileExt"]:
633 if wrd.endswith( ext ):
634 fIsMF = True
635 break
636 if fIsMF:
637 s_ModelFiles += ' ' + wrd
638 if 0<len( s_ModelFiles ):
639 self.params.instList.append( s_ModelFiles )
640
641 ## Result logs can be used to extract the instance list, if wished
642 def compileResultLogs(self):
643 self.cmpRes = cmp_result_logs.CompareLogs()
644 cmpFileList = []
645 if None!=self.params.args.compare: ### If -c used
646 cmpFileList += self.params.args.compare
647 ### If -l used, interpret pos arguments as comparison logs
648 if None!=self.params.args.l_InstLists and 0<len( self.params.args.l_InstLists ):
649 cmpFileList += self.params.args.instanceFiles
650 for sFlnLog in cmpFileList:
651 nEntries = 0
652 logCurrent, lLogNames = self.cmpRes.addLog( sFlnLog )
653 print( "Reading result log '", sFlnLog, "'... ", sep='', end='', flush=True )
654 with open( sFlnLog, 'r' ) as rf:
655 while True:
656 chJ = json_log.readLogJSONChunk( rf )
657 if None==chJ:
658 break
659 try:
660 logCurrent[ self.getIName( chJ["Inst_Files"] )[0] ] = chJ
661 if "TEST_NAME" in chJ:
662 lLogNames[1] = chJ[ "TEST_NAME" ]
663 print( " TEST NAME: '", chJ[ "TEST_NAME" ],
664 "'... ", sep='', end='', flush=True )
665 except:
666 print( "\n WARNING: unrecognized result chunk in file '", sFlnLog,
667 "' before position", rf.tell(), ", doesn't contain all keys")
668 else:
669 nEntries += 1
670 print( len(logCurrent), "different instances among", nEntries, "recognized entries." )
671
672 def runTheInstances(self):
673 logCurrent, lLogNames = self.cmpRes.addLog( self.params.args.result )
674 if self.params.sThisName!=lLogNames[0]:
675 lLogNames[1] = self.params.sThisName
676 ## TRUNCATING files first, then "a" - better on Win??
677 sNow = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
678 self.sStartTime = sNow
679 self.fileSol00 = utils.openFile_autoDir( self.params.args.resultUnchk, "w" )
680 self.fileSol = utils.openFile_autoDir(self.params.args.result.format(sNow +
681 ("" if self.params.args.name is None else "__"+utils.flnfy(self.params.args.name))), "w" )
682 self.fileFailName = self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0].format(sNow)
683 self.fileFail = None ## Not opening yet
684 self.nCheckedInstances = 0
685 self.nChecksFailed = 0
686 self.cmpRes.initListComparison()
687 for i_Inst in range( len(self.params.instList) ):
688 s_Inst = self.params.instList[ i_Inst ]
689 self.initInstance( i_Inst, s_Inst )
690 self.solveOriginal( s_Inst )
691 try:
692 if self.ifShouldCheck():
693 self.checkOriginal( s_Inst )
694 else:
695 print( "NO CHECK, total check-failed instances:", self.nChecksFailed,
696 "from", self.nCheckedInstances )
697 except:
698 print( " WARNING: failed to check instance solution. ", sys.exc_info() )
699 ## Saving to the main log:
700 self.saveSolution( self.fileSol )
701 ## Ranking:
702 sSet_Inst = self.getIName( self.result["Inst_Files"] )
703 logCurrent[ sSet_Inst[0] ] = self.result
704 try:
705 self.cmpRes.compareInstance( sSet_Inst )
706 if i_Inst < len(self.params.instList)-1:
707 print( "STATS: ", end='' )
708 self.cmpRes.summarizeCurrent( lLogNames )
709 print( "" )
710 except:
711 print( " WARNING: failed to compare/rank instance. ", sys.exc_info() )
712
713 def compareLogs(self):
714 self.cmpRes.initListComparison()
715 theList = [ lfn for lfn in self.params.instList ]
716 if 0==len( theList ):
717 theList = self.cmpRes.getInstanceUnion()
718 for sInst in theList:
719 try:
720 self.cmpRes.compareInstance( self.getIName( sInst ) )
721 except:
722 print( " ------ WARNING: failed to compare/rank instance. ", )
723 traceback.print_exc()
724 self.cmpRes.summarizeCmp()
725 print( self.cmpRes.summarizeFinalHdr(), end='' )
726
727 def summarize(self):
728 try:
729 ### Printing summary
730 print ('')
731 print ('='*50)
732 print(' SUMMARY')
733 print ('='*50)
734 stats = self.cmpRes.summarize()
735 print (stats)
736 except Exception as e:
737 exc_type, exc_obj, exc_tb = sys.exc_info()
738 fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
739 print(exc_type, exc_obj, fname, exc_tb.tb_lineno)
740 if not self.bCmpOnly:
741 print( "Printing stats log to: ", end="" )
742 sFlnSL = sFlnStatLog.format( utils.flnfy(
743 " ".join(self.params.args.l_InstLists if self.params.args.l_InstLists is not None else []) ) )
744 print( sFlnSL )
745 with utils.openFile_autoDir( sFlnSL, "a" ) as wf:
746 wf.write( "\nRUN " + self.sStartTime + "--" + datetime.datetime.now().__str__() + ": " )
747 wf.write( sys.argv.__str__() )
748 wf.write( "\n" )
749 wf.write( stats )
750 print( "\nResult logs saved to '", self.params.args.result,
751 "', with the unchecked log in '", self.params.args.resultUnchk, "'; failed solutions saved to '",
752 self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0],
753 "' in an \"appendable JSON\" format, cf. https://github.com/pvorb/jsml."
754 ## no more for shell: "\nSolver/checker stdout(err) outputs saved to '" + sDirResults + "/last_stdout(err)_...txt'."
755 , sep='')
756
757##############################################################################################
758######################### MZNTest average-level #############################
759
760 ## init
761 def initInstance( self, i_Inst, s_Inst ):
762 print( "\n--------------------- INSTANCE %d of %d: '" % (i_Inst+1, len( self.params.instList )),
763 s_Inst, "' ----------------------", sep='' )
764 self.result = OrderedDict()
765 if 0==i_Inst and None!=self.params.args.name:
766 self.result["TEST_NAME"] = self.params.args.name
767 self.result[ "Inst_Index_Of" ] = [ (i_Inst+1), len( self.params.instList ) ]
768 self.result["Inst_Files"] = s_Inst
769 self.solList = [] ## The solution list
770 ## TODO: instance name source?
771
772 ## Solve the original instance
773 def solveOriginal( self, s_Inst ):
774 self.result["__SOLVE__"] = self.solveInstance( s_Inst, self.params.slvBE, '_SOLVING', self.solList )
775 # print( " RESULT:\n", self.result )
776 self.saveSolution( self.fileSol00 )
777
778 ## Solve a given MZN instance and return the result map
779 ## Arguments: instance files in a string, backend parameter dictionary
780 ## slvName is used for screen output and last_std(out/err)_... naming, so better no spaces
781 ## solList provided <=> this is solving (not checking) and will use --checkDZN if opted
782
783 ## TODO
784 ## Because we cannot limit memory of the subprocesses directly AND there seem to be bugs in the Python 3.4 impl,
785 ## could replace the subprocess call by a call to an external which would run under given memory/time limits
786 ## and save output to given files.
787 ## OR: update from Python 3.4.2?
788 ## NAming output / model files: sort input filenames, replace spaces/punctuation
789 ### USE smth like
790 ## keepcharacters = (' ','.','_')
791 ## "".join(c for c in filename if c.isalnum() or c in keepcharacters else 'I').strip()
792 ## PARAMETERS
793 ## : solList: if not None, we are solving originally ( not checking )
794 def solveInstance(self, s_Inst, slvBE, slvName, solList=None):
795 resSlv = OrderedDict()
796 bChkDZN = True if None!=solList and None!=self.params.args.checkDZN else False
797 if bChkDZN:
798 print( "_PARSING '", self.params.args.checkDZN, sep='', end="'... " )
799 with open( self.params.args.checkDZN, 'r' ) as ro:
800 mzn_exec.parseStdout( ro, resSlv, slvBE["Stdout_Keylines"], slvBE["Stdout_Keyvalues"], solList )
801 print( ro.tell(), "bytes", end='' )
802 if None!=self.params.args.checkStderr:
803 print( " and '", self.params.args.checkStderr, sep='', end="'... " )
804 with open( self.params.args.checkStderr, 'r' ) as re:
805 mzn_exec.parseStderr( re, resSlv, slvBE["Stderr_Keylines"], slvBE["Stderr_Keyvalues"] )
806 print( re.tell(), "bytes", end='' )
807 else: #### Solving oneself
808 print( slvName, "... ", sep='', end='', flush=True )
809 s_Call = slvBE["EXE"]["s_SolverCall"][0] % s_Inst \
810 + ' ' + slvBE["EXE"]["s_ExtraCmdline"][0]
811 if self.params.args.addOption is not None:
812 for sOpt in self.params.args.addOption:
813 s_Call += ' ' + sOpt
814 if solList is not None:
815 if self.params.args.addSolverOption is not None:
816 for sOpt in self.params.args.addSolverOption:
817 s_Call += ' ' + sOpt
818 else:
819 if self.params.args.addCheckerOption is not None:
820 for sOpt in self.params.args.addCheckerOption:
821 s_Call += ' ' + sOpt
822 s_InstMerged = s_Inst.strip()
823 ## s_InstMerged = regex.sub( r"[.\\/:~]", "", s_InstMerged );
824 ## s_InstMerged = regex.sub( r"[ ]", "-", s_InstMerged );
825 keepcharacters = ('-','_')
826 s_InstMerged = "".join(c if c.isalnum() or c in keepcharacters else 'I' for c in s_InstMerged).strip()
827
828 if solList is not None and self.params.args.useJoinedName is not None:
829 for sUseJN in self.params.args.useJoinedName:
830 s_UsingOpt = sUseJN % s_InstMerged
831 s_Call += ' ' + s_UsingOpt
832 if solList is not None: ## solving the original instance
833 sFlnStdout = sFlnStdoutBase.format( s_InstMerged )
834 sFlnStderr = sFlnStderrBase.format( s_InstMerged )
835 if self.params.args.debug is not None and ( self.params.args.debug & 1 ):
836 print( " CALL: \"", s_Call, "\"", sep='', flush=True )
837 else:
838 sFlnStdout = 'last_stdout' + slvName + '.txt'
839 sFlnStderr = 'last_stderr' + slvName + '.txt'
840 if self.params.args.debug is not None and ( self.params.args.debug & 2 ):
841 print( " CALL: \"", s_Call, "\"", sep='', flush=True )
842 resSlv["Solver_Call"] = s_Call
843 resSlv["DateTime_Start"] = datetime.datetime.now().__str__()
844 if 1==self.params.args.psutils:
845 completed, tmAll = \
846 mzn_exec.runCmd(
847 s_Call,
848 slvBE["EXE"]["b_ThruShell"][0],
849 slvBE["EXE"]["n_TimeoutRealHard"][0],
850 slvBE["EXE"]["n_VMEMLIMIT_SoftHard"]
851 )
852 with utils.openFile_autoDir( sFlnStdout, "w" ) as tf:
853 tf.write( completed.stdout )
854 with utils.openFile_autoDir( sFlnStderr, "w" ) as tf:
855 tf.write( completed.stderr )
856 print( "STDOUT/ERR: ", len(completed.stdout), '/',
857 len(completed.stderr), " bytes", sep='', end=', ' )
858 mzn_exec.parseStderr( io.StringIO( completed.stderr ), resSlv, slvBE["Stderr_Keylines"], slvBE["Stderr_Keyvalues"] )
859 mzn_exec.parseStdout( io.StringIO( completed.stdout ), resSlv, slvBE["Stdout_Keylines"], slvBE["Stdout_Keyvalues"], solList )
860 ## Adding the outputs to the log
861 ## resSlv["StdErr"] = completed.stderr
862 ## resSlv["StdOut"] = completed.stdout
863 else: ## use the 'system' call
864 with utils.openFile_autoDir( sFlnStdout, "w" ) as tf:
865 tf.write( "% EMPTY\n" )
866 with utils.openFile_autoDir( sFlnStderr, "w" ) as tf:
867 tf.write( "% EMPTY" )
868 tmAll = mzn_exec.runCmdCmdline(
869 s_Call,
870 sFlnStdout, sFlnStderr,
871 self.params.cfg["COMMON_OPTIONS"]["runCommand"],
872 slvBE["EXE"]["n_TimeoutRealHard"][0],
873 (self.params.args.verbose) if solList is not None else (self.params.args.vc),
874 slvBE["EXE"]["n_VMEMLIMIT_SoftHard"]
875 )
876 with open( sFlnStderr, "r" ) as rf:
877 mzn_exec.parseStderr( rf, resSlv, slvBE["Stderr_Keylines"], slvBE["Stderr_Keyvalues"] )
878 with open( sFlnStdout, "r" ) as rf:
879 mzn_exec.parseStdout( rf, resSlv, slvBE["Stdout_Keylines"], slvBE["Stdout_Keyvalues"], solList )
880
881 print( " t: {:.3f}".format( tmAll ), end=' s, ' )
882 resSlv["DateTime_Finish"] = datetime.datetime.now().__str__()
883 resSlv["TimeReal_All"] = tmAll
884 resSlv["TimeReal_LastStatus"] = 0
885 resSlv["Hostname"] = platform.uname()[1]
886 ### For all cases, some postprocessing #############################
887 if "Sol_Status" not in resSlv:
888 resSlv["Sol_Status"] = [-50, " !!!!! STATUS TOTALLY UNKNOWN - NO STATUS LINE PARSED."]
889 if "Time_Flt" not in resSlv or utils.try_float( resSlv.get( "Time_Flt" ) ) is None:
890 resSlv["NOFZN"] = [" !!!!! No flattening finish time registered or successfully parsed"]
891 dTmLast = utils.try_float( resSlv.get( "RealTime_Solns2Out" ) )
892 if None!=dTmLast:
893 resSlv["TimeReal_LastStatus"] = dTmLast / 1000.0
894 resSlv.pop( "RealTime_Solns2Out" )
895 ## if "SolutionLast" in resSlv:
896 ## print( " SOLUTION_LAST:\n", resSlv["SolutionLast"], sep='' )
897 if None!=solList:
898 print( " Nsol:", len(solList), end=',' )
899 print( " STATUS:", resSlv["Sol_Status"] )
900 return resSlv
901
902 ## Append the unchecked solution of the instance to a temp. file
903 def saveSolution(self, wf):
904 json_log.writeLogChunk( wf, json.dumps( self.result, indent=json_config.n_JSON_Indent ) )
905
906 ## Return the necessity to check solutions.
907 ## Can be false if we only want to compare different solvers.
908 def ifShouldCheck(self):
909 return \
910 0<self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0] and \
911 0!=self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0] and \
912 0<len( self.params.chkBEs ) and \
913 self.ifCheckableStatus( self.result["__SOLVE__"]["Sol_Status"][0] )
914
915 ##
916 def ifCheckableStatus( self, status ):
917 return status>0
918
919 ## Check selected solutions of the instance
920 def checkOriginal( self, s_Inst ):
921 nCM = self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0]
922 nFSM = self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0]
923 assert 0!=nCM
924 assert 0<len(self.solList)
925 rng = None
926 if 0<nCM:
927 rng = range( 0, min( nCM, len(self.solList ) ) )
928 else:
929 rng = range( -1, max( nCM-1, -len(self.solList)-1 ), -1 )
930 self.result["SOLUTION_CHECKS_DONE"] = 0
931 self.result["SOLUTION_CHECKS_FAILED"] = 0
932 fFailed = 0
933 ## Iterate over the solution range
934 for iSol in rng:
935 ## Try modify
936 # self.solList[ iSol ] = self.solList[iSol][:20] + '1' + self.solList[iSol][21:]
937 print ( "CHK SOL ", iSol if iSol<0 else (iSol+1), '/', len(rng), sep='', end='... ' )
938 with utils.openFile_autoDir( sFlnSolLastDzn, "w" ) as wf: ## Save the selected solution
939 wf.write( self.solList[ iSol ] )
940 s_IC = s_Inst + ' ' + sFlnSolLastDzn
941 bCheckOK = True
942 chkFlSt = []
943 # self.result["__CHECKS__"] = []
944 for iChk in range ( len ( self.params.chkBEs ) ):
945 chkBE = self.params.chkBEs[ iChk ]
946 chkRes = self.solveInstance( s_IC, chkBE, '__Checker_'+str(iChk+1) )
947 # self.result["__CHECKS__"].append( chkRes )
948 if ( ## -51==chkRes["Sol_Status"][0] or ## NOFZN? No, flattener should report INFEAS.
949 ( 0>chkRes["Sol_Status"][0] and -3<=chkRes["Sol_Status"][0] ) ): ## INFEAS
950 bCheckOK = False
951 chkFlSt = chkRes["Sol_Status"]
952 self.result["SOLUTION_CHECKS_DONE"] += 1
953 if not bCheckOK:
954 fFailed = 1
955 self.result["SOLUTION_CHECKS_FAILED"] += 1
956 self.result["SOLUTION_FAILED_LAST"] = self.solList[ iSol ]
957 # self.result["SOLUTION_FAILED_LAST__CHKSTATUS"] = chkRes["Sol_Status"]
958 if self.fileFail is None:
959 self.fileFail = utils.openFile_autoDir( self.fileFailName, "w" )
960 self.saveSolution( self.fileFail )
961 if nFSM<=self.result["SOLUTION_CHECKS_FAILED"]:
962 print ( self.result["SOLUTION_CHECKS_FAILED"], "failed solution(s) saved, go on" )
963 break
964
965 self.nCheckedInstances += 1
966 self.nChecksFailed += fFailed
967 print( " CHECK FAILS on this instance: ", self.result["SOLUTION_CHECKS_FAILED"],
968 ", total check-failed instances: ", self.nChecksFailed,
969 " from ", self.nCheckedInstances, sep='' )
970
971 def __init__(self):
972 ## Default params
973 self.params = MZT_Param()
974
975##############################################################################################
976######################### MZNTest public #############################
977##############################################################################################
978 def run(self):
979 self.obtainParams()
980 self.compileExplicitModelLists()
981 self.compileResultLogs()
982 if not self.bCmpOnly:
983 self.runTheInstances()
984 else:
985 self.compareLogs()
986 self.summarize()
987
988# def main
989mznTst = MznTest()
990mznTst.run()
991