patrick/tplp-planning-benchmark
1
0
Fork 0
Code Issues Releases Activity

Add option to print evaluation as plain text

Browse Source
This commit is contained in:
Patrick Lühne 2017-12-14 16:48:04 +01:00
parent 519338232c
commit 0ef8b69486
Signed by: patrick
GPG Key ID: 05F3611E97A70ABF
1 changed files with 56 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
evaluate.py
View File

@ -263,32 +263,58 @@ def renderResultsTable(configurations, instanceSetID, instanceSet, instances, re
print("</tr></thead></table>") print("</tr></thead></table>")
def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSet, instances, results, config): def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSet, instances, results, config, format = "html"):
aggregatedResults = aggregateResults(configurations, instanceSetID, instanceSet, instances, results) aggregatedResults = aggregateResults(configurations, instanceSetID, instanceSet, instances, results)
if not aggregatedResults: if not aggregatedResults:
print("<!-- error -->") print("<!-- error -->")
return return
print("<h2>" + instanceSetID + " (" + type + ")</h2><table><thead><tr><th>IPC</th><th>domain</th>") def compareConfigurations(configuration):
hasResults = instanceSetID in configuration["instanceSets"] and len(aggregatedResults[("total", "")][configuration["id"]]["results"]) > 0
for configuration in configurations: if not hasResults:
return (0, config["limits"]["time"])
return (-aggregatedResults[("total", "")][configuration["id"]]["instances solved"], aggregatedResults[("total", "")][configuration["id"]]["average runtime"])
sortedConfigurations = sorted(configurations, key = compareConfigurations)
if format == "html":
print("<h2>" + instanceSetID + " (" + type + ")</h2><table><thead><tr><th>IPC</th><th>domain</th>")
else:
print("# " + instanceSetID)
print("domain", end = "")
for configuration in sortedConfigurations:
if not instanceSetID in configuration["instanceSets"]: if not instanceSetID in configuration["instanceSets"]:
continue continue
print("<th><div title=\"" + str(configuration["options"]) + "\">" + configuration["id"] + "</div></th>") if format == "html":
print("<th><div title=\"" + str(configuration["options"]) + "\">" + configuration["id"] + "</div></th>")
else:
print("\t" + configuration["id"], end="")
print("</tr></thead><tbody>") if format == "html":
print("</tr></thead><tbody>")
else:
print("")
for ipcDomain, results in sorted(aggregatedResults.items()): for ipcDomain, results in sorted(aggregatedResults.items()):
print("<tr><td class=\"col-header\">" + ipcDomain[0] + "</td><td class=\"col-header\">" + ipcDomain[1] + "</td>") if format == "html":
print("<tr><td class=\"col-header\">" + ipcDomain[0] + "</td><td class=\"col-header\">" + ipcDomain[1] + "</td>")
else:
print((ipcDomain[0][4:] if ipcDomain[0][0:4] == "ipc-" else ipcDomain[0]) + ("/" + ipcDomain[1] if ipcDomain[1] else ""), end = "")
for configuration in configurations: for configuration in sortedConfigurations:
if not instanceSetID in configuration["instanceSets"]: if not instanceSetID in configuration["instanceSets"]:
continue continue
if len(results[configuration["id"]]["results"]) == 0: if len(results[configuration["id"]]["results"]) == 0:
print("<td class=\"tbd\"></td>") if format == "html":
print("<td class=\"tbd\"></td>")
else:
print("\t", end = "")
continue continue
r = results[configuration["id"]][type] r = results[configuration["id"]][type]
@ -297,9 +323,18 @@ def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSe
value = (r / config["limits"]["time"]) ** 0.2 if type == "average runtime" else 1.0 - r / max(1, len(results[configuration["id"]]["results"])) value = (r / config["limits"]["time"]) ** 0.2 if type == "average runtime" else 1.0 - r / max(1, len(results[configuration["id"]]["results"]))
classes = " result-" + str(int(value * 100)) classes = " result-" + str(int(value * 100))
print("<td class=\"result" + classes + "\">" + numberFormat % r + "</td>") if format == "html":
print("<td class=\"result" + classes + "\">" + numberFormat % r + "</td>")
else:
print("\t" + numberFormat % r, end = "")
print("</tr></tbody></thead></table>") if format != "html":
print("")
if format == "html":
print("</tr></tbody></thead></table>")
else:
print()
def main(): def main():
with open("config.yml", "r") as stream: with open("config.yml", "r") as stream:
@ -307,13 +342,18 @@ def main():
configurations, instances, results = collectResults(config) configurations, instances, results = collectResults(config)
print("<!DOCTYPE html><html lang=\"en\"><head><title>TPLP benchmark results</title><meta charset=\"UTF-8\"><link rel=\"stylesheet\" href=\"style.css?v=2\" type=\"text/css\"></head><body><main><h1>TPLP Benchmark Results</h1><div class=\"footnote\">last updated at " + time.strftime("%Y-%m-%d %H:%M:%S %z") + "</div>") if len(sys.argv) > 1 and sys.argv[1] == "--format=plain-text":
for instanceSetID, instanceSet in instances.items():
renderAggregatedResultsTable("instances solved", configurations, instanceSetID, instanceSet, instances, results, config, format = "plain")
renderAggregatedResultsTable("average runtime", configurations, instanceSetID, instanceSet, instances, results, config, format = "plain")
else:
print("<!DOCTYPE html><html lang=\"en\"><head><title>TPLP benchmark results</title><meta charset=\"UTF-8\"><link rel=\"stylesheet\" href=\"style.css?v=2\" type=\"text/css\"></head><body><main><h1>TPLP Benchmark Results</h1><div class=\"footnote\">last updated at " + time.strftime("%Y-%m-%d %H:%M:%S %z") + "</div>")
for instanceSetID, instanceSet in instances.items(): for instanceSetID, instanceSet in instances.items():
renderAggregatedResultsTable("instances solved", configurations, instanceSetID, instanceSet, instances, results, config) renderAggregatedResultsTable("instances solved", configurations, instanceSetID, instanceSet, instances, results, config)
renderAggregatedResultsTable("average runtime", configurations, instanceSetID, instanceSet, instances, results, config) renderAggregatedResultsTable("average runtime", configurations, instanceSetID, instanceSet, instances, results, config)
renderResultsTable(configurations, instanceSetID, instanceSet, instances, results) renderResultsTable(configurations, instanceSetID, instanceSet, instances, results)
print("</main></body></html>") print("</main></body></html>")
main() main()