tplp-planning-benchmark/evaluate.py

#!/usr/bin/python3

import math
import os
import re
import subprocess
import sys
import time
import yaml

import pprint

gray = (186, 189, 182)

def executeCommand(command, stdin = None, cwd = None):
	with subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, stdin = (subprocess.PIPE if stdin != None else None), cwd = cwd) as process:
		stdout, stderr = process.communicate(input = (stdin.encode("utf-8") if stdin != None else None))
		exitCode = process.returncode

	return stdout.decode("utf-8"), stderr.decode("utf-8"), exitCode

def git(command, cwd, enforce = False):
	stdout, stderr, exitCode = executeCommand(["git"] + command, cwd = cwd)

	if exitCode != 0:
		print(stderr, file = sys.stderr)

		if enforce:
			raise RuntimeError("git error")

def initRepo(config):
	dataDir = config["storage"]["local"]

	# clone repo if not existing
	if not os.path.isdir(config["storage"]["local"]):
		git(["clone", config["storage"]["remote"], dataDir], None, enforce = True)

	# fetch origin
	git(["fetch"], cwd = dataDir)

	# pull all branches
	for key, branch in config["storage"]["branches"].items():
		git(["checkout", branch], cwd = dataDir, enforce = True)
		git(["pull"], cwd = dataDir)

def readBenchmarkConfig(config):
	initRepo(config)

	dataDir = config["storage"]["local"]

	# checkout config branch
	git(["checkout", config["storage"]["branches"]["config"]], cwd = dataDir, enforce = True)

	# read instance list
	instancesFile = os.path.join(config["storage"]["local"], "instances.yml")

	with open(instancesFile, "r") as stream:
		instances = yaml.load(stream, Loader=yaml.CLoader)

	# read configurations to test
	configurationsFile = os.path.join(config["storage"]["local"], "configurations.yml")

	with open(configurationsFile, "r") as stream:
		configurations = yaml.load(stream, Loader=yaml.CLoader)

	# flatten lists of options
	for configuration in configurations["configurations"]:
		configuration["options"] = [item for sublist in configuration["options"] for item in sublist]

	return {"configurations": configurations, "instances": instances}

def outputFilenames(configuration, instance, config):
	instanceID = instance["ipc"] + "_" + instance["domain"] + "_" + str(instance["instance"])
	outputFile = os.path.join(configuration["id"], instanceID + ".out")
	errorFile = os.path.join(configuration["id"], instanceID + ".err")
	environmentFile = os.path.join(configuration["id"], instanceID + ".env")

	return {"outputFile": outputFile, "errorFile": errorFile, "environmentFile": environmentFile}

def jobKey(configuration, instance):
	return (configuration["id"], instance["ipc"], instance["domain"], instance["instance"])

def instanceKey(instance):
	return (instance["ipc"], instance["domain"], instance["instance"])

def addResult(results, configuration, instance, result):
	if not configuration["id"] in results:
		results[configuration["id"]] = {}

	results[configuration["id"]][instanceKey(instance)] = result

def result(results, configuration, instance):
	return results[configuration["id"]][instanceKey(instance)]

def mix(color1, color2, t):
	return (color1[0] * (1 - t) + color2[0] * t, color1[1] * (1 - t) + color2[1] * t, color1[2] * (1 - t) + color2[2] * t)

def resultColor(result, config):
	if result <= 0:
		return colors[0]
	elif result >= config["limits"]["time"]:
		return colors[-1]

	normalizedResult = (result / config["limits"]["time"]) ** 0.2
	normalizedResult *= (len(colors) - 1)

	c0 = min(math.floor(normalizedResult), len(colors) - 1)
	t = normalizedResult - c0

	if t <= 0:
		return colors[c0]
	elif t >= 1:
		return colors[c0 + 1]

	return mix(colors[c0], colors[c0 + 1], t)

def collectResults(config):
	benchmarkConfig = readBenchmarkConfig(config)

	dataDir = config["storage"]["local"]

	# checkout results branch
	git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir, enforce = True)

	configurations = benchmarkConfig["configurations"]["configurations"]
	instances = benchmarkConfig["instances"]

	results = {}

	for instanceSetName, instanceSet in instances.items():
		for instance in instanceSet:
			for configuration in configurations:
				filenames = outputFilenames(configuration, instance, config)
				outputFile = os.path.join(config["storage"]["local"], filenames["outputFile"])
				errorFile = os.path.join(config["storage"]["local"], filenames["errorFile"])
				environmentFile = os.path.join(config["storage"]["local"], filenames["environmentFile"])

				if not os.path.exists(outputFile) or not os.path.exists(errorFile) or not os.path.exists(environmentFile):
					addResult(results, configuration, instance, None)
					continue

				with open(errorFile, "r") as errorOutput:
					errors = errorOutput.read()

				finishedRE = re.compile("^FINISHED CPU", re.M)
				runtimeRE = re.compile("<time name=\"ALL\">(.*)</time>", re.M)
				timeoutRE = re.compile("^TIMEOUT CPU", re.M)
				memoutRE = re.compile("^MEM CPU", re.M)
				exitCodeRE = re.compile("^# exit code: (\d+)$", re.M)

				finished = finishedRE.search(errors)
				runtime = runtimeRE.search(errors)
				timeout = timeoutRE.search(errors)
				memout = memoutRE.search(errors)
				exitCode = exitCodeRE.search(errors)

				if exitCode and int(exitCode.group(1)) != 0:
					text = "error"
					color = None
				elif finished:
					value = float(runtime.group(1)) / 1000
					text = str(value)
					color = (value / config["limits"]["time"]) ** 0.2
				elif timeout:
					text = "> " + str(config["limits"]["time"])
					color = 1.0
				elif memout:
					text = "> " + str(config["limits"]["memory"] / 1000000) + " GB"
					color = 1.0

				result = {"text": text, "color": color}

				addResult(results, configuration, instance, result)

	return configurations, instances, results

def aggregateResults(configurations, instanceSetID, instanceSet, instances, results):
	aggregatedResults = {("total", ""): {}}

	for instance in instanceSet:
		ipcDomain = (instance["ipc"], instance["domain"])

		if not ipcDomain in aggregatedResults:
			aggregatedResults[ipcDomain] = {}

		for configuration in configurations:
			if not instanceSetID in configuration["instanceSets"]:
				continue

			if not configuration["id"] in aggregatedResults[ipcDomain]:
				aggregatedResults[ipcDomain][configuration["id"]] = {"instances solved": 0, "average runtime": None, "results": []}

			if not configuration["id"] in aggregatedResults[("total", "")]:
				aggregatedResults[("total", "")][configuration["id"]] = {"instances solved": 0, "average runtime": None, "results": []}

			r = result(results, configuration, instance)

			if r == None:
				continue

			value = 900.0

			try:
				value = float(r["text"])
				aggregatedResults[ipcDomain][configuration["id"]]["instances solved"] += 1
				aggregatedResults[("total", "")][configuration["id"]]["instances solved"] += 1
			except:
				pass

			aggregatedResults[ipcDomain][configuration["id"]]["results"].append(value)
			aggregatedResults[("total", "")][configuration["id"]]["results"].append(value)

	for ipcDomain, results in aggregatedResults.items():
		for configurationKey, configurationResults in aggregatedResults[ipcDomain].items():
			configurationResults["average runtime"] = sum(configurationResults["results"]) / max(1, len(configurationResults["results"]))

	return aggregatedResults

def requiresInstance(configuration, instance, instances):
	for requiredInstanceSet in configuration["instanceSets"]:
		if not requiredInstanceSet in instances:
			raise RuntimeError("undefined instance set “" + requiredInstanceSet + "”")

		if instance in instances[requiredInstanceSet]:
			return True

	return False

def renderResultsTable(configurations, instanceSetID, instanceSet, instances, results):
	print("<h2>" + instanceSetID + " (detailed results)</h2><table><thead><tr><th>IPC</th><th>domain</th><th>instance</th>")

	for configuration in configurations:
		if not instanceSetID in configuration["instanceSets"]:
			continue

		print("<th><div title=\"" + str(configuration["options"]) + "\">" + configuration["id"] + "</div></th>")

	print("<tbody>")

	for instance in instanceSet:
		print("<tr><td class=\"col-header\">" + instance["ipc"] + "</td><td class=\"col-header\">" + instance["domain"] + "</td><td class=\"col-header\">" + str(instance["instance"]) + "</td>")

		for configuration in configurations:
			if not instanceSetID in configuration["instanceSets"]:
				continue

			r = result(results, configuration, instance)

			if r and r["text"] != "error":
				print("<td class=\"result result-" + str(int(r["color"] * 100)) + "\">")
				print(r["text"])
			elif r and r["text"] == "error":
				print("<td class=\"error\">")
				print(r["text"])
			else:
				print("<td class=\"tbd\">")

			print("</td>")

		print ("</tr>")

	print("</tbody>")

	print("</tr></thead></table>")

def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSet, instances, results, config):
	aggregatedResults = aggregateResults(configurations, instanceSetID, instanceSet, instances, results)

	if not aggregatedResults:
		print("<!-- error -->")
		return

	print("<h2>" + instanceSetID + " (" + type + ")</h2><table><thead><tr><th>IPC</th><th>domain</th>")

	for configuration in configurations:
		if not instanceSetID in configuration["instanceSets"]:
			continue

		print("<th><div title=\"" + str(configuration["options"]) + "\">" + configuration["id"] + "</div></th>")

	print("</tr></thead><tbody>")

	for ipcDomain, results in sorted(aggregatedResults.items()):
		print("<tr><td class=\"col-header\">" + ipcDomain[0] + "</td><td class=\"col-header\">" + ipcDomain[1] + "</td>")

		for configuration in configurations:
			if not instanceSetID in configuration["instanceSets"]:
				continue

			if len(results[configuration["id"]]["results"]) == 0:
				print("<td class=\"tbd\"></td>")
				continue

			r = results[configuration["id"]][type]

			numberFormat = "%.2f" if type == "average runtime" else "%d/" + str(len(results[configuration["id"]]["results"]))
			value = (r / config["limits"]["time"]) ** 0.2 if type == "average runtime" else 1.0 - r / max(1, len(results[configuration["id"]]["results"]))
			classes = " result-" + str(int(value * 100))

			print("<td class=\"result" + classes + "\">" + numberFormat % r + "</td>")

	print("</tr></tbody></thead></table>")

def main():
	with open("config.yml", "r") as stream:
		config = yaml.load(stream, Loader=yaml.CLoader)

	configurations, instances, results = collectResults(config)

	print("<!DOCTYPE html><html lang=\"en\"><head><title>TPLP benchmark results</title><meta charset=\"UTF-8\"><link rel=\"stylesheet\" href=\"style.css?v=2\" type=\"text/css\"></head><body><main><h1>TPLP Benchmark Results</h1><div class=\"footnote\">last updated at " + time.strftime("%Y-%m-%d %H:%M:%S %z") + "</div>")

	for instanceSetID, instanceSet in instances.items():
		renderAggregatedResultsTable("instances solved", configurations, instanceSetID, instanceSet, instances, results, config)
		renderAggregatedResultsTable("average runtime", configurations, instanceSetID, instanceSet, instances, results, config)
		renderResultsTable(configurations, instanceSetID, instanceSet, instances, results)

	print("</main></body></html>")

main()