Add benchmark runner script with example config
This commit is contained in:
parent
e2cb68c8fb
commit
5d06d0127c
GPG Key ID:
05F3611E97A70ABF
@ -4,6 +4,7 @@
|
||||
|
||||
## Structure
|
||||
|
||||
- master branch: contains the [benchmark runner script](benchmark.py)
|
||||
- [config branch](https://git.luehne.de/patrick/tplp-planning-benchmark/src/config): contains the tested [configurations](https://git.luehne.de/patrick/tplp-planning-benchmark/src/config/configurations.yaml) and [instance sets](https://git.luehne.de/patrick/tplp-planning-benchmark/src/config/instances.yaml)
|
||||
- [status branch](https://git.luehne.de/patrick/tplp-planning-benchmark/src/status): contains the [benchmark status log](https://git.luehne.de/patrick/tplp-planning-benchmark/src/status/status.log) (updated after each instance)
|
||||
- [results branch](https://git.luehne.de/patrick/tplp-planning-benchmark/src/results): contains the raw result files
|
||||
|
||||
256
benchmark.py
Executable file
256
benchmark.py
Executable file
@ -0,0 +1,256 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import atexit
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import yaml
|
||||
|
||||
import pprint
|
||||
|
||||
def executeCommand(command, stdin = None, cwd = None):
|
||||
with subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, stdin = (subprocess.PIPE if stdin != None else None), cwd = cwd) as process:
|
||||
stdout, stderr = process.communicate(input = (stdin.encode("utf-8") if stdin != None else None))
|
||||
exitCode = process.returncode
|
||||
|
||||
return stdout.decode("utf-8"), stderr.decode("utf-8"), exitCode
|
||||
|
||||
def plaspVersion(config):
|
||||
version, _, _ = executeCommand([config["executables"]["plasp"]["binary"], "-v"])
|
||||
version = version.strip()
|
||||
|
||||
match = re.match(r'plasp version (.*?)$', version, re.M | re.I)
|
||||
|
||||
return match.group(1)
|
||||
|
||||
def clingoVersion(config):
|
||||
version, _, _ = executeCommand([config["executables"]["clingo"]["binary"], "-v"])
|
||||
version = version.strip()
|
||||
|
||||
match = re.match(r'clingo version (.*?)$', version, re.M | re.I)
|
||||
|
||||
return match.group(1)
|
||||
|
||||
def plannerVersion(config):
|
||||
version, _, _ = executeCommand(["git", "rev-parse", "HEAD"], cwd = config["executables"]["planner"]["directory"])
|
||||
date, _, _ = executeCommand(["git", "show", "-s", "--format=%ci"], cwd = config["executables"]["planner"]["directory"])
|
||||
|
||||
return version.strip() + " (" + date.strip() + ")"
|
||||
|
||||
def fastDownwardVersion(config):
|
||||
version, _, _ = executeCommand(["hg", "log", "-r.", "-T {rev}:{node} ({date|isodate})"], cwd = config["executables"]["fastDownward"]["directory"])
|
||||
|
||||
return version.strip()
|
||||
|
||||
def git(command, cwd):
|
||||
stdout, stderr, exitCode = executeCommand(["git"] + command, cwd = cwd)
|
||||
|
||||
if exitCode != 0:
|
||||
print(stderr, file = sys.stderr)
|
||||
raise RuntimeError("git error")
|
||||
|
||||
def initRepo(config):
|
||||
dataDir = config["storage"]["local"]
|
||||
|
||||
# clone repo if not existing
|
||||
if not os.path.isdir(config["storage"]["local"]):
|
||||
git(["clone", config["storage"]["remote"], dataDir], None)
|
||||
|
||||
# default settings
|
||||
git(["config", "--local", "user.name", config["storage"]["user"]], dataDir)
|
||||
git(["config", "--local", "user.email", config["storage"]["userEMail"]], dataDir)
|
||||
git(["config", "--local", "commit.gpgsign", "false"], dataDir)
|
||||
|
||||
# fetch origin
|
||||
git(["fetch"], cwd = dataDir)
|
||||
|
||||
# pull all branches
|
||||
for key, branch in config["storage"]["branches"].items():
|
||||
git(["checkout", branch], cwd = dataDir)
|
||||
git(["pull"], cwd = dataDir)
|
||||
|
||||
def readBenchmarkConfig(config):
|
||||
initRepo(config)
|
||||
|
||||
dataDir = config["storage"]["local"]
|
||||
|
||||
# checkout config branch
|
||||
git(["checkout", config["storage"]["branches"]["config"]], cwd = dataDir)
|
||||
|
||||
# read instance list
|
||||
instancesFile = os.path.join(config["storage"]["local"], "instances.yaml")
|
||||
|
||||
with open(instancesFile, "r") as stream:
|
||||
instances = yaml.load(stream, Loader=yaml.CLoader)
|
||||
|
||||
# read configurations to test
|
||||
configurationsFile = os.path.join(config["storage"]["local"], "configurations.yaml")
|
||||
|
||||
with open(configurationsFile, "r") as stream:
|
||||
configurations = yaml.load(stream, Loader=yaml.CLoader)
|
||||
|
||||
# flatten lists of options
|
||||
for configuration in configurations["configurations"]:
|
||||
configuration["options"] = [item for sublist in configuration["options"] for item in sublist]
|
||||
|
||||
return {"configurations": configurations, "instances": instances}
|
||||
|
||||
def inputFilenames(instance, config):
|
||||
pddlInstancesDir = config["input"]["pddlInstances"]
|
||||
|
||||
domainFile = os.path.join(pddlInstancesDir, instance["ipc"], "domains", instance["domain"], "domain.pddl")
|
||||
instanceFile = os.path.join(pddlInstancesDir, instance["ipc"], "domains", instance["domain"], "instances", "instance-" + str(instance["instance"]) + ".pddl")
|
||||
|
||||
return {"domainFile": domainFile, "instanceFile": instanceFile}
|
||||
|
||||
def outputFilenames(configuration, instance, config):
|
||||
instanceID = instance["ipc"] + "_" + instance["domain"] + "_" + str(instance["instance"])
|
||||
outputFile = os.path.join(configuration["id"], instanceID + ".out")
|
||||
errorFile = os.path.join(configuration["id"], instanceID + ".err")
|
||||
environmentFile = os.path.join(configuration["id"], instanceID + ".env")
|
||||
|
||||
return {"outputFile": outputFile, "errorFile": errorFile, "environmentFile": environmentFile}
|
||||
|
||||
def nextJob(config):
|
||||
benchmarkConfig = readBenchmarkConfig(config)
|
||||
|
||||
dataDir = config["storage"]["local"]
|
||||
|
||||
# checkout results branch
|
||||
git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir)
|
||||
|
||||
configurations = benchmarkConfig["configurations"]["configurations"]
|
||||
instances = benchmarkConfig["instances"]
|
||||
|
||||
for instanceSetName, instanceSet in instances.items():
|
||||
for instance in instanceSet:
|
||||
for configuration in configurations:
|
||||
filenames = outputFilenames(configuration, instance, config)
|
||||
outputFile = os.path.join(config["storage"]["local"], filenames["outputFile"])
|
||||
environmentFile = os.path.join(config["storage"]["local"], filenames["environmentFile"])
|
||||
|
||||
if not os.path.exists(outputFile) or not os.path.exists(environmentFile):
|
||||
return {"configuration": configuration, "instance": instance}
|
||||
|
||||
return None
|
||||
|
||||
def writeStatus(message, config):
|
||||
dataDir = config["storage"]["local"]
|
||||
|
||||
# checkout status branch
|
||||
git(["checkout", config["storage"]["branches"]["status"]], cwd = dataDir)
|
||||
|
||||
statusFilename = os.path.join(dataDir, "status.log")
|
||||
|
||||
if os.path.exists(statusFilename):
|
||||
with open(statusFilename, "r") as statusFile:
|
||||
content = statusFile.read()
|
||||
else:
|
||||
content = ""
|
||||
|
||||
with open(statusFilename, "w") as statusFile:
|
||||
print(time.strftime("%Y-%m-%d %H:%M:%S %z") + "\t" + message + "\n" + content, file = statusFile, end = "")
|
||||
|
||||
git(["add", "status.log"], dataDir)
|
||||
git(["commit", "-m Update status: " + message], dataDir)
|
||||
git(["push"], dataDir)
|
||||
|
||||
def runJob(configuration, instance, config):
|
||||
jobName = "[" + str(configuration["id"]) + " | " + instance["ipc"] + " | " + instance["domain"] + " | " + str(instance["instance"]) + "]"
|
||||
|
||||
writeStatus("started benchmark job " + jobName, config)
|
||||
|
||||
dataDir = config["storage"]["local"]
|
||||
|
||||
inputFiles = inputFilenames(instance, config)
|
||||
|
||||
# checkout results branch
|
||||
git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir)
|
||||
|
||||
command = \
|
||||
[
|
||||
config["executables"]["timeout"]["binary"],
|
||||
"-m=" + str(config["limits"]["memory"]),
|
||||
"-t=" + str(config["limits"]["time"]),
|
||||
config["executables"]["planner"]["binary"],
|
||||
"--domain=" + inputFiles["domainFile"],
|
||||
inputFiles["instanceFile"],
|
||||
]
|
||||
|
||||
command += configuration["options"]
|
||||
|
||||
# TODO: verify planner Git hash
|
||||
plannerDir = config["executables"]["planner"]["directory"]
|
||||
stdout, stderr, exitCode = executeCommand(command, cwd = plannerDir)
|
||||
|
||||
outputFiles = outputFilenames(configuration, instance, config)
|
||||
outputDir = os.path.dirname(os.path.join(config["storage"]["local"], outputFiles["outputFile"]))
|
||||
|
||||
if not os.path.isdir(outputDir):
|
||||
os.makedirs(outputDir)
|
||||
|
||||
with open(os.path.join(config["storage"]["local"], outputFiles["outputFile"]), "w") as outputFile, \
|
||||
open(os.path.join(config["storage"]["local"], outputFiles["errorFile"]), "w") as errorFile, \
|
||||
open(os.path.join(config["storage"]["local"], outputFiles["environmentFile"]), "w") as environmentFile:
|
||||
print(stdout, file = outputFile)
|
||||
print("# configuration: " + str(configuration), file = errorFile)
|
||||
print("# instance: " + str(instance), file = errorFile)
|
||||
print("# command: " + str(command), file = errorFile)
|
||||
print("# working directory: " + plannerDir, file = errorFile)
|
||||
print(stderr, file = errorFile)
|
||||
|
||||
if exitCode != 0:
|
||||
print(stderr)
|
||||
|
||||
environment = \
|
||||
{
|
||||
"configuration": configuration,
|
||||
"instance": instance,
|
||||
"command": command,
|
||||
"workingDirectory": plannerDir,
|
||||
"versions": \
|
||||
{
|
||||
"clingo": clingoVersion(config),
|
||||
"plasp": plaspVersion(config),
|
||||
"planner": plannerVersion(config),
|
||||
"fastDownward": fastDownwardVersion(config)
|
||||
}
|
||||
}
|
||||
|
||||
print(yaml.dump(environment, default_flow_style = False), file = environmentFile)
|
||||
|
||||
git(["add", outputFiles["outputFile"], outputFiles["errorFile"], outputFiles["environmentFile"]], dataDir)
|
||||
git(["commit", "-m Add benchmark result " + jobName], dataDir)
|
||||
git(["push"], dataDir)
|
||||
|
||||
if exitCode != 0:
|
||||
writeStatus("errors reported for benchmark job " + jobName, config)
|
||||
else:
|
||||
writeStatus("finished benchmark job " + jobName, config)
|
||||
|
||||
def main():
|
||||
with open("config.yaml", "r") as stream:
|
||||
config = yaml.load(stream, Loader=yaml.CLoader)
|
||||
|
||||
atexit.register(writeStatus, "benchmark runner exited", config)
|
||||
|
||||
performedJobs = 0
|
||||
|
||||
while True:
|
||||
job = nextJob(config)
|
||||
|
||||
if not job:
|
||||
break
|
||||
|
||||
configuration = job["configuration"]
|
||||
instance = job["instance"]
|
||||
|
||||
runJob(configuration, instance, config)
|
||||
performedJobs += 1
|
||||
|
||||
if performedJobs == 0:
|
||||
writeStats("finished benchmark series", config)
|
||||
|
||||
main()
|
||||
43
config.example.yaml
Normal file
43
config.example.yaml
Normal file
@ -0,0 +1,43 @@
|
||||
storage:
|
||||
# local directory where remote is cloned to
|
||||
local: storage
|
||||
# repote Git repository URL
|
||||
remote: git@git.luehne.de:patrick/tplp-planning-benchmark.git
|
||||
# user name for commits
|
||||
user: potassco-bot
|
||||
# user e-mail for commits
|
||||
userEMail: bot@potassco.org
|
||||
# data branches
|
||||
branches:
|
||||
results: results
|
||||
config: config
|
||||
status: status
|
||||
|
||||
executables:
|
||||
timeout:
|
||||
# path to timeout script (https://github.com/pshved/timeout)
|
||||
binary: /home/user/repos/timeout/timeout
|
||||
plasp:
|
||||
# path to plasp binary (https://github.com/potassco/plasp)
|
||||
binary: plasp
|
||||
clingo:
|
||||
# path to clingo binary (https://github.com/potassco/clingo)
|
||||
binary: clingo
|
||||
planner:
|
||||
# path to planner executable (https://github.com/javier-romero/plasp/tree/master/encodings/planner)
|
||||
directory: /home/user/repos/plasp-javier/encodings/planner
|
||||
binary: /home/user/repos/plasp-javier/encodings/planner/runplanner.py
|
||||
fastDownward:
|
||||
# path to Fast Downward executable (http://hg.fast-downward.org/)
|
||||
directory: /home/user/repos/fast-downward
|
||||
binary: /home/user/repos/fast-downward/fast-downward.py
|
||||
|
||||
input:
|
||||
# path to PDDL instances (https://github.com/potassco/pddl-instances)
|
||||
pddlInstances: /home/user/repos/pddl-instances
|
||||
|
||||
limits:
|
||||
# time limit per instance in seconds
|
||||
time: 900
|
||||
# memory limit per instance in kilobytes
|
||||
memory: 8000000
|
||||
Loading…
Reference in New Issue
Block a user