diff --git a/examples/test-job.sh b/examples/test-job.sh index 20efd9c..f0c43a7 100755 --- a/examples/test-job.sh +++ b/examples/test-job.sh @@ -8,10 +8,10 @@ tmp_dir=/home/pluehne/tmp echo "job ID: $JOB_ID" -echo "result repository: $GIT_REMOTE_URL" +echo "result repository: $JOB_RESULT_REPOSITORY_URL" mkdir -p "$tmp_dir" -dir="$tmp_dir"/"$JOB_BRANCH_NAME" +dir="$tmp_dir"/"job-$JOB_ID" if [[ -d $dir ]] then @@ -21,15 +21,16 @@ fi mkdir -p "$dir" pushd "$dir" - git clone "$GIT_REMOTE_URL" repository + git clone "$JOB_RESULT_REPOSITORY_URL" repository pushd repository git config user.name "Potassco Bot" git config user.email "bot@potassco.org" - echo "$JOB_BRANCH_NAME" > test-output + echo "start of benchmark output of job $JOB_ID" > test-output sleep 10 start_time=$(date +%s%N) + echo "end of benchmark output of job $JOB_ID" >> test-output git add test-output git commit -m "Test update" git push @@ -37,4 +38,4 @@ pushd "$dir" popd echo $(expr "$end_time" - "$start_time") >> "$tmp_dir"/overhead -popd +popd \ No newline at end of file diff --git a/examples/test.rs b/examples/test.rs index b55c43d..861214b 100644 --- a/examples/test.rs +++ b/examples/test.rs @@ -14,7 +14,7 @@ fn main() { pretty_env_logger::init(); - let benchmark_repository = BenchmarkRepository::new("git@git.luehne.de:patrick/tplp-planning-benchmark.git", Path::new("cache").to_path_buf(), "git", "Potassco Bot", "bot@potassco.org"); + let mut benchmark_repository = BenchmarkRepository::new("gitea@git.luehne.de:patrick/tplp-planning-benchmark.git", Path::new("cache").to_path_buf(), "gitea", "Potassco Bot", "bot@potassco.org"); let content = benchmark_repository.read_file(Path::new("configurations.yml"), "config").unwrap(); let configurations = &YamlLoader::load_from_str(&content).unwrap()[0]["configurations"]; @@ -45,12 +45,12 @@ fn main() continue; } - let result_repository = benchmark_repository.create_result_repository_for_job(i); + let job = benchmark_repository.create_result_repository(); Command::new("sbatch") .args(&["/home/pluehne/test-job.sh"]) - .env("GIT_REMOTE_URL", &format!("file://{}", fs::canonicalize(&result_repository).unwrap().display())) - .env("JOB_ID", format!("{}", i)) + .env("JOB_RESULT_REPOSITORY_URL", &format!("file://{}", fs::canonicalize(&job.result_repository_path).unwrap().display())) + .env("JOB_ID", format!("{}", job.id)) .output() .expect("Could not execute command"); diff --git a/src/lib.rs b/src/lib.rs index b790258..768654a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ use git2::{Cred, Error, FetchOptions, Index, IndexEntry, IndexTime, Oid, Progress, PushOptions, RemoteCallbacks, Repository, Signature}; use indicatif::{ProgressBar, ProgressStyle}; use log::{info, trace}; +use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::string::String; use std::str; @@ -22,6 +23,12 @@ pub struct TargetPath pub destination: PathBuf, } +pub struct Job +{ + pub id: u32, + pub result_repository_path: PathBuf, +} + impl BenchmarkRepository { fn progress_bar_style() -> ProgressStyle @@ -359,87 +366,88 @@ impl BenchmarkRepository Some(content.to_owned()) } - pub fn create_result_repository(&mut self) -> PathBuf + pub fn create_result_repository(&mut self) -> Job { let job_id = self.jobs; self.jobs += 1; - let repository_path = self.base_path.join(format!("job-{}", job_id)); + let result_repository_path = self.base_path.join(format!("job-{}", job_id)); - if repository_path.exists() + if result_repository_path.exists() { - match std::fs::remove_dir_all(&repository_path) + match std::fs::remove_dir_all(&result_repository_path) { Ok(_) => (), - Err(error) => panic!("failed to initialize result Git repository for job {} in “{}”: {}", job_id, repository_path.display(), error), + Err(error) => panic!("failed to initialize result Git repository for job {} in “{}”: {}", job_id, result_repository_path.display(), error), }; } - match Repository::init_bare(&repository_path) + match Repository::init_bare(&result_repository_path) { Ok(repository) => repository, - Err(error) => panic!("failed to initialize result Git repository for job {} in “{}”: {}", job_id, repository_path.display(), error), + Err(error) => panic!("failed to initialize result Git repository for job {} in “{}”: {}", job_id, result_repository_path.display(), error), }; - info!("Initialized result Git repository for job {} in “{}”", job_id, repository_path.display()); + info!("Initialized result Git repository for job {} in “{}”", job_id, result_repository_path.display()); - repository_path + Job{id: job_id, result_repository_path} + } + + fn tip_commit<'repository>(repository: &'repository git2::Repository, branch_name: &str) -> Result, git2::Error> + { + let tip_reference_name = format!("refs/remotes/origin/{}", branch_name); + let tip_reference = match repository.find_reference(&tip_reference_name) + { + Ok(value) => value, + Err(error) => panic!("Could not find reference “{}”: {}", tip_reference_name, error), + }; + + tip_reference.peel_to_commit() } pub fn join(&self) { - let mut Vec job_commit_ids; + let mut active_job_ids = HashSet::new(); + + for job_id in 0..self.jobs + { + active_job_ids.insert(job_id); + } loop { - for job_id in 0..self.jobs + while !active_job_ids.is_empty() { - let repository_path = self.base_path.join(format!("job-{}", job_id)); + active_job_ids.retain + ( + |job_id| + { + let job_repository_path = self.base_path.join(format!("job-{}", job_id)); - let repository = match Repository::open(&repository_path) - { - Ok(repository) => repository, - Err(error) => panic!("cannot access result repository for job {}: {}", job_id, error), - }; + let job_repository = match Repository::open(&job_repository_path) + { + Ok(value) => value, + Err(error) => panic!("cannot access result repository for job {}: {}", job_id, error), + }; - let tip_reference_name = "refs/remotes/origin/master"; - let tip_reference = match self.repository.find_reference(&tip_reference_name) - { - Ok(value) => value, - Err(error) => panic!("error reading result repository for job {}: {}", job_id, error), - }; + let job_commit = match Self::tip_commit(&job_repository, "master") + { + Ok(value) => value, + // Job is not done yet, so skip it until it is + Err(_) => return true, + }; - let commit = match tip_reference.peel_to_commit() - { - Ok(value) => value, - Err(error) => panic!("error reading result repository for job {}: {}", job_id, error), - }; + info!("job {} finished", job_id); - let tree = match tip_reference.peel_to_tree() - { - Ok(value) => value, - Err(error) => panic!("Could not peel reference to tree: {}", error), - }; + let remote_commit = match Self::tip_commit(&self.repository, "results") + { + Ok(value) => value, + Err(error) => panic!("could not access tip commit of “results” branch: {}", error), + }; - let object_id = match tree.get_path(file_path) - { - Ok(tree_entry) => tree_entry.id(), - Err(_) => return None, - }; - - let blob = match self.repository.find_blob(object_id) - { - Ok(blob) => blob, - Err(_) => return None, - }; - - let content = match std::str::from_utf8(blob.content()) - { - Ok(content) => content, - Err(error) => panic!("Could not interpret file “{}” as UTF-8: {}", file_path.display(), error), - }; - - Some(content.to_owned()) + false + } + ); } //let origin = self.inner.repository.find_remote("origin").expect("could not find remote “origin”");