Update to latest revision

This commit is contained in:
Patrick Lühne 2018-09-28 16:51:46 +02:00
parent 9676adc24d
commit a9f4b700ba
Signed by: patrick
GPG Key ID: 05F3611E97A70ABF
4 changed files with 159 additions and 50 deletions

21
Cargo.lock generated
View File

@ -35,6 +35,7 @@ dependencies = [
"notify 4.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"pretty_env_logger 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"walkdir 2.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"yaml-rust 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -248,8 +249,8 @@ dependencies = [
"inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -715,7 +716,7 @@ dependencies = [
[[package]]
name = "tokio-executor"
version = "0.1.5"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
@ -723,7 +724,7 @@ dependencies = [
[[package]]
name = "tokio-io"
version = "0.1.9"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
@ -733,7 +734,7 @@ dependencies = [
[[package]]
name = "tokio-reactor"
version = "0.1.6"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -744,8 +745,8 @@ dependencies = [
"num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -964,9 +965,9 @@ dependencies = [
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
"checksum termios 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "70226acdf12d182df757d9fb07c0257a1558ec48c8059f607d6b38145ce4e2fa"
"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
"checksum tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c117b6cf86bb730aab4834f10df96e4dd586eff2c3c27d3781348da49e255bde"
"checksum tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "8b8a85fffbec3c5ab1ab62324570230dcd37ee5996a7859da5caf7b9d45e3e8c"
"checksum tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4b26fd37f1125738b2170c80b551f69ff6fecb277e6e5ca885e53eec2b005018"
"checksum tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "84823b932d566bc3c6aa644df4ca36cb38593c50b7db06011fd4e12e31e4047e"
"checksum tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8d6cc2de7725863c86ac71b0b9068476fec50834f055a243558ef1655bbd34cb"
"checksum tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4bfbaf9f260635649ec26b6fb4aded03887295ffcd999f6e43fd2c4758f758ea"
"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
"checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25"

View File

@ -11,6 +11,7 @@ indicatif = "0.9"
yaml-rust = "0.4"
notify = "4"
tempfile = "3"
walkdir = "2"
[dependencies.cpython]
git = "https://github.com/dgrunwald/rust-cpython/"

View File

@ -16,10 +16,18 @@ fn main()
let mut benchmark_repository = BenchmarkRepository::new("git@git.luehne.de:patrick/tplp-planning-benchmark.git", Path::new("storage"), "git", "Potassco Bot", "bot@potassco.org");
let content = benchmark_repository.read_file(Path::new("configurations.yml"), "config").unwrap();
/*let files = vec!
[
TargetPath{source: &Path::new("/tmp/test"), destination: &Path::new("foobar/test")},
TargetPath{source: &Path::new("/tmp/test2"), destination: &Path::new("foobar/test-2")},
];
benchmark_repository.commit_files(&files[..], "test-results");*/
let content = benchmark_repository.read_file(Path::new("configurations.yml"), "test-config").unwrap();
let configurations = &YamlLoader::load_from_str(&content).unwrap()[0]["configurations"];
let content = benchmark_repository.read_file(Path::new("instances.yml"), "config").unwrap();
let content = benchmark_repository.read_file(Path::new("instances.yml"), "test-config").unwrap();
let instances = &YamlLoader::load_from_str(&content).unwrap()[0];
for configuration in configurations.as_vec().unwrap()
@ -37,13 +45,13 @@ fn main()
let file_name_base = format!("{}/{}/{}/{}", configuration_id, instance_ipc, instance_domain, instance_number);
let file_name_output = format!("{}.out", file_name_base);
if (benchmark_repository.file_exists(Path::new(&file_name_output), "results"))
if (benchmark_repository.file_exists(Path::new(&file_name_output), "test-results"))
{
println!("done: [{}, {}, {}/{}/{}]", configuration_id, instance_set_id, instance_ipc, instance_domain, instance_number);
continue;
}
let test = benchmark_repository.create_autocommit_directory(Path::new(&file_name_base), "results").unwrap();
let test = benchmark_repository.create_autocommit_directory(Path::new(&file_name_base), "test-results").unwrap();
//println!("{}", test.display());
//println!("to do: [{}, {}, {}/{}/{}]", configuration_id, instance_set_id, instance_ipc, instance_domain, instance_number);
@ -51,8 +59,5 @@ fn main()
}
}
let duration = time::Duration::from_secs(20);
thread::sleep(duration);
benchmark_repository.wait_for_autocommit_thread();
}

View File

@ -5,36 +5,45 @@ extern crate log;
extern crate indicatif;
extern crate notify;
extern crate tempfile;
extern crate walkdir;
use git2::{Cred, Error, FetchOptions, Index, IndexEntry, IndexTime, Progress, PushOptions, RemoteCallbacks, ResetType, Repository, Signature, TreeBuilder};
use git2::build::{CheckoutBuilder, RepoBuilder};
use std::fs::{File, create_dir_all, read};
use std::fs::{File, create_dir_all, read, remove_file};
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::string::String;
use std::str;
use std::sync::{Arc, Mutex};
use std::sync::mpsc::{channel, Sender, Receiver};
use std::thread::{spawn, JoinHandle};
use indicatif::{ProgressBar, ProgressStyle};
use notify::{raw_watcher, RecommendedWatcher, RawEvent, Op};
use notify::{raw_watcher, RecommendedWatcher, RawEvent, Op, RecursiveMode, Watcher};
use tempfile::TempDir;
use walkdir::WalkDir;
pub struct BenchmarkRepository
pub struct BenchmarkRepositoryInner
{
repository: Repository,
ssh_user: String,
user_name: String,
user_email: String,
}
pub struct BenchmarkRepository
{
inner: Arc<Mutex<BenchmarkRepositoryInner>>,
autocommit_channel: (Sender<RawEvent>, Receiver<RawEvent>),
autocommit_directory: Option<TempDir>,
autocommit_thread: Option<JoinHandle<()>>,
autocommit_watcher: Option<RecommendedWatcher>,
autocommit_locks: Arc<Mutex<usize>>,
}
pub struct TargetPath<'a>
pub struct TargetPath
{
pub source: &'a Path,
pub destination: &'a Path,
pub source: PathBuf,
pub destination: PathBuf,
}
impl BenchmarkRepository
@ -74,11 +83,13 @@ impl BenchmarkRepository
fn reset_origin(&self, remote_url: &str) -> &Self
{
let remote = match self.repository.find_remote("origin")
let inner = self.inner.lock().unwrap();
let remote = match inner.repository.find_remote("origin")
{
Ok(remote) => remote,
Err(_) =>
match self.repository.remote("origin", remote_url)
match inner.repository.remote("origin", remote_url)
{
Ok(remote) => remote,
Err(error) => panic!("Could not reset remote “origin”: {}", error),
@ -92,6 +103,8 @@ impl BenchmarkRepository
fn fetch_branch(&self, branch_name: &str) -> &Self
{
let inner = self.inner.lock().unwrap();
let mut progress_bar = ProgressBar::new(0);
progress_bar.set_style(BenchmarkRepository::progress_bar_style());
@ -100,10 +113,10 @@ impl BenchmarkRepository
remote_callbacks.credentials(
|_, _, _|
{
match Cred::ssh_key_from_agent(&self.ssh_user)
match Cred::ssh_key_from_agent(&inner.ssh_user)
{
Ok(credentials) => Ok(credentials),
Err(error) => panic!("could not retrieve key pair for SSH authentication as user “{}”: {}", self.ssh_user, error),
Err(error) => panic!("could not retrieve key pair for SSH authentication as user “{}”: {}", inner.ssh_user, error),
}
});
@ -113,7 +126,7 @@ impl BenchmarkRepository
let mut fetch_options = FetchOptions::new();
fetch_options.remote_callbacks(remote_callbacks);
let mut origin = self.repository.find_remote("origin").expect("could not find remote “origin”");
let mut origin = inner.repository.find_remote("origin").expect("could not find remote “origin”");
info!("Updating branch “{}”", branch_name);
@ -156,17 +169,24 @@ impl BenchmarkRepository
Err(_) => BenchmarkRepository::init(base_path),
};
let benchmark_repository =
BenchmarkRepository
let benchmark_repository_inner =
BenchmarkRepositoryInner
{
repository: repository,
ssh_user: ssh_user.to_string(),
user_name: user_name.to_string(),
user_email: user_email.to_string(),
};
let benchmark_repository =
BenchmarkRepository
{
inner: Arc::new(Mutex::new(benchmark_repository_inner)),
autocommit_channel: channel(),
autocommit_directory: None,
autocommit_thread: None,
autocommit_watcher: None,
autocommit_locks: Arc::new(Mutex::new(0)),
};
benchmark_repository
@ -178,10 +198,10 @@ impl BenchmarkRepository
benchmark_repository
}
pub fn read_file_as_index_entry(&self, file_path: &Path, result_file_path: &Path) -> IndexEntry
pub fn read_file_as_index_entry(inner: &BenchmarkRepositoryInner, file_path: &Path, result_file_path: &Path) -> IndexEntry
{
// create a new blob with the file contents
let object_id = match self.repository.blob_path(file_path)
let object_id = match inner.repository.blob_path(file_path)
{
Ok(object_id) => object_id,
Err(error) => panic!("Could not write blob for “{}”: {}", file_path.display(), error),
@ -206,10 +226,33 @@ impl BenchmarkRepository
}
}
pub fn commit_files(&self, file_paths: &[TargetPath], branch_name: &str)
pub fn commit_directory(inner: &BenchmarkRepositoryInner, directory_path: &TargetPath, branch_name: &str)
{
let mut file_paths = vec![];
for entry in WalkDir::new(&directory_path.source)
{
let entry = entry.unwrap();
if entry.path().file_name().unwrap() == ".lock" || entry.file_type().is_dir()
{
continue;
}
let relative_path = entry.path().strip_prefix(&directory_path.source).unwrap();
trace!("Adding “{}” (from “{}”) to autocommit list", relative_path.display(), entry.path().display());
file_paths.push(TargetPath{source: entry.path().to_owned(), destination: directory_path.destination.join(&relative_path)});
}
BenchmarkRepository::commit_files(&inner, &file_paths, branch_name);
}
pub fn commit_files(inner: &BenchmarkRepositoryInner, file_paths: &[TargetPath], branch_name: &str)
{
let tip_reference_name = format!("refs/remotes/origin/{}", branch_name);
let tip_reference = match self.repository.find_reference(&tip_reference_name)
let tip_reference = match inner.repository.find_reference(&tip_reference_name)
{
Ok(value) => value,
Err(error) => panic!("Could not find reference “{}”: {}", tip_reference_name, error),
@ -235,7 +278,7 @@ impl BenchmarkRepository
for target_path in file_paths
{
let index_entry = self.read_file_as_index_entry(target_path.source, target_path.destination);
let index_entry = BenchmarkRepository::read_file_as_index_entry(inner, &target_path.source, &target_path.destination);
if let Err(error) = index.add(&index_entry)
{
@ -243,13 +286,13 @@ impl BenchmarkRepository
}
}
let tree_object_id = match index.write_tree_to(&self.repository)
let tree_object_id = match index.write_tree_to(&inner.repository)
{
Ok(value) => value,
Err(error) => panic!("Could not write index to tree: {}", error),
};
let tree = match self.repository.find_tree(tree_object_id)
let tree = match inner.repository.find_tree(tree_object_id)
{
Ok(value) => value,
Err(error) => panic!("Could obtain tree: {}", error),
@ -257,7 +300,7 @@ impl BenchmarkRepository
info!("Created tree object “{}”", tree_object_id);
let signature = Signature::now(&self.user_name, &self.user_email).expect("Could not create signature");
let signature = Signature::now(&inner.user_name, &inner.user_email).expect("Could not create signature");
let message = format!("Add files");
let parent = match tip_reference.peel_to_commit()
@ -266,7 +309,7 @@ impl BenchmarkRepository
Err(error) => panic!("Could not peel reference: {}", error),
};
let commit_id = match self.repository.commit(Some(&tip_reference_name), &signature, &signature, &message, &tree, &[&parent])
let commit_id = match inner.repository.commit(Some(&tip_reference_name), &signature, &signature, &message, &tree, &[&parent])
{
Ok(value) => value,
Err(error) => panic!("Could not write commit: {}", error),
@ -280,10 +323,10 @@ impl BenchmarkRepository
remote_callbacks.credentials(
|_, _, _|
{
match Cred::ssh_key_from_agent(&self.ssh_user)
match Cred::ssh_key_from_agent(&inner.ssh_user)
{
Ok(value) => Ok(value),
Err(error) => panic!("could not retrieve key pair for SSH authentication as user “{}”: {}", self.ssh_user, error),
Err(error) => panic!("could not retrieve key pair for SSH authentication as user “{}”: {}", inner.ssh_user, error),
}
});
@ -293,14 +336,16 @@ impl BenchmarkRepository
let mut push_options = PushOptions::new();
push_options.remote_callbacks(remote_callbacks);
let mut remote = self.repository.find_remote("origin").expect("");
let mut remote = inner.repository.find_remote("origin").expect("");
remote.push(&[&push_refspec], Some(&mut push_options)).expect("couldnt push");
}
pub fn file_exists(&self, file_path: &Path, branch_name: &str) -> bool
{
let inner = self.inner.lock().unwrap();
let tip_reference_name = format!("refs/remotes/origin/{}", branch_name);
let tip_reference = match self.repository.find_reference(&tip_reference_name)
let tip_reference = match inner.repository.find_reference(&tip_reference_name)
{
Ok(value) => value,
Err(error) => panic!("Could not find reference “{}”: {}", tip_reference_name, error),
@ -318,7 +363,7 @@ impl BenchmarkRepository
Err(error) => return false,
};
let blob = match self.repository.find_blob(object_id)
let blob = match inner.repository.find_blob(object_id)
{
Ok(blob) => blob,
Err(error) => return false,
@ -329,8 +374,10 @@ impl BenchmarkRepository
pub fn read_file(&self, file_path: &Path, branch_name: &str) -> Option<String>
{
let inner = self.inner.lock().unwrap();
let tip_reference_name = format!("refs/remotes/origin/{}", branch_name);
let tip_reference = match self.repository.find_reference(&tip_reference_name)
let tip_reference = match inner.repository.find_reference(&tip_reference_name)
{
Ok(value) => value,
Err(error) => panic!("Could not find reference “{}”: {}", tip_reference_name, error),
@ -348,7 +395,7 @@ impl BenchmarkRepository
Err(error) => return None,
};
let blob = match self.repository.find_blob(object_id)
let blob = match inner.repository.find_blob(object_id)
{
Ok(blob) => blob,
Err(error) => return None,
@ -376,6 +423,8 @@ impl BenchmarkRepository
Err(error) => panic!("Could not create autocommit directory: {}", error),
};
trace!("Created temporary autocommit directory in {}", tmp_dir.path().display());
let lock_file_path = tmp_dir.path().join(".lock");
if let Err(error) = File::create(&lock_file_path)
@ -385,22 +434,27 @@ impl BenchmarkRepository
let (sender, receiver) = channel();
let watcher = match raw_watcher(sender)
let mut watcher = match raw_watcher(sender)
{
Ok(value) => value,
Err(error) => panic!("Could not create filesystem watcher: {}", error),
};
watcher.watch(&tmp_dir, RecursiveMode::Recursive);
self.autocommit_watcher = Some(watcher);
self.autocommit_directory = Some(tmp_dir);
trace!("{:#?}", receiver);
let autocommit_base_path = self.autocommit_directory.as_ref().unwrap().path().to_owned();
let inner = self.inner.clone();
let autocommit_locks = self.autocommit_locks.clone();
let thread = spawn(
move ||
{
trace!("Autocommit thread running");
trace!("{:#?}", receiver);
let mut active = true;
loop
{
@ -408,12 +462,48 @@ impl BenchmarkRepository
{
Ok(RawEvent{path: Some(path), op: Ok(Op::REMOVE), cookie: _}) =>
{
trace!("Received “remove” event for path “{}”", path.display());
if path == lock_file_path
{
trace!("Waiting for remaining autocommit locks to be released");
active = false;
}
else
{
trace!("Received “remove” event for path “{}”", path.display());
let stripped_path = match path.strip_prefix(&autocommit_base_path)
{
Ok(value) => value,
Err(error) => panic!("Cannot handle “remove” event for path “{}", path.display()),
};
let mut components = stripped_path.components();
let branch_name = components.next().unwrap();
let result_path = stripped_path.strip_prefix(&branch_name).unwrap().parent().unwrap();
info!("Autocommiting “{}”", result_path.display());
let inner = inner.lock().unwrap();
BenchmarkRepository::commit_directory(&inner, &TargetPath{source: path.parent().unwrap().to_owned(), destination: result_path.to_owned()}, &branch_name.as_os_str().to_str().unwrap());
let mut autocommit_locks = autocommit_locks.lock().unwrap();
*autocommit_locks -= 1;
}
},
Err(error) => panic!("Error handling notify event: {}", error),
_ => (),
}
let mut autocommit_locks = autocommit_locks.lock().unwrap();
if *autocommit_locks == 0
{
break;
}
}
trace!("Autocommit thread finished");
});
self.autocommit_thread = Some(thread);
@ -447,6 +537,9 @@ impl BenchmarkRepository
}
}
let mut autocommit_locks = self.autocommit_locks.lock().unwrap();
*autocommit_locks += 1;
Ok(result_directory_path)
}
@ -457,6 +550,15 @@ impl BenchmarkRepository
panic!("No autocommit thread started");
}
let lock_file_path = self.autocommit_directory.as_ref().unwrap().path().join(".lock");
if let Err(error) = remove_file(&lock_file_path)
{
panic!("Could not remove lock file “{}”: {}", lock_file_path.display(), error);
}
info!("Removed lock file");
let autocommit_thread = self.autocommit_thread;
if let Err(RecvError) = autocommit_thread.unwrap().join()