diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 38d3452..3de0185 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -3,6 +3,7 @@ on: [pull_request] env: RUST_LOG: debug + RUST_BACKTRACE: 1 jobs: test: @@ -13,4 +14,4 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - run: cargo test --all-features + - run: cargo test --all-features -- --nocapture diff --git a/Cargo.lock b/Cargo.lock index 3b9a493..717949d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.11" @@ -147,6 +162,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets 0.52.0", +] + [[package]] name = "chwd" version = "0.2.0" @@ -359,7 +387,7 @@ version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" dependencies = [ - "nix", + "nix 0.27.1", "windows-sys 0.52.0", ] @@ -450,6 +478,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -457,6 +500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -465,12 +509,34 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -489,8 +555,11 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -567,6 +636,20 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +[[package]] +name = "homedir" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22074da8bba2ef26fc1737ae6c777b5baab5524c2dc403b5c6a76166766ccda5" +dependencies = [ + "cfg-if", + "nix 0.26.4", + "serde", + "widestring", + "windows-sys 0.48.0", + "wmi", +] + [[package]] name = "http" version = "0.2.11" @@ -645,6 +728,29 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.5.0" @@ -741,12 +847,14 @@ dependencies = [ "filetime", "flate2", "git2", + "homedir", "indicatif", "log", "paste", "reqwest", "rusty-fork", "serde", + "serde_json", "similar", "tar", "tempfile", @@ -860,6 +968,15 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -896,6 +1013,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", + "pin-utils", +] + [[package]] name = "nix" version = "0.27.1" @@ -907,6 +1037,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -1247,9 +1386,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.113" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ "itoa", "ryu", @@ -1773,6 +1912,12 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "widestring" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" + [[package]] name = "winapi" version = "0.3.9" @@ -1804,6 +1949,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-implement", + "windows-interface", + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-implement" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12168c33176773b86799be25e2a2ba07c7aab9968b37541f1094dbd7a60c8946" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "windows-interface" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d8dc32e0095a7eeccebd0e3f09e9509365ecb3fc6ac4d6f5f14a3f6392942d1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -1955,6 +2143,20 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wmi" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f0a4062ca522aad4705a2948fd4061b3857537990202a8ddd5af21607f79a" +dependencies = [ + "chrono", + "futures", + "log", + "serde", + "thiserror", + "windows", +] + [[package]] name = "xattr" version = "1.3.1" diff --git a/Cargo.toml b/Cargo.toml index d230e3c..2f1b6c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,10 +40,12 @@ ctrlc = "^3.4" env_logger = "^0.10" filetime = "^0.2" flate2 = "1.0.28" +homedir = "0.2.1" indicatif = "^0.17" log = "^0.4" reqwest = { version = "^0.11", default-features = false, features = ["json", "blocking", "rustls-tls"] } serde = { version = "^1.0", features = ["derive"] } +serde_json = "1.0.115" tar = "0.4.40" tempfile = "^3.8" termimad = "0.29.0" diff --git a/docs/src/tutorial/run.md b/docs/src/tutorial/run.md index 1b0783e..5755630 100644 --- a/docs/src/tutorial/run.md +++ b/docs/src/tutorial/run.md @@ -114,6 +114,25 @@ In this way, you can detect from inside the pipeline if you are in a profile or This is useful if you want to keep the outputs of different profiles separate, for instance. +### File modification times when using profiles +`make` tracks file creation times to determine if it has to re-run pipelines again. +This means that if you move files around, like Kerblam! does when it applies +profiles, `make` will always re-run your pipelines, even if you run the same +pipeline with the same profile back-to-back. + +To avoid this, Kerblam! will keep track of the last-run profile in your +projects and update the timestamps of the moved files +**only when strictly necessary**. + +This means that the profile files will get updated timestamps only when they +actually need to be updated, which is: +- When you use a profile for the first time; +- When you switch from one profile to a different one; +- When you don't use a profile, but you just used one the previous run; + +To track what was the last profile used, Kerblam! creates a file in +`$HOME/.cache/kerblam/` for each of your projects. + ## Sending additional arguments to make or bash You can send additional arguments to either `make` or `bash` after what Kerblam! sets by default by specifying them after kerblam's own `run` arguments: @@ -147,4 +166,3 @@ kerblam run my_pipe -- -- arg_1 arg_2 ... The first `--` is "eaten up" by Kerblam!, and the second one is passed to the containerization engine, telling it to pass `arg_1`, `arg_2` etc... as-is to the entrypoint. - diff --git a/docs/src/tutorial/run_containers.md b/docs/src/tutorial/run_containers.md index cedafe3..b358480 100644 --- a/docs/src/tutorial/run_containers.md +++ b/docs/src/tutorial/run_containers.md @@ -127,3 +127,11 @@ In this way, Kerblam! will run the containers with the proper paths. > > There is currently no way to configure a different working directory for every > specific dockerfile. + +### Skipping using cache +Sometimes, you want to skip using the build cache when executing a pipeline +with a container executable. + +Using `kerblam run my_pipeline --no-build-cache` will do just that: the +build backend will be told not to use the cached layers for that build (with +[the `--no-cache` flag](https://docs.docker.com/reference/cli/docker/image/build/#options)). diff --git a/src/cache.rs b/src/cache.rs new file mode 100644 index 0000000..d1a1a7c --- /dev/null +++ b/src/cache.rs @@ -0,0 +1,132 @@ +use std::env::current_dir; +use std::fs::File; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::path::PathBuf; +use std::str::FromStr; + +use anyhow::Result; +use homedir::get_my_home; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone)] +pub struct Cache { + pub last_executed_profile: Option, +} + +impl Default for Cache { + fn default() -> Self { + Cache { + last_executed_profile: None, + } + } +} + +/// Return the cache file for the current directory +/// +/// The location of the cache file is dependent on the hash of the path +/// of the current directory, which generally is where the kerblam.toml +/// file is. +/// +/// This causes each project to have its own cache file. +pub fn get_cache_path() -> Result { + let cache_dir = match get_my_home() { + Ok(dir) => dir + .unwrap_or_else(|| PathBuf::from_str("/tmp").unwrap()) + .join(".cache/kerblam"), + Err(_) => PathBuf::from_str("/tmp/.cache/kerblam").unwrap(), + }; + if !cache_dir.exists() { + std::fs::create_dir_all(&cache_dir)?; + }; + let path_hash = calc_hash(¤t_dir().unwrap()); + Ok(cache_dir.join(format!("{}", path_hash))) +} + +/// Read the content of the cache file to a string +/// Returns None if a cache cannot be found. +pub fn get_cache() -> Option { + let cache_file = get_cache_path().unwrap(); + if !cache_file.exists() { + return None; + } + let conn = match File::open(cache_file.clone()) { + Ok(file) => file, + Err(_) => { + log::warn!("Cache file cannot be read. Returning 'None' for cache content."); + return None; + } + }; + + let cache_content = match serde_json::from_reader(conn) { + Ok(content) => content, + Err(e) => { + log::error!( + "Failed to parse cache content: {e:?} Returning None. Consider deleting {cache_file:?}." + ); + return None; + } + }; + + Some(cache_content) +} + +/// Save content to the cache +pub fn write_cache(content: Cache) -> Result<()> { + let cache_file = get_cache_path().unwrap(); + std::fs::write(cache_file, serde_json::to_string_pretty(&content)?)?; + + Ok(()) +} + +/// Check the name of the last profile run in this project and return +/// True if it's the same as the current one, False otherwise. +/// Returns None if there is no cache. +/// +/// Also updates the cache to the new profile +pub fn check_last_profile(current_profile: String) -> Option { + let last_profile = get_cache(); + + let result = last_profile.clone().is_some_and(|x| { + x.last_executed_profile + .is_some_and(|x| x == current_profile) + }); + + let new_cache = Cache { + last_executed_profile: Some(current_profile), + ..last_profile.clone().unwrap_or_default() + }; + match write_cache(new_cache) { + Ok(_) => {} + Err(_) => { + log::error!("New cache was generated but could not be written. Silently continuing.") + } + }; + + if last_profile.is_none() { + None + } else { + Some(result) + } +} + +/// Delete from the cache the last profile used. +/// +/// This currently just deletes the cache. +pub fn delete_last_profile() -> Result<()> { + let cache = get_cache(); + + let new_cache = Cache { + last_executed_profile: None, + ..cache.unwrap_or_default() + }; + + write_cache(new_cache)?; + + Ok(()) +} + +fn calc_hash(t: &T) -> u64 { + let mut s = DefaultHasher::new(); + t.hash(&mut s); + s.finish() +} diff --git a/src/commands/package.rs b/src/commands/package.rs index 0846af1..19228b5 100644 --- a/src/commands/package.rs +++ b/src/commands/package.rs @@ -70,7 +70,7 @@ pub fn package_pipe(config: KerblamTomlOptions, pipe: Pipe, package_name: &str) }; let sigint_rec = setup_ctrlc_hook().expect("Failed to setup SIGINT hook!"); let backend: String = config.execution.backend.clone().into(); - let base_container = executor.build_env(sigint_rec, &backend)?; + let base_container = executor.build_env(sigint_rec, &backend, false)?; log::debug!("Base container name: {base_container:?}"); // We now have the empty container. We can add our own layers. @@ -132,7 +132,7 @@ pub fn package_pipe(config: KerblamTomlOptions, pipe: Pipe, package_name: &str) // Create the 'name' file let name_file = temp_package.path().join("name"); - let mut name_file_conn = File::create(&name_file)?; + let mut name_file_conn = File::create(name_file)?; write!(name_file_conn, "{}", package_name)?; let package = here.join(format!("{}.kerblam.tar", pipe_name)); diff --git a/src/commands/replay.rs b/src/commands/replay.rs index cfe9724..ddfb260 100644 --- a/src/commands/replay.rs +++ b/src/commands/replay.rs @@ -48,8 +48,8 @@ pub fn replay( let tag_name = match tag { Some(x) => x, None => { - if read_name.is_ok() { - read_name.unwrap() + if let Ok(name) = read_name { + name } else { bail!("Could not read container name. Try running with --tag") } @@ -79,7 +79,7 @@ pub fn replay( package_config.input_data_dir() ); let data = gunzip_file(&data_archive, &data_archive)?; - let data_archive_conn = File::open(&data)?; + let data_archive_conn = File::open(data)?; let mut data_archive = tar::Archive::new(data_archive_conn); data_archive.unpack(package_config.input_data_dir())?; }; diff --git a/src/commands/run.rs b/src/commands/run.rs index 9829c9b..80e6ffc 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -1,12 +1,13 @@ -use log; use std::collections::HashMap; use std::path::{Path, PathBuf}; +use crate::cache::{check_last_profile, delete_last_profile, get_cache}; use crate::execution::{setup_ctrlc_hook, Executor, FileMover}; use crate::options::KerblamTomlOptions; use crate::options::Pipe; use anyhow::{anyhow, bail, Result}; +use filetime::{set_file_mtime, FileTime}; /// Push a bit of a string to the end of this path /// @@ -145,6 +146,7 @@ pub fn kerblam_run_project( runtime_dir: &PathBuf, profile: Option, ignore_container: bool, + skip_build_cache: bool, extra_args: Option>, ) -> Result { let pipe = if ignore_container { @@ -165,9 +167,20 @@ pub fn kerblam_run_project( // This should mean that there is a profile with the same name in the // config... let profile_paths = extract_profile_paths(&config, profile.as_str())?; + + // Check the cache (if there) what the last profile was. + // If it was this one, we should not update the file creation time + // as we move them around, or the make pipelines re-run from the + // beginning even if we did nothing to them + let cached_run = check_last_profile(profile); + let cached_run = cached_run.unwrap_or(false); + log::debug!("Checked cached profile: {}", cached_run); + // Rename the paths that we found - let move_results: Vec> = - profile_paths.into_iter().map(|x| x.rename()).collect(); + let move_results: Vec> = profile_paths + .into_iter() + .map(|x| x.rename(!cached_run)) + .collect(); // If they are all ok, return the vec if move_results.iter().all(|x| x.is_ok()) { move_results.into_iter().map(|x| x.unwrap()).collect() @@ -183,7 +196,7 @@ pub fn kerblam_run_project( .collect(); for mover in unwindable { // I don't use the result for the same reason. - let _ = mover.rename(); + let _ = mover.rename(!cached_run); } let failed: Vec = @@ -199,6 +212,29 @@ pub fn kerblam_run_project( ) } } else { + // If we are not in a profile now, but we were before, we should + // re-touch all the old profile paths just to be safe that the + // whole pipeline is re-run again with the new data + let last_cache = get_cache(); + if last_cache + .clone() + .is_some_and(|x| x.last_executed_profile.is_some()) + { + log::debug!("Should re-touch profile files."); + let profile_paths = extract_profile_paths( + &config, + &last_cache.unwrap().last_executed_profile.unwrap(), + )?; + + for mover in profile_paths { + log::debug!("Touching {:?}", &mover.clone().get_from()); + set_file_mtime(&mover.get_from(), FileTime::now())?; + } + + // We are done. We can delete the last profile. + let _ = delete_last_profile(); + } + vec![] }; @@ -210,7 +246,8 @@ pub fn kerblam_run_project( }; // Execute the executor - let runtime_result = executor.execute(sigint_rec, &config, env_vars, extra_args); + let runtime_result = + executor.execute(sigint_rec, &config, env_vars, skip_build_cache, extra_args); // Undo the input file renaming if !unwinding_paths.is_empty() { @@ -219,7 +256,9 @@ pub fn kerblam_run_project( // If this worked before, it should work now, that is why I discard the // result... // TODO: This might be a bad idea. - let _ = item.rename(); + // + // We can skip updating timestamps at this stage + let _ = item.rename(false); } } @@ -233,7 +272,7 @@ pub fn kerblam_run_project( if res.success() { Ok("Done!".into()) } else { - Err(anyhow!("Process exited with error.")) + Err(anyhow!("Process exited with error: {res:?}")) } } None => Err(anyhow!("Process killed.")), diff --git a/src/execution.rs b/src/execution.rs index 959b5b8..d7d18a9 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -10,7 +10,7 @@ use crate::options::KerblamTomlOptions; use anyhow::{anyhow, bail, Context, Result}; use crossbeam_channel::{bounded, Receiver}; -use ctrlc; + use filetime::{set_file_mtime, FileTime}; // TODO: I think we can add all cleanup code to `Drop`, so that a lot of these @@ -117,6 +117,7 @@ impl Executor { signal_receiver: Receiver, config: &KerblamTomlOptions, env_vars: HashMap, + skip_build_cache: bool, extra_args: Option>, ) -> Result> { let mut cleanup: Vec = vec![]; @@ -124,7 +125,8 @@ impl Executor { let mut command_args = if self.env.is_some() { // This is a containerized run let backend: String = config.execution.backend.clone().into(); - let runtime_name = self.build_env(signal_receiver.clone(), &backend)?; + let runtime_name = + self.build_env(signal_receiver.clone(), &backend, skip_build_cache)?; let mut partial: Vec = if stdout().is_terminal() { // We are in a terminal. Run interactively stringify![vec![&backend, "run", "--rm", "-it"]] @@ -151,7 +153,7 @@ impl Executor { "make", &runtime_name, "-f", - &format!("{}/executor", workdir), + "executor", "-C", &workdir ]), @@ -183,7 +185,7 @@ impl Executor { if extra_args.is_some() { log::debug!("Appending extra command arguments..."); - command_args.extend(extra_args.unwrap().into_iter()); + command_args.extend(extra_args.unwrap()); } log::debug!("Executor command arguments: {:?}", command_args); @@ -225,7 +227,12 @@ impl Executor { /// Build the context of this executor and return its tag. /// /// If the executor has no environment file, this function fails. - pub fn build_env(&self, signal_receiver: Receiver, backend: &str) -> Result { + pub fn build_env( + &self, + signal_receiver: Receiver, + backend: &str, + no_cache: bool, + ) -> Result { let mut cleanup: Vec = vec![]; if self.env.is_none() { @@ -246,17 +253,31 @@ impl Executor { let containerfile_path = containerfile_path.as_os_str().to_string_lossy().to_string(); + let build_args: Vec<&str> = if no_cache { + vec![ + "build", + "-f", + containerfile_path.as_str(), + "--tag", + env_name.as_str(), + "--no-cache", + ".", + ] + } else { + vec![ + "build", + "-f", + containerfile_path.as_str(), + "--tag", + env_name.as_str(), + ".", + ] + }; + let builder = || { Command::new(backend) // If the `self.env` path is not UTF-8 I'll eat my hat. - .args([ - "build", - "-f", - containerfile_path.as_str(), - "--tag", - env_name.as_str(), - ".", - ]) + .args(&build_args) .stdout(Stdio::inherit()) .stdin(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -380,10 +401,12 @@ impl FileMover { /// /// Works identically to `mv`, but returns a new `FileMover` that can be /// used to quickly undo the move. - pub fn rename(self) -> Result { + pub fn rename(self, update_time: bool) -> Result { log::debug!("Moving {:?} to {:?}", self.from, self.to); fs::rename(&self.from, &self.to)?; - set_file_mtime(&self.to, FileTime::now())?; + if update_time { + set_file_mtime(&self.to, FileTime::now())?; + } Ok(self.invert()) } @@ -418,6 +441,14 @@ impl FileMover { to: self.from, } } + + pub fn get_from(self) -> PathBuf { + self.from.clone() + } + #[allow(dead_code)] + pub fn get_to(self) -> PathBuf { + self.to.clone() + } } impl, T: Into> From<(F, T)> for FileMover { diff --git a/src/lib.rs b/src/lib.rs index 965fd62..3f642f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ use std::env::set_current_dir; use std::ffi::OsString; use std::{env::current_dir, path::PathBuf}; +mod cache; mod commands; mod execution; mod options; @@ -49,6 +50,9 @@ enum Command { /// Do not run in container even if a container is available #[arg(long, short, action)] local: bool, + /// Skip using build cache if running in containers. + #[arg(long = "no-build-cache", action)] + skip_build_cache: bool, /// Command line arguments to be passed to child process #[clap(last = true, allow_hyphen_values = true)] extra_args: Option>, @@ -121,7 +125,7 @@ enum DataCommands { } /// Run Kerblam! with a certain arguments list. -pub fn kerblam<'a, I, T>(arguments: I) -> anyhow::Result<()> +pub fn kerblam(arguments: I) -> anyhow::Result<()> where I: Iterator, T: Into + Clone, @@ -181,6 +185,7 @@ where profile, local, desc, + skip_build_cache, extra_args, } => { let pipe = find_pipe_by_name(&config, module_name)?; @@ -194,6 +199,7 @@ where ¤t_dir().unwrap(), profile, local, + skip_build_cache, extra_args, )?; } diff --git a/src/options.rs b/src/options.rs index f75b1e7..bacaec8 100644 --- a/src/options.rs +++ b/src/options.rs @@ -216,12 +216,10 @@ impl Display for Pipe { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let container_prefix = if self.env_path.is_none() { "◾" + } else if self.env_path.clone().unwrap().file_stem().unwrap() == "default" { + "🐟" } else { - if self.env_path.clone().unwrap().file_stem().unwrap() == "default" { - "🐟" - } else { - "🐋" - } + "🐋" }; let desc_prefix = if self .description() @@ -416,7 +414,7 @@ impl KerblamTomlOptions { let default_dockerfile: Option = envs_names .iter() .find(|(name, _)| name == "default") - .and_then(|(_, path)| Some(path.to_owned())); + .map(|(_, path)| path.to_owned()); for (pipe_name, pipe_path) in pipes_names { let mut found = false; diff --git a/src/utils.rs b/src/utils.rs index 75b8062..20dc36a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -432,7 +432,7 @@ pub fn tar_files(files: Vec, strip: impl AsRef, target: PathBuf) let mut data_tarball = tar::Builder::new(data_conn); for item in files { - let inner = item.strip_prefix(&strip)?; + let inner = item.strip_prefix(strip)?; log::debug!("Adding {item:?} as {inner:?} to {data_tar:?}..."); data_tarball.append_path_with_name(&item, inner)?; } diff --git a/tests/run_local_examples.rs b/tests/run_local_examples.rs index 77e6638..4a9cc5c 100644 --- a/tests/run_local_examples.rs +++ b/tests/run_local_examples.rs @@ -1,5 +1,3 @@ -use git2; -use paste; use std::collections::HashMap; use std::env::set_current_dir; use std::mem::drop; @@ -8,7 +6,7 @@ use std::process::Command; use std::{fs, io}; use tempfile::TempDir; -const EXAMPLE_REPO: &'static str = "https://github.com/MrHedmad/kerblam-examples.git"; +const EXAMPLE_REPO: &str = "https://github.com/MrHedmad/kerblam-examples.git"; /// Setup local tests by cloning the example repo. fn setup_test() -> TempDir { @@ -22,7 +20,7 @@ fn setup_test() -> TempDir { fetcher.fetch_options(clone_options); fetcher - .clone(EXAMPLE_REPO, &target.path()) + .clone(EXAMPLE_REPO, target.path()) .expect("Could not clone remote repo."); target diff --git a/tests/test_run.rs b/tests/test_run.rs index 833372b..849192a 100644 --- a/tests/test_run.rs +++ b/tests/test_run.rs @@ -1,5 +1,5 @@ use crate::utils::{assert_ok, init_log, setup_workdir, File}; -use chwd; + use kerblam::kerblam; use rusty_fork::rusty_fork_test; @@ -18,7 +18,7 @@ use rusty_fork::rusty_fork_test; // // Keep that in mind. -static TEST_KERBLAM_TOML: &'static str = r#" +static TEST_KERBLAM_TOML: &str = r#" [data.remote] "https://raw.githubusercontent.com/MrHedmad/kerblam/main/README.md" = "input_data.txt" "https://raw.githubusercontent.com/BurntSushi/ripgrep/master/README.md" = "alternate_input_data.txt" @@ -28,18 +28,18 @@ static TEST_KERBLAM_TOML: &'static str = r#" "#; -static TEST_SHELL_PIPE: &'static str = r#" +static TEST_SHELL_PIPE: &str = r#" echo "Running shell pipe" mkdir -p ./data/out/ cat ./data/in/input_data.txt | wc -l > ./data/out/line_count.txt "#; -static TEST_ERROR_SHELL_PIPE: &'static str = r#" +static TEST_ERROR_SHELL_PIPE: &str = r#" echo "Running error shell pipe" exit 1 "#; -static TEST_MAKE_PIPE: &'static str = r#" +static TEST_MAKE_PIPE: &str = r#" .RECIPEPREFIX = > all: ./data/out/line_count.txt @@ -50,7 +50,7 @@ all: ./data/out/line_count.txt "#; -static TEST_DOCKER_FILE: &'static str = r#" +static TEST_DOCKER_FILE: &str = r#" FROM ubuntu:latest COPY . . @@ -74,8 +74,8 @@ rusty_fork_test! { let temp_dir = setup_workdir(default_files.iter()); let _flag = chwd::ChangeWorkingDirectory::change(&temp_dir).unwrap(); - assert_ok(kerblam(vec!["", "data", "fetch"].iter())); - assert_ok(kerblam(vec!["", "run", "shell_pipe", "--local"].iter())); + assert_ok(kerblam(["", "data", "fetch"].iter())); + assert_ok(kerblam(["", "run", "shell_pipe", "--local"].iter())); } } @@ -87,8 +87,8 @@ rusty_fork_test! { let temp_dir = setup_workdir(default_files.iter()); let _flag = chwd::ChangeWorkingDirectory::change(&temp_dir).unwrap(); - assert_ok(kerblam(vec!["", "data", "fetch"].iter())); - assert_ok(kerblam(vec!["", "run", "make_pipe", "--local"].iter())); + assert_ok(kerblam(["", "data", "fetch"].iter())); + assert_ok(kerblam(["", "run", "make_pipe", "--local"].iter())); } } @@ -101,7 +101,7 @@ rusty_fork_test! { let temp_dir = setup_workdir(default_files.iter()); let _flag = chwd::ChangeWorkingDirectory::change(&temp_dir).unwrap(); - assert_ok(kerblam(vec!["", "data", "fetch"].iter())); - assert_ok(kerblam(vec!["", "run", "error", "--local"].iter())); + assert_ok(kerblam(["", "data", "fetch"].iter())); + assert_ok(kerblam(["", "run", "error", "--local"].iter())); } }