From ed11a39167899de16d5f4277290509e61a01ac6f Mon Sep 17 00:00:00 2001 From: Luca Visentin Date: Tue, 26 Nov 2024 10:28:39 +0100 Subject: [PATCH 1/4] feat: add experimental windows compatibility --- src/commands/data/describe.rs | 15 ++++++++++++++- src/execution.rs | 21 ++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/commands/data/describe.rs b/src/commands/data/describe.rs index cc66c4e..f4d472f 100644 --- a/src/commands/data/describe.rs +++ b/src/commands/data/describe.rs @@ -1,11 +1,16 @@ use core::iter::Sum; use std::fmt::Display; use std::fs; -use std::os::unix::fs::MetadataExt; use std::path::PathBuf; use crate::options::KerblamTomlOptions; +#[cfg(target_family = "windows")] +use std::os::windows::fs::MetadataExt; + +#[cfg(target_family = "unix")] +use std::os::unix::fs::MetadataExt; + #[derive(Debug, Clone)] /// Wrapper to interpret a usize as a File size. pub struct FileSize { @@ -19,6 +24,14 @@ impl TryFrom for FileSize { fn try_from(value: PathBuf) -> std::result::Result { let meta = fs::metadata(value)?; + #[cfg(target_family = "windows")] + { + Ok(FileSize { + size: meta.file_size() as usize, + }) + } + + #[cfg(target_family = "unix")] Ok(FileSize { size: meta.size() as usize, }) diff --git a/src/execution.rs b/src/execution.rs index fd5002f..c491e21 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -13,6 +13,12 @@ use anyhow::{anyhow, bail, Context, Result}; use crossbeam_channel::{bounded, Receiver}; use lazy_static::lazy_static; +#[cfg(target_family = "unix")] +static SHELL: &str = "bash"; + +#[cfg(target_family = "windows")] +static SHELL: &str = "powershell"; + // TODO: I think we can add all cleanup code to `Drop`, so that a lot of these // functions can be simplified a lot. // E.g. make a "cleanup: Option>" to the Executor and remove @@ -109,7 +115,7 @@ impl Executor { /// Execute this executor based on its data /// /// Either builds and runs a container, or executes make and/or - /// bash, depending on the execution strategy. + /// bash (powershell on windows), depending on the execution strategy. /// /// Needs the kerblam config to work, as we need to bind-mount the local /// paths in the containers as locally needed and follow other settings. @@ -161,7 +167,7 @@ impl Executor { ]), ExecutionStrategy::Shell => stringify!(vec![ "--entrypoint", - "bash", + SHELL, &runtime_name, &format!("{}/executor", workdir) ]), @@ -180,7 +186,7 @@ impl Executor { stringify![vec!["make", "-f", self.target.to.to_str().unwrap()]] } ExecutionStrategy::Shell => { - stringify![vec!["bash", self.target.to.to_str().unwrap()]] + stringify![vec![SHELL, self.target.to.to_str().unwrap()]] } } }; @@ -414,8 +420,17 @@ impl FileMover { fn _link(&self) -> Result { // TODO: Make this compatible with other operating systems. log::debug!("Linking {:?} to {:?}", self.from, self.to); + + #[cfg(target_family = "unix")] std::os::unix::fs::symlink(self.from.clone(), self.to.clone())?; + #[cfg(target_family = "windows")] + if self.from.is_dir() { + std::os::windows::fs::symlink_dir(self.from.clone(), self.to.clone())?; + } else { + std::os::windows::fs::symlink_file(self.from.clone(), self.to.clone())?; + } + Ok(self.to.clone()) } From 90951908652108090f3fe1d25765e4a9b5b81d85 Mon Sep 17 00:00:00 2001 From: Luca Visentin Date: Tue, 26 Nov 2024 11:27:11 +0100 Subject: [PATCH 2/4] fix: fix windows idiotic execution policies Powershell does not support running files (with `-File`) that do not end in the `.ps1` extension. This is obviously dumb for interoperability: you'd have kerblam shell workflows working on linux but not on windows, and vice-versa. A workaround is to use `Invoke-Command`, and source the file inside the "CommandBlock" that Invoke-Command supports. This is fine, but then if you find out that `Invoke-Command` is *not* a program, *it's a powershell internal "magic" command*. So when Kerblam! tries to spawn the `Invoke-Command` executable it fails. So, we combine both approaches: we use `powershell -Command` to spawn a powershell, and pass a command to it. This command is `Invoke-Command`, which takes care of reading the content of whatever workflow file we are using, and execute it (in the current shell, so our sub-shell). Fuck me! --- src/execution.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/execution.rs b/src/execution.rs index c491e21..598ddbb 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -13,12 +13,6 @@ use anyhow::{anyhow, bail, Context, Result}; use crossbeam_channel::{bounded, Receiver}; use lazy_static::lazy_static; -#[cfg(target_family = "unix")] -static SHELL: &str = "bash"; - -#[cfg(target_family = "windows")] -static SHELL: &str = "powershell"; - // TODO: I think we can add all cleanup code to `Drop`, so that a lot of these // functions can be simplified a lot. // E.g. make a "cleanup: Option>" to the Executor and remove @@ -167,7 +161,7 @@ impl Executor { ]), ExecutionStrategy::Shell => stringify!(vec![ "--entrypoint", - SHELL, + "bash", &runtime_name, &format!("{}/executor", workdir) ]), @@ -186,7 +180,19 @@ impl Executor { stringify![vec!["make", "-f", self.target.to.to_str().unwrap()]] } ExecutionStrategy::Shell => { - stringify![vec![SHELL, self.target.to.to_str().unwrap()]] + #[cfg(target_family = "unix")] + { + stringify![vec!["bash", self.target.to.to_str().unwrap()]] + } + + #[cfg(target_family = "windows")] + { + // The double {{ and }} are the escaped forms of { and } for format! + let powershell_slug = format!("& {{Invoke-command -ScriptBlock ([ScriptBlock]::Create((Get-Content {})))}}", + self.target.to.into_os_string().into_string().unwrap() + ); + stringify![vec!["powershell", "-Command", &powershell_slug]] + } } } }; From b34aed534bc1183265c9dd114bdfba28275ff70e Mon Sep 17 00:00:00 2001 From: Luca Visentin Date: Tue, 26 Nov 2024 11:38:52 +0100 Subject: [PATCH 3/4] chore: update cargo dist to support Windows targets --- Cargo.toml | 4 ++-- README.md | 2 +- docs/src/manual/run.md | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 66c5e56..e511e8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,9 +77,9 @@ cargo-dist-version = "0.16.0" # CI backends to support ci = "github" # The installers to generate for each app -installers = ["shell"] +installers = ["shell", "powershell"] # Target platforms to build apps for (Rust target-triple syntax) -targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl"] +targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "x86_64-pc-windows-msvc"] # Publish jobs to run in CI pr-run-mode = "plan" # Whether to install an updater program diff --git a/README.md b/README.md index 784cdbd..17bdadb 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ It provides an overview of what Kerblam! is and does with an hands-on project. > binaries in the releases tab, the command below or compile directly from git with > `cargo install --git https://github.com/MrHedmad/kerblam`. -In short, use a unix-compatible OS and either: +In short, use: ```bash # Install a prebuilt binary curl --proto '=https' --tlsv1.2 -LsSf https://github.com/MrHedmad/kerblam/releases/latest/download/kerblam-installer.sh | sh diff --git a/docs/src/manual/run.md b/docs/src/manual/run.md index 60ac114..681e755 100644 --- a/docs/src/manual/run.md +++ b/docs/src/manual/run.md @@ -8,7 +8,10 @@ It automatically uses the proper execution strategy based on what extension the file is saved as: either `make` or `bash`. > [!IMPORTANT] -> Shell scripts are always executed in `bash`. +> Shell scripts are always executed in `bash` for Linux/Unix, and as Powershell +> scripts for Windows. +> +> Note that the extension for shell files is *always* `.sh`, even on windows. You can use any workflow manager that is installed on your system through Kerblam! (e.g. `snakemake` or `nextflow`) by writing thin shell wrappers From 9ec633c04f71aa6a7dc56422aecb655b8525a7fb Mon Sep 17 00:00:00 2001 From: Luca Visentin Date: Tue, 26 Nov 2024 12:34:13 +0100 Subject: [PATCH 4/4] docs: update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 400384c..4f22185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ the top. Non pre-release versions sometimes have an associated name. fetching the file with `kerblam data fetch`, but will still treat it as remote for all other purposes (e.g. for `kerblam data clean`). This allows the user to use non-canonical fetching methods. +- Experimental: Kerblam! is now compatible with Windows. + In Windows systems, shell scripts are executed with PowerShell instead of Bash. ### Changed - The way that profiles are handled was changed.