diff --git a/Cargo.lock b/Cargo.lock index f6f5d45a74..5d9bcd6d2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,6 +713,32 @@ dependencies = [ "cc", ] +[[package]] +name = "cmd_lib" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "371c15a3c178d0117091bd84414545309ca979555b1aad573ef591ad58818d41" +dependencies = [ + "cmd_lib_macros", + "env_logger", + "faccess", + "lazy_static", + "log", + "os_pipe", +] + +[[package]] +name = "cmd_lib_macros" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb844bd05be34d91eb67101329aeba9d3337094c04fd8507d821db7ebb488eaf" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "colored" version = "2.1.0" @@ -1317,6 +1343,19 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + [[package]] name = "environmental" version = "1.1.4" @@ -1477,6 +1516,17 @@ dependencies = [ "sha3", ] +[[package]] +name = "faccess" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ae66425802d6a903e268ae1a08b8c38ba143520f227a205edf4e9c7e3e26d5" +dependencies = [ + "bitflags 1.3.2", + "libc", + "winapi", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1984,6 +2034,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "1.5.2" @@ -3150,6 +3206,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "os_pipe" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "overload" version = "0.1.1" @@ -3488,6 +3554,28 @@ dependencies = [ "toml_edit 0.22.22", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "proc-macro2" version = "1.0.92" @@ -3885,11 +3973,16 @@ dependencies = [ [[package]] name = "rofl-containers" -version = "0.1.0" +version = "0.2.0" dependencies = [ + "anyhow", "base64", + "cmd_lib", + "hex", + "nix", "oasis-runtime-sdk", "rofl-appd", + "tokio", ] [[package]] diff --git a/rofl-appd/src/lib.rs b/rofl-appd/src/lib.rs index 9b5bc5037b..7faa846b94 100644 --- a/rofl-appd/src/lib.rs +++ b/rofl-appd/src/lib.rs @@ -1,7 +1,7 @@ //! REST API daemon accessible by ROFL apps. mod routes; -pub(crate) mod services; +pub mod services; pub(crate) mod state; use std::sync::Arc; @@ -10,26 +10,29 @@ use rocket::{figment::Figment, routes}; use oasis_runtime_sdk::modules::rofl::app::{App, Environment}; +/// API server configuration. +#[derive(Clone)] +pub struct Config<'a> { + /// Address where the service should listen on. + pub address: &'a str, + /// Key management service to use. + pub kms: Arc, +} + /// Start the REST API server. -pub async fn start(address: &str, env: Environment) -> Result<(), rocket::Error> +pub async fn start(cfg: Config<'_>, env: Environment) -> Result<(), rocket::Error> where A: App, { - // KMS service. - let kms_service: Arc = - Arc::new(services::kms::OasisKmsService::new(env.clone())); - let kms_service_task = kms_service.clone(); - tokio::spawn(async move { kms_service_task.start().await }); - // Oasis runtime environment. let env: Arc = Arc::new(state::EnvImpl::new(env)); // Server configuration. - let cfg = Figment::new().join(("address", address)); + let rocket_cfg = Figment::new().join(("address", cfg.address)); - rocket::custom(cfg) + rocket::custom(rocket_cfg) .manage(env) - .manage(kms_service) + .manage(cfg.kms) .mount("/rofl/v1/app", routes![routes::app::id,]) .mount("/rofl/v1/keys", routes![routes::keys::generate,]) .launch() diff --git a/rofl-appd/src/services/kms.rs b/rofl-appd/src/services/kms.rs index fccd9b3d82..ae4d3e851a 100644 --- a/rofl-appd/src/services/kms.rs +++ b/rofl-appd/src/services/kms.rs @@ -4,6 +4,7 @@ use std::sync::{ }; use sp800_185::KMac; +use tokio::sync::Notify; use oasis_runtime_sdk::{ core::common::logger::get_logger, @@ -17,6 +18,9 @@ pub trait KmsService: Send + Sync { /// Start the KMS service. async fn start(&self) -> Result<(), Error>; + /// Waits for the service to become ready to accept requests. + async fn wait_ready(&self) -> Result<(), Error>; + /// Generate a key based on the passed parameters. async fn generate(&self, request: &GenerateRequest<'_>) -> Result; } @@ -94,6 +98,7 @@ pub struct OasisKmsService { root_key: Arc>>>, env: Environment, logger: slog::Logger, + ready_notify: Notify, } impl OasisKmsService { @@ -103,6 +108,7 @@ impl OasisKmsService { root_key: Arc::new(Mutex::new(None)), env, logger: get_logger("appd/services/kms"), + ready_notify: Notify::new(), } } } @@ -146,11 +152,25 @@ impl KmsService for OasisKmsService { // Store the key in memory. *self.root_key.lock().unwrap() = Some(root_key.key); + self.ready_notify.notify_waiters(); + slog::info!(self.logger, "KMS service initialized"); Ok(()) } + async fn wait_ready(&self) -> Result<(), Error> { + let handle = self.ready_notify.notified(); + + if self.root_key.lock().unwrap().is_some() { + return Ok(()); + } + + handle.await; + + Ok(()) + } + async fn generate(&self, request: &GenerateRequest<'_>) -> Result { let root_key_guard = self.root_key.lock().unwrap(); let root_key = root_key_guard.as_ref().ok_or(Error::NotInitialized)?; @@ -173,6 +193,10 @@ impl KmsService for MockKmsService { Ok(()) } + async fn wait_ready(&self) -> Result<(), Error> { + Ok(()) + } + async fn generate(&self, request: &GenerateRequest<'_>) -> Result { let key = Kdf::derive_key( INSECURE_MOCK_ROOT_KEY, diff --git a/rofl-containers/Cargo.toml b/rofl-containers/Cargo.toml index 51e2250ca3..ae6170e3b4 100644 --- a/rofl-containers/Cargo.toml +++ b/rofl-containers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rofl-containers" -version = "0.1.0" +version = "0.2.0" edition = "2021" [dependencies] @@ -9,4 +9,9 @@ oasis-runtime-sdk = { path = "../runtime-sdk", features = ["tdx"] } rofl-appd = { path = "../rofl-appd" } # Third party. +anyhow = "1.0.86" base64 = "0.22.1" +cmd_lib = "1.9.5" +hex = "0.4.3" +nix = { version = "0.29.0", features = ["signal"] } +tokio = { version = "1.38", features = ["rt", "rt-multi-thread", "sync", "time", "macros"] } diff --git a/rofl-containers/src/main.rs b/rofl-containers/src/main.rs index 5f65213d4a..e3552ba022 100644 --- a/rofl-containers/src/main.rs +++ b/rofl-containers/src/main.rs @@ -11,7 +11,15 @@ use std::env; use base64::prelude::*; -use oasis_runtime_sdk::{cbor, modules::rofl::app::prelude::*}; +use oasis_runtime_sdk::{ + cbor, + core::common::{logger::get_logger, process}, + modules::rofl::app::prelude::*, +}; +use rofl_appd::services; + +mod reaper; +mod storage; /// UNIX socket address where the REST API server will listen on. const ROFL_APPD_ADDRESS: &str = "unix:/run/rofl-appd.sock"; @@ -42,12 +50,39 @@ impl App for ContainersApp { .expect("Corrupted ROFL_CONSENSUS_TRUST_ROOT (must be Base64-encoded CBOR).") } - async fn run(self: Arc, env: Environment) { + async fn post_registration_init(self: Arc, env: Environment) { + // Temporarily disable the default process reaper as it interferes with scripts. + let _guard = reaper::disable_default_reaper(); + let logger = get_logger("post_registration_init"); + + // Start the key management service and wait for it to initialize. + let kms: Arc = + Arc::new(services::kms::OasisKmsService::new(env.clone())); + let kms_task = kms.clone(); + tokio::spawn(async move { kms_task.start().await }); + let _ = kms.wait_ready().await; + + // Initialize storage when configured in the kernel cmdline. + if let Err(err) = storage::init(kms.clone()).await { + slog::error!(logger, "failed to initialize stage 2 storage"; "err" => ?err); + process::abort(); + } + // Start the REST API server. - let _ = rofl_appd::start(ROFL_APPD_ADDRESS, env).await; + let cfg = rofl_appd::Config { + address: ROFL_APPD_ADDRESS, + kms, + }; + let _ = rofl_appd::start(cfg, env).await; } } fn main() { + // Configure the binary search path. + // SAFETY: This is safe as no other threads are running yet. + unsafe { + env::set_var("PATH", "/usr/sbin:/usr/bin:/sbin:/bin"); + } + ContainersApp.start(); } diff --git a/rofl-containers/src/reaper.rs b/rofl-containers/src/reaper.rs new file mode 100644 index 0000000000..d5ed5dc370 --- /dev/null +++ b/rofl-containers/src/reaper.rs @@ -0,0 +1,33 @@ +use nix::sys::signal::{sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal}; + +/// Guard that re-enables the default process reaper when dropped. +pub struct DisableReaperGuard { + _internal: (), +} + +impl Drop for DisableReaperGuard { + fn drop(&mut self) { + // Re-enable default kernel process reaper. + unsafe { + let _ = sigaction( + Signal::SIGCHLD, + &SigAction::new(SigHandler::SigIgn, SaFlags::empty(), SigSet::empty()), + ); + } + } +} + +/// Temporarily disables the default process reaper. When the returned guard gets out of scope, the +/// default reaper is re-enabled. +/// +/// This assumes that the default reaper has been previously configured by core init. +pub fn disable_default_reaper() -> DisableReaperGuard { + unsafe { + let _ = sigaction( + Signal::SIGCHLD, + &SigAction::new(SigHandler::SigDfl, SaFlags::empty(), SigSet::empty()), + ); + } + + DisableReaperGuard { _internal: () } +} diff --git a/rofl-containers/src/storage.rs b/rofl-containers/src/storage.rs new file mode 100644 index 0000000000..e5df76fe86 --- /dev/null +++ b/rofl-containers/src/storage.rs @@ -0,0 +1,92 @@ +use std::{fs, sync::Arc}; + +use anyhow::Result; +use cmd_lib::run_cmd; + +use rofl_appd::services::{ + self, + kms::{GenerateRequest, KeyKind}, +}; + +/// Storage encryption key identifier. +const STORAGE_ENCRYPTION_KEY_ID: &str = + "oasis-runtime-sdk/rofl-containers: storage encryption key v1"; + +/// Initialize stage 2 storage based on configuration. +pub async fn init(kms: Arc) -> Result<()> { + // Parse kernel command line to determine relevant features. + let cmdline = fs::read_to_string("/proc/cmdline")?; + let storage_mode = cmdline + .split(' ') + .filter_map(|s| { + if !s.is_empty() { + Some(s.split_once('=')?) + } else { + None + } + }) + .filter(|(k, _)| *k == "oasis.stage2.storage_mode") + .map(|(_, v)| v) + .next(); + if storage_mode != Some("custom") { + return Ok(()); // Ignore non-custom storage mode. + } + + // Derive storage key. + let storage_key = kms + .generate(&GenerateRequest { + key_id: STORAGE_ENCRYPTION_KEY_ID, + kind: KeyKind::Raw384, + }) + .await?; + let storage_key = hex::encode(&storage_key.key); + + // Ensure all device mapper devices are present. + run_cmd!(dmsetup mknodes)?; + + // Open or re-format storage. + let result = open_storage(&storage_key); + if result.is_err() { + format_storage(&storage_key)?; + } + + // Mount filesystem as /storage. + run_cmd!(mount "/dev/mapper/storage" "/storage")?; + + // Setup /run and /var. + run_cmd!( + mkdir "/storage/run"; + mkdir -p "/storage/var/lib"; + mkdir -p "/storage/var/cache"; + mount --bind "/storage/run" "/run"; + mount --bind "/storage/var" "/var"; + )?; + + Ok(()) +} + +/// Attempt to open the storage partition block device using the given storage key. +fn open_storage(storage_key: &str) -> Result<()> { + run_cmd!( + echo -n ${storage_key} | + cryptsetup open --type luks2 --disable-locks "/dev/mapper/part-storage" storage + )?; + + Ok(()) +} + +/// Format the storage partition block device using the given storage key. +fn format_storage(storage_key: &str) -> Result<()> { + // Format block device. + run_cmd!( + echo -n ${storage_key} | + cryptsetup luksFormat --type luks2 --integrity hmac-sha256 --disable-locks "/dev/mapper/part-storage" + )?; + + open_storage(storage_key)?; + + // Format filesystem. + run_cmd!(mkfs.ext4 "/dev/mapper/storage")?; + + Ok(()) +} diff --git a/runtime-sdk/src/modules/rofl/app/mod.rs b/runtime-sdk/src/modules/rofl/app/mod.rs index b0bf320681..b3f114a2c3 100644 --- a/runtime-sdk/src/modules/rofl/app/mod.rs +++ b/runtime-sdk/src/modules/rofl/app/mod.rs @@ -74,6 +74,18 @@ pub trait App: Send + Sync + 'static { tx } + /// Custom post-registration initialization. It runs before any image-specific scripts are + /// called by the runtime so it can be used to do things like set up custom storage after + /// successful registration. + /// + /// Until this function completes, no further initialization will happen. + async fn post_registration_init(self: Arc, env: Environment) + where + Self: Sized, + { + // Default implementation does nothing. + } + /// Main application processing loop. async fn run(self: Arc, env: Environment) where diff --git a/runtime-sdk/src/modules/rofl/app/processor.rs b/runtime-sdk/src/modules/rofl/app/processor.rs index 455acf9df2..e4936cf6ab 100644 --- a/runtime-sdk/src/modules/rofl/app/processor.rs +++ b/runtime-sdk/src/modules/rofl/app/processor.rs @@ -181,11 +181,22 @@ where tokio::spawn(self.state.app.clone().run(self.env.clone())); // Perform post-registration initialization. - slog::info!( - self.logger, - "performing additional post-registration initialization" - ); - init::post_registration_init(); + let app = self.state.app.clone(); + let env = self.env.clone(); + let logger = self.logger.clone(); + tokio::spawn(async move { + slog::info!( + logger, + "performing app-specific post-registration initialization" + ); + app.post_registration_init(env).await; + + slog::info!( + logger, + "performing additional post-registration initialization" + ); + init::post_registration_init(); + }); // Notify notifier task. self.tasks