diff --git a/Cargo.lock b/Cargo.lock index 831a277bc..8488c2406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -128,6 +128,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" +[[package]] +name = "ascii" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae7d751998c189c1d4468cf0a39bb2eae052a9c58d50ebb3b9591ee3813ad50" + [[package]] name = "async-trait" version = "0.1.72" @@ -160,6 +166,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + [[package]] name = "base64" version = "0.21.2" @@ -454,6 +466,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1645a65a99c7c8d345761f4b75a6ffe5be3b3b27a93ee731fccc5050ba6be97c" +dependencies = [ + "ascii", + "byteorder", +] + [[package]] name = "command-fds" version = "0.2.2" @@ -464,6 +486,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "const_fn" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" + [[package]] name = "containerd-shim" version = "0.4.0" @@ -489,7 +517,7 @@ dependencies = [ "serde_json", "signal-hook", "thiserror", - "time", + "time 0.3.25", "windows-sys 0.48.0", ] @@ -517,6 +545,7 @@ dependencies = [ "containerd-shim", "env_logger", "libc", + "libcontainer", "log", "nix 0.26.2", "oci-spec", @@ -954,6 +983,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + [[package]] name = "either" version = "1.9.0" @@ -1599,6 +1634,18 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +[[package]] +name = "libbpf-sys" +version = "1.2.1+v1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75adb4021282a72ca63ebbc0e4247750ad74ede68ff062d247691072d709ad8b" +dependencies = [ + "cc", + "nix 0.26.2", + "num_cpus", + "pkg-config", +] + [[package]] name = "libc" version = "0.2.147" @@ -1612,10 +1659,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12f6fef16f505466473eeeee906244e03a437beaf41ccd85c39355b4077890c9" dependencies = [ "dbus", + "errno", "fixedbitset 0.4.2", + "libbpf-sys", + "libc", "nix 0.26.2", "oci-spec", "procfs", + "rbpf", "serde", "thiserror", "tracing", @@ -2171,6 +2222,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.20+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" + [[package]] name = "proc-macro2" version = "1.0.66" @@ -2402,6 +2459,18 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rbpf" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b536dc5c7e3a730d06c578a41df1fbcccd66240a7a9bd5f150a0826291f01c66" +dependencies = [ + "byteorder", + "combine", + "libc", + "time 0.2.27", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2551,6 +2620,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustix" version = "0.36.15" @@ -2662,12 +2740,27 @@ dependencies = [ "untrusted", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.183" @@ -2736,6 +2829,21 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "sha1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.7" @@ -2849,12 +2957,70 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn 1.0.109", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn 1.0.109", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + [[package]] name = "strsim" version = "0.10.0" @@ -2958,6 +3124,21 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "time" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros 0.1.1", + "version_check", + "winapi", +] + [[package]] name = "time" version = "0.3.25" @@ -2967,7 +3148,7 @@ dependencies = [ "deranged", "serde", "time-core", - "time-macros", + "time-macros 0.2.11", ] [[package]] @@ -2976,6 +3157,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + [[package]] name = "time-macros" version = "0.2.11" @@ -2985,6 +3176,19 @@ dependencies = [ "time-core", ] +[[package]] +name = "time-macros-impl" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn 1.0.109", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -3479,7 +3683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29e3ac9b780c7dda0cac7a52a5d6d2d6707cc6e3451c9db209b6c758f40d7acb" dependencies = [ "indexmap 1.9.3", - "semver", + "semver 1.0.18", ] [[package]] @@ -3489,7 +3693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dfcdb72d96f01e6c85b6bf20102e7423bdbaad5c337301bab2bbf253d26413c" dependencies = [ "indexmap 2.0.0", - "semver", + "semver 1.0.18", ] [[package]] @@ -4147,7 +4351,7 @@ dependencies = [ "indexmap 1.9.3", "log", "pulldown-cmark", - "semver", + "semver 1.0.18", "unicode-xid", "url", ] diff --git a/Cargo.toml b/Cargo.toml index 110f82e19..4078bca95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ thiserror = "1.0" libc = "0.2.147" oci-spec = { version = "0.6.1", features = ["runtime"] } sha256 = "1.3.0" -libcontainer = "0.1" +libcontainer = { version = "0.1", default-features = false } [profile.release] panic = "abort" diff --git a/Makefile b/Makefile index 43d27269a..4e629165e 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ KIND_CLUSTER_NAME ?= containerd-wasm .PHONY: build build: cargo build -p containerd-shim-wasm --features generate_bindings $(RELEASE_FLAG) + cargo build -p containerd-shim-wasm --features libcontainer $(RELEASE_FLAG) cargo build $(RELEASE_FLAG) .PHONY: check diff --git a/crates/containerd-shim-wasm/Cargo.toml b/crates/containerd-shim-wasm/Cargo.toml index 49330d5fc..40d85e4c1 100644 --- a/crates/containerd-shim-wasm/Cargo.toml +++ b/crates/containerd-shim-wasm/Cargo.toml @@ -28,6 +28,7 @@ clone3 = "0.2" libc = { workspace = true } caps = "0.5" proc-mounts = "0.3" +libcontainer = { workspace = true, optional = true, default-features = false } cgroups-rs = "0.3.3" [build-dependencies] @@ -42,5 +43,11 @@ rand = "0.8" [features] default = [] +libcontainer = ["libcontainer/default"] generate_bindings = ["ttrpc-codegen"] -generate_doc = [] \ No newline at end of file +generate_doc = [] +libseccomp = ["libcontainer/libseccomp"] +systemd = ["libcontainer/systemd"] +cgroupsv2 = ["libcontainer/v2"] +cgroupsv1 = ["libcontainer/v1"] +cgroupsv2_devices = ["libcontainer/cgroupsv2_devices"] \ No newline at end of file diff --git a/crates/containerd-shim-wasm/src/sandbox/instance.rs b/crates/containerd-shim-wasm/src/sandbox/instance.rs index ce0af303b..0caf3a461 100644 --- a/crates/containerd-shim-wasm/src/sandbox/instance.rs +++ b/crates/containerd-shim-wasm/src/sandbox/instance.rs @@ -10,7 +10,7 @@ use chrono::{DateTime, Utc}; use super::error::Error; -type ExitCode = (Mutex)>>, Condvar); +pub type ExitCode = Arc<(Mutex)>>, Condvar)>; /// Generic options builder for creating a wasm instance. /// This is passed to the `Instance::new` method. @@ -159,7 +159,7 @@ impl Wait { /// code. When the child process exits, the shim will use the ExitCode /// to signal the exit status to the caller. This function returns so that /// the wait() function in the shim implementation API would not block. - pub fn set_up_exit_code_wait(&self, exit_code: Arc) -> Result<(), Error> { + pub fn set_up_exit_code_wait(&self, exit_code: ExitCode) -> Result<(), Error> { let sender = self.tx.clone(); let code = Arc::clone(&exit_code); thread::spawn(move || { @@ -180,7 +180,7 @@ impl Wait { pub struct Nop { /// Since we are faking the container, we need to keep track of the "exit" code/time /// We'll just mark it as exited when kill is called. - exit_code: Arc, + exit_code: ExitCode, } impl Instance for Nop { diff --git a/crates/containerd-shim-wasm/src/sandbox/libcontainer_instance.rs b/crates/containerd-shim-wasm/src/sandbox/libcontainer_instance.rs new file mode 100644 index 000000000..28dc99cba --- /dev/null +++ b/crates/containerd-shim-wasm/src/sandbox/libcontainer_instance.rs @@ -0,0 +1,187 @@ +//! Abstractions for running/managing a wasm/wasi instance that uses youki's libcontainer library. + +use anyhow::Context; +use chrono::Utc; +use libc::{SIGINT, SIGKILL}; +use libcontainer::{ + container::{Container, ContainerStatus}, + signal::Signal, +}; +use log::error; +use nix::{ + errno::Errno, + sys::wait::{waitid, Id as WaitID, WaitPidFlag, WaitStatus}, +}; + +use crate::sandbox::{ + instance::ExitCode, + instance_utils::{get_instance_root, instance_exists}, +}; + +use crate::sandbox::InstanceConfig; +use std::{path::PathBuf, thread}; + +use super::{error::Error, instance::Wait, Instance}; + +/// LibcontainerInstance is a trait that gets implemented by a WASI runtime that +/// uses youki's libcontainer library as the container runtime. +/// It provides default implementations for some of the Instance trait methods. +/// The implementor of this trait is expected to implement the +/// * `new_libcontainer()` +/// * `get_exit_code()` +/// * `get_id()` +/// * `get_root_dir()` +/// * `build_container()` +/// methods. +pub trait LibcontainerInstance { + /// The WASI engine type + type E: Send + Sync + Clone; + + /// Create a new instance + fn new_libcontainer(id: String, cfg: Option<&InstanceConfig>) -> Self; + + /// Get the exit code of the instance + fn get_exit_code(&self) -> ExitCode; + + /// Get the ID of the instance + fn get_id(&self) -> String; + + /// Get the root directory of the instance + fn get_root_dir(&self) -> Result; + + /// Build the container + fn build_container(&self) -> Result; +} + +/// Default implementation of the Instance trait for YoukiInstance +/// This implementation uses the libcontainer library to create and start +/// the container. +impl Instance for T { + type E = T::E; + + fn new(id: String, cfg: Option<&InstanceConfig>) -> Self { + Self::new_libcontainer(id, cfg) + } + + /// Start the instance + /// The returned value should be a unique ID (such as a PID) for the instance. + /// Nothing internally should be using this ID, but it is returned to containerd where a user may want to use it. + fn start(&self) -> Result { + let id = self.get_id(); + log::info!("starting instance: {}", id); + + let mut container = self.build_container()?; + let code = self.get_exit_code(); + let pid = container.pid().context("failed to get pid")?; + + container + .start() + .map_err(|err| Error::Any(anyhow::anyhow!("failed to start container: {}", err)))?; + + thread::spawn(move || { + let (lock, cvar) = &*code; + + let status = match waitid(WaitID::Pid(pid), WaitPidFlag::WEXITED) { + Ok(WaitStatus::Exited(_, status)) => status, + Ok(WaitStatus::Signaled(_, sig, _)) => sig as i32, + Ok(_) => 0, + Err(e) => { + if e == Errno::ECHILD { + log::info!("no child process"); + 0 + } else { + panic!("waitpid failed: {}", e); + } + } + } as u32; + let mut ec = lock.lock().unwrap(); + *ec = Some((status, Utc::now())); + drop(ec); + cvar.notify_all(); + }); + + Ok(pid.as_raw() as u32) + } + + /// Send a signal to the instance + fn kill(&self, signal: u32) -> Result<(), Error> { + let id = self.get_id(); + let root_dir = self.get_root_dir()?; + log::info!("killing instance: {}", id.clone()); + if signal as i32 != SIGKILL && signal as i32 != SIGINT { + return Err(Error::InvalidArgument( + "only SIGKILL and SIGINT are supported".to_string(), + )); + } + let signal = Signal::try_from(signal as i32) + .map_err(|err| Error::InvalidArgument(format!("invalid signal number: {}", err)))?; + let container_root = get_instance_root(root_dir, id.as_str())?; + let mut container = Container::load(container_root).with_context(|| { + format!("could not load state for container {id}", id = id.as_str()) + })?; + + match container.kill(signal, true) { + Ok(_) => Ok(()), + Err(e) => { + if container.status() == ContainerStatus::Stopped { + return Err(Error::Others("container not running".into())); + } + Err(Error::Others(e.to_string())) + } + } + } + + /// Delete any reference to the instance + /// This is called after the instance has exited. + fn delete(&self) -> Result<(), Error> { + let id = self.get_id(); + let root_dir = self.get_root_dir()?; + log::info!("deleting instance: {}", id.clone()); + match instance_exists(&root_dir, id.as_str()) { + Ok(exists) => { + if !exists { + return Ok(()); + } + } + Err(err) => { + error!("could not find the container, skipping cleanup: {}", err); + return Ok(()); + } + } + let container_root = get_instance_root(&root_dir, id.as_str())?; + let container = Container::load(container_root).with_context(|| { + format!( + "could not load state for container {id}", + id = id.clone().as_str() + ) + }); + match container { + Ok(mut container) => container.delete(true).map_err(|err| { + Error::Any(anyhow::anyhow!( + "failed to delete container {}: {}", + id, + err + )) + })?, + Err(err) => { + error!("could not find the container, skipping cleanup: {}", err); + return Ok(()); + } + } + Ok(()) + } + + /// Set up waiting for the instance to exit + /// The Wait struct is used to send the exit code and time back to the + /// caller. The recipient is expected to call function + /// set_up_exit_code_wait() implemented by Wait to set up exit code + /// processing. Note that the "wait" function doesn't block, but + /// it sets up the waiting channel. + fn wait(&self, waiter: &Wait) -> Result<(), Error> { + let id = self.get_id(); + let exit_code = self.get_exit_code(); + log::info!("waiting for instance: {}", id); + let code = exit_code; + waiter.set_up_exit_code_wait(code) + } +} diff --git a/crates/containerd-shim-wasm/src/sandbox/mod.rs b/crates/containerd-shim-wasm/src/sandbox/mod.rs index 603d2c28d..5f683d776 100644 --- a/crates/containerd-shim-wasm/src/sandbox/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/mod.rs @@ -7,8 +7,12 @@ pub mod error; // pub mod exec; pub mod instance; pub mod instance_utils; +#[cfg(feature = "libcontainer")] +pub mod libcontainer_instance; pub mod manager; pub mod shim; +#[cfg(feature = "libcontainer")] +pub use libcontainer_instance::LibcontainerInstance; pub use error::{Error, Result}; pub use instance::{EngineGetter, Instance, InstanceConfig}; diff --git a/crates/containerd-shim-wasmedge/Cargo.toml b/crates/containerd-shim-wasmedge/Cargo.toml index dadb87e2e..11451f06b 100644 --- a/crates/containerd-shim-wasmedge/Cargo.toml +++ b/crates/containerd-shim-wasmedge/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true [dependencies] containerd-shim = { workspace = true } -containerd-shim-wasm = { workspace = true } +containerd-shim-wasm = { workspace = true, features = ["libcontainer"]} log = { workspace = true } ttrpc = { workspace = true } wasmedge-sdk = { version = "0.11.2" } diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance.rs index 52ec2da82..0ba638e56 100644 --- a/crates/containerd-shim-wasmedge/src/instance.rs +++ b/crates/containerd-shim-wasmedge/src/instance.rs @@ -3,22 +3,14 @@ use std::io::prelude::*; use std::io::ErrorKind; use std::os::unix::io::RawFd; use std::sync::{Arc, Condvar, Mutex}; -use std::thread; use anyhow::Context; -use anyhow::{anyhow, Result}; -use chrono::{DateTime, Utc}; +use anyhow::Result; use containerd_shim_wasm::sandbox::error::Error; -use containerd_shim_wasm::sandbox::instance::Wait; -use containerd_shim_wasm::sandbox::instance_utils::{ - get_instance_root, instance_exists, maybe_open_stdio, -}; -use containerd_shim_wasm::sandbox::{EngineGetter, Instance, InstanceConfig}; -use libc::{dup2, SIGINT, SIGKILL, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; -use log::{debug, error}; -use nix::errno::Errno; -use nix::sys::signal::Signal as NixSignal; -use nix::sys::wait::{waitid, Id as WaitID, WaitPidFlag, WaitStatus}; +use containerd_shim_wasm::sandbox::instance::ExitCode; +use containerd_shim_wasm::sandbox::instance_utils::maybe_open_stdio; +use containerd_shim_wasm::sandbox::{EngineGetter, InstanceConfig, LibcontainerInstance}; +use libc::{dup2, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use nix::unistd::close; use serde::{Deserialize, Serialize}; use wasmedge_sdk::{ @@ -33,8 +25,7 @@ use std::{ }; use libcontainer::container::builder::ContainerBuilder; -use libcontainer::container::{Container, ContainerStatus}; -use libcontainer::signal::Signal; +use libcontainer::container::Container; use libcontainer::syscall::syscall::create_syscall; use crate::executor::WasmEdgeExecutor; @@ -45,11 +36,10 @@ static mut STDERR_FD: Option = None; static DEFAULT_CONTAINER_ROOT_DIR: &str = "/run/containerd/wasmedge"; -type ExitCode = (Mutex)>>, Condvar); pub struct Wasi { id: String, - exit_code: Arc, + exit_code: ExitCode, stdin: String, stdout: String, @@ -97,9 +87,10 @@ fn determine_rootdir>(bundle: P, namespace: String) -> Result>) -> Self { + + fn new_libcontainer(id: String, cfg: Option<&InstanceConfig>) -> Self { let cfg = cfg.unwrap(); // TODO: handle error let bundle = cfg.get_bundle().unwrap_or_default(); let namespace = cfg.get_namespace(); @@ -114,137 +105,45 @@ impl Instance for Wasi { } } - fn start(&self) -> Result { - debug!("preparing module"); - - fs::create_dir_all(&self.rootdir)?; - - let stdin = maybe_open_stdio(self.stdin.as_str()).context("could not open stdin")?; - let stdout = maybe_open_stdio(self.stdout.as_str()).context("could not open stdout")?; - let stderr = maybe_open_stdio(self.stderr.as_str()).context("could not open stderr")?; - let mut container = self.build_container(stdin, stdout, stderr)?; - - let code = self.exit_code.clone(); - let pid = container.pid().unwrap(); - - // Close the fds now that they have been passed to the container process - // so that we don't leak them. - stdin.map(close); - stdout.map(close); - stderr.map(close); - - container - .start() - .map_err(|err| Error::Any(anyhow!("failed to start container: {}", err)))?; - - thread::spawn(move || { - let (lock, cvar) = &*code; - - let status = match waitid(WaitID::Pid(pid), WaitPidFlag::WEXITED) { - Ok(WaitStatus::Exited(_, status)) => status, - Ok(WaitStatus::Signaled(_, sig, _)) => sig as i32, - Ok(_) => 0, - Err(e) => { - if e == Errno::ECHILD { - log::info!("no child process"); - 0 - } else { - panic!("waitpid failed: {}", e); - } - } - } as u32; - let mut ec = lock.lock().unwrap(); - *ec = Some((status, Utc::now())); - drop(ec); - cvar.notify_all(); - }); - - Ok(pid.as_raw() as u32) + fn get_exit_code(&self) -> ExitCode { + self.exit_code.clone() } - fn kill(&self, signal: u32) -> Result<(), Error> { - let signal: Signal = match signal as i32 { - SIGKILL => NixSignal::SIGKILL.into(), - SIGINT => NixSignal::SIGINT.into(), - _ => Err(Error::InvalidArgument( - "only SIGKILL and SIGINT are supported".to_string(), - ))?, - }; - - let container_root = get_instance_root(&self.rootdir, self.id.as_str())?; - let mut container = Container::load(container_root).with_context(|| { - format!( - "could not load state for container {id}", - id = self.id.as_str() - ) - })?; - match container.kill(signal, true) { - Ok(_) => Ok(()), - Err(e) => { - if container.status() == ContainerStatus::Stopped { - return Err(Error::Others("container not running".into())); - } - Err(Error::Others(e.to_string())) - } - } + fn get_id(&self) -> String { + self.id.clone() } - fn delete(&self) -> Result<(), Error> { - match instance_exists(&self.rootdir, self.id.as_str()) { - Ok(exists) => { - if !exists { - return Ok(()); - } - } - Err(err) => { - error!("could not find the container, skipping cleanup: {}", err); - return Ok(()); - } - } - let container_root = get_instance_root(&self.rootdir, self.id.as_str())?; - let container = Container::load(container_root).with_context(|| { - format!( - "could not load state for container {id}", - id = self.id.as_str() - ) - }); - match container { - Ok(mut container) => container.delete(true).map_err(|err| { - Error::Any(anyhow!("failed to delete container {}: {}", self.id, err)) - })?, - Err(err) => { - error!("could not find the container, skipping cleanup: {}", err); - return Ok(()); - } - } - - Ok(()) + fn get_root_dir(&self) -> std::result::Result { + Ok(self.rootdir.clone()) } - fn wait(&self, waiter: &Wait) -> Result<(), Error> { - let code = self.exit_code.clone(); - waiter.set_up_exit_code_wait(code) - } -} + fn build_container(&self) -> std::result::Result { + fs::create_dir_all(&self.rootdir)?; + let stdin = maybe_open_stdio(self.stdin.as_str()).context("could not open stdin")?; + let stdout = maybe_open_stdio(self.stdout.as_str()).context("could not open stdout")?; + let stderr = maybe_open_stdio(self.stderr.as_str()).context("could not open stderr")?; -impl Wasi { - fn build_container( - &self, - stdin: Option, - stdout: Option, - stderr: Option, - ) -> anyhow::Result { let syscall = create_syscall(); + let err_others = |err| Error::Others(format!("failed to create container: {}", err)); let container = ContainerBuilder::new(self.id.clone(), syscall.as_ref()) .with_executor(vec![Box::new(WasmEdgeExecutor { stdin, stdout, stderr, - })])? - .with_root_path(self.rootdir.clone())? + })]) + .map_err(err_others)? + .with_root_path(self.rootdir.clone()) + .map_err(err_others)? .as_init(&self.bundle) .with_systemd(false) - .build()?; + .build() + .map_err(err_others)?; + // Close the fds now that they have been passed to the container process + // so that we don't leak them. + stdin.map(close); + stdout.map(close); + stderr.map(close); + Ok(container) } } @@ -267,6 +166,7 @@ impl EngineGetter for Wasi { .with_config(config) .build() .map_err(anyhow::Error::msg)?; + Ok(vm) } } @@ -279,8 +179,12 @@ mod wasitest { use std::sync::mpsc::channel; use std::time::Duration; + use chrono::{DateTime, Utc}; use containerd_shim_wasm::function; + use containerd_shim_wasm::sandbox::instance::Wait; use containerd_shim_wasm::sandbox::testutil::{has_cap_sys_admin, run_test_with_sudo}; + use containerd_shim_wasm::sandbox::Instance; + use libc::SIGKILL; use oci_spec::runtime::{ProcessBuilder, RootBuilder, SpecBuilder}; use tempfile::{tempdir, TempDir}; diff --git a/crates/containerd-shim-wasmtime/Cargo.toml b/crates/containerd-shim-wasmtime/Cargo.toml index 24fa3a7b7..9da08bd2d 100644 --- a/crates/containerd-shim-wasmtime/Cargo.toml +++ b/crates/containerd-shim-wasmtime/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true [dependencies] containerd-shim = { workspace = true } -containerd-shim-wasm = { workspace = true } +containerd-shim-wasm = { workspace = true, features = ["libcontainer"]} log = { workspace = true } ttrpc = { workspace = true } diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index e7d12d1b3..ae44c5733 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -1,37 +1,27 @@ use anyhow::Result; -use containerd_shim_wasm::sandbox::instance_utils::{ - get_instance_root, instance_exists, maybe_open_stdio, -}; use libcontainer::container::builder::ContainerBuilder; -use libcontainer::container::{Container, ContainerStatus}; -use nix::errno::Errno; -use nix::sys::wait::waitid; +use libcontainer::container::Container; +use nix::unistd::close; use serde::{Deserialize, Serialize}; use std::fs::File; use std::io::{ErrorKind, Read}; use std::os::fd::RawFd; use std::path::{Path, PathBuf}; use std::sync::{Arc, Condvar, Mutex}; -use std::thread; use anyhow::Context; -use chrono::{DateTime, Utc}; use containerd_shim_wasm::sandbox::error::Error; -use containerd_shim_wasm::sandbox::instance::Wait; -use containerd_shim_wasm::sandbox::{EngineGetter, Instance, InstanceConfig}; +use containerd_shim_wasm::sandbox::instance::ExitCode; +use containerd_shim_wasm::sandbox::instance_utils::maybe_open_stdio; +use containerd_shim_wasm::sandbox::{EngineGetter, InstanceConfig, LibcontainerInstance}; use libc::{dup2, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; -use libc::{SIGINT, SIGKILL}; use libcontainer::syscall::syscall::create_syscall; -use log::error; -use nix::sys::wait::{Id as WaitID, WaitPidFlag, WaitStatus}; use wasmtime::Engine; use crate::executor::WasmtimeExecutor; -use libcontainer::signal::Signal; static DEFAULT_CONTAINER_ROOT_DIR: &str = "/run/containerd/wasmtime"; -type ExitCode = Arc<(Mutex)>>, Condvar)>; static mut STDIN_FD: Option = None; static mut STDOUT_FD: Option = None; @@ -92,9 +82,10 @@ fn determine_rootdir>(bundle: P, namespace: String) -> Result>) -> Self { + + fn new_libcontainer(id: String, cfg: Option<&InstanceConfig>) -> Self { // TODO: there are failure cases e.x. parsing cfg, loading spec, etc. // thus should make `new` return `Result` instead of `Self` log::info!("creating new instance: {}", id); @@ -112,145 +103,47 @@ impl Instance for Wasi { rootdir, } } - fn start(&self) -> Result { - log::info!("starting instance: {}", self.id); - let engine: Engine = self.engine.clone(); - - let mut container = self.build_container( - self.stdin.as_str(), - self.stdout.as_str(), - self.stderr.as_str(), - engine, - )?; - - log::info!("created container: {}", self.id); - let code = self.exit_code.clone(); - let pid = container.pid().unwrap(); - - container - .start() - .map_err(|err| Error::Any(anyhow::anyhow!("failed to start container: {}", err)))?; - - thread::spawn(move || { - let (lock, cvar) = &*code; - - let status = match waitid(WaitID::Pid(pid), WaitPidFlag::WEXITED) { - Ok(WaitStatus::Exited(_, status)) => status, - Ok(WaitStatus::Signaled(_, sig, _)) => sig as i32, - Ok(_) => 0, - Err(e) => { - if e == Errno::ECHILD { - log::info!("no child process"); - 0 - } else { - panic!("waitpid failed: {}", e); - } - } - } as u32; - let mut ec = lock.lock().unwrap(); - *ec = Some((status, Utc::now())); - drop(ec); - cvar.notify_all(); - }); - - Ok(pid.as_raw() as u32) - } - fn kill(&self, signal: u32) -> Result<(), Error> { - log::info!("killing instance: {}", self.id); - if signal as i32 != SIGKILL && signal as i32 != SIGINT { - return Err(Error::InvalidArgument( - "only SIGKILL and SIGINT are supported".to_string(), - )); - } - let container_root = get_instance_root(&self.rootdir, self.id.as_str())?; - let mut container = Container::load(container_root).with_context(|| { - format!( - "could not load state for container {id}", - id = self.id.as_str() - ) - })?; - let signal = Signal::try_from(signal as i32) - .map_err(|err| Error::InvalidArgument(format!("invalid signal number: {}", err)))?; - match container.kill(signal, true) { - Ok(_) => Ok(()), - Err(e) => { - if container.status() == ContainerStatus::Stopped { - return Err(Error::Others("container not running".into())); - } - Err(Error::Others(e.to_string())) - } - } + fn get_exit_code(&self) -> ExitCode { + self.exit_code.clone() } - fn delete(&self) -> Result<(), Error> { - log::info!("deleting instance: {}", self.id); - match instance_exists(&self.rootdir, self.id.as_str()) { - Ok(exists) => { - if !exists { - return Ok(()); - } - } - Err(err) => { - error!("could not find the container, skipping cleanup: {}", err); - return Ok(()); - } - } - let container_root = get_instance_root(&self.rootdir, self.id.as_str())?; - let container = Container::load(container_root).with_context(|| { - format!( - "could not load state for container {id}", - id = self.id.as_str() - ) - }); - match container { - Ok(mut container) => container.delete(true).map_err(|err| { - Error::Any(anyhow::anyhow!( - "failed to delete container {}: {}", - self.id, - err - )) - })?, - Err(err) => { - error!("could not find the container, skipping cleanup: {}", err); - return Ok(()); - } - } - - Ok(()) + fn get_id(&self) -> String { + self.id.clone() } - fn wait(&self, waiter: &Wait) -> Result<(), Error> { - log::info!("waiting for instance: {}", self.id); - let code = self.exit_code.clone(); - waiter.set_up_exit_code_wait(code) + fn get_root_dir(&self) -> std::result::Result { + Ok(self.rootdir.clone()) } -} -impl Wasi { - fn build_container( - &self, - stdin: &str, - stdout: &str, - stderr: &str, - engine: Engine, - ) -> anyhow::Result { + fn build_container(&self) -> std::result::Result { + let engine = self.engine.clone(); let syscall = create_syscall(); - let stdin = maybe_open_stdio(stdin).context("could not open stdin")?; - let stdout = maybe_open_stdio(stdout).context("could not open stdout")?; - let stderr = maybe_open_stdio(stderr).context("could not open stderr")?; - + let stdin = maybe_open_stdio(&self.stdin).context("could not open stdin")?; + let stdout = maybe_open_stdio(&self.stdout).context("could not open stdout")?; + let stderr = maybe_open_stdio(&self.stderr).context("could not open stderr")?; + let err_others = |err| Error::Others(format!("failed to create container: {}", err)); let container = ContainerBuilder::new(self.id.clone(), syscall.as_ref()) .with_executor(vec![Box::new(WasmtimeExecutor { stdin, stdout, stderr, engine, - })])? - .with_root_path(self.rootdir.clone())? + })]) + .map_err(err_others)? + .with_root_path(self.rootdir.clone()) + .map_err(err_others)? .as_init(&self.bundle) .with_systemd(false) - .build()?; + .build() + .map_err(err_others)?; + + // Close the fds now that they have been passed to the container process + // so that we don't leak them. + stdin.map(close); + stdout.map(close); + stderr.map(close); + Ok(container) } } @@ -273,6 +166,8 @@ mod wasitest { use containerd_shim_wasm::function; use containerd_shim_wasm::sandbox::instance::Wait; use containerd_shim_wasm::sandbox::testutil::{has_cap_sys_admin, run_test_with_sudo}; + use containerd_shim_wasm::sandbox::Instance; + use libc::SIGKILL; use oci_spec::runtime::{ProcessBuilder, RootBuilder, SpecBuilder}; use tempfile::{tempdir, TempDir};