diff --git a/.gitignore b/.gitignore index 01b1f255c90e130c9b95c734a69a96a71bf53cec..a544b8b6243876cb46d0e8276461706172828e86 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .vscode *.swp target +target-cover +vendor diff --git a/Cargo.lock b/Cargo.lock index f79ad85fa5d77101795f6f8f7d82e25e083a53b5..df6be98dd69cbcb955dff6c8c1ca6fcf5b4043fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,12 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec8ad6edb4840b78c5c3d88de606b22252d552b55f3a4699fbb10fc070ec3049" +[[package]] +name = "another_json_minimal" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ba8341e1396c8a379f62de1b47f31256f2fe0846f5f95e9c60014d2102d9bd" + [[package]] name = "ansi_term" version = "0.12.1" @@ -2762,6 +2768,7 @@ dependencies = [ name = "wasm_engine" version = "0.1.0" dependencies = [ + "another_json_minimal", "anyhow", "clap 3.2.8", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 716bb32cdf9b6f4035eb7326b8e1f91d3627662e..c92a4a1e44afdc291098882db89177f62cfb60ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,8 @@ anyhow = "1.0.44" [dev-dependencies] criterion = "0.3" +another_json_minimal = "0.0.2" + [[bench]] name = "benchmark" diff --git a/Makefile b/Makefile index 170d5b39b58e3d29a24dd37e34c53ce07fd0d734..de63f052a40a8df8722e9e6d3efe8d1ec2a91bce 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ RUST_SCRIPT = $(shell command -v rust-script) .PHONY: wasm-engine wasm-engine: @cargo build --release - @mv /usr/bin/wasm-engine /usr/bin/wasm-engine-bak - @cp ./target/release/wasm-engine /usr/bin + @mv /usr/bin/wasm_engine /usr/bin/wasm_engine-bak + @cp ./target/release/wasm_engine /usr/bin .PHONY: apps apps: @@ -15,4 +15,12 @@ endif @cp build.rs run.sh @chmod +x run.sh @./run.sh | grep -v cargo:rerun | grep -v cargo:warning | grep -v cargo:rustc-env || true - @rm -f run.sh \ No newline at end of file + @rm -f run.sh + +.PHONY: cover +cover: + cargo tarpaulin --target-dir target-cover --skip-clean -o Html --output-dir output --exclude-files src/main.rs benches/benchmark.rs target/* tests/ + +.PHONY: cover-clean +cover-clean: + cargo tarpaulin --target-dir target-cover -o Html --output-dir output --exclude-files src/main.rs benches/benchmark.rs target/* tests/ diff --git a/experiments/application/authentication-wasi/src/main.rs b/experiments/application/authentication-wasi/src/main.rs index df00eae3f609423de2de7173fb923ce3c1d76471..91b7a87e356064facb7beac862b35e690e40022c 100644 --- a/experiments/application/authentication-wasi/src/main.rs +++ b/experiments/application/authentication-wasi/src/main.rs @@ -10,14 +10,20 @@ struct Response { fn main() { let args: Vec = env::args().collect(); - if args.len() != 3 { - eprintln!("usage: authentication "); + if args.len() != 1 { + eprintln!("too many args"); return; } - let arg_uri = &args[0]; - let arg_body = &args[1]; - let arg_secret = &args[2]; + let json = match Json::parse(&args[0].as_bytes()) { + Ok(json) => json, + Err((position, message)) => { + panic!("`{}` at position `{}`!!!", position, message); + } + }; + let arg_uri = json.get("arg_uri").unwrap().print(); + let arg_body = json.get("arg_body").unwrap().print(); + let arg_secret = json.get("arg_secret").unwrap().print(); let arg_func = "argfunc"; let content = format!("{}#{}#{}", arg_uri, arg_body, arg_func); @@ -36,7 +42,7 @@ fn main() { let mut r: Response = Response::default(); let html: String; - if &hash == arg_secret { + if hash == arg_secret { r.status = "200".to_string(); html = "

Auth Pass!

hash ".to_owned() + &hash + "

"; r.body = html; diff --git a/src/function_store/local_store.rs b/src/function_store/local_store.rs index d68c45151c8ee7958cbaca26a2f89a7c35fd51ca..2afcd96a22ed96e2e283da7a9dd19b9d68f15b9c 100644 --- a/src/function_store/local_store.rs +++ b/src/function_store/local_store.rs @@ -1,5 +1,6 @@ use super::pull; use anyhow::{anyhow, Ok, Result}; +use oci_distribution::client::ClientConfig; use oci_distribution::{secrets::RegistryAuth, Client, Reference}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -100,11 +101,13 @@ impl FunctionStore { )); } - let mut client = Client::new(pull::build_client_config(true)); - let reference: Reference = image_name.parse().expect("Not a valid image reference"); + let mut client = Client::new(ClientConfig::default()); + let reference: Reference = image_name + .parse() + .map_err(|err| anyhow::format_err!("Not a valid image reference: {}", err))?; // pull the wasm image into the local func_store_path - pull::pull_wasm( + let pull_result = pull::pull_wasm( &mut client, &RegistryAuth::Anonymous, &reference, @@ -112,6 +115,24 @@ impl FunctionStore { ) .await; + if !pull_result.is_ok() { + let mut client = Client::new(pull::build_client_config(true)); + pull::pull_wasm( + &mut client, + &RegistryAuth::Anonymous, + &reference, + func_store_dir.as_str(), + ) + .await + .map_err(|err| { + anyhow::format_err!( + "Pull image both failed with https auth: {}, and insecure http: {}", + pull_result.unwrap_err(), + err + ) + })?; + } + // only one wasm module file should be in the func_store_dir if read_dir(func_store_dir.clone()).unwrap().count() != 1 { return Err(anyhow!( diff --git a/src/function_store/module_store.rs b/src/function_store/module_store.rs index cac3027fa85913acc0c91f62b2b1b0d5d51e678c..12616a632b30557ed0c6803c4ddef3f1e38e335f 100644 --- a/src/function_store/module_store.rs +++ b/src/function_store/module_store.rs @@ -1,8 +1,3 @@ -/*! -Registries allow you to define "well-known" modules in the &environment that can be looked up by -name and version. -*/ - use anyhow::{anyhow, Result}; use std::{collections::HashMap, sync::Arc, sync::RwLock}; use tracing::info; @@ -24,10 +19,7 @@ impl ModuleStore { } } - /// Insert module into the registry under a specific name, version and wasi capabilites. - /// - /// The version needs to be a correct semver string (e.g "1.2.3-alpha3") or the insertion will - /// fail. If the exact same version and name exists it will be overwritten. + /// Insert module into the ModuleStore under a specific name, module and wasi capabilites. pub fn insert(&self, name: &str, module: Module, wasi_cap: bool) -> Result<()> { let mut writer = self.module_store.write().unwrap(); @@ -44,9 +36,6 @@ impl ModuleStore { Ok(()) } - /// Remove module under name & version from registry - /// - /// Exact version matching is used for lookup. pub fn remove(&self, name: &str) -> Result<()> { let mut writer = self.module_store.write().unwrap(); @@ -113,8 +102,4 @@ impl ModuleEntry { pub fn capability(&self) -> bool { self.wasi_cap } - - fn all(&self) -> ModuleEntry { - self.clone() - } } diff --git a/src/function_store/pull.rs b/src/function_store/pull.rs index 40bbc1558193eb9702ad166f083f7154f150a6e5..2ba41858b7eaaeac1484a90917f2a592c3b5ca10 100644 --- a/src/function_store/pull.rs +++ b/src/function_store/pull.rs @@ -1,3 +1,4 @@ +use anyhow::Result; use flate2::read; use oci_distribution::{manifest, secrets::RegistryAuth, Client, Reference}; use tar::Archive; @@ -21,7 +22,7 @@ pub async fn pull_wasm( auth: &RegistryAuth, reference: &Reference, output: &str, -) { +) -> Result<()> { info!(?reference, ?output, "pulling wasm module"); let image_content = client @@ -31,19 +32,22 @@ pub async fn pull_wasm( vec![manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE], ) .await - .expect("Cannot pull Wasm module") + .map_err(|err| anyhow::format_err!("Cannot pull Wasm module {}", err))? .layers .into_iter() .next() .map(|layer| layer.data) - .expect("No data found"); + .ok_or(anyhow::format_err!("No data found"))?; // webassembly oci spec definition: https://github.com/solo-io/wasm/spec // for IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE, we need to unzip iamge_content // into raw wasm file let gz = read::GzDecoder::new(&image_content[..]); let mut archive = Archive::new(gz); - archive.unpack(output).expect("Cannot write to file"); + archive + .unpack(output) + .map_err(|err| anyhow::format_err!("Cannot write to file: {}", err))?; info!("Wasm module successfully written to {}", output); + Ok(()) } diff --git a/src/main.rs b/src/main.rs index c4e7ec0fb0a138daafb8e972dcd80cdd7df8e88d..547ade718c69daf12cdd31d672402eb67fae6dbe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,4 @@ use anyhow::Context; -use clap::Parser; use serde::Deserialize; use std::{collections::HashMap, error::Error}; use tracing::{info, instrument, Level}; @@ -23,22 +22,9 @@ lazy_static::lazy_static! { ]); } -#[derive(Parser, Debug)] -#[clap(about, version, author)] -struct Args { - /// Log level used in wasm-engine, TRACE: 0, DEBUG: 1, INFO: 2(Default), WARN: 3, ERROR: 4 - #[clap(short, long, default_value = "2")] - log_level: u8, - - /// dir that contains preload apps - #[clap(short, long)] - preload_apps: Option, -} - #[instrument] #[tokio::main] async fn main() -> Result<(), Box> { - let args = Args::parse(); tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) .try_init()?; @@ -254,7 +240,7 @@ mod handlers { Ok(Response { status: http::StatusCode::OK.as_u16(), - body: format!("delete function {} successfull!\n", func.function_name), + body: format!("delete function {} successfully!\n", func.function_name), }) } diff --git a/src/wrapper/environment.rs b/src/wrapper/environment.rs index b7ea873250403fb51120ec51afc6beffec876c88..bb9bde756d0df99aadf7669ec04c8f14b976f095 100644 --- a/src/wrapper/environment.rs +++ b/src/wrapper/environment.rs @@ -12,10 +12,7 @@ pub const UNIT_OF_COMPUTE_IN_INSTRUCTIONS: u64 = 100_000; /// Environments let us set limits on instances: /// * Memory limits /// * Compute limits -/// * Access to host functions -/// -/// They also define the set of plugins. Plugins can be used to modify loaded Wasm modules. -/// Plugins are WIP and not well documented. +/// * Access to modules #[derive(Clone)] pub struct Environment { runtime: WasmtimeRuntime, @@ -36,7 +33,7 @@ impl Environment { &self.runtime } - pub fn registry(&self) -> &ModuleStore { + pub fn store(&self) -> &ModuleStore { &self.store } } diff --git a/src/wrapper/wasmtime_runtime.rs b/src/wrapper/wasmtime_runtime.rs index 415dd57b81e77223ee72906b17d1e8709ccaf5e5..6d5e9cc1c18bb3c87dac380b158bc002bb7c8ac0 100644 --- a/src/wrapper/wasmtime_runtime.rs +++ b/src/wrapper/wasmtime_runtime.rs @@ -64,11 +64,8 @@ impl WasmtimeRuntime { } let stdout = WritePipe::new_in_memory(); wasi = wasi.stdout(Box::new(stdout.clone())); - let mut args: Vec = Vec::new(); - for (_, v) in data { - args.push(v); - } - wasi = wasi.args(&args)?; + let serialized = serde_json::to_string(&data)?; + wasi = wasi.args(&[serialized])?; for preopen_dir_path in self.config.preopened_dirs() { let preopen_dir = Dir::open_ambient_dir(preopen_dir_path, ambient_authority())?; wasi = wasi.preopened_dir(preopen_dir, preopen_dir_path)?; @@ -128,10 +125,11 @@ impl WasmtimeRuntime { //let wasm_function = instance.get_func(&mut store, function).unwrap(); let wasm_function = instance.get_typed_func::<(i32, i32), (i32, i32), _>(&mut store, function)?; + if serialized.len() > WASM_PAGE_SIZE as usize { return Err(anyhow!("input args size larger than {}", WASM_PAGE_SIZE)); } - + info!("serialized.len() is {}", serialized.len() as usize); let memory = instance .get_memory(&mut store, "memory") .ok_or(anyhow::format_err!("failed to find `memory` export"))?; diff --git a/tests/authentication-wasi.wasm b/tests/authentication-wasi.wasm new file mode 100755 index 0000000000000000000000000000000000000000..edc0e1c1e27da55c517532b8b1ed939845e557e4 Binary files /dev/null and b/tests/authentication-wasi.wasm differ diff --git a/tests/authentication.wasm b/tests/authentication.wasm new file mode 100755 index 0000000000000000000000000000000000000000..635bcceb7502a8b54e57258f43873939915ba831 Binary files /dev/null and b/tests/authentication.wasm differ diff --git a/tests/store.rs b/tests/store.rs new file mode 100644 index 0000000000000000000000000000000000000000..f8666ba226bb1e47f29b33aac5141fc69626b7ae --- /dev/null +++ b/tests/store.rs @@ -0,0 +1,102 @@ +use wasm_engine::{ + function_store::{ + local_store::{FunctionEntries, FunctionStore}, + module_store::ModuleStore, + }, + wrapper::{config::EnvConfig, environment::Environment}, +}; +use wasmtime::Module; + +#[test] +fn module_store() -> anyhow::Result<()> { + let wasm_runtime = Environment::new(EnvConfig::default()).unwrap(); + let runtime = wasm_runtime.runtime(); + let module = &Module::from_file(runtime.get_engine(), "./tests/authentication.wasm")?; + let module_wasi = &Module::from_file(runtime.get_engine(), "./tests/authentication-wasi.wasm")?; + let store = ModuleStore::new(); + + store.insert("authentication", module.clone(), false)?; + store.insert("authentication-wasi", module_wasi.clone(), true)?; + + assert!(store.exist("authentication")); + assert!(store.exist("authentication-wasi")); + + store.remove("authentication-wasi")?; + assert!(!store.exist("authentication-wasi")); + + let result = store.get("authentication"); + assert!(result.is_ok()); + let authentication_module = result.unwrap(); + assert_eq!(authentication_module.name(), "authentication"); + assert_eq!(authentication_module.capability(), false); + + Ok(()) +} + +#[tokio::test] +async fn local_store() -> anyhow::Result<()> { + struct FunctionInfo { + function_name: String, + function_image: String, + wasi_cap: bool, + } + let store: FunctionStore = FunctionStore::new("/var/lib/wasmengine-test/functions/"); + let authentication = FunctionInfo { + function_name: "authentication".to_string(), + function_image: "hub.oepkgs.net/library/authentication-wasm:latest".to_string(), + wasi_cap: false, + }; + let authentication_wasi = FunctionInfo { + function_name: "authentication-wasi".to_string(), + function_image: "hub.oepkgs.net/library/authentication-wasi:latest".to_string(), + wasi_cap: true, + }; + + let add_authentication = store + .add( + &authentication.function_name, + &authentication.function_image, + authentication.wasi_cap, + ) + .await; + assert!(add_authentication.is_ok()); + assert!(store.save().await.is_ok()); + + let add_authentication_wasi = store + .add( + &authentication_wasi.function_name, + &authentication_wasi.function_image, + authentication_wasi.wasi_cap, + ) + .await; + assert!(add_authentication_wasi.is_ok()); + assert!(store.save().await.is_ok()); + + drop(store); + let store: FunctionStore = FunctionStore::new("/var/lib/wasmengine-test/functions/"); + let entries = store.list().await; + assert!(entries.is_ok()); + assert_eq!(entries.unwrap().len(), 0); + + assert!(store.restore().await.is_ok()); + + let entries = store.list().await; + assert!(entries.is_ok()); + assert_eq!(entries.unwrap().len(), 2); + + let entries = store.list().await; + assert!(FunctionEntries(entries.unwrap()) + .to_string() + .contains("authentication")); + + assert!(store.query("authentication").await.is_ok()); + assert!(!store.query("authentication-none").await.is_ok()); + + assert!(store.exist("authentication").await); + assert!(!store.exist("authentication-none").await); + + store.delete("authentication-wasi").await?; + assert!(!store.exist("authentication-wasi").await); + + Ok(()) +} diff --git a/tests/wasmtime.rs b/tests/wasmtime.rs new file mode 100644 index 0000000000000000000000000000000000000000..bfe152473cc447003d60b46e6edb7016a09b3a89 --- /dev/null +++ b/tests/wasmtime.rs @@ -0,0 +1,74 @@ +use std::collections::HashMap; + +use wasm_engine::wrapper::{config::EnvConfig, environment::Environment}; +use wasmtime::Module; + +#[tokio::test(flavor = "multi_thread")] +async fn authentication() -> anyhow::Result<()> { + let wasm_runtime = Environment::new(EnvConfig::default()).unwrap(); + let runtime = wasm_runtime.runtime(); + let module = &Module::from_file(runtime.get_engine(), "./tests/authentication.wasm")?; + let module_wasi = &Module::from_file(runtime.get_engine(), "./tests/authentication-wasi.wasm")?; + + // init common arg_uri and arg_body args + let args = &mut HashMap::new(); + args.insert("arg_uri".to_string(), "uri".to_string()); + args.insert("arg_body".to_string(), "body".to_string()); + + struct Testcase { + name: String, + secret: String, + contains: String, + wasi: bool, + } + + let testcases = [ + Testcase { + name: "spawn with right secret".to_string(), + secret: "32af198911cb4a9727dca0aaf9149020".to_string(), + contains: "Auth Pass".to_string(), + wasi: false, + }, + Testcase { + name: "spawn with wrong secret".to_string(), + secret: "32af198911cb4a9727dca0aaf9149020-forbid".to_string(), + contains: "Auth Forbidden!".to_string(), + wasi: false, + }, + Testcase { + name: "spawn wasi with right secret".to_string(), + secret: "32af198911cb4a9727dca0aaf9149020".to_string(), + contains: "Auth Pass".to_string(), + wasi: true, + }, + Testcase { + name: "spawn wasi with wrong secret".to_string(), + secret: "32af198911cb4a9727dca0aaf9149020-forbid".to_string(), + contains: "Auth Forbidden!".to_string(), + wasi: true, + }, + ]; + + for t in testcases { + println!("start debuging: {}", t.name); + let mut t_args = args.clone(); + t_args.insert("arg_secret".to_string(), t.secret); + + let result: String; + if !t.wasi { + result = runtime + .spawn(module.clone(), "authentication", t_args) + .await?; + } else { + result = runtime.spawn_wasi(module_wasi.clone(), t_args).await?; + } + assert!( + result.contains(t.contains.as_str()), + "expect {}, get {}", + t.contains.as_str(), + result + ); + } + + Ok(()) +}