From 4b30709fbc1753dd11da82454d613e643725adf4 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Thu, 23 Apr 2026 23:47:07 +0800 Subject: [PATCH] Create loadable root module --- .github/workflows/test.yml | 39 ++++++ .mise.toml | 18 +-- README.md | 40 ++++-- libs/edge-toolkit/src/config.rs | 40 ++++++ .../dart-comm1/pkg/et_ws_dart_comm1.js | 2 +- services/ws-modules/face-detection/src/lib.rs | 2 +- services/ws-modules/har1/src/lib.rs | 2 +- services/ws-modules/har1/src/test_har1.rs | 27 ---- .../ws-modules/pydata1/pkg/et_ws_pydata1.js | 2 +- services/ws-server/src/config.rs | 17 +-- services/ws-server/src/lib.rs | 58 +++++++-- services/ws-server/static/app.js | 22 +++- services/ws-server/static/index.html | 118 +----------------- services/ws-server/static/package.json | 15 +++ services/ws-server/static/style.css | 112 +++++++++++++++++ services/ws-server/tests/api_modules.rs | 31 ++--- utilities/cli/src/lib.rs | 60 +++++---- utilities/cli/src/tests.rs | 78 +----------- .../facility-security-scenario/mise.toml | 18 ++- 19 files changed, 377 insertions(+), 324 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 services/ws-server/static/package.json create mode 100644 services/ws-server/static/style.css diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..4360478 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,39 @@ +--- +name: test + +"on": + pull_request: + workflow_dispatch: + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + rust: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install mise + uses: taiki-e/install-action@v2 + with: + tool: mise + + - name: Install mise tools + run: | + mise settings add idiomatic_version_file_enable_tools "[]" + mise settings experimental=true + mise settings set cargo.binstall true + mise use -g cargo-binstall + mise install + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: Build WASM modules + run: mise run build-modules + + - name: Run tests + run: mise run test diff --git a/.mise.toml b/.mise.toml index 666a1de..473e985 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,6 +1,7 @@ [tools] action-validator = "latest" cargo-binstall = "latest" +"cargo:aube" = "latest" "cargo:taplo-cli" = "latest" "cargo:wasm-pack" = "latest" "chromedriver" = "146" @@ -15,6 +16,7 @@ editorconfig-checker = "latest" gemini-cli = "latest" "github:block/goose" = "latest" "github:wasm-bindgen/wasm-bindgen" = "0.2.114" +"npm:onnxruntime-web" = "latest" ollama = "latest" osv-scanner = "latest" pipx = "latest" @@ -22,7 +24,10 @@ pipx = "latest" protoc = "latest" rclone = "latest" ruff = "latest" -rust = { version = "latest", components = "clippy" } +rust = [ + { version = "latest", components = "clippy", targets = "wasm32-unknown-unknown" }, + { version = "nightly", components = "rust-src,rustfmt", targets = "wasm32-unknown-unknown" }, +] typos = "latest" uv = "latest" yq = "latest" @@ -33,6 +38,7 @@ ec = "ec" editorconfig-check = "ec" ruff-check = "ruff check services/ws-modules/" ruff-fmt = "ruff format services/ws-modules/" +test = "cargo test --workspace" zig-check = "zig fmt --check services/ws-modules/" zig-fmt = "zig fmt services/ws-modules/" @@ -57,9 +63,6 @@ run = "dprint fmt" depends = ["cargo-clippy-fix", "cargo-fmt", "dart-fmt", "dotnet-fmt", "dprint-fmt", "taplo-fmt", "zig-fmt"] description = "Run repository formatters" -[tasks.install-nightly] -run = "cargo +nightly fmt --version >/dev/null 2>&1 || rustup toolchain install nightly --component rustfmt" - [tasks.dart-check] run = "dart analyze services/ws-modules/dart-comm1/" @@ -75,10 +78,10 @@ depends = [ "dotnet-check", "dprint-check", "editorconfig-check", - "generated-scenarios-check", "osv-scanner", "taplo-check", "typos", + "verification-check", "zig-check", ] description = "Run repository checks" @@ -87,11 +90,9 @@ description = "Run repository checks" run = "cargo check --workspace" [tasks.cargo-fmt] -depends = ["install-nightly"] run = "cargo +nightly fmt" [tasks.cargo-fmt-check] -depends = ["install-nightly"] run = "cargo +nightly fmt -- --check" [tasks.cargo-clippy] @@ -114,10 +115,11 @@ depends = ["cargo-check"] run = "osv-scanner --lockfile Cargo.lock" [tasks.regenerate-verification] +alias = "regen-verification" description = "Regenerate checked-in verification output files" run = "cargo run -p et-cli -- regen-verification" -[tasks.generated-scenarios-check] +[tasks.verification-check] depends = ["regenerate-verification"] description = "Fail if generated scenario output files are stale" run = "git diff --exit-code -- verification" diff --git a/README.md b/README.md index 17dcc12..fcd29b2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## mise -Please install [`mise`](https://mise.jdx.dev/). +Please install [`mise`](https://mise.jdx.dev/), including the shell integration. It is needed for all use of this repository. Configure it with @@ -16,14 +16,6 @@ mise settings set cargo.binstall true Use `mise run fmt` and `mise run check` to run formatters and checkers. -## Run e2e - -Run the end-to-end tests using Chrome: - -```bash -mise run ws-e2e-chrome -``` - ## Run ws agent in browser ### Build modules and run the WS server @@ -60,14 +52,36 @@ The server logs appear in the Logs section. The module list is dynamically populated from the modules in [services/ws-modules](services/ws-modules). -Each module must be a directory `pkg` containing a `package.json` that defines a `main` which contains a JavaScript file +Each module must have a `package.json` that defines a `main` which contains a JavaScript file that can load and run the module. +Under each module in `ws-modules`, the package can be found in a subdirectory `pkg`. + Most of the module are built from Rust using `wasm-pack build --target web`. -The module `pydata1` uses [pyodide](https://pyodide.org/) to run a Python script. +There are also modules written in: + +- Python, using [pyodide](https://pyodide.org/) +- .Net C# +- Dart +- Zig, including C code. + +## Root module + +The default UX in the web-browser is also a loadable module located in +[services/ws-server/static](services/ws-server/static). + +A custom UX module can be used by setting the `ws-server` environment variable `MODULES_ROOT`. + +## Run e2e + +Run the end-to-end tests using Chrome: + +```bash +mise run ws-e2e-chrome +``` -## Run an example demo scenario using et-cli +Run an example demo scenario using et-cli ```bash cargo install --path utilities/cli --force @@ -93,7 +107,7 @@ To regenerate all checked-in verification outputs from the matching `verification/*/output/` folder: ```bash -et-cli regen-verification +mise run regen-verification ``` ## Grant diff --git a/libs/edge-toolkit/src/config.rs b/libs/edge-toolkit/src/config.rs index 4845ca6..9aa44d3 100644 --- a/libs/edge-toolkit/src/config.rs +++ b/libs/edge-toolkit/src/config.rs @@ -28,6 +28,46 @@ pub fn get_project_root() -> PathBuf { } } +/// Returns the default module search paths for ws-server. +/// +/// Includes the standard workspace paths and any npm packages installed via mise. +#[must_use] +pub fn default_modules_folders() -> Vec { + let project_root = get_project_root(); + let mut paths = vec![ + project_root.join("services/ws-server/static"), + project_root.join("services/ws-wasm-agent"), + project_root.join("data/model-modules"), + project_root.join("services/ws-modules"), + ]; + if let Some(p) = mise_npm_modules_path("onnxruntime-web") { + paths.push(p); + } + paths +} + +/// Returns the install path for a `mise` tool, e.g. `mise where npm:onnxruntime-web`. +#[must_use] +pub fn mise_where(tool: &str) -> Option { + let output = std::process::Command::new("mise").args(["where", tool]).output().ok()?; + if !output.status.success() { + return None; + } + let s = std::str::from_utf8(&output.stdout).ok()?; + let p = PathBuf::from(s.trim()); + p.is_dir().then_some(p) +} + +/// Returns the `lib/node_modules` path within a mise npm tool install root. +/// +/// This directory contains all npm packages installed by mise and can be used +/// as a modules search path. +#[must_use] +pub fn mise_npm_modules_path(package: &str) -> Option { + let p = mise_where(&format!("npm:{package}"))?.join("lib/node_modules"); + p.is_dir().then_some(p) +} + /// Default port for the otlp http collector. #[must_use] const fn default_otlp_collector_port() -> u16 { diff --git a/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js b/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js index 8d54485..b7ef44a 100644 --- a/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js +++ b/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js @@ -16,7 +16,7 @@ export async function run() { } // Dart @JS() interop resolves against globalThis, so expose the wasm-agent // classes there for the duration of the call. - const { WsClient, WsClientConfig } = await import("/modules/ws-wasm-agent/et_ws_wasm_agent.js"); + const { WsClient, WsClientConfig } = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); globalThis.WsClient = WsClient; globalThis.WsClientConfig = WsClientConfig; try { diff --git a/services/ws-modules/face-detection/src/lib.rs b/services/ws-modules/face-detection/src/lib.rs index 49ba5be..b4df0b6 100644 --- a/services/ws-modules/face-detection/src/lib.rs +++ b/services/ws-modules/face-detection/src/lib.rs @@ -12,7 +12,7 @@ use wasm_bindgen_futures::{JsFuture, spawn_local}; use web_sys::MediaStreamConstraints; use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, HtmlVideoElement, ImageData, MediaStream}; -const FACE_MODEL_PATH: &str = "/modules/model-face1/video_cv.onnx"; +const FACE_MODEL_PATH: &str = "/modules/et-model-face1/video_cv.onnx"; const FACE_INPUT_WIDTH: usize = 640; const FACE_INPUT_HEIGHT: usize = 608; const FACE_INPUT_WIDTH_F64: f64 = FACE_INPUT_WIDTH as f64; diff --git a/services/ws-modules/har1/src/lib.rs b/services/ws-modules/har1/src/lib.rs index d8e58c0..271ce9b 100644 --- a/services/ws-modules/har1/src/lib.rs +++ b/services/ws-modules/har1/src/lib.rs @@ -13,7 +13,7 @@ use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; use web_sys::Event; -const HAR_MODEL_PATH: &str = "/modules/model-har-motion1/har-motion1.onnx"; +const HAR_MODEL_PATH: &str = "/modules/et-model-har-motion1/har-motion1.onnx"; const HAR_SEQUENCE_LENGTH: usize = 150; const HAR_FEATURE_COUNT: usize = 8; const HAR_FEAT_INPUT_SIZE: usize = 36; diff --git a/services/ws-modules/har1/src/test_har1.rs b/services/ws-modules/har1/src/test_har1.rs index b5f1e76..630adae 100644 --- a/services/ws-modules/har1/src/test_har1.rs +++ b/services/ws-modules/har1/src/test_har1.rs @@ -1,32 +1,5 @@ use super::*; -#[test] -fn softmax_distribution_preserves_order_and_normalizes() { - let logits = vec![2.0, 1.0, 0.1]; - let probs = softmax(&logits); - - assert_eq!(probs.len(), 3); - let sum: f64 = probs.iter().sum(); - assert!((sum - 1.0).abs() < 1e-6); - assert!(probs[0] > probs[1]); - assert!(probs[1] > probs[2]); -} - -#[test] -fn softmax_handles_empty_equal_and_large_values() { - assert!(softmax(&[]).is_empty()); - - let equal = softmax(&[7.0, 7.0, 7.0]); - assert!(equal.iter().all(|value| (*value - (1.0 / 3.0)).abs() < 1e-6)); - - let large = softmax(&[1000.0, 1001.0, 999.0]); - assert_eq!(large.len(), 3); - assert!(large.iter().all(|value| value.is_finite())); - assert!((large.iter().sum::() - 1.0).abs() < 1e-6); - assert!(large[1] > large[0]); - assert!(large[0] > large[2]); -} - #[test] fn gravity_and_rotation_conversions_handle_positive_negative_and_zero() { assert_eq!(to_g(0.0), 0.0); diff --git a/services/ws-modules/pydata1/pkg/et_ws_pydata1.js b/services/ws-modules/pydata1/pkg/et_ws_pydata1.js index 85ad427..e648e4c 100644 --- a/services/ws-modules/pydata1/pkg/et_ws_pydata1.js +++ b/services/ws-modules/pydata1/pkg/et_ws_pydata1.js @@ -41,7 +41,7 @@ export async function run() { const wsProtocol = window.location.protocol === "https:" ? "wss:" : "ws:"; const wsUrl = `${wsProtocol}//${window.location.host}/ws`; - const { WsClient, WsClientConfig } = await import("/modules/ws-wasm-agent/et_ws_wasm_agent.js"); + const { WsClient, WsClientConfig } = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); const client = new WsClient(new WsClientConfig(wsUrl)); let responseResolvers = []; diff --git a/services/ws-server/src/config.rs b/services/ws-server/src/config.rs index 9eca946..97b7d7d 100644 --- a/services/ws-server/src/config.rs +++ b/services/ws-server/src/config.rs @@ -1,19 +1,9 @@ use std::path::PathBuf; -use edge_toolkit::config::OtlpConfig; +use edge_toolkit::config::{OtlpConfig, default_modules_folders}; use serde::Deserialize; use serde_default::DefaultFromSerde; - -/// Default modules directory. -#[must_use] -pub fn default_modules_folders() -> Vec { - let project_root = edge_toolkit::config::get_project_root(); - vec![ - project_root.join("services/ws-wasm-agent"), - project_root.join("data").join("model-modules"), - project_root.join("services").join("ws-modules"), - ] -} +use serde_inline_default::serde_inline_default; /// Default modules directory. #[must_use] @@ -23,10 +13,13 @@ pub fn default_storage_folder() -> std::path::PathBuf { } /// Modules config. +#[serde_inline_default] #[derive(Clone, Debug, DefaultFromSerde, Deserialize)] pub struct ModulesConfig { #[serde(default = "default_modules_folders")] pub paths: Vec, + #[serde_inline_default(String::from("et-ws-server-static"))] + pub root: String, } /// Storage config. diff --git a/services/ws-server/src/lib.rs b/services/ws-server/src/lib.rs index 26ac528..0fc9f8d 100644 --- a/services/ws-server/src/lib.rs +++ b/services/ws-server/src/lib.rs @@ -775,17 +775,34 @@ pub async fn health() -> HttpResponse { })) } +/// Read the `name` field from a `package.json` file, returning `None` on any error. +fn read_package_name(package_json: &std::path::Path) -> Option { + let content = std::fs::read_to_string(package_json).ok()?; + let v: serde_json::Value = serde_json::from_str(&content).ok()?; + v.get("name")?.as_str().map(str::to_string) +} + /// Scan all configured module paths and return a sorted list of (name, pkg_dir) pairs. /// -/// Each entry is a module whose `/pkg/` subdirectory exists. +/// Each entry is a module whose `/pkg/` subdirectory exists, or whose directory +/// directly contains a `package.json`. pub fn list_modules(config: &ModulesConfig) -> Vec<(String, PathBuf)> { let mut modules: Vec<(String, PathBuf)> = Vec::new(); for path in &config.paths { let pkg_dir = path.join("pkg"); if pkg_dir.is_dir() { // This path is itself a single module (e.g. ws-wasm-agent). - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - modules.push((name.to_string(), pkg_dir)); + let name = read_package_name(&pkg_dir.join("package.json")) + .or_else(|| path.file_name().and_then(|n| n.to_str()).map(str::to_string)); + if let Some(name) = name { + modules.push((name, pkg_dir)); + } + } else if path.join("package.json").is_file() { + // This path directly contains a package.json. + let name = read_package_name(&path.join("package.json")) + .or_else(|| path.file_name().and_then(|n| n.to_str()).map(str::to_string)); + if let Some(name) = name { + modules.push((name, path.clone())); } } else if let Ok(entries) = std::fs::read_dir(path) { // This path is a directory of modules (e.g. ws-modules). @@ -793,11 +810,21 @@ pub fn list_modules(config: &ModulesConfig) -> Vec<(String, PathBuf)> { if let Ok(file_type) = entry.file_type() && file_type.is_dir() && !config.paths.contains(&entry.path()) - && let Some(name) = entry.file_name().to_str() { - let pkg_dir = entry.path().join("pkg"); + let entry_path = entry.path(); + let pkg_dir = entry_path.join("pkg"); if pkg_dir.is_dir() { - modules.push((name.to_string(), pkg_dir)); + let name = read_package_name(&pkg_dir.join("package.json")) + .or_else(|| entry.file_name().to_str().map(str::to_string)); + if let Some(name) = name { + modules.push((name, pkg_dir)); + } + } else if entry_path.join("package.json").is_file() { + let name = read_package_name(&entry_path.join("package.json")) + .or_else(|| entry.file_name().to_str().map(str::to_string)); + if let Some(name) = name { + modules.push((name, entry_path)); + } } } } @@ -811,7 +838,6 @@ async fn api_list_modules(config: web::Data) -> HttpResponse { let names: Vec = list_modules(&config.modules) .into_iter() .map(|(name, _)| name) - .filter(|name| name != "ws-wasm-agent") .collect(); HttpResponse::Ok().json(names) } @@ -859,6 +885,18 @@ pub async fn agent_put_file( pub fn configure_app(cfg: &mut web::ServiceConfig, agent_registry: web::Data, config: Config) { let modules = list_modules(&config.modules); let storage_dir = config.storage.path.clone(); + + let root_module_dir: PathBuf = modules + .iter() + .find(|(name, _)| name == &config.modules.root) + .map(|(_, path)| path.clone()) + .unwrap_or_else(|| { + panic!( + "Root module '{}' not found among configured modules paths", + config.modules.root + ); + }); + cfg.app_data(agent_registry) .app_data(web::Data::new(config)) .route("/favicon.ico", web::get().to(no_content)) @@ -872,11 +910,13 @@ pub fn configure_app(cfg: &mut web::ServiceConfig, agent_registry: web::Data { + const s = document.createElement("script"); + s.src = "/modules/onnxruntime-web/dist/ort.min.js"; + s.onload = resolve; + s.onerror = reject; + document.head.appendChild(s); +}); + const logEl = document.getElementById("log"); const moduleSelect = document.getElementById("module-select"); const runModuleButton = document.getElementById("run-module-button"); const agentStatusEl = document.getElementById("agent-status"); const agentIdEl = document.getElementById("agent-id"); -const STORED_AGENT_ID_KEY = "ws_wasm_agent.agent_id"; +const STORED_AGENT_ID_KEY = "et_ws_wasm_agent.agent_id"; let currentAgentId = null; const append = (line) => { @@ -35,6 +43,14 @@ const populateModuleDropdown = async () => { for (const name of moduleNames) { try { + if (name === "et-ws-wasm-agent") { + append(`Skipping ${name}: already loaded as the main WASM agent module`); + continue; + } + if (name === "onnxruntime-web") { + append(`Skipping ${name}: already loaded as a dependency`); + continue; + } const pkgResp = await fetch(`/modules/${name}/package.json`); if (!pkgResp.ok) { append(`Skipping ${name}: no package.json (${pkgResp.status})`); @@ -163,7 +179,7 @@ const wsUrl = `${wsProtocol}//${window.location.host}/ws`; const retainedAgentId = readStoredAgentId(); logEl.textContent = - `Initializing WASM from /modules/ws-wasm-agent/et_ws_wasm_agent_bg.wasm\nWebSocket endpoint: ${wsUrl}`; + `Initializing WASM from /modules/et-ws-wasm-agent/et_ws_wasm_agent_bg.wasm\nWebSocket endpoint: ${wsUrl}`; updateAgentCard( retainedAgentId ? "Found retained agent ID in local storage. It will be re-used on connect." diff --git a/services/ws-server/static/index.html b/services/ws-server/static/index.html index db0518a..bc596d2 100644 --- a/services/ws-server/static/index.html +++ b/services/ws-server/static/index.html @@ -5,121 +5,7 @@ WASM web agent - +
@@ -154,8 +40,6 @@

WASM web agent

Booting…
- - diff --git a/services/ws-server/static/package.json b/services/ws-server/static/package.json new file mode 100644 index 0000000..a5c124d --- /dev/null +++ b/services/ws-server/static/package.json @@ -0,0 +1,15 @@ +{ + "description": "edge-toolkit static assets", + "name": "et-ws-server-static", + "version": "0.1.0", + "files": [ + "app.js", + "favicon.png", + "index.html", + "style.css" + ], + "dependencies": { + "et-ws-wasm-agent": "*", + "onnxruntime-web": "*" + } +} diff --git a/services/ws-server/static/style.css b/services/ws-server/static/style.css new file mode 100644 index 0000000..14d71ca --- /dev/null +++ b/services/ws-server/static/style.css @@ -0,0 +1,112 @@ +:root { + color-scheme: light; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + background: #f4efe6; + color: #182028; +} + +body { + margin: 0; + min-height: 100vh; + display: grid; + place-items: center; + background: + radial-gradient( + circle at top, + rgba(213, 155, 66, 0.22), + transparent 35% + ), + linear-gradient(160deg, #f4efe6 0%, #e6dfd1 100%); +} + +main { + width: min(760px, calc(100vw - 32px)); + padding: 24px; + border: 1px solid rgba(24, 32, 40, 0.14); + background: rgba(255, 250, 242, 0.88); + box-shadow: 0 24px 80px rgba(24, 32, 40, 0.12); + backdrop-filter: blur(10px); +} + +h1 { + margin: 0 0 8px; + font-size: clamp(2rem, 5vw, 3.4rem); + line-height: 0.95; +} + +p { + margin: 0 0 12px; + max-width: 65ch; +} + +.agent-card { + margin: 0 0 18px; + padding: 14px 16px; + border: 1px solid rgba(24, 32, 40, 0.14); + background: rgba(255, 253, 250, 0.92); +} + +.agent-card strong { + display: block; + margin-bottom: 6px; +} + +.agent-card code { + display: block; + overflow-wrap: anywhere; +} + +code { + font-size: 0.95em; +} + +pre { + margin: 18px 0 0; + padding: 16px; + min-height: 220px; + overflow: auto; + background: #182028; + color: #f8f5f0; +} + +textarea { + box-sizing: border-box; + width: 100%; + margin: 18px 0 0; + padding: 16px; + min-height: 220px; + resize: vertical; + border: 1px solid rgba(24, 32, 40, 0.18); + background: #fffdfa; + color: #182028; + font: inherit; +} + +.status-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); + gap: 18px; + margin-top: 18px; +} + +.status-grid textarea { + margin: 0; + min-height: 180px; +} + +video, +canvas { + box-sizing: border-box; + display: block; + width: auto; + height: auto; + max-width: 100%; + max-height: min(42vh, 420px); + margin: 12px auto 0; + background: #182028; + object-fit: contain; +} + +[hidden] { + display: none !important; +} diff --git a/services/ws-server/tests/api_modules.rs b/services/ws-server/tests/api_modules.rs index 431f720..4acaf00 100644 --- a/services/ws-server/tests/api_modules.rs +++ b/services/ws-server/tests/api_modules.rs @@ -1,30 +1,25 @@ -use std::path::PathBuf; - use actix_web::{App, test, web}; -use et_ws_server::config::ModulesConfig; +use et_ws_server::config::Config; use et_ws_server::{AgentRegistry, configure_app}; #[actix_rt::test] -async fn test_list_modules() { +async fn list_modules() { let agent_registry = web::Data::new(AgentRegistry::default()); - let storage_dir = PathBuf::from("/tmp/et-ws-test-storage"); - let app = test::init_service(App::new().configure(|cfg| { - configure_app( - cfg, - agent_registry.clone(), - storage_dir.clone(), - ModulesConfig::default(), - ) - })) - .await; + let app = + test::init_service(App::new().configure(|cfg| configure_app(cfg, agent_registry.clone(), Config::default()))) + .await; let req = test::TestRequest::get().uri("/api/modules").to_request(); let resp: Vec = test::call_and_read_body_json(&app, req).await; // We expect at least the modules we know exist - assert!(resp.contains(&"comm1".to_string())); - assert!(resp.contains(&"data1".to_string())); - assert!(resp.contains(&"har1".to_string())); - assert!(resp.contains(&"face-detection".to_string())); + assert!(resp.contains(&"et-ws-server-static".to_string())); + assert!(resp.contains(&"et-ws-wasm-agent".to_string())); + assert!(resp.contains(&"et-ws-comm1".to_string())); + assert!(resp.contains(&"et-ws-data1".to_string())); + assert!(resp.contains(&"et-ws-har1".to_string())); + assert!(resp.contains(&"et-ws-face-detection".to_string())); + assert!(resp.contains(&"et-model-har-motion1".to_string())); + assert!(resp.contains(&"onnxruntime-web".to_string())); } diff --git a/utilities/cli/src/lib.rs b/utilities/cli/src/lib.rs index 23d6369..89a3201 100644 --- a/utilities/cli/src/lib.rs +++ b/utilities/cli/src/lib.rs @@ -172,12 +172,18 @@ fn generate_mise_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result let output_abs = absolute_from(&workspace_root, output_dir); let ws_server_dir = workspace_root.join("services/ws-server"); let workspace_rel = relative_path_from(&output_abs, &workspace_root).display().to_string(); - let openobserve_env_file_rel = relative_path_from(&output_abs, &workspace_root.join("config/o2.env")) - .display() - .to_string(); + let openobserve_env_file_rel = "config/o2.env"; let module_names = cluster_module_names(cluster); - let module_paths = scenario_module_paths(&ws_server_dir, &workspace_root, &module_names); - let module_paths_value = module_paths.join(","); + let module_paths = scenario_module_paths(&ws_server_dir, &module_names); + let module_paths_lines = module_paths + .iter() + .map(|p| format!(" {p}")) + .collect::>() + .join(",\\\n"); + let ws_server_run = format!( + "MODULES_PATHS=\"\\\n{},\\\n $(mise where npm:onnxruntime-web)/lib/node_modules\"\ncargo run\n", + module_paths_lines + ); let ws_server_rel = relative_path_from(&output_abs, &ws_server_dir).display().to_string(); let mut root = Table::new(); @@ -203,9 +209,9 @@ fn generate_mise_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result None, Some("Run the WebSocket server"), Some(&ws_server_rel), - Some("cargo run"), + Some(&ws_server_run), None, - Some(mise_env(&module_paths_value)), + Some(mise_env()), )), ); tasks.insert( @@ -235,7 +241,7 @@ fn generate_mise_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result let content = format_mise_toml( toml::to_string(&Value::Table(root)).context("Failed to serialize mise TOML")?, - &openobserve_env_file_rel, + openobserve_env_file_rel, ); fs::write(&output_path, content).with_context(|| format!("Failed to write output file: {:?}", output_path))?; fs::write(&readme_path, generated_readme(cluster, &module_names)) @@ -285,9 +291,9 @@ fn format_mise_toml(content: String, openobserve_env_file_rel: &str) -> String { let wrapped_openobserve_run = format!( concat!( "run = \"\"\"\n", - "docker run --rm -it --name openobserve -p 5080:5080 \\\n", - "--env-file {} \\\n", - "openobserve/openobserve:v0.70.3\n", + "docker run --rm --name openobserve -p 5080:5080 \\\n", + " --env-file {} \\\n", + " openobserve/openobserve:v0.70.3\n", "\"\"\"" ), openobserve_env_file_rel @@ -327,9 +333,8 @@ fn mise_task( task } -fn mise_env(module_paths: &str) -> Table { +fn mise_env() -> Table { let mut env = Table::new(); - env.insert("MODULES_PATHS".to_string(), Value::String(module_paths.to_string())); env.insert("OTLP_AUTH_PASSWORD".to_string(), Value::String("1234".to_string())); env.insert( "OTLP_AUTH_USERNAME".to_string(), @@ -352,26 +357,19 @@ fn mise_depends(depends: [&str; N]) -> Table { extra } -fn scenario_module_paths(ws_server_dir: &Path, workspace_root: &Path, module_names: &[String]) -> Vec { - let mut paths = Vec::with_capacity(module_names.len() + 2); - paths.push( - relative_path_from(ws_server_dir, &workspace_root.join("services/ws-wasm-agent")) - .display() - .to_string(), - ); - paths.push( - relative_path_from(ws_server_dir, &workspace_root.join("data/model-modules")) - .display() - .to_string(), - ); +fn scenario_module_paths(ws_server_dir: &Path, module_names: &[String]) -> Vec { + let project_root = edge_toolkit::config::get_project_root(); + let ws_modules_dir = project_root.join("services/ws-modules"); + let mut paths: Vec = edge_toolkit::config::default_modules_folders() + .into_iter() + .filter(|p| p != &ws_modules_dir && p.starts_with(&project_root)) + .map(|p| relative_path_from(ws_server_dir, &p).display().to_string()) + .collect(); for module_name in module_names { paths.push( - relative_path_from( - ws_server_dir, - &workspace_root.join("services/ws-modules").join(module_name), - ) - .display() - .to_string(), + relative_path_from(ws_server_dir, &ws_modules_dir.join(module_name)) + .display() + .to_string(), ); } paths diff --git a/utilities/cli/src/tests.rs b/utilities/cli/src/tests.rs index 1ce9ee2..4db67bd 100644 --- a/utilities/cli/src/tests.rs +++ b/utilities/cli/src/tests.rs @@ -2,83 +2,7 @@ use std::fs; use tempfile::tempdir; -use crate::{generate_deployment, regenerate_verification, relative_path_from}; - -#[test] -fn generate_mise_deployment_writes_mise_tasks() { - let test_root = tempdir().unwrap(); - let input_dir = test_root.path().join("input"); - let output_dir = test_root.path().join("output"); - fs::create_dir_all(&input_dir).unwrap(); - - let input_file = input_dir.join("cluster.yaml"); - fs::write( - &input_file, - r#"cluster_name: "test-cluster" -deployment_type: "mise" -agents: - - name: "camera" - resources: - - type: "custom-camera-module" - - name: "tracker" - resources: - - type: "har1" -"#, - ) - .unwrap(); - - let summary = generate_deployment(&input_file, &output_dir, None).unwrap(); - - assert_eq!(summary.cluster_name, "test-cluster"); - assert_eq!(summary.agent_templates, 2); - assert_eq!( - summary.module_names, - vec!["custom-camera-module".to_string(), "har1".to_string()] - ); - - let content = fs::read_to_string(output_dir.join("mise.toml")).unwrap(); - let readme = fs::read_to_string(output_dir.join("README.md")).unwrap(); - assert!(!content.contains("[env]")); - assert!(!content.contains("EDGE_CLUSTER_NAME")); - assert!(!content.contains("WS_SERVER_ALLOWED_MODULES")); - assert!(!content.contains("[tasks.build-ws-wasm-agent]")); - assert!(!content.contains("[tasks.build-modules]")); - assert!(!content.contains("wasm-pack build")); - assert!(content.contains("[tasks.openobserve]")); - assert!(content.contains("alias = \"o2\"")); - let expected_openobserve_env_file = relative_path_from( - &std::env::current_dir().unwrap().join(&output_dir), - &std::env::current_dir().unwrap().join("config/o2.env"), - ); - assert!(content.contains(&format!( - concat!( - "docker run --rm -it --name openobserve -p 5080:5080 \\\n", - "--env-file {} \\\n", - "openobserve/openobserve:v0.70.3\n", - "\"\"\"" - ), - expected_openobserve_env_file.display() - ))); - assert!(content.contains("[tasks.ws-server]")); - assert!(content.contains("run = \"cargo run\"")); - assert!(content.contains("[tasks.ws-server.env]")); - assert!(content.contains(concat!( - "MODULES_PATHS = ", - "\"../ws-wasm-agent,", - "../../data/model-modules,", - "../ws-modules/custom-camera-module,", - "../ws-modules/har1\"" - ))); - assert!(content.contains("OTLP_AUTH_PASSWORD = \"1234\"")); - assert!(content.contains("OTLP_AUTH_USERNAME = \"root@example.com\"")); - assert!(content.contains("[tasks.generated-scenario]")); - assert!(content.contains("depends = [\"openobserve\", \"ws-server\"]")); - assert!(content.contains("[tasks.open-o2]")); - assert!(content.contains("run = \"open http://localhost:5080/\"")); - assert!(readme.contains("# test-cluster")); - assert!(readme.contains("mise run generated-scenario")); - assert!(readme.contains("mise run open-o2")); -} +use crate::{generate_deployment, regenerate_verification}; #[test] fn generate_deployment_rejects_unsupported_deployment_type() { diff --git a/verification/local/output/facility-security-scenario/mise.toml b/verification/local/output/facility-security-scenario/mise.toml index 619768e..8782be7 100644 --- a/verification/local/output/facility-security-scenario/mise.toml +++ b/verification/local/output/facility-security-scenario/mise.toml @@ -10,17 +10,25 @@ run = "open http://localhost:5080/" alias = "o2" dir = "../../../.." run = """ -docker run --rm -it --name openobserve -p 5080:5080 \ ---env-file ../../../../config/o2.env \ -openobserve/openobserve:v0.70.3 +docker run --rm --name openobserve -p 5080:5080 \ + --env-file config/o2.env \ + openobserve/openobserve:v0.70.3 """ [tasks.ws-server] description = "Run the WebSocket server" dir = "../../../../services/ws-server" -run = "cargo run" +run = ''' +MODULES_PATHS="\ + static,\ + ../ws-wasm-agent,\ + ../../data/model-modules,\ + ../ws-modules/face-detection,\ + ../ws-modules/har1,\ + $(mise where npm:onnxruntime-web)/lib/node_modules" +cargo run +''' [tasks.ws-server.env] -MODULES_PATHS = "../ws-wasm-agent,../../data/model-modules,../ws-modules/face-detection,../ws-modules/har1" OTLP_AUTH_PASSWORD = "1234" OTLP_AUTH_USERNAME = "root@example.com"