diff --git a/.gitignore b/.gitignore index 555c018..782ea70 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ target/ services/ws-wasm-agent/pkg/ services/ws-server/static/models/ .zig-cache/ +*.pem diff --git a/Cargo.lock b/Cargo.lock index f2ee374..565a0da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -540,9 +540,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.60" +version = "1.2.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -790,9 +790,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "der-parser" @@ -968,6 +968,9 @@ dependencies = [ "actix-rt", "actix-web", "edge-toolkit", + "serde", + "serde-inline-default", + "serde_default", "serde_json", "tracing", ] @@ -988,6 +991,9 @@ dependencies = [ "actix-web", "edge-toolkit", "futures-util", + "serde", + "serde-inline-default", + "serde_default", "tokio", "tracing", ] @@ -1590,9 +1596,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +checksum = "08d46837a0ed51fe95bd3b05de33cd64a1ee88fc797477ca48446872504507c5" dependencies = [ "typenum", ] @@ -1882,9 +1888,9 @@ checksum = "e8052b3d5cfa8bae8af3b44aae11a43e9fa48ce0ae477c4a39733a8deff34059" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" @@ -2762,9 +2768,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" dependencies = [ "aws-lc-rs", "log", @@ -2777,9 +2783,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "zeroize", ] diff --git a/libs/edge-toolkit/src/ws_server.rs b/libs/edge-toolkit/src/ws_server.rs index 78ae80d..8e903e8 100644 --- a/libs/edge-toolkit/src/ws_server.rs +++ b/libs/edge-toolkit/src/ws_server.rs @@ -1,52 +1,10 @@ use std::collections::BTreeMap; -use std::path::PathBuf; use std::sync::{Arc, Mutex}; use serde::{Deserialize, Serialize}; -use serde_default::DefaultFromSerde; -use serde_inline_default::serde_inline_default; -use crate::config::{OtlpConfig, default_modules_folders}; use crate::ws::{AgentConnectionState, AgentSummary, ConnectStatus}; -/// Default storage directory. -#[must_use] -pub fn default_storage_folder() -> PathBuf { - let project_root = crate::config::get_project_root(); - project_root.join("services/ws-server/storage") -} - -/// Modules config. -#[serde_inline_default] -#[derive(Clone, Debug, DefaultFromSerde, Deserialize)] -pub struct ModulesConfig { - #[serde(default = "default_modules_folders")] - pub paths: Vec, - #[serde_inline_default(String::from("et-ws-server-static"))] - pub root: String, -} - -/// Storage config. -#[derive(Clone, Debug, DefaultFromSerde, Deserialize)] -pub struct StorageConfig { - #[serde(default = "default_storage_folder")] - pub path: PathBuf, -} - -/// Application config shared across ws-server services. -#[derive(Clone, Debug, DefaultFromSerde, Deserialize)] -pub struct Config { - /// OpenTelemetry config. - #[serde(default)] - pub otlp: Option, - /// Modules config. - #[serde(default)] - pub modules: ModulesConfig, - /// Storage config. - #[serde(default)] - pub storage: StorageConfig, -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PendingDirectMessage { pub message_id: String, diff --git a/services/modules/Cargo.toml b/services/modules/Cargo.toml index dc75f95..7ab854f 100644 --- a/services/modules/Cargo.toml +++ b/services/modules/Cargo.toml @@ -9,6 +9,9 @@ repository.workspace = true actix-files = "0.6" actix-web = "4" edge-toolkit = { path = "../../libs/edge-toolkit" } +serde.workspace = true +serde-inline-default.workspace = true +serde_default.workspace = true serde_json.workspace = true tracing.workspace = true diff --git a/services/modules/src/lib.rs b/services/modules/src/lib.rs index b45e178..42dec4c 100644 --- a/services/modules/src/lib.rs +++ b/services/modules/src/lib.rs @@ -2,7 +2,20 @@ use std::path::PathBuf; use actix_files::Files; use actix_web::{HttpResponse, web}; -use edge_toolkit::ws_server::{Config, ModulesConfig}; +use edge_toolkit::config::default_modules_folders; +use serde::Deserialize; +use serde_default::DefaultFromSerde; +use serde_inline_default::serde_inline_default; + +/// Modules config. +#[serde_inline_default] +#[derive(Clone, Debug, DefaultFromSerde, Deserialize)] +pub struct ModulesConfig { + #[serde(default = "default_modules_folders")] + pub paths: Vec, + #[serde_inline_default(String::from("et-ws-server-static"))] + pub root: String, +} fn read_package_name(package_json: &std::path::Path) -> Option { let content = std::fs::read_to_string(package_json).ok()?; @@ -56,24 +69,21 @@ pub fn list_modules(config: &ModulesConfig) -> Vec<(String, PathBuf)> { modules } -async fn list_modules_handler(config: web::Data) -> HttpResponse { - let names: Vec = list_modules(&config.modules) - .into_iter() - .map(|(name, _)| name) - .collect(); +async fn list_modules_handler(config: web::Data) -> HttpResponse { + let names: Vec = list_modules(&config).into_iter().map(|(name, _)| name).collect(); HttpResponse::Ok().json(names) } /// Register `GET /modules/` (JSON list), `GET /modules/{name}/...` (static files), /// and `GET /` (root module). -pub fn configure(cfg: &mut web::ServiceConfig, config: &Config) { - let modules = list_modules(&config.modules); +pub fn configure(cfg: &mut web::ServiceConfig, config: &ModulesConfig) { + let modules = list_modules(config); let root_module_dir = modules .iter() - .find(|(name, _)| name == &config.modules.root) + .find(|(name, _)| name == &config.root) .map(|(_, path)| path.clone()) - .unwrap_or_else(|| panic!("Root module '{}' not found", config.modules.root)); + .unwrap_or_else(|| panic!("Root module '{}' not found", config.root)); cfg.route("/modules/", web::get().to(list_modules_handler)); for (name, pkg_dir) in &modules { diff --git a/services/modules/tests/api_modules.rs b/services/modules/tests/api_modules.rs index 3d44d13..d08af0d 100644 --- a/services/modules/tests/api_modules.rs +++ b/services/modules/tests/api_modules.rs @@ -1,10 +1,10 @@ use actix_web::{App, test, web}; -use edge_toolkit::ws_server::{AgentRegistry, Config}; -use et_modules_service::configure; +use edge_toolkit::ws_server::AgentRegistry; +use et_modules_service::{ModulesConfig, configure}; #[actix_rt::test] async fn list_modules_api() { - let config = Config::default(); + let config = ModulesConfig::default(); let app = test::init_service( App::new() .app_data(web::Data::new(AgentRegistry::<()>::default())) diff --git a/services/storage/Cargo.toml b/services/storage/Cargo.toml index f959502..d0d4c7b 100644 --- a/services/storage/Cargo.toml +++ b/services/storage/Cargo.toml @@ -10,5 +10,8 @@ actix-files = "0.6" actix-web = "4" edge-toolkit = { path = "../../libs/edge-toolkit" } futures-util = "0.3" +serde.workspace = true +serde-inline-default.workspace = true +serde_default.workspace = true tokio = { version = "1", features = ["full"] } tracing.workspace = true diff --git a/services/storage/src/lib.rs b/services/storage/src/lib.rs index 06758aa..02babff 100644 --- a/services/storage/src/lib.rs +++ b/services/storage/src/lib.rs @@ -2,15 +2,31 @@ use std::path::PathBuf; use actix_files::Files; use actix_web::{Error, HttpRequest, HttpResponse, web}; -use edge_toolkit::ws_server::{AgentRegistry, Config}; +use edge_toolkit::ws_server::AgentRegistry; use futures_util::StreamExt; +use serde::Deserialize; +use serde_default::DefaultFromSerde; use tracing::info; +/// Default storage directory. +#[must_use] +pub fn default_storage_folder() -> PathBuf { + let project_root = edge_toolkit::config::get_project_root(); + project_root.join("services/ws-server/storage") +} + +/// Storage config. +#[derive(Clone, Debug, DefaultFromSerde, Deserialize)] +pub struct StorageConfig { + #[serde(default = "default_storage_folder")] + pub path: PathBuf, +} + pub async fn agent_put_file( req: HttpRequest, mut payload: web::Payload, registry: web::Data>, - config: web::Data, + config: web::Data, ) -> Result { let agent_id: String = req.match_info().query("agent_id").parse().unwrap(); let filename: PathBuf = req @@ -30,7 +46,7 @@ pub async fn agent_put_file( return Err(actix_web::error::ErrorBadRequest("invalid filename")); } - let storage_dir = &config.storage.path; + let storage_dir = &config.path; let agent_dir = storage_dir.join(&agent_id); std::fs::create_dir_all(&agent_dir)?; @@ -47,8 +63,8 @@ pub async fn agent_put_file( } /// Register `PUT /storage/{agent_id}/{filename}` and `GET /storage/...` (static file serving). -pub fn configure(cfg: &mut web::ServiceConfig, config: &Config) { - let storage_dir = config.storage.path.clone(); +pub fn configure(cfg: &mut web::ServiceConfig, config: &StorageConfig) { + let storage_dir = config.path.clone(); cfg.route("/storage/{agent_id}/{filename}", web::put().to(agent_put_file::)) .service( Files::new("/storage", storage_dir) diff --git a/services/ws-server/src/config.rs b/services/ws-server/src/config.rs index ebbf7d9..804174d 100644 --- a/services/ws-server/src/config.rs +++ b/services/ws-server/src/config.rs @@ -1 +1,35 @@ -pub use edge_toolkit::ws_server::{Config, ModulesConfig, StorageConfig}; +use std::path::PathBuf; + +use edge_toolkit::config::OtlpConfig; +pub use et_modules_service::ModulesConfig; +pub use et_storage_service::StorageConfig; +use serde::Deserialize; +use serde_default::DefaultFromSerde; +use serde_inline_default::serde_inline_default; + +/// TLS certificate and key paths. +#[serde_inline_default] +#[derive(Clone, Debug, DefaultFromSerde, Deserialize)] +pub struct TlsConfig { + #[serde_inline_default(PathBuf::from("cert.pem"))] + pub cert_file: PathBuf, + #[serde_inline_default(PathBuf::from("key.pem"))] + pub key_file: PathBuf, +} + +/// Application config shared across ws-server services. +#[derive(Clone, Debug, DefaultFromSerde, Deserialize)] +pub struct Config { + /// OpenTelemetry config. + #[serde(default)] + pub otlp: Option, + /// Modules config. + #[serde(default)] + pub modules: ModulesConfig, + /// Storage config. + #[serde(default)] + pub storage: StorageConfig, + /// TLS config. + #[serde(default)] + pub tls: TlsConfig, +} diff --git a/services/ws-server/src/lib.rs b/services/ws-server/src/lib.rs index 235c868..789dd84 100644 --- a/services/ws-server/src/lib.rs +++ b/services/ws-server/src/lib.rs @@ -1,15 +1,9 @@ -pub mod config; - -use std::path::{Path, PathBuf}; - use actix_web::{HttpResponse, web}; pub use et_ws_service::{WebSocketActor, WsAgentRegistry}; -use crate::config::Config; +pub mod config; -pub fn browser_static_dir() -> PathBuf { - Path::new(".").join("static") -} +use crate::config::Config; pub async fn no_content() -> HttpResponse { HttpResponse::NoContent().finish() @@ -22,14 +16,16 @@ pub async fn health() -> HttpResponse { })) } -pub fn configure_app(cfg: &mut web::ServiceConfig, agent_registry: web::Data, config: Config) { +pub fn configure_app(cfg: &mut web::ServiceConfig, agent_registry: web::Data, config: &Config) { cfg.app_data(agent_registry) .app_data(web::Data::new(config.clone())) + .app_data(web::Data::new(config.modules.clone())) + .app_data(web::Data::new(config.storage.clone())) .route("/favicon.ico", web::get().to(no_content)) .route("/health", web::get().to(health)); - et_ws_service::configure(cfg, &config); - et_storage_service::configure::>(cfg, &config); + et_ws_service::configure(cfg); + et_storage_service::configure::>(cfg, &config.storage); // Must be last: registers a catch-all Files::new("/", ...) for the root module. - et_modules_service::configure(cfg, &config); + et_modules_service::configure(cfg, &config.modules); } diff --git a/services/ws-server/src/main.rs b/services/ws-server/src/main.rs index 12b8868..0789a9b 100644 --- a/services/ws-server/src/main.rs +++ b/services/ws-server/src/main.rs @@ -5,12 +5,13 @@ use actix_web::{App, HttpServer, web}; use clap::Parser; use et_modules_service::list_modules; use et_ws_server::config::Config; -use et_ws_server::{browser_static_dir, configure_app}; +use et_ws_server::configure_app; use et_ws_service::load_registry; use tracing::{error, info}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; mod otlp; +mod tls; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -20,25 +21,8 @@ struct Args { agent_registry: PathBuf, } -fn tls_config() -> std::io::Result { - let certified = rcgen::generate_simple_self_signed(vec![ - "localhost".to_string(), - "127.0.0.1".to_string(), - "::1".to_string(), - ]) - .map_err(|e| std::io::Error::other(format!("failed to generate dev certificate: {e}")))?; - - let cert_der: rustls::pki_types::CertificateDer<'static> = certified.cert.der().clone(); - let key_der = rustls::pki_types::PrivatePkcs8KeyDer::from(certified.signing_key.serialize_der()); - - rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(vec![cert_der], key_der.into()) - .map_err(|e| std::io::Error::other(format!("failed to configure TLS: {e}"))) -} - #[actix_web::main] -async fn main() -> std::io::Result<()> { +async fn main() -> Result<(), std::io::Error> { let args = Args::parse(); let env = serde_env::from_env::().unwrap(); @@ -59,11 +43,24 @@ async fn main() -> std::io::Result<()> { .init(); } - let tls_config = tls_config()?; - let network_ip = local_ip_address::local_ip() .map(|ip| ip.to_string()) .unwrap_or_else(|_| "127.0.0.1".to_string()); + + let cert_filename = &env.tls.cert_file; + let key_filename = &env.tls.key_file; + let (cert_der, key_der) = if cert_filename.exists() && key_filename.exists() { + info!("Loading TLS certificate from {:?}", cert_filename); + tls::load_tls_certs(cert_filename, key_filename) + } else { + info!( + "Generated self-signed localhost certificate to {:?} and key to {:?}", + cert_filename, key_filename + ); + tls::generate_tls_certs(cert_filename, key_filename) + }; + let rustls_config = tls::build_tls_server_config(cert_der, key_der); + let https_url = format!( "https://{}:{}", network_ip, @@ -79,14 +76,12 @@ async fn main() -> std::io::Result<()> { if let Err(e) = qr2term::print_qr(&https_url) { error!("Failed to generate QR code: {}", e); } - info!("Serving browser assets from {:?}", browser_static_dir()); - info!("HTTPS uses an in-memory self-signed localhost certificate for development"); - let agent_registry = web::Data::new(load_registry(&args.agent_registry)?); + let agent_registry = web::Data::new(load_registry(&args.agent_registry).unwrap()); let registry_clone = agent_registry.clone(); let registry_path = args.agent_registry.clone(); - std::fs::create_dir_all(&env.storage.path)?; + std::fs::create_dir_all(&env.storage.path).unwrap(); for (name, pkg_dir) in list_modules(&env.modules) { info!("Loading module {name} at {}", pkg_dir.display()); @@ -101,12 +96,12 @@ async fn main() -> std::io::Result<()> { .add(("Cross-Origin-Opener-Policy", "same-origin")) .add(("Cross-Origin-Embedder-Policy", "require-corp")), ) - .configure(|cfg| configure_app(cfg, registry, config)) + .configure(|cfg| configure_app(cfg, registry, &config)) }) .bind(("0.0.0.0", edge_toolkit::ports::Services::InsecureWebSocketServer.port()))? .bind_rustls_0_23( ("0.0.0.0", edge_toolkit::ports::Services::SecureWebSocketServer.port()), - tls_config, + rustls_config, )? .run(); diff --git a/services/ws-server/src/tls.rs b/services/ws-server/src/tls.rs new file mode 100644 index 0000000..da2f195 --- /dev/null +++ b/services/ws-server/src/tls.rs @@ -0,0 +1,40 @@ +use std::path::Path; + +use rustls::pki_types::pem::PemObject; + +type CertKeyPair = ( + rustls::pki_types::CertificateDer<'static>, + rustls::pki_types::PrivateKeyDer<'static>, +); + +pub fn load_tls_certs(cert_filename: &Path, key_filename: &Path) -> CertKeyPair { + let cert_der = rustls::pki_types::CertificateDer::from_pem_file(cert_filename).unwrap(); + let key_der = rustls::pki_types::PrivateKeyDer::from_pem_file(key_filename).unwrap(); + (cert_der, key_der) +} + +pub fn generate_tls_certs(cert_filename: &Path, key_filename: &Path) -> CertKeyPair { + let certified = rcgen::generate_simple_self_signed(vec![ + "localhost".to_string(), + "127.0.0.1".to_string(), + "::1".to_string(), + ]) + .unwrap(); + std::fs::write(cert_filename, certified.cert.pem()).unwrap(); + std::fs::write(key_filename, certified.signing_key.serialize_pem()).unwrap(); + let cert_der = certified.cert.der().clone(); + let key_der = rustls::pki_types::PrivateKeyDer::from(rustls::pki_types::PrivatePkcs8KeyDer::from( + certified.signing_key.serialize_der(), + )); + (cert_der, key_der) +} + +pub fn build_tls_server_config( + cert_der: rustls::pki_types::CertificateDer<'static>, + key_der: rustls::pki_types::PrivateKeyDer<'static>, +) -> rustls::ServerConfig { + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![cert_der], key_der) + .unwrap() +} diff --git a/services/ws/src/lib.rs b/services/ws/src/lib.rs index 9756f85..9fae7ce 100644 --- a/services/ws/src/lib.rs +++ b/services/ws/src/lib.rs @@ -21,7 +21,7 @@ pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(1); pub type WsAgentRegistry = AgentRegistry>; /// Load a registry from disk. Sessions are not persisted, so they are initialised to `None`. -pub fn load_registry(path: &std::path::Path) -> std::io::Result { +pub fn load_registry(path: &std::path::Path) -> Result { use edge_toolkit::ws::AgentConnectionState; if !path.exists() { warn!("Registry file {:?} does not exist, starting with empty registry", path); @@ -562,6 +562,6 @@ pub async fn ws_handler( result } -pub fn configure(cfg: &mut web::ServiceConfig, _config: &edge_toolkit::ws_server::Config) { +pub fn configure(cfg: &mut web::ServiceConfig) { cfg.route("/ws", web::get().to(ws_handler)); } diff --git a/utilities/onnx/src/main.rs b/utilities/onnx/src/main.rs index 878ffdc..42263d6 100644 --- a/utilities/onnx/src/main.rs +++ b/utilities/onnx/src/main.rs @@ -11,7 +11,6 @@ struct Args { } fn main() { - //} -> std::io::Result<()> { let args = Args::parse(); let model = onnx_extractor::OnnxModel::load_from_file(&args.filename.to_string_lossy()).unwrap();