diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 8e51a73932..6cf942c237 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -238,6 +238,14 @@ Common findings: - Current gateways reuse the primary listener when it covers Podman's callback address. If the primary does not cover that address, inspect the gateway startup logs for the additional callback-only listener and its provenance. +- In a nested Linux container with rootful Podman, a missing bridge address can + trigger `listener_purpose="nested-podman-callback-fallback"`. Confirm the + failed exact address is the built-in driver's discovered managed bridge (not + rootless Podman, an explicit `host_gateway_ip`, or an external driver), the + bind error is `EADDRNOTAVAIL`, and the wildcard socket is callback-only off + loopback. The callback RPC surface is + reachable on every IPv4 interface in the outer container namespace for that + gateway process, so also inspect the surrounding container-network boundary. - Rootless slirp4netns, another named helper, or missing helper metadata requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip` cannot bypass slirp4netns host-loopback isolation. Do not work around @@ -589,6 +597,7 @@ openshell logs | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | | Gateway exits while resolving compute-driver listener requirements | Callback alias topology is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route | +| Nested rootful Podman gateway logs `nested-podman-callback-fallback` | Podman reported a private bridge gateway before netavark assigned it | Confirm the exact bind failed with `EADDRNOTAVAIL`; verify non-loopback destinations receive callback-only scope and restrict the outer container network | | Admin, health, reflection, or HTTP request is denied on an additional Docker/Podman callback-only listener | Additional callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected | | Docker or Podman sandbox never registers | Wrong callback endpoint or supervisor startup failure | Gateway logs and sandbox container logs | | Docker GPU e2e fails before GPU sandbox comparison | NVIDIA CDI specs are missing or Docker has not discovered them | `docker info --format '{{json .DiscoveredDevices}}'`, `/etc/cdi`, `/var/run/cdi`, `nvidia-cdi-refresh.service` | diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 8d5d6d157e..f4844f4a96 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -82,6 +82,29 @@ Note `install.sh` is pulled from `raw.githubusercontent.com/NVIDIA/OpenShell/${h To smoke-test the chart for a specific dev build, dispatch `Release Dev` on the branch first, then run the kind canary steps locally pointed at the SHA-pinned chart (see "Local kind reproduction" below). The release-canary workflow itself does not currently expose `chart_version` / `image_tag` inputs. +## PR-built Fedora callback reproduction + +For a PR-built reproduction of the Fedora callback topology, run the +`e2e:podman:nested-container` task with `OPENSHELL_BIN`, +`OPENSHELL_GATEWAY_BIN`, and `OPENSHELL_SUPERVISOR_IMAGE` set to Linux +artifacts from the same revision. The `test:e2e` PR label enables this lane in +`Branch E2E Checks`. It injects those artifacts into the same +Docker-to-Fedora-to-rootful-Podman topology as the canary and verifies both +gateway startup and a supervisor callback before published RPMs exist. The +test also requires the future bridge address to be absent at gateway startup +and asserts the `nested-podman-callback-fallback` listener and IPv4 wildcard +socket before it creates the sandbox. + +On macOS, use `nix run .#test-guest -- --distro ubuntu --with docker` as the +outer host when the local Docker daemon is unavailable. Cross-build matching +Linux CLI and gateway binaries, copy the nested E2E script plus the gateway +service/default-config assets with `--copy`, and set +`OPENSHELL_E2E_GATEWAY_SERVICE_FILE` and +`OPENSHELL_E2E_GATEWAY_DEFAULT_CONFIG` to their guest paths. Pin +`OPENSHELL_SUPERVISOR_IMAGE` to the same revision as the binaries; using +`supervisor:latest` can produce protocol or SSH behavior skew unrelated to the +listener under test. + ## Local kind reproduction The `kubernetes` job can be reproduced on any machine with Docker and `mise install`-provided `kubectl` + `helm`: diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 2b4d9d5d46..1f81b3ca1f 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -137,6 +137,7 @@ jobs: cli-artifact-prefix: rust-binary-cli gateway-artifact-prefix: rust-binary-gateway vm-driver-artifact-name: driver-vm-linux-amd64 + nested-podman-callback: true gpu-e2e: needs: [pr_metadata, build-gateway, build-supervisor, build-cli] diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 006f5cf440..cc69165135 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -32,6 +32,11 @@ on: required: false type: string default: "" + nested-podman-callback: + description: "Run the Fedora-in-Docker/rootful-Podman callback regression lane (requires prebuilt CLI and gateway artifacts)" + required: false + type: boolean + default: false permissions: actions: read @@ -39,6 +44,44 @@ permissions: packages: read jobs: + e2e-nested-podman-callback: + name: E2E (nested Fedora/rootful Podman callback) + if: inputs['nested-podman-callback'] + runs-on: linux-amd64-cpu8 + timeout-minutes: 30 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /home/runner/_work:/home/runner/_work + env: + OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs['checkout-ref'] || github.sha }} + persist-credentials: false + + - name: Use prebuilt OpenShell CLI + uses: ./.github/actions/setup-e2e-cli + with: + artifact-prefix: ${{ inputs.cli-artifact-prefix }} + + - name: Use prebuilt OpenShell gateway + uses: ./.github/actions/setup-e2e-gateway + with: + artifact-prefix: ${{ inputs.gateway-artifact-prefix }} + + - name: Log in to GHCR with Docker + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Run nested Podman callback E2E + run: e2e/nested-podman-callback.sh + e2e: name: "E2E (${{ matrix.suite }})" runs-on: ${{ inputs.runner }} diff --git a/CI.md b/CI.md index aae22f4c4f..50a02b18a5 100644 --- a/CI.md +++ b/CI.md @@ -15,13 +15,14 @@ Merge queue validation is a second integration gate for `main`. After a PR has p Three opt-in labels enable the long-running E2E suites: - `test:e2e` runs the standard Docker, rootless Podman, Kubernetes, and VM E2E - suites in `Branch E2E Checks` + suites plus the Fedora-in-Docker/rootful-Podman callback regression lane in + `Branch E2E Checks` - `test:e2e-gpu` runs GPU E2E in `Branch E2E Checks` - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay (`replicaCount: 2` and bundled PostgreSQL) and the credential-driver suite (Kubernetes Secrets plus Vault) in `Branch E2E Checks` -When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once, builds one CLI artifact per runner architecture, builds the Linux VM driver artifact once, and fans out all enabled suites in parallel. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E jobs reuse the matching prebuilt gateway and CLI binaries instead of compiling additional debug binaries in each job; Kubernetes E2E consumes the gateway image directly and reuses the prebuilt CLI. VM E2E also reuses the prebuilt VM driver artifact and falls back to local VM-driver/runtime preparation for local runs or workflow invocations that omit the artifact. +When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once, builds one CLI artifact per runner architecture, builds the Linux VM driver artifact once, and fans out all enabled suites in parallel. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E jobs reuse the matching prebuilt gateway and CLI binaries instead of compiling additional debug binaries in each job; Kubernetes E2E consumes the gateway image directly and reuses the prebuilt CLI. The nested Podman lane injects those artifacts into a Fedora systemd container and creates a rootful Podman sandbox, reproducing the Release Canary callback topology without waiting for an RPM publication. VM E2E also reuses the prebuilt VM driver artifact and falls back to local VM-driver/runtime preparation for local runs or workflow invocations that omit the artifact. The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while Kubernetes HA and credential-driver behavior are under active iteration: failures are visible in the workflow run but do not publish a required CI gate status. The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly. diff --git a/architecture/gateway.md b/architecture/gateway.md index 32bca6a1f6..3b58b4ae11 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -56,6 +56,21 @@ reflection, non-callback inference APIs, and HTTP routes before normal request authentication. The operator-configured primary listener retains the full multiplexed API surface. +One Linux container startup race requires a narrower exception. Rootful Podman +can report a private bridge gateway before netavark assigns that address. If an +exact built-in Podman callback bind fails with `EADDRNOTAVAIL` while the gateway +itself runs in a container, the gateway replaces the loopback primary and +missing bridge sockets with one IPv4 wildcard socket. The wildcard defaults to +callback-only authorization; only traffic addressed to the configured loopback +endpoint receives primary scope. This keeps user and administrator APIs off the +container's non-loopback interfaces, but it does make the sandbox-callable gRPC +surface reachable on every IPv4 interface in that container namespace for the +lifetime of the gateway process. Sandbox mTLS/JWT authentication and the RPC +allowlist remain mandatory defenses. This exception does not apply on a host, +to rootless Podman, an explicit `host_gateway_ip`, Docker or external drivers, +to public callback addresses, or to bind errors +other than `EADDRNOTAVAIL`. + The `rpc_auth` classification is also the source of truth for negotiated listener exposure: marking an RPC as `sandbox` or `dual` makes it callable on these listeners. Review such changes as both authorization and network-surface diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 33acf1a2c6..0edaecf0a7 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1529,6 +1529,7 @@ impl ComputeDriver for DockerComputeDriver { DockerGatewayRoute::HostGateway => "docker host-gateway IPv4 loopback", } .to_string(), + allow_nested_container_wildcard_fallback: false, selector: Some(Selector::ExactBindAddress(bind_address.to_string())), }] }); diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 567abcbfcd..958474c018 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -281,6 +281,20 @@ the supervisor's RPCs. Otherwise, it creates an additional listener that exposes only the gateway's sandbox-callable gRPC methods. Operator, health, reflection, and HTTP requests must use the primary listener. +When both the gateway and rootful Podman run inside another Linux container, +netavark may not assign the managed bridge gateway until the first sandbox +joins the network. If that exact private callback address fails to bind with +`EADDRNOTAVAIL`, the gateway uses one scoped IPv4 wildcard listener for that +process. Only the in-process built-in Podman driver can mark its discovered +rootful managed-bridge address as eligible; rootless Podman, explicit +`host_gateway_ip` values, and external drivers cannot activate the fallback. +Connections addressed to loopback retain primary scope; connections +addressed to the Podman bridge or any other IPv4 interface are callback-only. +This exposes the sandbox-callable gRPC surface on every IPv4 interface in the +outer container namespace, so deployments should still restrict that namespace +at the container-network boundary. It does not expose operator, health, +reflection, or HTTP routes on those interfaces. + ### Layer 3 Inner Sandbox Network Namespace Inside the container, the supervisor creates another network namespace for the diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 973b2ececa..987f1d501d 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -517,6 +517,7 @@ impl PodmanComputeDriver { reason: "Podman rootless pasta callback uses the host default-route interface" .to_string(), + allow_nested_container_wildcard_fallback: false, selector: Some(Selector::DefaultRouteInterface( GatewayDefaultRouteInterfaceRequirement {}, )), @@ -542,6 +543,11 @@ impl PodmanComputeDriver { })?; Ok(vec![GatewayListenerRequirement { reason: format!("Podman network '{}' host gateway", self.config.network_name), + // A rootful managed bridge can be created after gateway + // startup. An explicit override is operator-owned, and + // rootless networking must never broaden the listener. + allow_nested_container_wildcard_fallback: !self.rootless + && self.config.host_gateway_ip.trim().is_empty(), selector: Some(Selector::ExactBindAddress( SocketAddr::new(gateway_ip, callback_port).to_string(), )), @@ -552,6 +558,7 @@ impl PodmanComputeDriver { Ok(vec![GatewayListenerRequirement { reason: "Podman machine callback forwarding terminates on gateway loopback" .to_string(), + allow_nested_container_wildcard_fallback: false, selector: Some(Selector::LoopbackInterface( GatewayLoopbackInterfaceRequirement {}, )), @@ -2121,6 +2128,7 @@ mod tests { let requirements = driver.gateway_listener_requirements().unwrap(); assert_eq!(requirements.len(), 1); + assert!(requirements[0].allow_nested_container_wildcard_fallback); assert_eq!( requirements[0].selector, Some(Selector::ExactBindAddress("10.89.1.1:17670".to_string())) @@ -2145,6 +2153,22 @@ mod tests { requirements[0].selector, Some(Selector::ExactBindAddress("10.90.1.1:17670".to_string())) ); + assert!(!requirements[0].allow_nested_container_wildcard_fallback); + } + + #[test] + #[cfg(target_os = "linux")] + fn rootful_explicit_host_gateway_does_not_allow_wildcard_fallback() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.containers.internal:17670".to_string(), + host_gateway_ip: "10.90.1.1".to_string(), + ..PodmanComputeConfig::default() + }); + driver.network_gateway_ip = Some("10.89.1.1".to_string()); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert!(!requirements[0].allow_nested_container_wildcard_fallback); } #[test] @@ -2163,6 +2187,7 @@ mod tests { requirements[0].selector, Some(Selector::DefaultRouteInterface(_)) )); + assert!(!requirements[0].allow_nested_container_wildcard_fallback); } #[test] diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 30a1303bd5..04da5cd5b9 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -157,6 +157,7 @@ pub enum GatewayListenerRequirement { address: SocketAddr, driver_name: String, reason: String, + allow_nested_container_wildcard_fallback: bool, }, DefaultRouteInterface { driver_name: String, @@ -168,6 +169,12 @@ pub enum GatewayListenerRequirement { }, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GatewayListenerFallbackPolicy { + Deny, + TrustedBuiltinPodman, +} + impl GatewayListenerRequirement { pub fn driver_name(&self) -> &str { match self { @@ -591,6 +598,7 @@ impl ComputeRuntime { driver_name: String, driver: SharedComputeDriver, driver_process: Option>, + listener_fallback_policy: GatewayListenerFallbackPolicy, store: Arc, sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, @@ -646,6 +654,9 @@ impl ComputeRuntime { address, driver_name: driver_name.clone(), reason: requirement.reason, + allow_nested_container_wildcard_fallback: listener_fallback_policy + == GatewayListenerFallbackPolicy::TrustedBuiltinPodman + && requirement.allow_nested_container_wildcard_fallback, }) } Selector::DefaultRouteInterface(_) => { @@ -733,6 +744,7 @@ impl ComputeRuntime { ComputeDriverKind::Docker.as_str().to_string(), driver, None, + GatewayListenerFallbackPolicy::Deny, store, sandbox_index, sandbox_watch_bus, @@ -761,6 +773,7 @@ impl ComputeRuntime { ComputeDriverKind::Kubernetes.as_str().to_string(), driver, None, + GatewayListenerFallbackPolicy::Deny, store, sandbox_index, sandbox_watch_bus, @@ -784,6 +797,7 @@ impl ComputeRuntime { endpoint.name, driver, endpoint.driver_process, + GatewayListenerFallbackPolicy::Deny, store, sandbox_index, sandbox_watch_bus, @@ -810,6 +824,7 @@ impl ComputeRuntime { ComputeDriverKind::Podman.as_str().to_string(), driver, None, + GatewayListenerFallbackPolicy::TrustedBuiltinPodman, store, sandbox_index, sandbox_watch_bus, @@ -9027,6 +9042,7 @@ mod tests { "test-driver".to_string(), Arc::new(TestDriver::default()), None, + GatewayListenerFallbackPolicy::Deny, store, SandboxIndex::new(), SandboxWatchBus::new(), @@ -9040,6 +9056,41 @@ mod tests { test_exporter::assert_is_root(&initialization); } + #[tokio::test] + async fn trusted_builtin_podman_preserves_wildcard_fallback_marker() { + use crate::test_support::FakeComputeDriver; + + let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); + let driver = FakeComputeDriver::new() + .with_nested_container_wildcard_fallback_listener_requirement( + "10.89.0.1:17670", + "Podman managed bridge", + ); + let runtime = ComputeRuntime::from_driver( + "podman".to_string(), + Arc::new(driver), + None, + GatewayListenerFallbackPolicy::TrustedBuiltinPodman, + store, + SandboxIndex::new(), + SandboxWatchBus::new(), + TracingLogBus::new(), + Arc::new(SupervisorSessionRegistry::new()), + ) + .await + .unwrap(); + + assert_eq!( + runtime.gateway_listener_requirements(), + &[GatewayListenerRequirement::Exact { + address: "10.89.0.1:17670".parse().unwrap(), + driver_name: "podman".to_string(), + reason: "Podman managed bridge".to_string(), + allow_nested_container_wildcard_fallback: true, + }] + ); + } + #[tokio::test] #[cfg(unix)] async fn remote_compute_driver_interceptor_propagates_every_rpc() { @@ -9177,6 +9228,44 @@ mod tests { ); } + #[tokio::test] + #[cfg(unix)] + async fn external_driver_named_podman_cannot_authorize_wildcard_fallback() { + use crate::test_support::FakeComputeDriver; + + let dir = tempfile::tempdir().unwrap(); + let socket_path = dir.path().join("compute-driver.sock"); + let driver = FakeComputeDriver::new() + .with_nested_container_wildcard_fallback_listener_requirement( + "10.89.0.1:17670", + "external Podman bridge", + ); + let _server = driver.serve_uds(&socket_path).unwrap(); + let endpoint = connect_remote_compute_driver("podman", &socket_path) + .await + .unwrap(); + let runtime = ComputeRuntime::new_remote_driver( + endpoint, + Arc::new(Store::connect("sqlite::memory:").await.unwrap()), + SandboxIndex::new(), + SandboxWatchBus::new(), + TracingLogBus::new(), + Arc::new(SupervisorSessionRegistry::new()), + ) + .await + .unwrap(); + + assert_eq!( + runtime.gateway_listener_requirements(), + &[GatewayListenerRequirement::Exact { + address: "10.89.0.1:17670".parse().unwrap(), + driver_name: "podman".to_string(), + reason: "external Podman bridge".to_string(), + allow_nested_container_wildcard_fallback: false, + }] + ); + } + #[tokio::test] #[cfg(unix)] async fn remote_compute_driver_forwards_lifecycle_calls_over_uds() { @@ -9188,7 +9277,7 @@ mod tests { .with_driver_name("fake-remote-driver") .with_default_image("openshell/sandbox:remote") .with_gateway_manages_lifecycle() - .with_gateway_listener_requirement( + .with_nested_container_wildcard_fallback_listener_requirement( "172.19.0.1:17670", "external driver managed bridge", ); @@ -9214,6 +9303,7 @@ mod tests { address: "172.19.0.1:17670".parse().unwrap(), driver_name: "docker".to_string(), reason: "external driver managed bridge".to_string(), + allow_nested_container_wildcard_fallback: false, }] ); diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index 640757bfb3..3b2883673c 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -4,9 +4,12 @@ use crate::compute::GatewayListenerRequirement; use openshell_core::{Error, Result}; use socket2::{Domain, Protocol, Socket, Type}; +use std::io::ErrorKind; use std::net::{IpAddr, SocketAddr}; +#[cfg(target_os = "linux")] +use std::path::Path; use tokio::net::TcpListener; -use tracing::info; +use tracing::{info, warn}; /// Authorization scope associated with a gateway listener. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -27,6 +30,7 @@ pub struct GatewayListenerSpec { pub scope: GatewayListenerScope, covered_addresses: Vec, provenance: Option, + allows_nested_container_wildcard_fallback: bool, } /// Diagnostic source of a driver-requested listener. @@ -49,6 +53,7 @@ impl GatewayListenerSpec { scope, covered_addresses: Vec::new(), provenance: None, + allows_nested_container_wildcard_fallback: false, } } @@ -177,6 +182,13 @@ fn callback_listener_spec( address: SocketAddr, requirement: &GatewayListenerRequirement, ) -> GatewayListenerSpec { + let allows_nested_container_wildcard_fallback = matches!( + requirement, + GatewayListenerRequirement::Exact { + allow_nested_container_wildcard_fallback: true, + .. + } + ); GatewayListenerSpec { address, scope: GatewayListenerScope::ComputeDriverCallback, @@ -185,6 +197,7 @@ fn callback_listener_spec( driver_name: requirement.driver_name().to_string(), reason: requirement.reason().to_string(), }), + allows_nested_container_wildcard_fallback, } } @@ -272,9 +285,49 @@ pub async fn bind_gateway_listeners( ) && specs.iter().any(|candidate| { candidate.address.port() == spec.address.port() && candidate.address.is_ipv4() }); - let listener = bind_gateway_listener(spec.address, ipv6_only) - .await - .map_err(|e| Error::transport(format!("failed to bind to {}: {e}", spec.address)))?; + let listener = bind_gateway_listener(spec.address, ipv6_only).await; + let listener = match listener { + Ok(listener) => listener, + Err(err) => { + let Some(fallback_spec) = nested_podman_wildcard_fallback_spec( + &specs, + spec, + err.kind(), + running_in_linux_container(), + ) else { + return Err(Error::transport(format!( + "failed to bind to {}: {err}", + spec.address + ))); + }; + + // The wildcard cannot coexist with the already-bound loopback + // socket on the same port. Dropping the partial listener set is + // safe because none of it has been returned to the server yet. + drop(listeners); + let listener = bind_gateway_listener(fallback_spec.address, false) + .await + .map_err(|fallback_err| { + Error::transport(format!( + "failed to bind Podman callback address {} ({err}); scoped wildcard fallback {} also failed: {fallback_err}", + spec.address, fallback_spec.address + )) + })?; + let local_addr = listener.local_addr().unwrap_or(fallback_spec.address); + let fallback_spec = fallback_spec.bind_to(local_addr); + warn!( + address = %local_addr, + unavailable_callback_address = %spec.address, + listener_purpose = "nested-podman-callback-fallback", + authorization_scope = "primary-on-loopback; sandbox-callable-grpc-only-on-other-ipv4-interfaces", + "Podman bridge address is not available yet; gateway callback listener is exposed on all container IPv4 interfaces for this gateway process" + ); + return Ok(vec![BoundGatewayListener { + listener, + spec: fallback_spec, + }]); + } + }; let local_addr = listener.local_addr().unwrap_or(spec.address); match spec.scope { GatewayListenerScope::Primary => { @@ -308,6 +361,62 @@ pub async fn bind_gateway_listeners( Ok(listeners) } +fn nested_podman_wildcard_fallback_spec( + specs: &[GatewayListenerSpec], + failed_spec: &GatewayListenerSpec, + error_kind: ErrorKind, + running_in_container: bool, +) -> Option { + if !running_in_container || error_kind != ErrorKind::AddrNotAvailable || specs.len() != 2 { + return None; + } + + let primary = specs + .iter() + .find(|spec| spec.scope == GatewayListenerScope::Primary)?; + let callback = specs.iter().find(|spec| { + spec.scope == GatewayListenerScope::ComputeDriverCallback && *spec == failed_spec + })?; + let callback_provenance = callback.provenance.as_ref()?; + let (IpAddr::V4(primary_ip), IpAddr::V4(callback_ip)) = + (primary.address.ip(), callback.address.ip()) + else { + return None; + }; + if !primary_ip.is_loopback() + || !callback_ip.is_private() + || primary.address.port() == 0 + || primary.address.port() != callback.address.port() + || !callback.allows_nested_container_wildcard_fallback + || callback_provenance.driver_name != "podman" + { + return None; + } + + Some(GatewayListenerSpec { + address: SocketAddr::from((std::net::Ipv4Addr::UNSPECIFIED, primary.address.port())), + // The broader socket is callback-only by default. Only connections + // addressed to the original loopback endpoint retain primary scope. + scope: GatewayListenerScope::ComputeDriverCallback, + covered_addresses: vec![CoveredGatewayAddress { + address: primary.address, + scope: GatewayListenerScope::Primary, + }], + provenance: Some(callback_provenance.clone()), + allows_nested_container_wildcard_fallback: true, + }) +} + +#[cfg(target_os = "linux")] +fn running_in_linux_container() -> bool { + Path::new("/.dockerenv").exists() || Path::new("/run/.containerenv").exists() +} + +#[cfg(not(target_os = "linux"))] +fn running_in_linux_container() -> bool { + false +} + fn resolve_bound_covered_addresses( covered_addresses: &[CoveredGatewayAddress], requested_listener_addr: SocketAddr, @@ -376,9 +485,10 @@ mod tests { use super::{ GatewayListenerProvenance, GatewayListenerScope, GatewayListenerSpec, bind_gateway_listeners, gateway_listener_specs, - gateway_listener_specs_with_default_route_ip, + gateway_listener_specs_with_default_route_ip, nested_podman_wildcard_fallback_spec, }; use crate::compute::GatewayListenerRequirement; + use std::io::ErrorKind; use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::net::TcpListener; @@ -435,6 +545,7 @@ mod tests { scope: GatewayListenerScope::Primary, covered_addresses: Vec::new(), provenance: None, + allows_nested_container_wildcard_fallback: false, }, GatewayListenerSpec { address: docker, @@ -444,6 +555,7 @@ mod tests { driver_name: "docker".to_string(), reason: "managed bridge".to_string(), }), + allows_nested_container_wildcard_fallback: false, }, ] ); @@ -456,6 +568,7 @@ mod tests { address: "172.18.0.1:8080".parse().unwrap(), driver_name: "external-test".to_string(), reason: "external bridge".to_string(), + allow_nested_container_wildcard_fallback: false, }; let specs = gateway_listener_specs(primary, &[requirement]).unwrap(); @@ -624,6 +737,112 @@ mod tests { assert_eq!(specs[1].scope, GatewayListenerScope::ComputeDriverCallback); } + #[test] + fn nested_podman_fallback_is_callback_only_off_loopback() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let podman_gateway: SocketAddr = "10.89.0.1:8080".parse().unwrap(); + let specs = gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)]) + .unwrap(); + + let fallback = nested_podman_wildcard_fallback_spec( + &specs, + &specs[1], + ErrorKind::AddrNotAvailable, + true, + ) + .expect("eligible nested Podman bind failure should use the scoped fallback"); + + assert_eq!(fallback.address, "0.0.0.0:8080".parse().unwrap()); + assert_eq!( + fallback.scope_for_local_addr(primary), + GatewayListenerScope::Primary + ); + assert_eq!( + fallback.scope_for_local_addr(podman_gateway), + GatewayListenerScope::ComputeDriverCallback + ); + assert_eq!( + fallback.scope_for_local_addr("172.17.0.2:8080".parse().unwrap()), + GatewayListenerScope::ComputeDriverCallback + ); + } + + #[test] + fn nested_podman_fallback_rejects_unrelated_bind_failures() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let podman_gateway: SocketAddr = "10.89.0.1:8080".parse().unwrap(); + let specs = gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)]) + .unwrap(); + + assert!( + nested_podman_wildcard_fallback_spec( + &specs, + &specs[1], + ErrorKind::AddrNotAvailable, + false, + ) + .is_none(), + "the gateway must be running in a Linux container" + ); + assert!( + nested_podman_wildcard_fallback_spec(&specs, &specs[1], ErrorKind::AddrInUse, true,) + .is_none(), + "only a not-yet-present interface is eligible" + ); + + let docker_specs = + gateway_listener_specs(primary, &[docker_listener_requirement(podman_gateway)]) + .unwrap(); + assert!( + nested_podman_wildcard_fallback_spec( + &docker_specs, + &docker_specs[1], + ErrorKind::AddrNotAvailable, + true, + ) + .is_none(), + "the fallback must remain specific to Podman" + ); + + let untrusted_podman_requirement = GatewayListenerRequirement::Exact { + address: podman_gateway, + driver_name: "podman".to_string(), + reason: "external or rootless Podman address".to_string(), + allow_nested_container_wildcard_fallback: false, + }; + let untrusted_podman_specs = + gateway_listener_specs(primary, &[untrusted_podman_requirement]).unwrap(); + assert!( + nested_podman_wildcard_fallback_spec( + &untrusted_podman_specs, + &untrusted_podman_specs[1], + ErrorKind::AddrNotAvailable, + true, + ) + .is_none(), + "the trusted rootful managed-bridge marker is required" + ); + } + + #[test] + fn nested_podman_fallback_rejects_non_private_callback_address() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let public_callback: SocketAddr = "203.0.113.1:8080".parse().unwrap(); + let specs = + gateway_listener_specs(primary, &[podman_listener_requirement(public_callback)]) + .unwrap(); + + assert!( + nested_podman_wildcard_fallback_spec( + &specs, + &specs[1], + ErrorKind::AddrNotAvailable, + true, + ) + .is_none() + ); + } + #[tokio::test] async fn failed_bind_does_not_return_partially_bound_listeners() { let occupied_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -680,6 +899,7 @@ mod tests { address, driver_name: "docker".to_string(), reason: "managed bridge".to_string(), + allow_nested_container_wildcard_fallback: false, } } @@ -688,6 +908,7 @@ mod tests { address, driver_name: "podman".to_string(), reason: "Podman managed bridge".to_string(), + allow_nested_container_wildcard_fallback: true, } } @@ -711,6 +932,7 @@ mod tests { scope: GatewayListenerScope::Primary, covered_addresses: Vec::new(), provenance: None, + allows_nested_container_wildcard_fallback: false, } } @@ -727,6 +949,7 @@ mod tests { driver_name: driver_name.to_string(), reason: reason.to_string(), }), + allows_nested_container_wildcard_fallback: reason == "Podman managed bridge", } } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index fe889d7445..ce3c3cad28 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -848,11 +848,11 @@ async fn serve_gateway_listener( continue; } }; - let listener_scope = match stream.local_addr() { - Ok(local_addr) => spec.scope_for_local_addr(local_addr), + let (accepted_local_addr, listener_scope) = match stream.local_addr() { + Ok(local_addr) => (local_addr, spec.scope_for_local_addr(local_addr)), Err(e) => { debug!(error = %e, client = %addr, listen = %listen_addr, "Failed to inspect accepted local address"); - spec.scope + (listen_addr, spec.scope) } }; @@ -861,7 +861,7 @@ async fn serve_gateway_listener( spawn_gateway_connection( stream, addr, - listen_addr, + accepted_local_addr, listener_scope, service.clone(), tls_acceptor.clone(), @@ -1399,6 +1399,7 @@ impl ComputeDriverBuildContext<'_> { self.driver_name, driver, None, + compute::GatewayListenerFallbackPolicy::Deny, self.store, self.sandbox_index, self.sandbox_watch_bus, @@ -2553,6 +2554,7 @@ operator_namespace_label = "openshell.ai/workspace=true" address, driver_name: "docker".to_string(), reason: "managed bridge".to_string(), + allow_nested_container_wildcard_fallback: false, } } } diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index 957db46fab..278a914e07 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -140,6 +140,25 @@ impl FakeComputeDriver { .gateway_listener_requirements .push(GatewayListenerRequirement { reason: reason.into(), + allow_nested_container_wildcard_fallback: false, + selector: Some(Selector::ExactBindAddress(bind_address.into())), + }); + }); + self + } + + #[must_use] + pub fn with_nested_container_wildcard_fallback_listener_requirement( + self, + bind_address: impl Into, + reason: impl Into, + ) -> Self { + self.with_state(|state| { + state + .gateway_listener_requirements + .push(GatewayListenerRequirement { + reason: reason.into(), + allow_nested_container_wildcard_fallback: true, selector: Some(Selector::ExactBindAddress(bind_address.into())), }); }); diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 656ae43bb6..3552a49750 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -138,6 +138,19 @@ HTTP requests. A `PermissionDenied` response from an additional callback-only listener is expected for those requests. Do not broaden the primary listener to `0.0.0.0` solely to make sandbox callbacks reachable. +For rootful Podman nested inside a Linux container, the bridge gateway may not +exist until the first sandbox joins. If the exact private bridge bind fails +with `EADDRNOTAVAIL`, OpenShell falls back to one IPv4 wildcard socket for that +gateway process. This applies only to the managed bridge discovered by the +built-in Podman driver; rootless Podman, explicit `host_gateway_ip` values, and +external drivers remain exact-bind only. Loopback traffic still receives +primary scope; every other +IPv4 interface receives callback-only scope. The fallback therefore exposes +the sandbox-callable gRPC surface—not administrator, health, reflection, or +HTTP routes—on all IPv4 interfaces in the outer container namespace. Restrict +access to that namespace with the surrounding container network, and retain +mTLS and sandbox JWT authentication. + ## Docker Driver [Docker](https://www.docker.com/get-started/)-backed sandboxes run as containers on the gateway host. Use Docker for local development, single-machine gateways, and hosts that already use Docker Desktop or Docker Engine. diff --git a/e2e/nested-podman-callback.sh b/e2e/nested-podman-callback.sh new file mode 100755 index 0000000000..a02ac3ad73 --- /dev/null +++ b/e2e/nested-podman-callback.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Exercise the Release Canary callback topology with artifacts from the current +# checkout: Docker -> Fedora systemd container -> rootful Podman sandbox. + +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +CLI_BIN="${OPENSHELL_BIN:-}" +GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-}" +SUPERVISOR_IMAGE="${OPENSHELL_SUPERVISOR_IMAGE:-}" +SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" +FEDORA_IMAGE="${OPENSHELL_E2E_NESTED_FEDORA_IMAGE:-fedora:latest}" +CONTAINER_NAME="${OPENSHELL_E2E_NESTED_CONTAINER_NAME:-openshell-e2e-nested-podman-$$}" +SANDBOX_NAME="${OPENSHELL_E2E_NESTED_SANDBOX_NAME:-nested-podman-cb}" +GATEWAY_SERVICE_FILE="${OPENSHELL_E2E_GATEWAY_SERVICE_FILE:-${ROOT}/deploy/deb/openshell-gateway.service}" +GATEWAY_DEFAULT_CONFIG="${OPENSHELL_E2E_GATEWAY_DEFAULT_CONFIG:-${ROOT}/deploy/rpm/gateway.toml.default}" + +die() { + echo "ERROR: $*" >&2 + exit 2 +} + +command -v docker >/dev/null 2>&1 || die "docker is required" +[ -x "${CLI_BIN}" ] || die "OPENSHELL_BIN must point to an executable Linux CLI artifact" +[ -x "${GATEWAY_BIN}" ] || die "OPENSHELL_GATEWAY_BIN must point to an executable Linux gateway artifact" +[ -n "${SUPERVISOR_IMAGE}" ] || die "OPENSHELL_SUPERVISOR_IMAGE is required" +[ -f "${GATEWAY_SERVICE_FILE}" ] || die "gateway service file not found: ${GATEWAY_SERVICE_FILE}" +[ -f "${GATEWAY_DEFAULT_CONFIG}" ] || die "gateway default config not found: ${GATEWAY_DEFAULT_CONFIG}" + +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/openshell-e2e-nested-podman.XXXXXX")" +IMAGE_ARCHIVE="${WORKDIR}/images.tar" + +root_exec() { + docker exec --interactive "${CONTAINER_NAME}" env \ + HOME=/root \ + XDG_RUNTIME_DIR=/run/user/0 \ + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \ + "$@" +} + +diagnostics() { + if ! docker inspect "${CONTAINER_NAME}" >/dev/null 2>&1; then + return + fi + + echo "=== nested Fedora container ===" >&2 + docker inspect --format '{{json .State}}' "${CONTAINER_NAME}" >&2 || true + docker logs "${CONTAINER_NAME}" >&2 || true + echo "=== nested gateway journal ===" >&2 + docker exec "${CONTAINER_NAME}" \ + journalctl --no-pager -n 200 _SYSTEMD_USER_UNIT=openshell-gateway.service >&2 || true + echo "=== nested Podman state ===" >&2 + root_exec podman info >&2 || true + root_exec podman network inspect openshell-e2e-nested >&2 || true + root_exec podman ps --all >&2 || true +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + if [ "${status}" -ne 0 ]; then + diagnostics + fi + docker rm --force "${CONTAINER_NAME}" >/dev/null 2>&1 || true + rm -rf -- "${WORKDIR}" + exit "${status}" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "==> Pulling images used by nested Podman" +docker pull "${SUPERVISOR_IMAGE}" +docker pull "${SANDBOX_IMAGE}" +docker save --output "${IMAGE_ARCHIVE}" "${SUPERVISOR_IMAGE}" "${SANDBOX_IMAGE}" + +echo "==> Starting Fedora systemd container" +docker run --detach \ + --name "${CONTAINER_NAME}" \ + --privileged \ + --cgroupns=host \ + --tmpfs /run \ + --tmpfs /tmp \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:rw \ + "${FEDORA_IMAGE}" \ + bash -lc 'dnf install -y dbus-daemon iproute podman systemd && exec /usr/sbin/init' \ + >/dev/null + +for _ in $(seq 1 120); do + if docker exec "${CONTAINER_NAME}" systemctl list-units --no-pager >/dev/null 2>&1; then + break + fi + if [ "$(docker inspect --format '{{.State.Running}}' "${CONTAINER_NAME}")" != true ]; then + die "Fedora systemd container exited before systemd became reachable" + fi + sleep 1 +done +docker exec "${CONTAINER_NAME}" systemctl list-units --no-pager >/dev/null 2>&1 \ + || die "Fedora systemd container did not become reachable within 120 seconds" + +echo "==> Installing PR artifacts in nested Fedora" +docker exec "${CONTAINER_NAME}" install -d /usr/lib/systemd/user /usr/share/openshell +docker cp "${CLI_BIN}" "${CONTAINER_NAME}:/usr/bin/openshell" +docker cp "${GATEWAY_BIN}" "${CONTAINER_NAME}:/usr/bin/openshell-gateway" +docker cp "${GATEWAY_SERVICE_FILE}" \ + "${CONTAINER_NAME}:/usr/lib/systemd/user/openshell-gateway.service" +docker cp "${GATEWAY_DEFAULT_CONFIG}" \ + "${CONTAINER_NAME}:/usr/share/openshell/gateway.toml.default" +docker cp "${IMAGE_ARCHIVE}" "${CONTAINER_NAME}:/var/lib/openshell-e2e-images.tar" + +root_exec bash -s -- "${SUPERVISOR_IMAGE}" "${SANDBOX_IMAGE}" "${SANDBOX_NAME}" <<'EOF' +set -euo pipefail + +supervisor_image=$1 +sandbox_image=$2 +sandbox_name=$3 + +test -f /.dockerenv || { + echo "ERROR: nested Podman E2E must run inside a Docker container" >&2 + exit 1 +} + +# The RPM service is a root systemd user unit. A container has no login +# session, so start root's user manager explicitly, just as Release Canary does. +mkdir -p "${XDG_RUNTIME_DIR}" +chmod 0700 "${XDG_RUNTIME_DIR}" +systemctl start user-runtime-dir@0.service || true +systemctl start user@0.service +for _ in $(seq 1 30); do + if systemctl --user daemon-reload; then + break + fi + sleep 1 +done +systemctl --user daemon-reload +systemctl --user enable --now podman.socket + +for _ in $(seq 1 30); do + if podman info >/dev/null 2>&1; then + break + fi + sleep 1 +done +podman info >/dev/null +test "$(podman info --format '{{.Host.Security.Rootless}}')" = false +podman load --input /var/lib/openshell-e2e-images.tar >/dev/null + +install -d -m 0700 "${HOME}/.config/openshell" +install -m 0600 /usr/share/openshell/gateway.toml.default \ + "${HOME}/.config/openshell/gateway.toml" +cat >>"${HOME}/.config/openshell/gateway.toml" <"${HOME}/.config/openshell/gateway.env" +install -d -m 0700 "${HOME}/.config/openshell/gateways/openshell" +cat >"${HOME}/.config/openshell/gateways/openshell/metadata.json" <<'METADATA' +{ + "name": "openshell", + "gateway_endpoint": "https://127.0.0.1:17670", + "is_remote": false, + "gateway_port": 17670 +} +METADATA +printf 'openshell' >"${HOME}/.config/openshell/active_gateway" + +# Capture the startup ordering that caused the canary regression: Podman can +# report its future bridge gateway before that address exists in this namespace. +podman network create openshell-e2e-nested >/dev/null +gateway_ip="$(podman network inspect openshell-e2e-nested --format '{{(index .Subnets 0).Gateway}}')" +if ip -4 address show | grep -Fq "${gateway_ip}"; then + echo "ERROR: Podman bridge gateway ${gateway_ip} unexpectedly exists before sandbox creation" >&2 + exit 1 +fi +echo "==> Confirmed future Podman bridge gateway ${gateway_ip} is not assigned yet" + +systemctl --user enable --now openshell-gateway.service +for _ in $(seq 1 90); do + if openshell status 2>/dev/null | grep -q Connected; then + break + fi + sleep 1 +done +openshell status +journalctl --user -u openshell-gateway.service --no-pager \ + | grep -F 'listener_purpose="nested-podman-callback-fallback"' +ss -ltn | grep -F '0.0.0.0:17670' + +echo "==> Creating sandbox through the nested rootful Podman driver" +openshell sandbox create \ + --name "${sandbox_name}" \ + --detach \ + --no-auto-providers \ + --from "${sandbox_image}" + +callback_output="$(openshell sandbox exec -n "${sandbox_name}" -- printf nested-callback-ok)" +test "${callback_output}" = nested-callback-ok +ip -4 address show | grep -F "${gateway_ip}" +echo "==> Nested rootful Podman callback succeeded" +EOF diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index afa93f1b18..4bee7a0cd2 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -86,6 +86,12 @@ message GatewayListenerRequirement { // Untrusted human-readable driver rationale for diagnostics. string reason = 1; + // Indicates that an exact address belongs to a driver-managed bridge that + // may not exist until the first sandbox joins it. Gateways must honor this + // only for a trusted built-in driver and only while running in a nested + // container; external drivers cannot authorize a wildcard listener. + bool allow_nested_container_wildcard_fallback = 5; + oneof selector { // Concrete IP:port address requested by the driver. The port must match // the gateway's configured primary listener port. diff --git a/tasks/test.toml b/tasks/test.toml index 9f47d0dcda..6852f60232 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -110,6 +110,10 @@ run = "e2e/with-docker-gateway.sh uv run pytest -o python_files='test_*.py *_tes description = "Run Rust CLI e2e tests against a Podman-backed gateway" run = "e2e/rust/e2e-podman.sh" +["e2e:podman:nested-container"] +description = "Run the Release Canary Fedora/rootful-Podman callback topology with prebuilt PR artifacts" +run = "e2e/nested-podman-callback.sh" + ["e2e:oidc-pkce"] description = "Run Linux browser PKCE and RBAC e2e tests against Keycloak and a Podman gateway" run = [