fix(gateway): handle delayed Podman bridge listeners

.agents/skills/debug-openshell-cluster/SKILL.md

@@ -238,6 +238,14 @@ Common findings:
 - Current gateways reuse the primary listener when it covers Podman's callback
   address. If the primary does not cover that address, inspect the gateway
   startup logs for the additional callback-only listener and its provenance.
+- In a nested Linux container with rootful Podman, a missing bridge address can
+  trigger `listener_purpose="nested-podman-callback-fallback"`. Confirm the
+  failed exact address is the built-in driver's discovered managed bridge (not
+  rootless Podman, an explicit `host_gateway_ip`, or an external driver), the
+  bind error is `EADDRNOTAVAIL`, and the wildcard socket is callback-only off
+  loopback. The callback RPC surface is
+  reachable on every IPv4 interface in the outer container namespace for that
+  gateway process, so also inspect the surrounding container-network boundary.
 - Rootless slirp4netns, another named helper, or missing helper metadata
   requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip`
   cannot bypass slirp4netns host-loopback isolation. Do not work around
@@ -589,6 +597,7 @@ openshell logs <sandbox-name>
 | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs |
 | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs |
 | Gateway exits while resolving compute-driver listener requirements | Callback alias topology is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route |
+| Nested rootful Podman gateway logs `nested-podman-callback-fallback` | Podman reported a private bridge gateway before netavark assigned it | Confirm the exact bind failed with `EADDRNOTAVAIL`; verify non-loopback destinations receive callback-only scope and restrict the outer container network |
 | Admin, health, reflection, or HTTP request is denied on an additional Docker/Podman callback-only listener | Additional callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected |
 | Docker or Podman sandbox never registers | Wrong callback endpoint or supervisor startup failure | Gateway logs and sandbox container logs |
 | Docker GPU e2e fails before GPU sandbox comparison | NVIDIA CDI specs are missing or Docker has not discovered them | `docker info --format '{{json .DiscoveredDevices}}'`, `/etc/cdi`, `/var/run/cdi`, `nvidia-cdi-refresh.service` |

.agents/skills/test-release-canary/SKILL.md

@@ -82,6 +82,29 @@ Note `install.sh` is pulled from `raw.githubusercontent.com/NVIDIA/OpenShell/${h
 
 To smoke-test the chart for a specific dev build, dispatch `Release Dev` on the branch first, then run the kind canary steps locally pointed at the SHA-pinned chart (see "Local kind reproduction" below). The release-canary workflow itself does not currently expose `chart_version` / `image_tag` inputs.
 
+## PR-built Fedora callback reproduction
+
+For a PR-built reproduction of the Fedora callback topology, run the
+`e2e:podman:nested-container` task with `OPENSHELL_BIN`,
+`OPENSHELL_GATEWAY_BIN`, and `OPENSHELL_SUPERVISOR_IMAGE` set to Linux
+artifacts from the same revision. The `test:e2e` PR label enables this lane in
+`Branch E2E Checks`. It injects those artifacts into the same
+Docker-to-Fedora-to-rootful-Podman topology as the canary and verifies both
+gateway startup and a supervisor callback before published RPMs exist. The
+test also requires the future bridge address to be absent at gateway startup
+and asserts the `nested-podman-callback-fallback` listener and IPv4 wildcard
+socket before it creates the sandbox.
+
+On macOS, use `nix run .#test-guest -- --distro ubuntu --with docker` as the
+outer host when the local Docker daemon is unavailable. Cross-build matching
+Linux CLI and gateway binaries, copy the nested E2E script plus the gateway
+service/default-config assets with `--copy`, and set
+`OPENSHELL_E2E_GATEWAY_SERVICE_FILE` and
+`OPENSHELL_E2E_GATEWAY_DEFAULT_CONFIG` to their guest paths. Pin
+`OPENSHELL_SUPERVISOR_IMAGE` to the same revision as the binaries; using
+`supervisor:latest` can produce protocol or SSH behavior skew unrelated to the
+listener under test.
+
 ## Local kind reproduction
 
 The `kubernetes` job can be reproduced on any machine with Docker and `mise install`-provided `kubectl` + `helm`:

.github/workflows/branch-e2e.yml

@@ -137,6 +137,7 @@ jobs:
       cli-artifact-prefix: rust-binary-cli
       gateway-artifact-prefix: rust-binary-gateway
       vm-driver-artifact-name: driver-vm-linux-amd64
+      nested-podman-callback: true
 
   gpu-e2e:
     needs: [pr_metadata, build-gateway, build-supervisor, build-cli]

.github/workflows/e2e-test.yml

@@ -32,13 +32,56 @@ on:
         required: false
         type: string
         default: ""
+      nested-podman-callback:
+        description: "Run the Fedora-in-Docker/rootful-Podman callback regression lane (requires prebuilt CLI and gateway artifacts)"
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   actions: read
   contents: read
   packages: read
 
 jobs:
+  e2e-nested-podman-callback:
+    name: E2E (nested Fedora/rootful Podman callback)
+    if: inputs['nested-podman-callback']
+    runs-on: linux-amd64-cpu8
+    timeout-minutes: 30
+    container:
+      image: ghcr.io/nvidia/openshell/ci:latest
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      options: --privileged
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
+        - /home/runner/_work:/home/runner/_work
+    env:
+      OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs['checkout-ref'] || github.sha }}
+          persist-credentials: false
+
+      - name: Use prebuilt OpenShell CLI
+        uses: ./.github/actions/setup-e2e-cli
+        with:
+          artifact-prefix: ${{ inputs.cli-artifact-prefix }}
+
+      - name: Use prebuilt OpenShell gateway
+        uses: ./.github/actions/setup-e2e-gateway
+        with:
+          artifact-prefix: ${{ inputs.gateway-artifact-prefix }}
+
+      - name: Log in to GHCR with Docker
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+      - name: Run nested Podman callback E2E
+        run: e2e/nested-podman-callback.sh
+
   e2e:
     name: "E2E (${{ matrix.suite }})"
     runs-on: ${{ inputs.runner }}

CI.md

@@ -15,13 +15,14 @@ Merge queue validation is a second integration gate for `main`. After a PR has p
 Three opt-in labels enable the long-running E2E suites:
 
 - `test:e2e` runs the standard Docker, rootless Podman, Kubernetes, and VM E2E
-  suites in `Branch E2E Checks`
+  suites plus the Fedora-in-Docker/rootful-Podman callback regression lane in
+  `Branch E2E Checks`
 - `test:e2e-gpu` runs GPU E2E in `Branch E2E Checks`
 - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay
   (`replicaCount: 2` and bundled PostgreSQL) and the credential-driver suite
   (Kubernetes Secrets plus Vault) in `Branch E2E Checks`
 
-When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once, builds one CLI artifact per runner architecture, builds the Linux VM driver artifact once, and fans out all enabled suites in parallel. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E jobs reuse the matching prebuilt gateway and CLI binaries instead of compiling additional debug binaries in each job; Kubernetes E2E consumes the gateway image directly and reuses the prebuilt CLI. VM E2E also reuses the prebuilt VM driver artifact and falls back to local VM-driver/runtime preparation for local runs or workflow invocations that omit the artifact.
+When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once, builds one CLI artifact per runner architecture, builds the Linux VM driver artifact once, and fans out all enabled suites in parallel. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E jobs reuse the matching prebuilt gateway and CLI binaries instead of compiling additional debug binaries in each job; Kubernetes E2E consumes the gateway image directly and reuses the prebuilt CLI. The nested Podman lane injects those artifacts into a Fedora systemd container and creates a rootful Podman sandbox, reproducing the Release Canary callback topology without waiting for an RPM publication. VM E2E also reuses the prebuilt VM driver artifact and falls back to local VM-driver/runtime preparation for local runs or workflow invocations that omit the artifact.
 The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while Kubernetes HA and credential-driver behavior are under active iteration: failures are visible in the workflow run but do not publish a required CI gate status.
 
 The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly.

architecture/gateway.md

@@ -56,6 +56,21 @@ reflection, non-callback inference APIs, and HTTP routes before normal request
 authentication. The operator-configured primary listener retains the full
 multiplexed API surface.
 
+One Linux container startup race requires a narrower exception. Rootful Podman
+can report a private bridge gateway before netavark assigns that address. If an
+exact built-in Podman callback bind fails with `EADDRNOTAVAIL` while the gateway
+itself runs in a container, the gateway replaces the loopback primary and
+missing bridge sockets with one IPv4 wildcard socket. The wildcard defaults to
+callback-only authorization; only traffic addressed to the configured loopback
+endpoint receives primary scope. This keeps user and administrator APIs off the
+container's non-loopback interfaces, but it does make the sandbox-callable gRPC
+surface reachable on every IPv4 interface in that container namespace for the
+lifetime of the gateway process. Sandbox mTLS/JWT authentication and the RPC
+allowlist remain mandatory defenses. This exception does not apply on a host,
+to rootless Podman, an explicit `host_gateway_ip`, Docker or external drivers,
+to public callback addresses, or to bind errors
+other than `EADDRNOTAVAIL`.
+
 The `rpc_auth` classification is also the source of truth for negotiated
 listener exposure: marking an RPC as `sandbox` or `dual` makes it callable on
 these listeners. Review such changes as both authorization and network-surface

crates/openshell-driver-docker/src/lib.rs

@@ -1529,6 +1529,7 @@ impl ComputeDriver for DockerComputeDriver {
                             DockerGatewayRoute::HostGateway => "docker host-gateway IPv4 loopback",
                         }
                         .to_string(),
+                        allow_nested_container_wildcard_fallback: false,
                         selector: Some(Selector::ExactBindAddress(bind_address.to_string())),
                     }]
                 });

crates/openshell-driver-podman/NETWORKING.md

@@ -281,6 +281,20 @@ the supervisor's RPCs. Otherwise, it creates an additional listener that
 exposes only the gateway's sandbox-callable gRPC methods. Operator, health,
 reflection, and HTTP requests must use the primary listener.
 
+When both the gateway and rootful Podman run inside another Linux container,
+netavark may not assign the managed bridge gateway until the first sandbox
+joins the network. If that exact private callback address fails to bind with
+`EADDRNOTAVAIL`, the gateway uses one scoped IPv4 wildcard listener for that
+process. Only the in-process built-in Podman driver can mark its discovered
+rootful managed-bridge address as eligible; rootless Podman, explicit
+`host_gateway_ip` values, and external drivers cannot activate the fallback.
+Connections addressed to loopback retain primary scope; connections
+addressed to the Podman bridge or any other IPv4 interface are callback-only.
+This exposes the sandbox-callable gRPC surface on every IPv4 interface in the
+outer container namespace, so deployments should still restrict that namespace
+at the container-network boundary. It does not expose operator, health,
+reflection, or HTTP routes on those interfaces.
+
 ### Layer 3 Inner Sandbox Network Namespace
 
 Inside the container, the supervisor creates another network namespace for the

crates/openshell-driver-podman/src/driver.rs

@@ -517,6 +517,7 @@ impl PodmanComputeDriver {
                         reason:
                             "Podman rootless pasta callback uses the host default-route interface"
                                 .to_string(),
+                        allow_nested_container_wildcard_fallback: false,
                         selector: Some(Selector::DefaultRouteInterface(
                             GatewayDefaultRouteInterfaceRequirement {},
                         )),
@@ -542,6 +543,11 @@ impl PodmanComputeDriver {
             })?;
             Ok(vec![GatewayListenerRequirement {
                 reason: format!("Podman network '{}' host gateway", self.config.network_name),
+                // A rootful managed bridge can be created after gateway
+                // startup. An explicit override is operator-owned, and
+                // rootless networking must never broaden the listener.
+                allow_nested_container_wildcard_fallback: !self.rootless
+                    && self.config.host_gateway_ip.trim().is_empty(),
                 selector: Some(Selector::ExactBindAddress(
                     SocketAddr::new(gateway_ip, callback_port).to_string(),
                 )),
@@ -552,6 +558,7 @@ impl PodmanComputeDriver {
             Ok(vec![GatewayListenerRequirement {
                 reason: "Podman machine callback forwarding terminates on gateway loopback"
                     .to_string(),
+                allow_nested_container_wildcard_fallback: false,
                 selector: Some(Selector::LoopbackInterface(
                     GatewayLoopbackInterfaceRequirement {},
                 )),
@@ -2121,6 +2128,7 @@ mod tests {
         let requirements = driver.gateway_listener_requirements().unwrap();
 
         assert_eq!(requirements.len(), 1);
+        assert!(requirements[0].allow_nested_container_wildcard_fallback);
         assert_eq!(
             requirements[0].selector,
             Some(Selector::ExactBindAddress("10.89.1.1:17670".to_string()))
@@ -2145,6 +2153,22 @@ mod tests {
             requirements[0].selector,
             Some(Selector::ExactBindAddress("10.90.1.1:17670".to_string()))
         );
+        assert!(!requirements[0].allow_nested_container_wildcard_fallback);
+    }
+
+    #[test]
+    #[cfg(target_os = "linux")]
+    fn rootful_explicit_host_gateway_does_not_allow_wildcard_fallback() {
+        let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig {
+            grpc_endpoint: "http://host.containers.internal:17670".to_string(),
+            host_gateway_ip: "10.90.1.1".to_string(),
+            ..PodmanComputeConfig::default()
+        });
+        driver.network_gateway_ip = Some("10.89.1.1".to_string());
+
+        let requirements = driver.gateway_listener_requirements().unwrap();
+
+        assert!(!requirements[0].allow_nested_container_wildcard_fallback);
     }
 
     #[test]
@@ -2163,6 +2187,7 @@ mod tests {
             requirements[0].selector,
             Some(Selector::DefaultRouteInterface(_))
         ));
+        assert!(!requirements[0].allow_nested_container_wildcard_fallback);
     }
 
     #[test]