From c521ef0d1d83558e10451323b1844b54b35d5b7f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 8 Oct 2026 12:56:41 +0200 Subject: [PATCH] fix(podman): discover CDI GPUs from server info Signed-off-by: Evan Lezar --- crates/openshell-driver-podman/README.md | 18 ++ crates/openshell-driver-podman/src/client.rs | 47 ++++ crates/openshell-driver-podman/src/driver.rs | 235 +++++++++++++++++-- docs/how-it-works/gateways/configuration.mdx | 8 + docs/how-it-works/sandboxes/overview.mdx | 8 + docs/how-it-works/sandboxes/runtimes.mdx | 4 + skills/debug-openshell-cluster/SKILL.md | 1 + 7 files changed, 298 insertions(+), 23 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index f7ceed653b..e4f075b837 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -149,3 +149,21 @@ authenticated health signal. Gateway OTLP configuration continues to export compute-driver spans under the `openshell-driver-podman` service, preserving gateway trace context. + +## CDI GPU discovery + +The driver reads `host.cdiSpecDirs` and `host.discoveredDevices` from +`GET /v5.0.0/libpod/info` during initialization and before each default GPU +preflight or create selection. Only devices with `source: "cdi"` enter the CDI +inventory. The shared GPU selector retains its NVIDIA class and count policy; +collecting other CDI IDs does not add GPU workload support for other vendors. + +Presence of either CDI field means server discovery is authoritative. Podman +can omit `discoveredDevices` when its inventory is empty while still reporting +`cdiSpecDirs`; this yields an empty inventory rather than probing local devices. +When both fields are absent, legacy discovery scans local `/dev/nvidia` +paths and retains the `/dev/dxg` all-device fallback. Spec directories are +server paths for diagnostics, not directories to scan on the gateway host. +Refresh API or decoding failures fail the default GPU request without updating +the selector or selecting from stale inventory. Explicit CDI selectors continue +to be validated and passed to Podman without inventory lookup. diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 96b483a2da..26f3206a25 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -311,6 +311,11 @@ pub struct SystemInfo { #[derive(Debug, Clone, serde::Deserialize)] #[serde(rename_all = "camelCase")] pub struct HostInfo { + /// Presence marks servers that expose CDI discovery, including empty inventories. + #[serde(default)] + pub cdi_spec_dirs: Option>, + #[serde(default)] + pub discovered_devices: Option>, #[serde(default)] pub cgroup_version: String, #[serde(default)] @@ -321,6 +326,13 @@ pub struct HostInfo { pub security: SecurityInfo, } +/// A device discovered by the Podman server, independent of GPU selection policy. +#[derive(Debug, Clone, serde::Deserialize)] +pub struct DiscoveredDevice { + pub source: String, + pub id: String, +} + /// Security-related fields from the Podman system info response. /// /// Podman returns `host.security.rootless: true` when the daemon is @@ -1158,6 +1170,41 @@ mod tests { assert!(validate_name(&exact_name).is_ok()); } + #[tokio::test] + async fn system_info_retains_cdi_inventory_and_spec_directories() { + let (socket, requests, handle) = spawn_podman_stub( + "info-cdi", + vec![ + StubResponse::new( + StatusCode::OK, + r#"{"host":{"cdiSpecDirs":["/etc/cdi","/var/run/cdi"],"discoveredDevices":[{"source":"cdi","id":"example.com/gpu=test0"}]}}"#, + ), + StubResponse::new(StatusCode::OK, r#"{"host":{"cdiSpecDirs":["/etc/cdi"]}}"#), + StubResponse::new(StatusCode::OK, r#"{"host":{}}"#), + ], + ); + let client = PodmanClient::new(socket); + let host = client.system_info().await.unwrap().host; + assert_eq!( + host.cdi_spec_dirs.unwrap(), + vec!["/etc/cdi", "/var/run/cdi"] + ); + let devices = host.discovered_devices.unwrap(); + assert_eq!(devices[0].source, "cdi"); + assert_eq!(devices[0].id, "example.com/gpu=test0"); + let empty = client.system_info().await.unwrap().host; + assert!(empty.cdi_spec_dirs.is_some()); + assert!(empty.discovered_devices.is_none()); + let legacy = client.system_info().await.unwrap().host; + assert!(legacy.cdi_spec_dirs.is_none()); + assert!(legacy.discovered_devices.is_none()); + handle.await.unwrap(); + assert_eq!( + *requests.lock().unwrap(), + vec!["GET /v5.0.0/libpod/info"; 3] + ); + } + #[test] fn system_info_parses_rootless_network_helper() { let info: SystemInfo = serde_json::from_str( diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index b92ce99c95..83f60eaa1c 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -3,7 +3,7 @@ //! Podman compute driver. -use crate::client::{ContainerListEntry, PodmanApiError, PodmanClient, VolumeInspect}; +use crate::client::{ContainerListEntry, HostInfo, PodmanApiError, PodmanClient, VolumeInspect}; use crate::config::{PodmanComputeConfig, podman_image_pull_policy}; use crate::container::{self, LABEL_MANAGED_FILTER, LABEL_SANDBOX_ID, PodmanSandboxDriverConfig}; use crate::watcher::{ @@ -141,7 +141,8 @@ pub struct PodmanComputeDriver { client: PodmanClient, config: PodmanComputeConfig, gpu_selector: Arc, - gpu_inventory_refresh: Arc (CdiGpuInventory, bool) + Send + Sync>, + #[cfg(test)] + gpu_inventory_refresh: Option (CdiGpuInventory, bool) + Send + Sync>>, lifecycle_event_fences: LifecycleEventFences, } @@ -408,6 +409,26 @@ fn local_podman_gpu_selector_state() -> (CdiGpuInventory, bool) { ) } +fn podman_gpu_selector_state( + host: &HostInfo, + legacy_discovery: impl FnOnce() -> (CdiGpuInventory, bool), + allow_all_default_gpu: bool, +) -> (CdiGpuInventory, bool) { + if host.cdi_spec_dirs.is_none() && host.discovered_devices.is_none() { + return legacy_discovery(); + } + // New servers omit discoveredDevices when no devices were found. Never + // replace that authoritative empty inventory with guessed local devices. + let inventory = CdiGpuInventory::new( + host.discovered_devices + .iter() + .flatten() + .filter(|device| device.source == "cdi") + .map(|device| &device.id), + ); + (inventory, allow_all_default_gpu) +} + fn podman_gpu_selection_error(err: CdiGpuSelectionError) -> ComputeDriverError { ComputeDriverError::Precondition(err.to_string()) } @@ -497,7 +518,7 @@ impl PodmanComputeDriver { } // Verify cgroups v2, detect rootless mode, and log system info. - match client.system_info().await { + let (gpu_inventory, allow_all_default_gpu) = match client.system_info().await { Ok(info) => { if info.host.cgroup_version != "v2" { return Err(PodmanApiError::Connection(format!( @@ -517,15 +538,22 @@ impl PodmanComputeDriver { rootless = info.host.security.rootless, rootless_network_cmd = %info.host.rootless_network_cmd, apparmor_enabled = info.host.security.apparmor_enabled, + cdi_spec_dirs = ?info.host.cdi_spec_dirs, "Connected to Podman" ); + let (inventory, allow_all) = podman_gpu_selector_state( + &info.host, + local_podman_gpu_selector_state, + local_podman_all_gpu_default_supported(), + ); + (inventory, allow_all) } Err(e) => { return Err(PodmanApiError::Connection(format!( "failed to query Podman system info: {e}" ))); } - } + }; // Rootless pre-flight: warn if subuid/subgid ranges look missing. // Not a hard error because some systems configure these via LDAP or @@ -552,11 +580,10 @@ impl PodmanComputeDriver { client.ensure_network(&config.network_name).await?; info!(network = %config.network_name, "Podman network ready"); - let (gpu_inventory, allow_all_default_gpu) = local_podman_gpu_selector_state(); if !gpu_inventory.is_empty() { info!( device_count = gpu_inventory.as_slice().len(), - "Discovered local Podman NVIDIA CDI GPU devices" + "Discovered Podman NVIDIA CDI GPU devices" ); } @@ -567,7 +594,8 @@ impl PodmanComputeDriver { gpu_inventory, allow_all_default_gpu, )), - gpu_inventory_refresh: Arc::new(local_podman_gpu_selector_state), + #[cfg(test)] + gpu_inventory_refresh: None, lifecycle_event_fences: LifecycleEventFences::default(), }; let reconciler = driver.clone(); @@ -649,11 +677,13 @@ impl PodmanComputeDriver { driver_config.admit_mount_types(&self.config.resource_admission)?; Self::validate_gpu_request(gpu_requirements, &driver_config)?; self.validate_user_volume_mounts_available(sandbox).await?; - let _ = self.resolve_gpu_cdi_devices( - gpu_requirements, - &driver_config, - CdiGpuDefaultSelector::peek_device_ids, - )?; + let _ = self + .resolve_gpu_cdi_devices( + gpu_requirements, + &driver_config, + CdiGpuDefaultSelector::peek_device_ids, + ) + .await?; Ok(ValidatedPodmanSandbox { driver_config, gpu_requirements, @@ -678,12 +708,26 @@ impl PodmanComputeDriver { Ok(()) } - fn refresh_gpu_inventory(&self) { - let (inventory, allow_all_default_gpu) = (self.gpu_inventory_refresh)(); + async fn refresh_gpu_inventory(&self) -> Result<(), ComputeDriverError> { + #[cfg(test)] + if let Some(refresh) = &self.gpu_inventory_refresh { + let (inventory, allow_all) = refresh(); + self.gpu_selector.refresh(inventory, allow_all); + return Ok(()); + } + let info = self.client.system_info().await.map_err(|error| { + ComputeDriverError::Message(format!("failed to refresh Podman CDI inventory: {error}")) + })?; + let (inventory, allow_all_default_gpu) = podman_gpu_selector_state( + &info.host, + local_podman_gpu_selector_state, + local_podman_all_gpu_default_supported(), + ); self.gpu_selector.refresh(inventory, allow_all_default_gpu); + Ok(()) } - fn resolve_gpu_cdi_devices( + async fn resolve_gpu_cdi_devices( &self, gpu_requirements: Option<&GpuResourceRequirements>, driver_config: &PodmanSandboxDriverConfig, @@ -708,7 +752,7 @@ impl PodmanComputeDriver { return Ok(None); }; - self.refresh_gpu_inventory(); + self.refresh_gpu_inventory().await?; select_default_devices(&self.gpu_selector, count) .map(Some) .map_err(podman_gpu_selection_error) @@ -1152,11 +1196,14 @@ impl PodmanComputeDriver { async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { - let gpu_devices = match self.resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::next_device_ids, - ) { + let gpu_devices = match self + .resolve_gpu_cdi_devices( + validated.gpu_requirements, + &validated.driver_config, + CdiGpuDefaultSelector::next_device_ids, + ) + .await + { Ok(devices) => devices, Err(e) => { cleanup_created().await; @@ -1854,9 +1901,9 @@ impl PodmanComputeDriver { gpu_inventory, allow_all_default_gpu, )), - gpu_inventory_refresh: Arc::new(move || { + gpu_inventory_refresh: Some(Arc::new(move || { (refresh_inventory.clone(), allow_all_default_gpu) - }), + })), lifecycle_event_fences: LifecycleEventFences::default(), } } @@ -2840,6 +2887,148 @@ mod tests { .expect("an omitted profile preserves Podman's runtime behavior"); } + #[test] + fn podman_cdi_presence_controls_legacy_fallback() { + for (json, legacy) in [ + ("{}", true), + (r#"{"cdiSpecDirs":[]}"#, false), + (r#"{"cdiSpecDirs":["/etc/cdi"]}"#, false), + (r#"{"discoveredDevices":[]}"#, false), + ] { + let host: HostInfo = serde_json::from_str(json).unwrap(); + let (inventory, allow_all) = podman_gpu_selector_state( + &host, + || { + assert!(legacy, "supported empty inventory must not scan /dev"); + (CdiGpuInventory::new([CDI_GPU_DEVICE_ALL]), true) + }, + false, + ); + assert_eq!(inventory.is_empty(), !legacy); + assert_eq!(allow_all, legacy); + } + } + + #[tokio::test] + async fn legacy_podman_info_refresh_uses_local_inventory() { + let (socket, requests, handle) = spawn_podman_stub( + "cdi-legacy", + vec![StubResponse::new( + StatusCode::OK, + r#"{"host":{"cgroupVersion":"v2"}}"#, + )], + ); + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + socket_path: Some(socket), + ..PodmanComputeConfig::default() + }); + driver.gpu_inventory_refresh = None; + driver.refresh_gpu_inventory().await.unwrap(); + let (inventory, allow_all) = local_podman_gpu_selector_state(); + assert_eq!( + driver.gpu_selector.peek_device_ids(1), + CdiGpuDefaultSelector::new(inventory, allow_all).peek_device_ids(1) + ); + handle.await.unwrap(); + assert_eq!(*requests.lock().unwrap(), vec!["GET /v5.0.0/libpod/info"]); + } + + #[test] + fn server_all_only_inventory_preserves_platform_selection_policy() { + let host: HostInfo = serde_json::from_str(r#"{"cdiSpecDirs":["/etc/cdi"],"discoveredDevices":[{"source":"cdi","id":"nvidia.com/gpu=all"}]}"#).unwrap(); + for allow_all in [false, true] { + let (inventory, allowed) = podman_gpu_selector_state( + &host, + || panic!("unexpected legacy fallback"), + allow_all, + ); + let selected = CdiGpuDefaultSelector::new(inventory, allowed).peek_device_ids(1); + assert_eq!(selected.is_ok(), allow_all); + } + } + + #[tokio::test] + async fn podman_info_refreshes_default_gpu_selection_and_propagates_errors() { + let bodies = [ + r#"{"host":{"cdiSpecDirs":["/etc/cdi"],"discoveredDevices":[{"source":"cdi","id":"nvidia.com/gpu=0"},{"source":"other","id":"nvidia.com/gpu=9"},{"source":"cdi","id":"example.com/gpu=test0"}]}}"#, + r#"{"host":{"cdiSpecDirs":["/var/run/cdi"],"discoveredDevices":[{"source":"cdi","id":"nvidia.com/gpu=2"}]}}"#, + r#"{"host":{"cdiSpecDirs":["/etc/cdi"]}}"#, + r#"{"host":{"cdiSpecDirs":[],"discoveredDevices":[]}}"#, + ]; + let mut responses = bodies + .into_iter() + .map(|body| StubResponse::new(StatusCode::OK, body)) + .collect::>(); + responses.push(StubResponse::new( + StatusCode::OK, + r#"{"host":{"discoveredDevices":[{"source":"cdi","id":"nvidia.com/gpu=3"}]}}"#, + )); + responses.push(StubResponse::new( + StatusCode::INTERNAL_SERVER_ERROR, + "info unavailable", + )); + responses.push(StubResponse::new(StatusCode::OK, "invalid json")); + let (socket, requests, handle) = spawn_podman_stub("cdi-info", responses); + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + socket_path: Some(socket), + ..PodmanComputeConfig::default() + }); + driver.gpu_inventory_refresh = None; + let gpu = GpuResourceRequirements { count: Some(1) }; + let config = PodmanSandboxDriverConfig::default(); + // Preflight peeks at inventory; create refreshes and consumes selection. + let peek: fn(&CdiGpuDefaultSelector, u32) -> Result, CdiGpuSelectionError> = + CdiGpuDefaultSelector::peek_device_ids; + for (selector, expected) in [ + (peek, "nvidia.com/gpu=0"), + (CdiGpuDefaultSelector::next_device_ids, "nvidia.com/gpu=2"), + ] { + let selected = driver + .resolve_gpu_cdi_devices(Some(&gpu), &config, selector) + .await + .unwrap() + .unwrap(); + assert_eq!(selected, vec![expected.to_string()]); + } + for _ in 0..2 { + let error = driver + .resolve_gpu_cdi_devices( + Some(&gpu), + &config, + CdiGpuDefaultSelector::peek_device_ids, + ) + .await + .unwrap_err(); + assert!(matches!(error, ComputeDriverError::Precondition(_))); + } + driver.refresh_gpu_inventory().await.unwrap(); + for _ in 0..2 { + let error = driver + .resolve_gpu_cdi_devices( + Some(&gpu), + &config, + CdiGpuDefaultSelector::peek_device_ids, + ) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("failed to refresh Podman CDI inventory") + ); + // A failed query does not mutate inventory or permit stale selection. + assert_eq!( + driver.gpu_selector.peek_device_ids(1).unwrap(), + vec!["nvidia.com/gpu=3"] + ); + } + handle.await.unwrap(); + assert_eq!( + *requests.lock().unwrap(), + vec!["GET /v5.0.0/libpod/info"; 7] + ); + } + #[test] fn local_podman_cdi_gpu_inventory_maps_nvidia_device_nodes() { let root = std::env::temp_dir().join(format!( diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index f7f1df8b5c..bdd0e84423 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1018,6 +1018,14 @@ path is never mounted into or exposed to the workload container. ### Podman +Default GPU selection reads CDI devices from Podman's server info at startup +and refreshes them before validation and creation. `host.cdiSpecDirs` identifies +the server's spec directories; OpenShell does not scan these paths locally. +Presence of either `host.cdiSpecDirs` or `host.discoveredDevices` makes the +server inventory authoritative, including an empty result. Only older servers +that omit both fields use local device-path discovery. An info API failure +fails the default GPU request instead of selecting stale devices. + Each Podman sandbox uses two containers. The workload container runs `openshell-sandbox` with `network=none`; the supervisor container runs on the host network and initiates policy-approved upstream connections. A private volume carries their authenticated Unix-domain socket. Configure the supervisor gateway CA once under `[openshell.gateway]`; the gateway validates and injects it into the selected local driver. The supervisor authenticates RPCs with a sandbox bearer token. Caller `cdi_devices` entries must be valid CDI qualified names diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 2e824c730e..739ea599a3 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -154,6 +154,14 @@ be reflected in later sandbox creates. On WSL2 all-only runtimes, the default can fall back to `nvidia.com/gpu=all`; that fallback counts as one selectable device. +Podman uses the server's `host.discoveredDevices` CDI inventory and reports its +`host.cdiSpecDirs` in gateway connection logs. An empty server inventory remains +empty, including when Podman omits `discoveredDevices` but returns `cdiSpecDirs`. +Servers that omit both fields use legacy discovery from local `/dev/nvidia` +and `/dev/dxg` paths. Inventory query failures fail the default GPU request; +OpenShell retries discovery on subsequent requests. Automatic selection retains +the NVIDIA GPU policy. + Exact GPU device selection is driver-specific and still requires `--gpu`. For Docker or Podman, pass CDI IDs through `cdi_devices`. The top-level key must match the active driver; replace `docker` with `podman` when using Podman. CDI diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 382e800e27..0226900c0e 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -191,6 +191,10 @@ On macOS, set `host_gateway_ip` only if your Podman machine uses a non-standard For networks that require a corporate proxy, set `https_proxy`, `no_proxy`, and related `proxy_*` options. See the [Gateway Configuration File](/how-it-works/gateways/configuration) reference. +Default GPU requests use Podman's reported CDI inventory when available. +See [GPU resources](/how-it-works/sandboxes/overview#gpu-resources) for refresh +behavior and compatibility with older Podman servers. + ### Podman Mounts Podman supports the same `volume`, `tmpfs`, and `bind` mounts as [Docker](#docker-mounts), using the `podman` key in driver config, and the same bind-mount warning applies. Podman `volume` mounts do not support `subpath`. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 635e7fe932..8c365c16cb 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -380,6 +380,7 @@ openshell status Common findings: +- For default GPU discovery, inspect the server inventory with `podman info --format '{{json .Host}}'`. Check `cdiSpecDirs` and devices whose `source` is `cdi` in `discoveredDevices`; the directories belong to the Podman server. An omitted device list with present spec directories means no discovered devices. OpenShell refreshes info before default GPU validation and creation and fails that request if info cannot be queried. Only servers omitting both CDI fields use legacy local `/dev/nvidia` and `/dev/dxg` discovery. Automatic GPU selection retains the NVIDIA class policy. Check generated specs in the reported server directories when devices are missing; later requests pick up spec changes. - Podman socket unavailable: start or expose the user socket. - Rootless networking unavailable: inspect Podman network configuration. - Sandbox image missing or pull denied: verify image reference and registry credentials.