feat(supervisor): export OTLP traces from sandbox supervisors

When the gateway exports OTLP traces, compute drivers pass the gateway's
endpoint to supervisors as OPENSHELL_OTLP_ENDPOINT, along with TRACEPARENT
from the operation that launched them. The Kubernetes, Docker, Podman, and
VM drivers all receive the endpoint from the gateway; driver TOML cannot
set it. On Podman Machine, the Podman driver points a loopback endpoint
at host.containers.internal, since supervisor loopback is the VM's.
The supervisor exports spans as openshell-supervisor, tagged with
the sandbox ID, and flushes them before exiting.

supervisor.startup joins the sandbox's creation trace and covers image
policy discovery, policy load, and boundary attach, confirm, agent start,
and access start. Supervisor calls to the gateway carry W3C trace context,
so the gateway's server spans nest under them.

Each egress connection emits supervisor.egress.connect with authorize,
resolve, and dial children. These spans use DEBUG level because every
outbound connection starts its own trace. OpenShell spans export at INFO,
or at the sandbox log level when it is debug or trace; spans from other
libraries export at INFO.

Signed-off-by: Kris Hicks <khicks@nvidia.com>

Author: krishicks

Cargo.lock

@@ -40,6 +40,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "std"
 reqwest = { workspace = true, features = ["blocking", "rustls-tls-native-roots-no-provider"], optional = true }
 tar = { version = "0.4", optional = true }
 tempfile = { version = "3", optional = true }
+openshell-otel = { path = "../openshell-otel", optional = true }
 
 [target.'cfg(unix)'.dependencies]
 nix = { workspace = true }
@@ -56,6 +57,8 @@ telemetry = ["dep:reqwest", "dep:chrono", "reqwest?/blocking"]
 oauth = ["dep:reqwest"]
 ## Strict Ed25519 JWT issuance and verification for authenticated sandbox sessions.
 jwt = ["dep:jsonwebtoken", "dep:zeroize"]
+## Propagate the active W3C trace context on supervisor gateway calls.
+trace-context = ["dep:openshell-otel"]
 
 [build-dependencies]
 tonic-prost-build = { workspace = true }

crates/openshell-core/Cargo.toml

@@ -195,6 +195,9 @@ impl tonic::service::Interceptor for AuthInterceptor {
             .expect("auth interceptor token slot poisoned")
             .clone();
         req.metadata_mut().insert("authorization", bearer);
+        #[cfg(feature = "trace-context")]
+        let req =
+            tonic::service::Interceptor::call(&mut openshell_otel::TraceContextInterceptor, req)?;
         Ok(req)
     }
 }
@@ -263,6 +266,34 @@ async fn build_plain_channel(endpoint: &str) -> Result<Channel> {
         .wrap_err("failed to connect to OpenShell server")
 }
 
+/// Marks the current client span failed unless the call finishes successfully.
+struct ClientSpanStatus {
+    span: tracing::Span,
+    finished: bool,
+}
+
+impl ClientSpanStatus {
+    fn current() -> Self {
+        Self {
+            span: tracing::Span::current(),
+            finished: false,
+        }
+    }
+
+    fn finish<T>(mut self, result: Result<T>) -> Result<T> {
+        self.finished = result.is_ok();
+        result
+    }
+}
+
+impl Drop for ClientSpanStatus {
+    fn drop(&mut self) {
+        if !self.finished {
+            self.span.record("otel.status_code", "ERROR");
+        }
+    }
+}
+
 /// Build a Bearer-authenticated channel to the gateway.
 ///
 /// First call per process resolves the sandbox JWT via the three-step
@@ -901,13 +932,19 @@ pub async fn fetch_policy(
 /// this snapshot instead of re-fetching metadata after policy construction.
 /// The snapshot also carries the external middleware registrations required
 /// by the policy.
+#[tracing::instrument(
+    name = "supervisor.gateway.fetch_settings_snapshot",
+    skip_all,
+    fields(otel.kind = "client", otel.status_code = tracing::field::Empty)
+)]
 pub async fn fetch_settings_snapshot(
     endpoint: &str,
     sandbox_name: &str,
 ) -> Result<SettingsPollResult> {
+    let status = ClientSpanStatus::current();
     debug!(endpoint = %endpoint, sandbox_name = %sandbox_name, "Connecting to fetch OpenShell settings snapshot");
     let mut client = connect(endpoint).await?;
-    fetch_settings_snapshot_with_client(&mut client, sandbox_name, None).await
+    status.finish(fetch_settings_snapshot_with_client(&mut client, sandbox_name, None).await)
 }
 
 async fn fetch_settings_snapshot_with_client(
@@ -1014,19 +1051,30 @@ pub async fn sync_policy(
 }
 
 /// Sync an enriched policy and return the authoritative revision snapshot.
+#[tracing::instrument(
+    name = "supervisor.gateway.sync_policy_and_fetch_snapshot",
+    skip_all,
+    fields(otel.kind = "client", otel.status_code = tracing::field::Empty)
+)]
 pub async fn sync_policy_and_fetch_snapshot(
     endpoint: &str,
     sandbox: &str,
     policy: &ProtoSandboxPolicy,
     workspace: &str,
 ) -> Result<SettingsPollResult> {
+    let status = ClientSpanStatus::current();
     let mut client = connect(endpoint).await?;
     sync_policy_with_client(&mut client, sandbox, policy, workspace).await?;
-    fetch_settings_snapshot_with_client(&mut client, sandbox, Some(workspace)).await
+    status.finish(fetch_settings_snapshot_with_client(&mut client, sandbox, Some(workspace)).await)
 }
 
 /// Report an exact runtime configuration generation. Pending registration uses
 /// the snapshot's instance fence; retain that snapshot across registration retries.
+#[tracing::instrument(
+    name = "supervisor.gateway.report_sandbox_configuration",
+    skip_all,
+    fields(otel.kind = "client", otel.status_code = tracing::field::Empty)
+)]
 pub async fn report_sandbox_configuration(
     endpoint: &str,
     sandbox_id: &str,
@@ -1035,6 +1083,7 @@ pub async fn report_sandbox_configuration(
     state: crate::proto::ConfigurationAdmissionState,
     error: &str,
 ) -> Result<()> {
+    let status = ClientSpanStatus::current();
     let mut client = connect(endpoint).await?;
     client
         .report_sandbox_configuration(crate::proto::ReportSandboxConfigurationRequest {
@@ -1056,18 +1105,24 @@ pub async fn report_sandbox_configuration(
         })
         .await
         .map_err(grpc_status_error)?;
-    Ok(())
+    status.finish(Ok(()))
 }
 
 /// Fetch provider environment variables for a sandbox from `OpenShell` server via gRPC.
 ///
 /// Returns the credential snapshot and its exact readiness identity. An empty
 /// environment represents a sandbox without provider credentials. Transport
 /// failure returns an error so callers can revoke credentials and retry.
+#[tracing::instrument(
+    name = "supervisor.gateway.fetch_provider_environment",
+    skip_all,
+    fields(otel.kind = "client", otel.status_code = tracing::field::Empty)
+)]
 pub async fn fetch_provider_environment(
     endpoint: &str,
     sandbox_id: &str,
 ) -> Result<ProviderEnvironmentResult> {
+    let status = ClientSpanStatus::current();
     debug!(endpoint = %endpoint, sandbox_id = %sandbox_id, "Fetching provider environment");
 
     let mut client = connect(endpoint).await?;
@@ -1080,7 +1135,7 @@ pub async fn fetch_provider_environment(
         .await
         .map_err(grpc_status_error)?;
 
-    provider_environment_result(response.into_inner())
+    status.finish(provider_environment_result(response.into_inner()))
 }
 
 /// Preserve snapshot authority and reject invalid credential expiration times.
@@ -1181,13 +1236,19 @@ mod provider_environment_tests {
     }
 }
 
+#[tracing::instrument(
+    name = "supervisor.gateway.exchange_provider_subject_token",
+    skip_all,
+    fields(otel.kind = "client", otel.status_code = tracing::field::Empty)
+)]
 pub async fn exchange_provider_subject_token(
     endpoint: &str,
     sandbox_id: &str,
     provider: &str,
     credential_key: &str,
     supervisor_jwt_svid: &str,
 ) -> Result<ProviderSubjectTokenExchangeResult> {
+    let status = ClientSpanStatus::current();
     debug!(
         endpoint = %endpoint,
         sandbox_id = %sandbox_id,
@@ -1216,11 +1277,11 @@ pub async fn exchange_provider_subject_token(
         .map_or(0, |value| {
             i64::try_from(value.as_secs()).unwrap_or(i64::MAX)
         });
-    Ok(ProviderSubjectTokenExchangeResult {
+    status.finish(Ok(ProviderSubjectTokenExchangeResult {
         access_token: inner.access_token,
         expires_in,
         token_type: inner.token_type,
-    })
+    }))
 }
 
 fn provider_subject_token_exchange_status(status: Status) -> miette::Report {

crates/openshell-core/src/grpc_client.rs

@@ -26,6 +26,9 @@ pub const SSH_SOCKET_PATH: &str = "OPENSHELL_SSH_SOCKET_PATH";
 /// Log level for the sandbox supervisor (e.g. `"debug"`, `"info"`, `"warn"`).
 pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL";
 
+/// OTLP/gRPC collector endpoint for supervisor trace export.
+pub const OTLP_ENDPOINT: &str = "OPENSHELL_OTLP_ENDPOINT";
+
 /// Versioned specification for the exact canonical main process.
 ///
 /// Most drivers use JSON directly. Transports that cannot preserve spaces in

crates/openshell-core/src/sandbox_env.rs

@@ -178,6 +178,11 @@ pub struct DockerComputeConfig {
     /// Gateway gRPC endpoint the sandbox connects back to.
     pub grpc_endpoint: String,
 
+    /// OTLP/gRPC collector endpoint passed to supervisors. The gateway
+    /// supplies its own export endpoint; driver TOML cannot set it.
+    #[serde(skip)]
+    pub supervisor_otlp_endpoint: Option<String>,
+
     /// Image containing the trusted `openshell-sandbox` binary.
     pub sandbox_runtime_image: Option<String>,
 
@@ -276,6 +281,7 @@ impl Default for DockerComputeConfig {
             image_pull_policy: ImagePullPolicy::default(),
             sandbox_label: "default".to_string(),
             grpc_endpoint: String::new(),
+            supervisor_otlp_endpoint: None,
             sandbox_runtime_image: None,
             supervisor_bin: None,
             supervisor_image: None,
@@ -310,6 +316,7 @@ struct DockerDriverRuntimeConfig {
     sandbox_binary: Arc<Vec<u8>>,
     supervisor_image_id: String,
     supervisor_grpc_endpoint: String,
+    supervisor_otlp_endpoint: Option<String>,
     ssh_socket_path: String,
     guest_tls: Option<DockerGuestTlsPaths>,
     gpu: DockerGpuRuntimeCapabilities,
@@ -939,6 +946,7 @@ impl DockerComputeDriver {
                 sandbox_binary,
                 supervisor_image_id,
                 supervisor_grpc_endpoint,
+                supervisor_otlp_endpoint: docker_config.supervisor_otlp_endpoint.clone(),
                 ssh_socket_path: docker_config.ssh_socket_path.clone(),
                 guest_tls,
                 gpu,
@@ -5093,6 +5101,7 @@ async fn spawn_docker_control_process(
             openshell_core::telemetry::enabled_env_value()
         ),
     ];
+    environment.extend(supervisor_tracing_environment(config));
     if config.guest_tls.is_some() {
         environment.push(format!(
             "{}={SUPERVISOR_STATE_MOUNT_PATH}/tls/ca.pem",
@@ -5534,6 +5543,17 @@ fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap<String, String>
     environment
 }
 
+/// Environment that lets the supervisor export spans and join the current trace.
+fn supervisor_tracing_environment(config: &DockerDriverRuntimeConfig) -> Vec<String> {
+    let Some(endpoint) = &config.supervisor_otlp_endpoint else {
+        return Vec::new();
+    };
+    std::iter::once((openshell_core::sandbox_env::OTLP_ENDPOINT, endpoint.clone()))
+        .chain(openshell_otel::current_trace_context_environment())
+        .map(|(name, value)| format!("{name}={value}"))
+        .collect()
+}
+
 fn build_boundary_environment(
     sandbox: &DriverSandbox,
     config: &DockerDriverRuntimeConfig,

crates/openshell-driver-docker/src/lib.rs

@@ -77,6 +77,7 @@ async fn main() -> Result<()> {
     if let Some(image) = args.supervisor_image {
         docker_config.supervisor_image = Some(image);
     }
+    docker_config.supervisor_otlp_endpoint = args.otlp_endpoint.clone();
     let driver = DockerComputeDriver::new(args.gateway_bind, &args.log_level, &docker_config)
         .await
         .into_diagnostic()?;

crates/openshell-driver-docker/src/main.rs

@@ -210,6 +210,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig {
         sandbox_binary: Arc::new(b"\x7fELFtest".to_vec()),
         supervisor_image_id: "sha256:supervisor-test".to_string(),
         supervisor_grpc_endpoint: "https://host.openshell.internal:8443".to_string(),
+        supervisor_otlp_endpoint: None,
         ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(),
         guest_tls: Some(DockerGuestTlsPaths {
             ca: PathBuf::from("/tmp/ca.crt"),
@@ -1389,6 +1390,21 @@ fn docker_child_environment_strips_supervisor_control_keys() {
     assert!(!env.values().any(|value| value == "spoofed"));
 }
 
+#[test]
+fn supervisor_tracing_environment_requires_an_endpoint() {
+    let mut config = runtime_config();
+    assert!(supervisor_tracing_environment(&config).is_empty());
+
+    config.supervisor_otlp_endpoint = Some("http://127.0.0.1:4317".to_string());
+    assert_eq!(
+        supervisor_tracing_environment(&config),
+        [format!(
+            "{}=http://127.0.0.1:4317",
+            openshell_core::sandbox_env::OTLP_ENDPOINT
+        )]
+    );
+}
+
 #[test]
 fn boundary_environment_contains_only_driver_owned_values() {
     let env = build_boundary_environment(&test_sandbox(), &runtime_config());

crates/openshell-driver-docker/src/tests.rs

@@ -79,6 +79,12 @@ The supervisor Pod has a direct, non-controller owner reference to the Sandbox
 resource. This links its garbage-collection lifecycle to the sandbox without
 competing with the Agent Sandbox controller for workload-Pod ownership.
 
+When the gateway exports OTLP traces, the driver sets
+`OPENSHELL_OTLP_ENDPOINT` on the supervisor Pod to the gateway's endpoint and
+`TRACEPARENT` to the trace context of the operation that created the Pod. The
+supervisor exports its spans there and parents its startup span on that
+context. The endpoint is not configurable in driver TOML.
+
 The driver creates one namespace-wide `NetworkPolicy` before it releases any
 workload Pod. It selects every OpenShell workload, denies all workload egress,
 and permits OpenShell supervisor Pods to reach the sandbox TLS port. The

crates/openshell-driver-kubernetes/README.md

@@ -244,6 +244,10 @@ pub struct KubernetesComputeConfig {
     /// contain at least one usable trust anchor.
     pub proxy_ca_bundle: Option<String>,
     pub grpc_endpoint: String,
+    /// OTLP/gRPC collector endpoint passed to supervisor pods. The gateway
+    /// supplies its own export endpoint; driver TOML cannot set it.
+    #[serde(skip)]
+    pub supervisor_otlp_endpoint: Option<String>,
     pub ssh_socket_path: String,
     pub client_tls_secret_name: String,
     pub host_gateway_ip: String,
@@ -352,6 +356,7 @@ impl Default for KubernetesComputeConfig {
             proxy_connect_by_hostname: None,
             proxy_ca_bundle: None,
             grpc_endpoint: String::new(),
+            supervisor_otlp_endpoint: None,
             ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(),
             client_tls_secret_name: String::new(),
             host_gateway_ip: String::new(),