fix(supervisor): report isolation backend Attach refusals to the gateway

When an isolation backend refused at Attach, the supervisor exited without
telling the gateway. The configuration it had reported Accepted before Attach
stayed accepted, so the sandbox showed ConfigurationReady: True, and the
revision it wrote at startup stayed Pending with no error.

ReportSandboxConfigurationRequest gains refused_before_start. On a Denied or
Attach refusal the supervisor reports the generation it accepted as Rejected
with the flag set, and the gateway records the configuration as rejected with
gateway-authored text. Unflagged reports keep today's behavior, so a runtime
that fails to apply the generation it is running stays accepted. The
supervisor also marks the revision this start wrote as Failed with the
backend's message. A revision the user wrote and transient errors are left
alone.

Closes #4739

Signed-off-by: Shiju <shiju@nvidia.com>

Author: shiju-nv

crates/openshell-core/src/grpc_client.rs

@@ -1074,11 +1074,6 @@ pub async fn sync_policy_and_fetch_snapshot(
 
 /// Report an exact runtime configuration generation. Pending registration uses
 /// the snapshot's instance fence; retain that snapshot across registration retries.
-#[tracing::instrument(
-    name = "supervisor.gateway.report_sandbox_configuration",
-    skip_all,
-    fields(otel.kind = "client", otel.status_code = tracing::field::Empty)
-)]
 pub async fn report_sandbox_configuration(
     endpoint: &str,
     sandbox_id: &str,
@@ -1087,10 +1082,9 @@ pub async fn report_sandbox_configuration(
     state: crate::proto::ConfigurationAdmissionState,
     error: &str,
 ) -> Result<()> {
-    let status = ClientSpanStatus::current();
-    let mut client = connect(endpoint).await?;
-    client
-        .report_sandbox_configuration(crate::proto::ReportSandboxConfigurationRequest {
+    report_sandbox_configuration_request(
+        endpoint,
+        crate::proto::ReportSandboxConfigurationRequest {
             sandbox_id: sandbox_id.to_string(),
             expected_instance_id: snapshot.map_or_else(String::new, |snapshot| {
                 snapshot.configuration_instance_id.clone()
@@ -1106,7 +1100,27 @@ pub async fn report_sandbox_configuration(
                     .map_or(0, |snapshot| snapshot.provider_env_revision),
                 error: error.to_string(),
             }),
-        })
+            refused_before_start: false,
+        },
+    )
+    .await
+}
+
+/// Send a configuration report the caller built itself, such as a refusal of
+/// the generation this instance already accepted.
+#[tracing::instrument(
+    name = "supervisor.gateway.report_sandbox_configuration",
+    skip_all,
+    fields(otel.kind = "client", otel.status_code = tracing::field::Empty)
+)]
+pub async fn report_sandbox_configuration_request(
+    endpoint: &str,
+    request: crate::proto::ReportSandboxConfigurationRequest,
+) -> Result<()> {
+    let status = ClientSpanStatus::current();
+    let mut client = connect(endpoint).await?;
+    client
+        .report_sandbox_configuration(request)
         .await
         .map_err(grpc_status_error)?;
     status.finish(Ok(()))

crates/openshell-server/src/grpc/policy.rs

@@ -4990,6 +4990,26 @@ fn configuration_failure_diagnostic(error: &Status) -> &'static str {
     }
 }
 
+/// Both admissions name the same instance and configuration generation.
+fn same_generation(
+    current: &openshell_core::proto::SandboxConfigurationAdmission,
+    reported: &openshell_core::proto::SandboxConfigurationAdmission,
+) -> bool {
+    (
+        &current.instance_id,
+        current.policy_version,
+        &current.policy_hash,
+        current.config_revision,
+        current.provider_env_revision,
+    ) == (
+        &reported.instance_id,
+        reported.policy_version,
+        &reported.policy_hash,
+        reported.config_revision,
+        reported.provider_env_revision,
+    )
+}
+
 fn configuration_generation_matches(
     admission: &openshell_core::proto::SandboxConfigurationAdmission,
     config: &GetSandboxConfigResponse,
@@ -5027,6 +5047,12 @@ pub(super) async fn handle_report_sandbox_configuration(
     if reported == ConfigurationAdmissionState::Unspecified {
         return Err(Status::invalid_argument("admission state is required"));
     }
+    let refused_before_start = request.get_ref().refused_before_start;
+    if refused_before_start && reported != ConfigurationAdmissionState::Rejected {
+        return Err(Status::invalid_argument(
+            "refused_before_start requires a rejected admission",
+        ));
+    }
     let Some(_mutation_guard) = state
         .compute
         .sandbox_mutation_guard_by_id(&sandbox_id)
@@ -5111,10 +5137,16 @@ pub(super) async fn handle_report_sandbox_configuration(
         if let Some(current) = current
             && current.state == i32::from(ConfigurationAdmissionState::Accepted)
         {
-            // A rejected desired update does not invalidate an accepted runtime.
-            let error = admission.error;
-            admission = current.clone();
-            admission.error = error;
+            if refused_before_start && same_generation(current, &admission) {
+                // The runtime refused what it accepted, so the workload never
+                // started on it. Runtime text stays out of public status.
+                admission.error = "The isolation backend refused this configuration before the workload started; create the sandbox again with a policy the backend accepts".to_string();
+            } else {
+                // A rejected desired update does not invalidate an accepted runtime.
+                let error = admission.error;
+                admission = current.clone();
+                admission.error = error;
+            }
         }
     } else {
         admission.error.clear();
@@ -8651,6 +8683,7 @@ mod tests {
                     sandbox_id: sandbox_id.to_string(),
                     admission: Some(admission),
                     expected_instance_id: String::new(),
+                    refused_before_start: false,
                 }),
                 sandbox_id,
             )
@@ -8788,6 +8821,134 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn configuration_admission_records_refusal_of_accepted_generation() {
+        use openshell_core::proto::{
+            ConfigurationAdmissionState as Admission, ReportSandboxConfigurationRequest,
+            SandboxConfigurationAdmission,
+        };
+        let state = test_server_state().await;
+        let sandbox_id = "sb-refused";
+        state
+            .store
+            .put_message(&test_sandbox(
+                sandbox_id,
+                "refused",
+                openshell_policy::restrictive_default_policy(),
+                Vec::new(),
+            ))
+            .await
+            .unwrap();
+        let instance_id = uuid::Uuid::new_v4().to_string();
+        let report = |admission, refused_before_start| {
+            with_sandbox(
+                Request::new(ReportSandboxConfigurationRequest {
+                    sandbox_id: sandbox_id.to_string(),
+                    admission: Some(admission),
+                    expected_instance_id: String::new(),
+                    refused_before_start,
+                }),
+                sandbox_id,
+            )
+        };
+        handle_report_sandbox_configuration(
+            &state,
+            report(
+                SandboxConfigurationAdmission {
+                    instance_id: instance_id.clone(),
+                    state: Admission::Pending.into(),
+                    ..Default::default()
+                },
+                false,
+            ),
+        )
+        .await
+        .unwrap();
+        let config = handle_get_sandbox_config(
+            &state,
+            with_sandbox(
+                Request::new(GetSandboxConfigRequest {
+                    name: "refused".to_string(),
+                    workspace_scope: Some(openshell_core::proto::workspace_selector(
+                        "default".to_string(),
+                    )),
+                }),
+                sandbox_id,
+            ),
+        )
+        .await
+        .unwrap()
+        .into_inner();
+        let accepted = SandboxConfigurationAdmission {
+            instance_id,
+            state: Admission::Accepted.into(),
+            policy_version: config.version,
+            policy_hash: config.policy_hash,
+            config_revision: config.config_revision,
+            provider_env_revision: config.provider_env_revision,
+            error: String::new(),
+        };
+        handle_report_sandbox_configuration(&state, report(accepted.clone(), false))
+            .await
+            .unwrap();
+        let readiness = || async {
+            let persisted = state
+                .store
+                .get_message::<Sandbox>(sandbox_id)
+                .await
+                .unwrap()
+                .unwrap();
+            let status = persisted.status.unwrap();
+            let ready = status
+                .conditions
+                .into_iter()
+                .find(|condition| condition.r#type == "ConfigurationReady")
+                .unwrap();
+            (status.configuration_admission.unwrap().state, ready)
+        };
+
+        // A runtime that fails to apply the generation it runs, such as a
+        // reconcile of unreachable middleware, keeps the accepted state.
+        let mut refused = accepted;
+        refused.state = Admission::Rejected.into();
+        refused.error = "backend text is not shown".to_string();
+        handle_report_sandbox_configuration(&state, report(refused.clone(), false))
+            .await
+            .unwrap();
+        let (admission_state, ready) = readiness().await;
+        assert_eq!(admission_state, i32::from(Admission::Accepted));
+        assert_eq!(ready.status, "True");
+
+        let mut flagged_acceptance = refused.clone();
+        flagged_acceptance.state = Admission::Accepted.into();
+        assert_eq!(
+            handle_report_sandbox_configuration(&state, report(flagged_acceptance, true))
+                .await
+                .unwrap_err()
+                .code(),
+            Code::InvalidArgument
+        );
+        let mut other_generation = refused.clone();
+        other_generation.policy_hash.push('x');
+        assert!(same_generation(&refused, &refused));
+        assert!(!same_generation(&refused, &other_generation));
+
+        handle_report_sandbox_configuration(&state, report(refused, true))
+            .await
+            .unwrap();
+        let (admission_state, ready) = readiness().await;
+        assert_eq!(admission_state, i32::from(Admission::Rejected));
+        assert_eq!(
+            (ready.status.as_str(), ready.reason.as_str()),
+            ("False", "ConfigurationInvalid")
+        );
+        assert!(
+            ready.message.contains("isolation backend refused"),
+            "{ready:?}"
+        );
+        assert!(!ready.message.contains("backend text"), "{ready:?}");
+    }
+
     #[test]
     fn configuration_diagnostic_is_bounded_and_removes_control_characters() {
         let diagnostic = bounded_configuration_diagnostic(&format!("rule\n{}", "é".repeat(1000)));
@@ -8829,6 +8990,7 @@ mod tests {
                         state: Admission::Pending.into(),
                         ..Default::default()
                     }),
+                    refused_before_start: false,
                 }),
                 sandbox_id,
             ),

crates/openshell-server/src/storage_proto.rs

@@ -156,8 +156,10 @@ mod tests {
     // Sandbox-wide grant reporting adds TokenGrantFailureObservation to both
     // closures and TokenGrantStatus to public reports. Older rows have no grant
     // record; absent report extensions retain compatibility without a rewrite.
+    // ReportSandboxConfigurationRequest field 4 adds a public-only refusal
+    // flag. Older supervisors omit it; durable schemas remain unchanged.
     const PUBLIC_RPC_SCHEMA_SHA256: &str =
-        "a91e2244a2c1a8e369efe85bd0e4f2769f0f29f6263137280fe38e8a8da3cc6e";
+        "dab6deeb6c4ef051efea068dd74d6807fe2b1212d9c21c40bd0dc8708b16ba28";
     const DURABLE_SCHEMA_SHA256: &str =
         "d4dc7abdb91b919db1f7be0d4652e30cbfda86f3b2ed9b59d5de1f76a2bf691d";
     const PUBLIC_DURABLE_OVERLAP_SHA256: &str =

crates/openshell-supervisor/src/backend_setup.rs

@@ -81,6 +81,10 @@ pub trait PreparedBackend: Send + Sync {
     ) -> std::result::Result<BuiltBackend, BackendError>;
 }
 
+/// Diagnostic code of an Attach error that is a backend refusal, as opposed to
+/// a transient failure the next start can retry.
+pub(crate) const ATTACH_REFUSED: &str = "openshell::backend::attach_refused";
+
 /// Created only after name and launch identity checks. Consuming attachment
 /// prevents reusing one prepared startup to build or attach a second client.
 pub struct SelectedBackend {
@@ -196,7 +200,12 @@ impl SelectedBackend {
                 },
             )
             .await
-            .map_err(|error| miette::miette!(error.to_string()))?;
+            .map_err(|error| match error {
+                BackendError::Denied(_) | BackendError::Attach(_) => {
+                    miette::miette!(code = ATTACH_REFUSED, "{error}")
+                }
+                error => miette::miette!("{error}"),
+            })?;
         Ok((bound, built.proxy_listener))
     }
 }

crates/openshell-supervisor/src/backend_setup/tests.rs

@@ -521,6 +521,23 @@ async fn attachment_receives_live_supervisor_services() {
     assert!(setup.observed.services.lock().unwrap().upgrade().is_none());
 }
 
+#[tokio::test]
+async fn attach_refusal_is_marked_as_a_refusal() {
+    let setup = TestSetup::new();
+    setup.observed.active.store(true, Ordering::SeqCst);
+    let error = setup
+        .select()
+        .attach(setup.services(), policy(), agent())
+        .await
+        .err()
+        .expect("an already attached boundary refuses attachment");
+    assert_eq!(
+        error.code().map(|code| code.to_string()).as_deref(),
+        Some(ATTACH_REFUSED)
+    );
+    assert!(error.to_string().contains("already attached"));
+}
+
 #[tokio::test]
 async fn constructed_backend_must_keep_selected_name() {
     let mut setup = TestSetup::new();