fix(api): scope workspace request IDs by target

Include the requested workspace name in create/delete admission keys while leaving workspace UUID guards unset. Cover cross-target UUID reuse, replay, and missing targets with server and live gateway regressions.

Merge the latest phase-two SDK fixes and preserve the approved interceptor replay changes.

Refs #3051.

Signed-off-by: Mrunal Patel <mrunalp@gmail.com>

Author: mrunalp

architecture/gateway.md

@@ -59,6 +59,8 @@ write conflicts attach `google.rpc.ErrorInfo` with a stable reason and current
 version when available. `google.rpc.RetryInfo` expresses a minimum retry delay;
 it does not establish that a mutation is safe to repeat. SDKs retain the original
 transport status, metadata, and unknown details alongside decoded fields.
+SDK deletion waits recognize missing-resource status through typed error wrappers
+without suppressing other failures.
 
 Ordinary user-callable unary mutations explicitly opt into durable request
 admission when the client supplies a UUID. Typed adapters
@@ -412,8 +414,10 @@ Public delete, membership-removal, and SSH-revocation responses use
 logical gateway deletion or revocation; it does not guarantee that downstream
 platform garbage collection has finished. Sandbox deletion returns `ACCEPTED`
 while its captured object ID remains in the store, and returns that ID so callers
-can distinguish the original sandbox from a same-name replacement. The existing
-owned deletion worker continues after request cancellation.
+can distinguish the original sandbox from a same-name replacement. Identity-aware
+SDK deletion waits complete on absence or a different observed ID; name-only waits
+continue until the name is absent. The existing owned deletion worker continues
+after request cancellation.
 
 Missing targets return `NOT_FOUND` unless `allow_missing` explicitly requests
 `ALREADY_ABSENT`. Authorization, parent resolution, preconditions, and backend
@@ -476,6 +480,9 @@ Mutation admission uses a private, version-tagged JSON envelope in the same
 object store. Its identity namespace stays stable across format changes, and an
 unknown format fails closed. It contains explicit typed receipts, not arbitrary
 public response payloads, and is not part of the protobuf storage closure.
+Workspace create/delete admissions include the requested workspace name in the
+key, but omit a workspace UUID guard. Different names have independent request-ID
+namespaces; deletion receipts remain replayable after the target disappears.
 
 Each sandbox policy revision stores the complete provenance annotation map
 supplied with that update. The revision payload is the authoritative immutable

crates/openshell-sdk/README.md

@@ -59,6 +59,22 @@ Curated calls without a workspace argument explicitly select the `default`
 workspace. Cross-workspace listing uses the separate `*_all_workspaces`
 methods and requires Platform Admin access.
 
+For an accepted sandbox deletion, pass its original ID to `wait_deleted` so a
+same-name replacement does not extend the wait. Both the default and
+workspace-scoped clients accept the optional third argument; pass `None` to wait
+for name absence instead.
+
+```rust
+let deletion = client.delete_sandbox(name, openshell_sdk::DeleteOptions::default()).await?;
+if deletion.outcome == openshell_sdk::DeletionOutcome::Accepted {
+    client.wait_deleted(
+        name,
+        std::time::Duration::from_secs(60),
+        deletion.sandbox_id.as_deref(),
+    ).await?;
+}
+```
+
 Curated `list_*` methods return a lazy `Pager<T>`. Each `next_page()` call
 issues at most one RPC and returns a `Page<T>` with its opaque continuation
 token. The explicit `list_all_*` conveniences exhaust that pager; `page_size`

crates/openshell-sdk/src/client.rs

@@ -422,16 +422,26 @@ impl OpenShellClient {
         .await
     }
 
-    /// Poll until the sandbox is gone (gRPC `NotFound`) or the `timeout`
-    /// elapses.
-    pub async fn wait_deleted(&self, name: &str, timeout: Duration) -> Result<()> {
+    /// Poll until the sandbox is gone (gRPC `NotFound`) or the timeout elapses.
+    ///
+    /// Pass the deletion result's `sandbox_id` as `expected_sandbox_id` to also
+    /// complete when the name resolves to a different sandbox. With `None`,
+    /// waits for the name to be absent, including any same-name replacement.
+    pub async fn wait_deleted(
+        &self,
+        name: &str,
+        timeout: Duration,
+        expected_sandbox_id: Option<&str>,
+    ) -> Result<()> {
         let deadline = Instant::now() + timeout;
         let mut delay = Duration::from_millis(250);
         loop {
             match self.get_sandbox(name).await {
                 Err(SdkError::NotFound { .. }) => return Ok(()),
                 Err(other) => return Err(other),
-                Ok(snapshot) if snapshot.phase == SandboxPhase::Deleting => {}
+                Ok(snapshot) if expected_sandbox_id.is_some_and(|id| snapshot.id != id) => {
+                    return Ok(());
+                }
                 Ok(_) => {}
             }
             if Instant::now() >= deadline {
@@ -1021,13 +1031,25 @@ impl WorkspaceScopedClient {
     }
 
     /// Poll until the sandbox is gone (`NotFound`) or the timeout elapses.
-    pub async fn wait_deleted(&self, name: &str, timeout: Duration) -> Result<()> {
+    ///
+    /// Pass the deletion result's `sandbox_id` as `expected_sandbox_id` to also
+    /// complete when the name resolves to a different sandbox. With `None`,
+    /// waits for the name to be absent, including any same-name replacement.
+    pub async fn wait_deleted(
+        &self,
+        name: &str,
+        timeout: Duration,
+        expected_sandbox_id: Option<&str>,
+    ) -> Result<()> {
         let deadline = Instant::now() + timeout;
         let mut delay = Duration::from_millis(250);
         loop {
             match self.get_sandbox(name).await {
                 Err(SdkError::NotFound { .. }) => return Ok(()),
                 Err(other) => return Err(other),
+                Ok(snapshot) if expected_sandbox_id.is_some_and(|id| snapshot.id != id) => {
+                    return Ok(());
+                }
                 Ok(_) => {}
             }
             if Instant::now() >= deadline {

crates/openshell-sdk/tests/client_mock.rs

@@ -60,6 +60,9 @@ struct MockState {
     last_workspace_request: Mutex<Option<String>>,
     get_calls: AtomicU32,
     phase_sequence: Vec<proto::SandboxPhase>,
+    get_sandbox_id: Option<String>,
+    get_error: Option<Status>,
+    delete_response: Option<proto::DeleteSandboxResponse>,
     get_returns_not_found: bool,
     not_found_after: Option<u32>,
     paginate_list: bool,
@@ -337,6 +340,9 @@ impl OpenShell for TestOpenShell {
             selected_workspace(&req.workspace_scope).map(str::to_string);
         let count = self.state.get_calls.fetch_add(1, Ordering::SeqCst);
 
+        if let Some(error) = &self.state.get_error {
+            return Err(error.clone());
+        }
         if self.state.get_returns_not_found {
             return Err(Status::not_found(format!("sandbox '{name}' not found")));
         }
@@ -354,8 +360,12 @@ impl OpenShell for TestOpenShell {
             .or_else(|| self.state.phase_sequence.last().copied())
             .unwrap_or(proto::SandboxPhase::Ready);
 
+        let mut sandbox = sandbox_with_phase(&name, phase);
+        if let Some(id) = &self.state.get_sandbox_id {
+            sandbox.metadata.as_mut().unwrap().id.clone_from(id);
+        }
         Ok(Response::new(proto::SandboxResponse {
-            sandbox: Some(sandbox_with_phase(&name, phase)),
+            sandbox: Some(sandbox),
         }))
     }
 
@@ -429,6 +439,9 @@ impl OpenShell for TestOpenShell {
         *self.state.last_delete_name.lock().await = Some(req.name);
         *self.state.last_delete_workspace.lock().await =
             selected_workspace(&req.workspace_scope).map(str::to_string);
+        if let Some(response) = &self.state.delete_response {
+            return Ok(Response::new(response.clone()));
+        }
         Ok(Response::new(proto::DeleteSandboxResponse {
             sandbox_id: String::new(),
             outcome: proto::DeletionOutcome::Completed.into(),
@@ -1282,19 +1295,154 @@ async fn wait_ready_surfaces_error_phase() {
 
 #[tokio::test]
 async fn wait_deleted_returns_when_get_reports_not_found() {
-    let state = Arc::new(MockState {
-        phase_sequence: vec![proto::SandboxPhase::Deleting],
-        not_found_after: Some(2),
-        ..Default::default()
-    });
-    let endpoint = start_mock(state.clone()).await;
-    let client = connect(&endpoint).await;
+    for scoped in [false, true] {
+        for expected_id in [None, Some("id-my-box")] {
+            let state = Arc::new(MockState {
+                phase_sequence: vec![proto::SandboxPhase::Deleting],
+                not_found_after: Some(1),
+                ..Default::default()
+            });
+            let endpoint = start_mock(state.clone()).await;
+            let client = connect(&endpoint).await;
+            let timeout = std::time::Duration::from_secs(5);
+            if scoped {
+                client
+                    .workspace("team")
+                    .wait_deleted("my-box", timeout, expected_id)
+                    .await
+            } else {
+                client.wait_deleted("my-box", timeout, expected_id).await
+            }
+            .unwrap();
+            assert_eq!(state.get_calls.load(Ordering::SeqCst), 2);
+        }
+    }
+}
 
-    client
-        .wait_deleted("my-box", std::time::Duration::from_secs(5))
-        .await
+#[tokio::test]
+async fn wait_deleted_completes_on_replacement_after_accepted_deletion() {
+    for scoped in [false, true] {
+        let state = Arc::new(MockState {
+            get_sandbox_id: Some("replacement-id".to_string()),
+            delete_response: Some(proto::DeleteSandboxResponse {
+                outcome: proto::DeletionOutcome::Accepted.into(),
+                sandbox_id: "old-id".to_string(),
+            }),
+            ..Default::default()
+        });
+        let endpoint = start_mock(state.clone()).await;
+        let client = connect(&endpoint).await;
+        let timeout = std::time::Duration::from_millis(50);
+        let result = if scoped {
+            client
+                .workspace("team")
+                .delete_sandbox("my-box", openshell_sdk::DeleteOptions::default())
+                .await
+        } else {
+            client
+                .delete_sandbox("my-box", openshell_sdk::DeleteOptions::default())
+                .await
+        }
         .unwrap();
-    assert!(state.get_calls.load(Ordering::SeqCst) >= 3);
+        assert_eq!(result.outcome, openshell_sdk::DeletionOutcome::Accepted);
+        assert_eq!(result.sandbox_id.as_deref(), Some("old-id"));
+        if scoped {
+            client
+                .workspace("team")
+                .wait_deleted("my-box", timeout, result.sandbox_id.as_deref())
+                .await
+        } else {
+            client
+                .wait_deleted("my-box", timeout, result.sandbox_id.as_deref())
+                .await
+        }
+        .unwrap();
+        assert_eq!(state.get_calls.load(Ordering::SeqCst), 1);
+        let workspace = if scoped { "team" } else { "default" };
+        assert_eq!(
+            state.last_get_workspace.lock().await.as_deref(),
+            Some(workspace)
+        );
+        assert_eq!(
+            state.last_delete_workspace.lock().await.as_deref(),
+            Some(workspace)
+        );
+    }
+}
+
+#[tokio::test]
+async fn wait_deleted_does_not_complete_while_expected_identity_remains() {
+    for scoped in [false, true] {
+        let state = Arc::new(MockState {
+            get_sandbox_id: Some("old-id".to_string()),
+            ..Default::default()
+        });
+        let endpoint = start_mock(state.clone()).await;
+        let client = connect(&endpoint).await;
+        let timeout = std::time::Duration::ZERO;
+        let error = if scoped {
+            client
+                .workspace("team")
+                .wait_deleted("my-box", timeout, Some("old-id"))
+                .await
+        } else {
+            client.wait_deleted("my-box", timeout, Some("old-id")).await
+        }
+        .unwrap_err();
+        assert_eq!(error.code(), "connect");
+        assert!(error.to_string().contains("timed out waiting"));
+        assert_eq!(state.get_calls.load(Ordering::SeqCst), 1);
+    }
+}
+
+#[tokio::test]
+async fn wait_deleted_without_identity_waits_for_replacement_to_disappear() {
+    for scoped in [false, true] {
+        let state = Arc::new(MockState {
+            get_sandbox_id: Some("replacement-id".to_string()),
+            not_found_after: Some(1),
+            ..Default::default()
+        });
+        let endpoint = start_mock(state.clone()).await;
+        let client = connect(&endpoint).await;
+        let timeout = std::time::Duration::from_secs(5);
+        if scoped {
+            client
+                .workspace("team")
+                .wait_deleted("my-box", timeout, None)
+                .await
+        } else {
+            client.wait_deleted("my-box", timeout, None).await
+        }
+        .unwrap();
+        assert_eq!(state.get_calls.load(Ordering::SeqCst), 2);
+    }
+}
+
+#[tokio::test]
+async fn wait_deleted_propagates_errors_other_than_not_found() {
+    for scoped in [false, true] {
+        for code in [tonic::Code::PermissionDenied, tonic::Code::Unavailable] {
+            let state = Arc::new(MockState {
+                get_error: Some(Status::new(code, "lookup failed")),
+                ..Default::default()
+            });
+            let endpoint = start_mock(state.clone()).await;
+            let client = connect(&endpoint).await;
+            let timeout = std::time::Duration::from_secs(5);
+            let error = if scoped {
+                client
+                    .workspace("team")
+                    .wait_deleted("my-box", timeout, Some("old-id"))
+                    .await
+            } else {
+                client.wait_deleted("my-box", timeout, Some("old-id")).await
+            }
+            .unwrap_err();
+            assert_eq!(error.grpc_status().unwrap().code(), code);
+            assert_eq!(state.get_calls.load(Ordering::SeqCst), 1);
+        }
+    }
 }
 
 #[tokio::test]

crates/openshell-server/src/grpc/mutation_replay.rs

@@ -646,17 +646,21 @@ resource_mutation!(
     "CreateWorkspace",
     workspace::handle_create_workspace,
     workspace,
-    async |_req: &CreateWorkspaceRequest, state: &ServerState, principal: &Principal| {
-        global_scope(state, principal)
+    async |req: &CreateWorkspaceRequest, state: &ServerState, principal: &Principal| {
+        let mut scope = global_scope(state, principal)?;
+        scope.name.clone_from(&req.name);
+        Ok(scope)
     }
 );
 deletion_mutation!(
     DeleteWorkspaceRequest,
     DeleteWorkspaceResponse,
     "DeleteWorkspace",
     workspace::handle_delete_workspace,
-    async |_req: &DeleteWorkspaceRequest, state: &ServerState, principal: &Principal| {
-        global_scope(state, principal)
+    async |req: &DeleteWorkspaceRequest, state: &ServerState, principal: &Principal| {
+        let mut scope = global_scope(state, principal)?;
+        scope.name.clone_from(&req.name);
+        Ok(scope)
     }
 );
 resource_mutation!(