Skip to content

Commit 490275a

Browse files
committed
fix(sandbox): check every backend confirmation against the launch
The delegated client compared a backend's Confirmed with the launch binding only on the first confirmation. Credential renewal, the liveness probe and reconnect accepted any Confirmed, and reconnect ignored the replayed Attach reply. A backend that lost its launch state could answer for another session and keep serving the sandbox. Check Attach and Confirm replies against the client's runtime descriptor on every path. Close the connection when a same-epoch renewal returns another launch. Document that Attach and Confirm must stay replayable for the whole generation. Closes #4703 Signed-off-by: Shiju <shiju@nvidia.com>
1 parent 55bf786 commit 490275a

4 files changed

Lines changed: 163 additions & 54 deletions

File tree

‎crates/openshell-sandbox-backend/src/runtime.rs‎

Lines changed: 75 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -194,10 +194,6 @@ impl OpenShellRuntimeBackend {
194194
};
195195
validate_runtime_descriptor(&runtime_descriptor, &sandbox)?;
196196
let host_gateway_ip = runtime_descriptor.host_gateway_ip;
197-
let resource_claims = runtime_descriptor.resource_claims.clone();
198-
let generation = runtime_descriptor.generation.clone();
199-
let session_id = runtime_descriptor.session_id;
200-
let provisioning_evidence = runtime_descriptor.provisioning_evidence.clone();
201197
let mut client = BoundaryClient::with_backend(
202198
runtime_descriptor,
203199
self.sandbox_bearer.clone(),
@@ -212,14 +208,7 @@ impl OpenShellRuntimeBackend {
212208
policy: Box::new(SandboxPolicyWire::from(sandbox.policy.clone())),
213209
})
214210
.await?;
215-
let Response::Attached { snapshot } = response else {
216-
return Err(unexpected_response("attached", &response));
217-
};
218-
if snapshot.generation != generation {
219-
return Err(BackendError::Confirm(
220-
"sandbox session snapshot generation does not match runtime descriptor".to_string(),
221-
));
222-
}
211+
client.check_attached(response)?;
223212
Ok(Box::new(BoundBoundary {
224213
client: client.clone(),
225214
agent: sandbox.agent,
@@ -229,10 +218,6 @@ impl OpenShellRuntimeBackend {
229218
ca_file_paths: self.ca_file_paths.clone(),
230219
provider_credentials: self.provider_credentials.clone(),
231220
identity: sandbox.identity,
232-
generation,
233-
session_id,
234-
resource_claims,
235-
provisioning_evidence,
236221
}))
237222
}
238223
}
@@ -339,10 +324,6 @@ pub struct BoundBoundary {
339324
ca_file_paths: Arc<std::sync::Mutex<Option<(PathBuf, PathBuf)>>>,
340325
provider_credentials: openshell_core::provider_credentials::ProviderCredentialState,
341326
identity: ResolvedWorkloadIdentity,
342-
generation: String,
343-
session_id: openshell_core::SandboxSessionId,
344-
resource_claims: std::collections::BTreeMap<String, String>,
345-
provisioning_evidence: openshell_isolation_interface::contract::ProvisioningEvidence,
346327
}
347328

348329
impl BoundBoundary {
@@ -356,20 +337,7 @@ impl BoundBoundary {
356337

357338
pub async fn confirm(self: Box<Self>) -> Result<ConfirmedBoundary, BackendError> {
358339
let response = self.client.call_idempotent(Request::Confirm).await?;
359-
let Response::Confirmed { confirmation } = response else {
360-
return Err(unexpected_response("confirmed", &response));
361-
};
362-
if confirmation.generation != self.generation
363-
|| confirmation.session_id != self.session_id
364-
|| confirmation.resource_claims != self.resource_claims
365-
|| confirmation.provisioning_evidence_digest
366-
!= self.provisioning_evidence.evidence_digest
367-
{
368-
return Err(BackendError::Confirm(
369-
"sandbox confirmation generation, session, resource claims, or provisioning evidence do not match runtime descriptor"
370-
.to_string(),
371-
));
372-
}
340+
let confirmation = self.client.launch_confirmation(response)?;
373341
if self.client.backend_name == crate::BACKEND_NAME {
374342
crate::audit::validate_native_linux_audit(&confirmation.backend_audit)?;
375343
}
@@ -1270,6 +1238,48 @@ impl BoundaryClient {
12701238
}
12711239
}
12721240

1241+
/// Check an Attach reply, including replays on reconnect, against the launch.
1242+
fn check_attached(&self, response: Response) -> Result<(), BackendError> {
1243+
let Response::Attached { snapshot } = response else {
1244+
return Err(unexpected_response("attached", &response));
1245+
};
1246+
if snapshot.generation != self.runtime_descriptor.generation {
1247+
return Err(BackendError::Confirm(
1248+
"sandbox session snapshot generation does not match runtime descriptor".to_string(),
1249+
));
1250+
}
1251+
Ok(())
1252+
}
1253+
1254+
/// Check a Confirm reply against the launch. Reconnect, credential renewal
1255+
/// and liveness probes replay Confirm, so every reply gets the same check.
1256+
fn launch_confirmation(
1257+
&self,
1258+
response: Response,
1259+
) -> Result<Box<BoundaryConfirmation>, BackendError> {
1260+
let Response::Confirmed { confirmation } = response else {
1261+
return Err(unexpected_response("confirmed", &response));
1262+
};
1263+
let launch = &self.runtime_descriptor;
1264+
confirmation.validate(&launch.workload_identity)?;
1265+
let mismatch = if confirmation.generation != launch.generation {
1266+
"generation"
1267+
} else if confirmation.session_id != launch.session_id {
1268+
"session"
1269+
} else if confirmation.resource_claims != launch.resource_claims {
1270+
"resource claims"
1271+
} else if confirmation.provisioning_evidence_digest
1272+
!= launch.provisioning_evidence.evidence_digest
1273+
{
1274+
"provisioning evidence"
1275+
} else {
1276+
return Ok(confirmation);
1277+
};
1278+
Err(BackendError::Confirm(format!(
1279+
"sandbox confirmation {mismatch} does not match runtime descriptor"
1280+
)))
1281+
}
1282+
12731283
async fn acquire_network_stream(&self) -> tokio::sync::OwnedSemaphorePermit {
12741284
let acquire = self.network_streams.clone().acquire_owned();
12751285
tokio::pin!(acquire);
@@ -1562,7 +1572,12 @@ impl BoundaryClient {
15621572
.map_err(|_| {
15631573
BackendError::Unavailable("boundary credential renewal timed out".to_string())
15641574
})??;
1565-
expect_response(response, "confirmed")?;
1575+
if let Err(error) = self.launch_confirmation(response) {
1576+
// A backend answering for another launch keeps no live streams.
1577+
cached.transport.abort();
1578+
*self.mediation.lock().await = None;
1579+
return Err(error);
1580+
}
15661581
if let Some(current) = self.grpc_channel.lock().await.as_mut()
15671582
&& current.generation == cached.generation
15681583
{
@@ -1684,14 +1699,15 @@ impl BoundaryClient {
16841699
let Ok(credential) = BoundaryCredential::capture(&self.sandbox_bearer) else {
16851700
return false;
16861701
};
1687-
matches!(
1688-
tokio::time::timeout(
1689-
CONNECTION_PROBE_TIMEOUT,
1690-
self.exchange_on_channel(cached.channel.clone(), confirm, &credential),
1691-
)
1692-
.await,
1693-
Ok(Ok(Response::Confirmed { .. }))
1702+
let Ok(Ok(response)) = tokio::time::timeout(
1703+
CONNECTION_PROBE_TIMEOUT,
1704+
self.exchange_on_channel(cached.channel.clone(), confirm, &credential),
16941705
)
1706+
.await
1707+
else {
1708+
return false;
1709+
};
1710+
self.launch_confirmation(response).is_ok()
16951711
}
16961712

16971713
/// Open a new physical connection and replay the authenticated lifecycle,
@@ -1704,11 +1720,15 @@ impl BoundaryClient {
17041720
) -> Result<(tonic::transport::Channel, Arc<TransportAbort>), BackendError> {
17051721
let (channel, transport) = self.build_grpc_channel(credential).await?;
17061722
let result = async {
1707-
self.exchange_on_channel(channel.clone(), attach, credential)
1723+
let attached = self
1724+
.exchange_on_channel(channel.clone(), attach, credential)
17081725
.await?;
1726+
self.check_attached(attached)?;
17091727
if let Some(confirm) = confirm {
1710-
self.exchange_on_channel(channel.clone(), confirm, credential)
1728+
let confirmed = self
1729+
.exchange_on_channel(channel.clone(), confirm, credential)
17111730
.await?;
1731+
self.launch_confirmation(confirmed)?;
17121732
}
17131733
Ok(())
17141734
}
@@ -2271,9 +2291,7 @@ fn connect_host_vsock(
22712291
fn expect_response(response: Response, expected: &str) -> Result<(), BackendError> {
22722292
let matches = matches!(
22732293
(&response, expected),
2274-
(Response::Attached { .. }, "attached")
2275-
| (Response::Confirmed { .. }, "confirmed")
2276-
| (Response::Signaled, "signaled")
2294+
(Response::Signaled, "signaled")
22772295
| (Response::Terminated, "terminated")
22782296
| (Response::BoundaryTerminated, "boundary_terminated")
22792297
);
@@ -2604,6 +2622,7 @@ mod tests {
26042622

26052623
#[derive(Clone)]
26062624
struct TestGrpcBoundary {
2625+
attach_generation: String,
26072626
wait_for_half_close: bool,
26082627
expected_token: String,
26092628
requests: Arc<std::sync::atomic::AtomicUsize>,
@@ -2637,6 +2656,7 @@ mod tests {
26372656
let mediation_ready = self.mediation_ready;
26382657
let provider_environment_generation = self.provider_environment_generation;
26392658
let confirmation = self.confirmation.clone();
2659+
let attach_generation = self.attach_generation.clone();
26402660
let (outbound, outbound_rx) = tokio::sync::mpsc::channel(1);
26412661
tokio::spawn(async move {
26422662
let mut frame = Vec::new();
@@ -2679,7 +2699,7 @@ mod tests {
26792699
},
26802700
Request::Attach { .. } => Response::Attached {
26812701
snapshot: crate::boundary_protocol::SessionSnapshotWire {
2682-
generation: "test-generation".to_string(),
2702+
generation: attach_generation,
26832703
processes: Vec::new(),
26842704
},
26852705
},
@@ -2806,6 +2826,7 @@ mod tests {
28062826
let address = listener.local_addr().unwrap();
28072827
let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
28082828
let service = TestGrpcBoundary {
2829+
attach_generation: "test-generation".to_string(),
28092830
wait_for_half_close: false,
28102831
expected_token: "a".repeat(32),
28112832
requests: requests.clone(),
@@ -2880,6 +2901,7 @@ mod tests {
28802901
.unwrap();
28812902
server_accepted.fetch_add(1, Ordering::AcqRel);
28822903
let service = TestGrpcBoundary {
2904+
attach_generation: "test-generation".to_string(),
28832905
wait_for_half_close: false,
28842906
expected_token: "a".repeat(32),
28852907
requests: server_requests.clone(),
@@ -2991,6 +3013,7 @@ mod tests {
29913013
.await
29923014
.unwrap();
29933015
let service = TestGrpcBoundary {
3016+
attach_generation: "test-generation".to_string(),
29943017
wait_for_half_close: false,
29953018
expected_token: "a".repeat(32),
29963019
requests: server_requests.clone(),
@@ -3046,6 +3069,7 @@ mod tests {
30463069
let address = listener.local_addr().unwrap();
30473070
let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
30483071
let service = TestGrpcBoundary {
3072+
attach_generation: "test-generation".to_string(),
30493073
wait_for_half_close: true,
30503074
expected_token: "a".repeat(32),
30513075
requests,
@@ -3245,6 +3269,7 @@ mod tests {
32453269
confirmation: BoundaryConfirmation,
32463270
) {
32473271
let service = TestGrpcBoundary {
3272+
attach_generation: "test-generation".to_string(),
32483273
wait_for_half_close: false,
32493274
expected_token,
32503275
requests: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
@@ -3353,7 +3378,6 @@ mod tests {
33533378
});
33543379

33553380
let descriptor = tls_runtime_descriptor(address, certificate.client_tls);
3356-
let provisioning_evidence = descriptor.provisioning_evidence.clone();
33573381
let context = sandbox();
33583382
let client = Arc::new(BoundaryClient::new(
33593383
descriptor,
@@ -3376,10 +3400,6 @@ mod tests {
33763400
HashMap::new(),
33773401
),
33783402
identity: context.identity,
3379-
generation: "test-generation".to_string(),
3380-
session_id: test_session_id(),
3381-
resource_claims: std::collections::BTreeMap::new(),
3382-
provisioning_evidence,
33833403
};
33843404

33853405
assert!(matches!(
@@ -3763,6 +3783,7 @@ mod tests {
37633783
let accepted_by_server = accepted.clone();
37643784
let handled = Arc::new(std::sync::atomic::AtomicUsize::new(0));
37653785
let service = TestGrpcBoundary {
3786+
attach_generation: "test-generation".to_string(),
37663787
wait_for_half_close: false,
37673788
expected_token: "a".repeat(32),
37683789
requests: handled.clone(),
@@ -3836,6 +3857,7 @@ mod tests {
38363857
let address = listener.local_addr().unwrap();
38373858
let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
38383859
let service = TestGrpcBoundary {
3860+
attach_generation: "test-generation".to_string(),
38393861
wait_for_half_close: false,
38403862
expected_token: "a".repeat(32),
38413863
requests: requests.clone(),

0 commit comments

Comments
 (0)