Skip to content

Commit cecc53a

Browse files
committed
fix(server): end image preparation on streamed configuration admission
Image preparation deadlines (#4038) leave preparation when a supervisor's admission is recorded. ReportSandboxConfiguration did that, but admission delivered over the supervisor session in push mode only stored the admission, so every push-mode sandbox stayed Provisioning with "Waiting for an authenticated supervisor configuration report" until its deadline. Record the admission start, and any rejection, in the same CAS that stores streamed admission. Signed-off-by: Piotr Mlocek <pmlocek@nvidia.com>
1 parent a09a3bf commit cecc53a

2 files changed

Lines changed: 99 additions & 0 deletions

File tree

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4963,6 +4963,10 @@ impl ComputeRuntime {
49634963
{
49644964
return Err("restarting sandbox requires a fresh supervisor instance".into());
49654965
}
4966+
let provisioning = admission
4967+
.map(|admission| admitted_provisioning(&current, admission))
4968+
.transpose()?
4969+
.flatten();
49664970
let expected_resource_version = sandbox_resource_version(&current);
49674971
let result = self
49684972
.store
@@ -4979,6 +4983,9 @@ impl ComputeRuntime {
49794983
== i32::from(openshell_core::proto::ConfigurationAdmissionState::Accepted),
49804984
);
49814985
}
4986+
if provisioning.is_some() {
4987+
status.provisioning.clone_from(&provisioning);
4988+
}
49824989
// An admission-only update leaves restart state to
49834990
// the instance that last reported runtime readiness.
49844991
if runtime_ready {
@@ -6738,6 +6745,31 @@ fn apply_driver_snapshot(
67386745
apply_configuration_readiness(sandbox);
67396746
}
67406747

6748+
/// Admission evidence from a supervisor session ends image preparation, as a
6749+
/// polled configuration report does. Returns the updated provisioning record,
6750+
/// or `None` when the sandbox has none.
6751+
fn admitted_provisioning(
6752+
sandbox: &Sandbox,
6753+
admission: &openshell_core::proto::SandboxConfigurationAdmission,
6754+
) -> Result<Option<openshell_core::proto::SandboxProvisioning>, String> {
6755+
let Some(mut record) = sandbox
6756+
.status
6757+
.as_ref()
6758+
.and_then(|status| status.provisioning.clone())
6759+
else {
6760+
return Ok(None);
6761+
};
6762+
let now_ms = openshell_core::time::now_ms();
6763+
if !provisioning_deadline::allows_admission(&record, now_ms) {
6764+
return Err("provisioning deadline expired".to_string());
6765+
}
6766+
provisioning_deadline::record_admission_start(&mut record, now_ms)?;
6767+
if admission.state == i32::from(openshell_core::proto::ConfigurationAdmissionState::Rejected) {
6768+
provisioning_deadline::record_rejection(&mut record, now_ms)?;
6769+
}
6770+
Ok(Some(record))
6771+
}
6772+
67416773
/// Configuration readiness is independent of compute/container readiness.
67426774
pub fn apply_configuration_readiness(sandbox: &mut Sandbox) {
67436775
use openshell_core::proto::ConfigurationAdmissionState;

‎crates/openshell-server/src/grpc/policy.rs‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8386,6 +8386,73 @@ mod tests {
83868386
request
83878387
}
83888388

8389+
#[tokio::test]
8390+
async fn streamed_admission_ends_preparation_before_runtime_readiness() {
8391+
use crate::compute::provisioning_deadline::new_preparation_record;
8392+
use openshell_core::proto::{
8393+
ConfigurationAdmissionState, SandboxConfigurationAdmission, SandboxPhase,
8394+
};
8395+
let state = test_server_state().await;
8396+
let sandbox_id = "sb-streamed-admission";
8397+
let mut sandbox = test_sandbox(
8398+
sandbox_id,
8399+
"streamed-admission",
8400+
openshell_policy::restrictive_default_policy(),
8401+
Vec::new(),
8402+
);
8403+
sandbox.set_phase(SandboxPhase::Provisioning.into());
8404+
sandbox.status.as_mut().unwrap().provisioning =
8405+
Some(new_preparation_record(current_time_ms(), 1800));
8406+
state.store.put_message(&sandbox).await.unwrap();
8407+
let instance_id = uuid::Uuid::new_v4().to_string();
8408+
8409+
// Push mode reports admission over the supervisor session instead of
8410+
// ReportSandboxConfiguration, then reports runtime readiness.
8411+
state
8412+
.compute
8413+
.supervisor_session_admission(
8414+
sandbox_id,
8415+
&instance_id,
8416+
&SandboxConfigurationAdmission {
8417+
instance_id: instance_id.clone(),
8418+
state: ConfigurationAdmissionState::Accepted.into(),
8419+
..Default::default()
8420+
},
8421+
)
8422+
.await
8423+
.unwrap();
8424+
state
8425+
.compute
8426+
.supervisor_runtime_ready(sandbox_id, &instance_id)
8427+
.await
8428+
.unwrap();
8429+
8430+
let saved = state
8431+
.store
8432+
.get_message::<Sandbox>(sandbox_id)
8433+
.await
8434+
.unwrap()
8435+
.unwrap();
8436+
let status = saved.status.as_ref().unwrap();
8437+
assert!(
8438+
status
8439+
.provisioning
8440+
.as_ref()
8441+
.unwrap()
8442+
.admission_start_time
8443+
.is_some()
8444+
);
8445+
assert_eq!(saved.phase(), i32::from(SandboxPhase::Ready));
8446+
assert!(
8447+
status
8448+
.conditions
8449+
.iter()
8450+
.any(|condition| condition.r#type == "Ready" && condition.status == "True"),
8451+
"{:?}",
8452+
status.conditions
8453+
);
8454+
}
8455+
83898456
#[tokio::test]
83908457
async fn preparation_registration_starts_repair_once_after_a_cold_start() {
83918458
use crate::compute::provisioning_deadline::new_preparation_record;

0 commit comments

Comments
 (0)