fix(cli): retry sync during supervisor reconnect

Signed-off-by: Drew Newberry <anewberry@nvidia.com>

Author: drew

architecture/gateway.md

@@ -423,7 +423,9 @@ stays correct throughout because a non-owner relays to the owner.
 File upload and download use tar-over-SSH through the same relay path. A gateway
 pod termination drops the active SSH proxy byte stream, so the CLI retries the
 whole sync operation with a fresh SSH session instead of attempting mid-stream
-resume.
+resume. After a transport interruption, the bounded retry also tolerates a
+brief sandbox-not-ready response while the supervisor reconnects. An initial
+sandbox-not-ready response still fails immediately.
 
 Gateway peer RPCs authenticate with Kubernetes ServiceAccount identity rather
 than a shared secret. Helm mounts a projected, pod-bound token with audience

crates/openshell-cli/src/ssh.rs

@@ -44,7 +44,7 @@ const FORWARD_LISTENER_CONNECT_TIMEOUT: Duration = Duration::from_millis(200);
 /// command has already reported its terminal result.
 const TERMINAL_RELAY_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(5);
 const TERMINAL_RELAY_REGISTRATION_INTERVAL: Duration = Duration::from_millis(50);
-const SYNC_RETRY_ATTEMPTS: usize = 4;
+const SYNC_RETRY_ATTEMPTS: usize = 8;
 const SYNC_RETRY_DELAY: Duration = Duration::from_secs(2);
 
 #[derive(Clone, Copy, Debug)]
@@ -1288,10 +1288,15 @@ where
     Fut: Future<Output = Result<()>>,
 {
     let mut attempt = 1;
+    let mut saw_transport_failure = false;
     loop {
         match run().await {
             Ok(()) => return Ok(()),
-            Err(error) if attempt < SYNC_RETRY_ATTEMPTS && sync_error_is_retryable(&error) => {
+            Err(error)
+                if attempt < SYNC_RETRY_ATTEMPTS
+                    && sync_error_is_retryable_after_disruption(&error, saw_transport_failure) =>
+            {
+                saw_transport_failure |= sync_error_is_retryable(&error);
                 tracing::warn!(
                     operation,
                     attempt,
@@ -1307,6 +1312,16 @@ where
     }
 }
 
+fn sync_error_is_not_ready(error: &Report) -> bool {
+    format!("{error:?}")
+        .to_ascii_lowercase()
+        .contains("sandbox is not ready")
+}
+
+fn sync_error_is_retryable_after_disruption(error: &Report, saw_transport_failure: bool) -> bool {
+    sync_error_is_retryable(error) || (saw_transport_failure && sync_error_is_not_ready(error))
+}
+
 fn sync_error_is_retryable(error: &Report) -> bool {
     let message = format!("{error:?}").to_ascii_lowercase();
     [
@@ -1868,6 +1883,36 @@ mod tests {
         assert!(!sync_error_is_retryable(&error));
     }
 
+    #[test]
+    fn sync_error_retry_filter_requires_prior_transport_failure_for_readiness() {
+        let error = miette::miette!("sandbox is not ready");
+        assert!(!sync_error_is_retryable_after_disruption(&error, false));
+        assert!(sync_error_is_retryable_after_disruption(&error, true));
+        let validation_error = miette::miette!("sandbox source path is invalid");
+        assert!(!sync_error_is_retryable_after_disruption(
+            &validation_error,
+            true
+        ));
+    }
+
+    #[tokio::test]
+    async fn sync_retries_not_ready_after_transport_break() {
+        let mut attempts = 0;
+        retry_sandbox_sync("download", || {
+            attempts += 1;
+            async move {
+                match attempts {
+                    1 => Err(miette::miette!("connection reset by peer")),
+                    2 => Err(miette::miette!("sandbox is not ready")),
+                    _ => Ok(()),
+                }
+            }
+        })
+        .await
+        .expect("sync should recover after supervisor readiness returns");
+        assert_eq!(attempts, 3);
+    }
+
     #[test]
     #[allow(unsafe_code)] // Test-only: env vars require unsafe in Rust 2024.
     fn install_ssh_config_adds_include_once_and_updates_managed_file() {

docs/sandboxes/manage-sandboxes.mdx

@@ -764,6 +764,11 @@ Download files from the sandbox to your host:
 openshell sandbox download my-sandbox output ./local
 ```
 
+If a gateway connection drops during a transfer, the CLI retries the whole
+upload or download with a fresh sandbox connection. It can also wait briefly
+for the sandbox to become ready again after a gateway restart. A sandbox that
+is already stopped still returns an error immediately.
+
 When the sandbox-side source is a single file, the destination follows `cp`-style placement: if the destination already exists as a directory or ends with `/`, the file lands inside it as `<dest>/<basename>`; otherwise the file is written at the exact destination path.
 
 The CLI discovers the sandbox's canonical working directory and only allows