From f28e20f12deefeace1f64b94f052e552b27fd0d4 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 16 Sep 2026 17:42:44 -0700 Subject: [PATCH 1/5] feat(kubernetes): support HA gateway rebalancing Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 117 +- .github/workflows/branch-e2e.yml | 3 + .github/workflows/e2e-kubernetes-test.yml | 6 + architecture/compute-runtimes.md | 17 +- architecture/gateway.md | 54 +- crates/openshell-cli/src/ssh.rs | 150 +- .../tests/ensure_providers_integration.rs | 11 + .../openshell-cli/tests/mtls_integration.rs | 11 + .../tests/provider_commands_integration.rs | 11 + .../sandbox_create_lifecycle_integration.rs | 11 + .../sandbox_name_fallback_integration.rs | 11 + crates/openshell-sdk/tests/client_mock.rs | 10 + .../src/auth/descriptor_authz.rs | 1 + crates/openshell-server/src/auth/guard.rs | 5 +- .../openshell-server/src/auth/method_authz.rs | 12 +- crates/openshell-server/src/auth/mod.rs | 1 + crates/openshell-server/src/auth/peer.rs | 597 +++++++ crates/openshell-server/src/auth/principal.rs | 11 + .../src/auth/sandbox_methods.rs | 1 + .../src/auth/workspace_authz.rs | 6 + crates/openshell-server/src/compute/mod.rs | 151 +- crates/openshell-server/src/grpc/mod.rs | 14 +- crates/openshell-server/src/grpc/policy.rs | 7 +- .../src/grpc/policy/endpoint_status.rs | 12 +- .../src/grpc/policy/endpoint_status_tests.rs | 2 +- crates/openshell-server/src/grpc/provider.rs | 34 +- .../src/grpc/provider_readiness_tests.rs | 1 + crates/openshell-server/src/grpc/sandbox.rs | 55 +- crates/openshell-server/src/grpc/workspace.rs | 3 + crates/openshell-server/src/lib.rs | 98 ++ crates/openshell-server/src/multiplex.rs | 24 +- .../openshell-server/src/persistence/mod.rs | 22 + .../src/persistence/postgres.rs | 28 + .../openshell-server/src/provider_refresh.rs | 4 +- crates/openshell-server/src/sandbox_watch.rs | 112 +- .../openshell-server/src/service_routing.rs | 21 +- crates/openshell-server/src/storage_proto.rs | 10 +- .../openshell-server/src/supervisor_owner.rs | 344 +++++ .../src/supervisor_session.rs | 639 +++++++- crates/openshell-server/tests/common/mod.rs | 17 +- .../tests/supervisor_relay_integration.rs | 12 +- .../src/supervisor_session.rs | 4 +- deploy/helm/openshell/README.md | 22 + deploy/helm/openshell/README.md.gotmpl | 22 + deploy/helm/openshell/skaffold.yaml | 19 + .../openshell/templates/_gateway-workload.tpl | 61 + deploy/helm/openshell/templates/_helpers.tpl | 4 + .../helm/openshell/templates/clusterrole.yaml | 4 +- .../helm/openshell/templates/peer-role.yaml | 35 + .../openshell/templates/peer-service.yaml | 19 + .../helm/openshell/templates/statefulset.yaml | 2 +- .../tests/credential_drivers_test.yaml | 39 +- .../openshell/tests/gateway_config_test.yaml | 132 +- .../tests/statefulset_client_ca_test.yaml | 24 +- .../manifests/envoy-gateway-openshell.yaml | 21 + docs/kubernetes/setup.mdx | 20 +- e2e/rust/Cargo.toml | 7 +- e2e/rust/tests/kubernetes_ha_rebalancing.rs | 627 ++++++++ e2e/rust/tests/readyz_health.rs | 95 -- e2e/with-kube-gateway.sh | 180 ++- proto/openshell.proto | 35 + sdk/go/proto/openshellv1/openshell.pb.go | 1368 ++++++++++------- sdk/go/proto/openshellv1/openshell_grpc.pb.go | 48 +- skills/debug-openshell-cluster/SKILL.md | 67 +- tasks/test.toml | 6 + 65 files changed, 4606 insertions(+), 911 deletions(-) create mode 100644 crates/openshell-server/src/auth/peer.rs create mode 100644 crates/openshell-server/src/supervisor_owner.rs create mode 100644 deploy/helm/openshell/templates/peer-role.yaml create mode 100644 deploy/helm/openshell/templates/peer-service.yaml create mode 100644 e2e/rust/tests/kubernetes_ha_rebalancing.rs delete mode 100644 e2e/rust/tests/readyz_health.rs diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 2a6b9ad38c..fddbe8f1e4 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -113,10 +113,31 @@ trace-context annotation. The same command exposes OTLP/gRPC on `gateway:vm` tasks detect the collector listener at startup and enable trace export only while it is reachable. -**HA test deploy** (two gateway replicas + external PostgreSQL Secret): uncomment -`#- ci/values-high-availability.yaml` in `deploy/helm/openshell/skaffold.yaml`, -create the Secret named `openshell-ha-pg` with a `uri` key, then run -`mise run helm:skaffold:run` or `mise run helm:skaffold:dev`. +The Skaffold profile for HA reverse-proxy development is available from +`deploy/helm/openshell/`: + +```bash +# Two gateway replicas + external PostgreSQL Secret + Envoy Gateway + Gateway API route. +KUBECONFIG=../../../kubeconfig skaffold run -p high-availability +``` + +The `high-availability` profile expects a Secret named `openshell-ha-pg` in the `openshell` +namespace with a `uri` key. For local manual testing, either create your own +PostgreSQL Secret or use the e2e PostgreSQL fixture manifest in +`e2e/kubernetes/postgres-fixture.yaml`. + +For the `high-availability` profile, return to the repository root and apply the +GatewayClass and BackendTrafficPolicy manifest after Skaffold has installed +Envoy Gateway: + +```bash +KUBECONFIG=kubeconfig mise run helm:gateway:apply +``` + +The BackendTrafficPolicy disables Envoy request and stream-duration timeouts for +OpenShell's `GRPCRoute`. Keep that policy in `deploy/kube/manifests/envoy-gateway-openshell.yaml`, +not in the Helm chart; it is required for long-lived gRPC create/watch/exec/relay +streams during gateway rollouts and scale events. ### TLS behaviour @@ -187,23 +208,85 @@ but will point to a deleted cluster — safe to ignore or clean up manually. ## Optional Add-ons -Each add-on requires uncommenting the corresponding `valuesFiles` entry in -`deploy/helm/openshell/skaffold.yaml` before running `helm:skaffold:dev` or `helm:skaffold:run`. +Some add-ons can be enabled by uncommenting values in `skaffold.yaml`, but prefer +the dedicated Skaffold profiles when they exist. Profiles avoid leaving local +manual edits in the worktree. ### Envoy Gateway (Gateway API / GRPCRoute) -Envoy Gateway is already installed by Skaffold (the `envoy-gateway` Helm release in -`skaffold.yaml`). To activate routing: +Use the `high-availability` Skaffold profile for HA reverse-proxy testing. The +profile intentionally includes Envoy Gateway so multi-replica behavior is +exercised through the same Gateway API path used by reverse-proxy deployments: -1. Uncomment `#- values-gateway.yaml` in `skaffold.yaml` -2. Redeploy: `mise run helm:skaffold:run` -3. Apply the GatewayClass: `mise run helm:gateway:apply` -4. Access: `http://127.0.0.1:8080` +```bash +cd deploy/helm/openshell +KUBECONFIG=../../../kubeconfig skaffold run -p high-availability +cd ../../.. +KUBECONFIG=kubeconfig mise run helm:gateway:apply +``` + +`values-gateway.yaml` creates a `Gateway` (listener on port 80, class `eg`) and +`GRPCRoute` in the `openshell` namespace. The `high-availability` profile +installs the Envoy Gateway Helm chart and layers both +`values-high-availability.yaml` and `values-gateway.yaml` onto the OpenShell +release. + +`deploy/kube/manifests/envoy-gateway-openshell.yaml` creates: + +- `GatewayClass/eg` +- `BackendTrafficPolicy/openshell-grpc-timeouts` + +The Envoy Gateway proxy Service is usually exposed through the k3d load balancer +at `http://127.0.0.1:8080`. If the cluster was created with a different +`HELM_K3S_LB_HOST_PORT`, use that host port instead. + +For manual tests against an existing cluster, prefer forwarding the Envoy proxy +Service rather than `svc/openshell`. That keeps client traffic on the same path +as a real reverse proxy while gateway pods rotate behind it: + +```bash +KUBECONFIG=kubeconfig kubectl get svc -A \ + -l gateway.envoyproxy.io/owning-gateway-name=openshell +KUBECONFIG=kubeconfig kubectl -n port-forward \ + svc/ 8080:80 +openshell gateway add http://127.0.0.1:8080 --name openshell --local +``` + +When running e2e tests manually through Envoy, register gateway metadata (as +above) instead of relying only on `OPENSHELL_GATEWAY_ENDPOINT`; some tests call +`openshell gateway info` and expect metadata for the active gateway. + +### Kubernetes E2E Notes + +Use `mise run e2e:kubernetes` for the standard Helm-backed Kubernetes suite. +The kube e2e wrapper creates only one port-forward, to `svc/openshell`; it no +longer forwards the unauthenticated health listener or runs a `/readyz` e2e +target. `/readyz` remains covered by server unit/integration tests. + +Use `mise run e2e:kubernetes:ha-rebalancing` for full-suite HA coverage. The +task creates an external PostgreSQL fixture, installs Envoy Gateway, applies +`deploy/kube/manifests/envoy-gateway-openshell.yaml`, enables the chart +`GRPCRoute`, and runs the full Kubernetes e2e suite, including +`kubernetes_ha_rebalancing`. That coverage validates sandbox create/watch and +exec through the Envoy proxy while gateway replicas scale up, scale down, and +rotate. It also keeps a long-running sandbox alive and runs upload/download +operations while gateway pods roll, so file sync exercises the same relay retry +path as interactive sessions. + +If you reuse an existing Skaffold cluster for the full kube suite, make sure the +chart has `server.hostGatewayIP` set so sandbox pods can resolve +`host.openshell.internal` back to the test host. The e2e wrapper detects this on +chart installs; manual reuse may require: + +```bash +HOST_GATEWAY_IP="${OPENSHELL_E2E_HOST_GATEWAY_IP:?set host gateway IP}" +KUBECONFIG=kubeconfig helm upgrade openshell deploy/helm/openshell \ + --namespace openshell --reuse-values \ + --set "server.hostGatewayIP=${HOST_GATEWAY_IP}" \ + --wait --timeout 5m +``` -`values-gateway.yaml` creates a `Gateway` (listener on port 80, class `eg`) and a -`GRPCRoute` in the `openshell` namespace. Envoy Gateway provisions a LoadBalancer -service for the proxy; klipper-lb binds it to hostPort 80, reachable via the -`8080:80` load balancer port mapping. +Use the IP that pods in that cluster use to reach listeners on the test host. ### BackendTLSPolicy (end-to-end TLS) @@ -356,6 +439,6 @@ for dependencies still declared in `Chart.yaml`. | `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev | | `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) | | `deploy/helm/openshell/ci/values-credential-driver-vault.yaml` | Vault credential-driver validation overlay with HTTPS and private-CA trust | -| `deploy/kube/manifests/envoy-gateway-openshell.yaml` | GatewayClass for Envoy Gateway (`mise run helm:gateway:apply`) | +| `deploy/kube/manifests/envoy-gateway-openshell.yaml` | GatewayClass and BackendTrafficPolicy for Envoy Gateway (`mise run helm:gateway:apply`) | | `tasks/scripts/helm-k3s-local.sh` | k3d cluster create/delete/start/stop/status | | `tasks/scripts/keycloak-k8s-setup.sh` | Keycloak deploy, realm import, and development TLS trust anchor | diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 827823df42..0491fb3136 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -426,6 +426,9 @@ jobs: job-name: Kubernetes HA E2E (Rust smoke) extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml external-postgres-secret: openshell-ha-pg + test-name: kubernetes_ha_rebalancing + kubernetes-features: e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha + use-envoy-gateway: true conformance-artifact-prefix: openshell-conformance kubernetes-credential-drivers-e2e: diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index e109acf4f3..4aaa932644 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -52,6 +52,11 @@ on: required: false type: string default: "" + use-envoy-gateway: + description: "Route the test through Envoy Gateway instead of a direct gateway Service port-forward" + required: false + type: boolean + default: false mise-version: description: "mise version to install on the bare Kubernetes e2e runner" required: false @@ -140,6 +145,7 @@ jobs: OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET: ${{ inputs.external-postgres-secret }} OPENSHELL_E2E_KUBE_TEST: ${{ inputs.test-name }} OPENSHELL_E2E_KUBERNETES_FEATURES: ${{ inputs.kubernetes-features }} + OPENSHELL_E2E_KUBE_USE_ENVOY: ${{ inputs.use-envoy-gateway && '1' || '0' }} IMAGE_TAG: ${{ inputs.image-tag }} OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell E2E_TASK: ${{ inputs.e2e-task }} diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 6573f7641e..b0a2074356 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -98,15 +98,14 @@ shortly after carrying a stale `Provisioning` or `Unknown` backend phase. The composition rule treats a connected session as the stronger signal and keeps `Ready` in that case, preventing a lagging snapshot from undoing the session-driven promotion. -**Known HA limitation:** Supervisor sessions are process-local while the public -sandbox phase is shared. A replica that reconciles a driver snapshot without owning -the active supervisor session can demote the shared phase to `Provisioning`. The -session-owning replica may not receive another connection event to restore `Ready`, -so a usable sandbox can remain unavailable through the public phase gate. Reliable -HA readiness requires persisted or leased supervisor presence plus routing to the -session-owning replica. That work is deferred to GitHub issue #1868. Until then, -deployments that require reliable readiness composition must run a single gateway -replica. +**HA session composition:** Live relay handles remain process-local, while the +session-owning gateway publishes a short-lived owner record in shared PostgreSQL. +Driver reconciliation treats a fresh local or remote owner as connected, so a +non-owner replica cannot demote the shared sandbox phase merely because it lacks the +in-memory stream. Session-bound requests are forwarded to the owning gateway; a +supervisor reconnect publishes a higher connection epoch before stale-session cleanup +can demote readiness. Multi-replica deployments therefore require shared PostgreSQL +and the gateway peer Service configured by the Helm chart. **Extension point:** Driver-reported readiness is a capability, not an operator-configurable hook. A driver may enable it only when it owns workload diff --git a/architecture/gateway.md b/architecture/gateway.md index e77ed5c5cd..dfa49d85ad 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -352,6 +352,58 @@ authenticated sandbox ID with any sandbox ID or name resolved from the request. Supervisor control and relay streams require a matching sandbox principal before the gateway registers the session or bridges relay bytes. +## HA Supervisor Ownership + +In multi-replica Kubernetes deployments, every gateway pod can accept client +RPCs, but a sandbox supervisor maintains one active stream to one gateway +replica at a time. The connected replica publishes a short-lived supervisor +owner record in the shared Postgres object store with its replica id, peer DNS +endpoint, supervisor instance id, and connection epoch. Ownership does not move +because another gateway receives a client request. It changes only when the +supervisor opens a new control stream, usually after the previous owner pod is +terminated or the stream breaks. A reconnect from the same supervisor instance +with a newer epoch can supersede the previous owner before the TTL expires, and +heartbeats from the active connection renew that current owner record. +Cleanup from an older connection checks the shared owner record before and +after changing sandbox readiness. It cannot demote a sandbox after a newer +replica has published replacement ownership. + +Session-bound operations such as exec, TCP forwarding, file sync, and sandbox +service routing first check the local session registry. If the supervisor is +owned by another gateway replica, the serving gateway opens an internal +`PeerRelay` stream to that owner and asks it to open the supervisor relay. This +keeps client traffic working when a Kubernetes Service routes the client to a +non-owner gateway pod. If a peer owner is stale or unreachable during a rollout, +the serving gateway retries ownership lookup until the normal relay wait +deadline. Each retry re-reads the owner record, so a supervisor reconnect or +heartbeat can surface a new owner; if no fresh reachable owner appears before +the deadline, the client operation fails rather than electing an owner itself. + +File upload and download use tar-over-SSH through the same relay path. A gateway +pod termination drops the active SSH proxy byte stream, so the CLI retries the +whole sync operation with a fresh SSH session instead of attempting mid-stream +resume. + +Gateway peer RPCs authenticate with Kubernetes ServiceAccount identity rather +than a shared secret. Helm mounts a projected, pod-bound token with audience +`openshell-gateway-peer`; the receiving gateway validates it through +TokenReview, checks the live pod UID and chart selector labels, and authorizes +only the internal peer relay method. When gateway TLS is enabled, peer clients +also trust the chart CA, present the chart-generated client certificate for +mTLS, and verify the stable gateway Service DNS name even when connecting to a +Deployment pod IP. + +`WatchSandbox` uses the local update bus for same-replica writes. One shared +poller per gateway observes resource-version changes made by other replicas and +feeds that bus for all local watchers, avoiding a database poll per client +stream. + +Mutations whose invariants span sandbox, provider-profile, policy, or provider +records take a process-local mutex and a shared PostgreSQL advisory lock. The +database session remains dedicated to the request and closes when the guard is +dropped, which releases the lock on normal completion, cancellation, or error. +SQLite deployments use only the local mutex because they are single-replica. + ## API Surface The gateway API is organized around platform objects and operational streams: @@ -751,7 +803,7 @@ migrations backfill existing rows with version 1. Provider profile imports, updates, and deletes hold the sandbox synchronization guard while checking attached-sandbox dynamic token grant ambiguity or in-use state and writing the profile record. Sandbox creation with initial providers and -sandbox provider attach/detach use the same guard, so one gateway process cannot +sandbox provider attach/detach use the same guard, so gateway replicas cannot interleave a profile mutation with a sandbox provider-set mutation that would leave an ambiguous final dynamic-token state or a deleted custom profile that is still referenced by a sandbox. diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index e6db263210..947c84cedc 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -5,7 +5,7 @@ use crate::color::Colorize; use crate::tls::{TlsOptions, grpc_client}; -use miette::{IntoDiagnostic, Result, WrapErr}; +use miette::{IntoDiagnostic, Report, Result, WrapErr}; #[cfg(unix)] use nix::sys::signal::{SaFlags, SigAction, SigHandler, SigSet, Signal, sigaction}; use openshell_core::driver_mounts; @@ -44,6 +44,8 @@ const FORWARD_LISTENER_CONNECT_TIMEOUT: Duration = Duration::from_millis(200); /// command has already reported its terminal result. const TERMINAL_RELAY_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(5); const TERMINAL_RELAY_REGISTRATION_INTERVAL: Duration = Duration::from_millis(50); +const SYNC_RETRY_ATTEMPTS: usize = 4; +const SYNC_RETRY_DELAY: Duration = Duration::from_secs(2); #[derive(Clone, Copy, Debug)] pub enum Editor { @@ -659,6 +661,7 @@ pub async fn sandbox_exec( } /// What to pack into the tar archive streamed to the sandbox. +#[derive(Clone)] enum UploadSource { /// A single local file or directory. `tar_name` controls the entry name /// inside the archive (e.g. the target basename for file-to-file uploads). @@ -1036,18 +1039,15 @@ pub async fn sandbox_sync_up_files( if files.is_empty() { return Ok(()); } - ssh_tar_upload( - server, - name, - dest, - UploadSource::FileList { - base_dir: base_dir.to_path_buf(), - files: files.to_vec(), - archive_prefix: file_list_archive_prefix(local_path), - }, - tls, - workspace, - ) + let source = UploadSource::FileList { + base_dir: base_dir.to_path_buf(), + files: files.to_vec(), + archive_prefix: file_list_archive_prefix(local_path), + }; + retry_sandbox_sync("upload", || { + let source = source.clone(); + async move { ssh_tar_upload(server, name, dest, source, tls, workspace).await } + }) .await } @@ -1082,17 +1082,16 @@ pub async fn sandbox_sync_up( { let (parent, target_name) = split_sandbox_path(path); if parent != "/" { - return ssh_tar_upload( - server, - name, - Some(parent), - UploadSource::SinglePath { - local_path: local_path.to_path_buf(), - tar_name: target_name.into(), - }, - tls, - workspace, - ) + let source = UploadSource::SinglePath { + local_path: local_path.to_path_buf(), + tar_name: target_name.into(), + }; + return retry_sandbox_sync("upload", || { + let source = source.clone(); + async move { + ssh_tar_upload(server, name, Some(parent), source, tls, workspace).await + } + }) .await; } } @@ -1110,17 +1109,14 @@ pub async fn sandbox_sync_up( directory_upload_prefix(local_path) }; - ssh_tar_upload( - server, - name, - sandbox_path, - UploadSource::SinglePath { - local_path: local_path.to_path_buf(), - tar_name, - }, - tls, - workspace, - ) + let source = UploadSource::SinglePath { + local_path: local_path.to_path_buf(), + tar_name, + }; + retry_sandbox_sync("upload", || { + let source = source.clone(); + async move { ssh_tar_upload(server, name, sandbox_path, source, tls, workspace).await } + }) .await } @@ -1259,6 +1255,20 @@ pub async fn sandbox_sync_down( dest: &str, tls: &TlsOptions, workspace: &str, +) -> Result<()> { + retry_sandbox_sync("download", || async { + sandbox_sync_down_once(server, name, sandbox_path, dest, tls, workspace).await + }) + .await +} + +async fn sandbox_sync_down_once( + server: &str, + name: &str, + sandbox_path: &str, + dest: &str, + tls: &TlsOptions, + workspace: &str, ) -> Result<()> { let session = ssh_session_config(server, name, tls, workspace, None).await?; let sandbox_path = resolve_sandbox_source_path(&session, sandbox_path).await?; @@ -1272,6 +1282,54 @@ pub async fn sandbox_sync_down( } } +async fn retry_sandbox_sync(operation: &str, mut run: F) -> Result<()> +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut attempt = 1; + loop { + match run().await { + Ok(()) => return Ok(()), + Err(error) if attempt < SYNC_RETRY_ATTEMPTS && sync_error_is_retryable(&error) => { + tracing::warn!( + operation, + attempt, + max_attempts = SYNC_RETRY_ATTEMPTS, + error = %error, + "sandbox sync operation failed; retrying" + ); + tokio::time::sleep(SYNC_RETRY_DELAY).await; + attempt += 1; + } + Err(error) => return Err(error), + } + } +} + +fn sync_error_is_retryable(error: &Report) -> bool { + let message = format!("{error:?}").to_ascii_lowercase(); + [ + "broken pipe", + "connection", + "early eof", + "http2", + "h2 protocol", + "reset before headers", + "service is currently unavailable", + "transport error", + "unexpected eof", + "unavailable", + "upstream connect error", + "ssh probe exited with status exit status: 255", + "ssh tar create exited", + "ssh tar extract exited", + "failed to extract tar archive from sandbox", + ] + .iter() + .any(|needle| message.contains(needle)) +} + /// Stream a tar archive from the sandbox and extract it into a fresh /// destination directory. The source is always wrapped on the sandbox side so /// the host can pick a basename when needed. @@ -1788,6 +1846,28 @@ mod tests { assert_eq!(output.matches("Host openshell-demo").count(), 1); } + #[test] + fn sync_error_retry_filter_accepts_transport_failures() { + let error = miette::miette!("transport error: connection reset by peer"); + assert!(sync_error_is_retryable(&error)); + } + + #[test] + fn sync_error_retry_filter_accepts_transient_ssh_probe_failures() { + let error = Err::<(), _>(miette::miette!( + "ssh probe exited with status exit status: 255" + )) + .wrap_err("failed to resolve sandbox source path '/sandbox/ha-sync/ha-sync-upload'") + .unwrap_err(); + assert!(sync_error_is_retryable(&error)); + } + + #[test] + fn sync_error_retry_filter_rejects_validation_failures() { + let error = miette::miette!("sandbox source path '/etc/passwd' resolves outside /sandbox"); + assert!(!sync_error_is_retryable(&error)); + } + #[test] #[allow(unsafe_code)] // Test-only: env vars require unsafe in Rust 2024. fn install_ssh_config_adds_include_once_and_updates_managed_file() { diff --git a/crates/openshell-cli/tests/ensure_providers_integration.rs b/crates/openshell-cli/tests/ensure_providers_integration.rs index 9541d38bf4..374301c961 100644 --- a/crates/openshell-cli/tests/ensure_providers_integration.rs +++ b/crates/openshell-cli/tests/ensure_providers_integration.rs @@ -708,6 +708,17 @@ impl OpenShell for TestOpenShell { Err(Status::unimplemented("not implemented in test")) } + type PeerRelayStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn peer_relay( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + type ForwardTcpStream = tokio_stream::wrappers::ReceiverStream< Result, >; diff --git a/crates/openshell-cli/tests/mtls_integration.rs b/crates/openshell-cli/tests/mtls_integration.rs index 59ba2d287d..cff6a46f5c 100644 --- a/crates/openshell-cli/tests/mtls_integration.rs +++ b/crates/openshell-cli/tests/mtls_integration.rs @@ -560,6 +560,17 @@ impl OpenShell for TestOpenShell { Err(Status::unimplemented("not implemented in test")) } + type PeerRelayStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn peer_relay( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + type ForwardTcpStream = tokio_stream::wrappers::ReceiverStream< Result, >; diff --git a/crates/openshell-cli/tests/provider_commands_integration.rs b/crates/openshell-cli/tests/provider_commands_integration.rs index fac1bc7c05..ace3827d2e 100644 --- a/crates/openshell-cli/tests/provider_commands_integration.rs +++ b/crates/openshell-cli/tests/provider_commands_integration.rs @@ -1436,6 +1436,17 @@ impl OpenShell for TestOpenShell { Err(Status::unimplemented("not implemented in test")) } + type PeerRelayStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn peer_relay( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + type ForwardTcpStream = tokio_stream::wrappers::ReceiverStream< Result, >; diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index 25bc9ca6e5..e46cd4345a 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -1019,6 +1019,17 @@ impl OpenShell for TestOpenShell { Err(Status::unimplemented("not implemented in test")) } + type PeerRelayStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn peer_relay( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + type ForwardTcpStream = tokio_stream::wrappers::ReceiverStream< Result, >; diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs index 9e2412db97..0f446d8f37 100644 --- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs +++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs @@ -651,6 +651,17 @@ impl OpenShell for TestOpenShell { Err(Status::unimplemented("not implemented in test")) } + type PeerRelayStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn peer_relay( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + type ForwardTcpStream = tokio_stream::wrappers::ReceiverStream< Result, >; diff --git a/crates/openshell-sdk/tests/client_mock.rs b/crates/openshell-sdk/tests/client_mock.rs index d46ff80db4..8e4ea7df32 100644 --- a/crates/openshell-sdk/tests/client_mock.rs +++ b/crates/openshell-sdk/tests/client_mock.rs @@ -563,6 +563,16 @@ impl OpenShell for TestOpenShell { Err(Status::unimplemented("unused")) } + type PeerRelayStream = + tokio_stream::wrappers::ReceiverStream>; + + async fn peer_relay( + &self, + _: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + type ExecSandboxInteractiveStream = tokio_stream::wrappers::ReceiverStream>; diff --git a/crates/openshell-server/src/auth/descriptor_authz.rs b/crates/openshell-server/src/auth/descriptor_authz.rs index baea879bb7..7558d14b12 100644 --- a/crates/openshell-server/src/auth/descriptor_authz.rs +++ b/crates/openshell-server/src/auth/descriptor_authz.rs @@ -123,6 +123,7 @@ impl DescriptorAuthTable { "sandbox" => AuthMode::Sandbox, "bearer" => AuthMode::Bearer, "dual" => AuthMode::Dual, + "peer" => AuthMode::Peer, other => { return Err(format!("method {path}: unknown auth_mode '{other}'")); } diff --git a/crates/openshell-server/src/auth/guard.rs b/crates/openshell-server/src/auth/guard.rs index edcd6bc013..a3d55eac71 100644 --- a/crates/openshell-server/src/auth/guard.rs +++ b/crates/openshell-server/src/auth/guard.rs @@ -30,6 +30,9 @@ use tracing::info; pub fn ensure_sandbox_scope(principal: &Principal, claimed_sandbox_id: &str) -> Result<(), Status> { match principal { Principal::User(_) => Ok(()), + Principal::Peer(_) => Err(Status::permission_denied( + "gateway peer principals may not call sandbox-scoped methods", + )), Principal::Sandbox(p) => { if p.sandbox_id == claimed_sandbox_id { Ok(()) @@ -84,7 +87,7 @@ pub fn ensure_sandbox_principal_scope( ensure_sandbox_scope(principal, claimed_sandbox_id)?; Ok(p.clone()) } - Principal::User(_) => Err(Status::permission_denied( + Principal::User(_) | Principal::Peer(_) => Err(Status::permission_denied( "supervisor RPCs require a sandbox principal", )), Principal::Anonymous => Err(Status::unauthenticated( diff --git a/crates/openshell-server/src/auth/method_authz.rs b/crates/openshell-server/src/auth/method_authz.rs index 92d3b0a56a..4d91408644 100644 --- a/crates/openshell-server/src/auth/method_authz.rs +++ b/crates/openshell-server/src/auth/method_authz.rs @@ -22,6 +22,8 @@ pub enum AuthMode { /// Either sandbox principal or Bearer; scope and role apply on /// the Bearer path only. Dual, + /// Only callable by a gateway peer principal. + Peer, } /// Coarse role mapping. Maps to the configured `admin_role` / @@ -86,11 +88,17 @@ pub fn is_sandbox_callable(method: &str) -> bool { #[must_use] pub fn is_user_callable(method: &str) -> bool { match lookup(method).map(|m| m.auth_mode) { - Some(AuthMode::Sandbox | AuthMode::Unauthenticated) => false, + Some(AuthMode::Sandbox | AuthMode::Unauthenticated | AuthMode::Peer) => false, Some(AuthMode::Bearer | AuthMode::Dual) | None => true, } } +/// `true` if the method is callable by a gateway peer. +#[must_use] +pub fn is_peer_callable(method: &str) -> bool { + matches!(lookup(method).map(|m| m.auth_mode), Some(AuthMode::Peer)) +} + #[cfg(test)] mod tests { use super::*; @@ -142,6 +150,8 @@ mod tests { "/openshell.v1.OpenShell/ConnectSupervisor" )); assert!(!is_user_callable("/openshell.v1.OpenShell/RelayStream")); + assert!(!is_user_callable("/openshell.v1.OpenShell/PeerRelay")); + assert!(is_peer_callable("/openshell.v1.OpenShell/PeerRelay")); // Unauthenticated methods are not "user callable" — they're // intercepted before principal evaluation. assert!(!is_user_callable("/openshell.v1.OpenShell/Health")); diff --git a/crates/openshell-server/src/auth/mod.rs b/crates/openshell-server/src/auth/mod.rs index d3433a557e..3994ee537b 100644 --- a/crates/openshell-server/src/auth/mod.rs +++ b/crates/openshell-server/src/auth/mod.rs @@ -18,6 +18,7 @@ mod http; pub mod identity; pub mod method_authz; pub mod oidc; +pub mod peer; pub mod principal; pub mod sandbox_jwt; pub mod sandbox_methods; diff --git a/crates/openshell-server/src/auth/peer.rs b/crates/openshell-server/src/auth/peer.rs new file mode 100644 index 0000000000..b92c77bd31 --- /dev/null +++ b/crates/openshell-server/src/auth/peer.rs @@ -0,0 +1,597 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway peer authentication for internal replica-to-replica RPCs. +//! +//! Peer calls use Kubernetes projected `ServiceAccount` tokens, not an +//! OpenShell-managed shared secret. The caller presents its pod-bound gateway +//! `ServiceAccount` token with the peer audience; the receiver validates it with +//! the apiserver `TokenReview` API, checks the live pod UID and required labels, +//! and only then produces a [`Principal::Peer`]. + +use super::authenticator::Authenticator; +use super::principal::{PeerPrincipal, Principal}; +use async_trait::async_trait; +use k8s_openapi::api::{ + authentication::v1::{TokenReview, TokenReviewSpec, TokenReviewStatus, UserInfo}, + core::v1::Pod, +}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use kube::api::{Api, PostParams}; +use std::path::PathBuf; +use std::sync::Arc; +use tonic::Status; +use tracing::{debug, info, warn}; + +/// gRPC path for internal gateway relay forwarding. +pub const PEER_RELAY_PATH: &str = "/openshell.v1.OpenShell/PeerRelay"; +/// Audience used for gateway-to-gateway projected `ServiceAccount` tokens. +pub const DEFAULT_PEER_TOKEN_AUDIENCE: &str = "openshell-gateway-peer"; +/// Environment variable overriding the expected peer token audience. +pub const PEER_TOKEN_AUDIENCE_ENV: &str = "OPENSHELL_PEER_TOKEN_AUDIENCE"; +/// Environment variable carrying the projected peer `ServiceAccount` token path. +/// Uses the `*_TOKEN_FILE` convention from sandbox token env vars. +pub const PEER_SA_TOKEN_FILE_ENV: &str = "OPENSHELL_PEER_SERVICE_ACCOUNT_TOKEN_FILE"; +/// Default mount path for the projected peer `ServiceAccount` token. +pub const DEFAULT_PEER_SA_TOKEN_FILE: &str = "/var/run/secrets/openshell-peer/token"; +/// Environment variable with comma-separated `key=value` pod labels required +/// on authenticated gateway peer pods. +pub const PEER_REQUIRED_POD_LABELS_ENV: &str = "OPENSHELL_PEER_POD_LABELS"; +const POD_NAME_EXTRA: &str = "authentication.kubernetes.io/pod-name"; +const POD_UID_EXTRA: &str = "authentication.kubernetes.io/pod-uid"; + +#[derive(Debug, Clone)] +pub struct ResolvedGatewayPeerIdentity { + pub pod_name: String, + pub pod_uid: String, +} + +#[async_trait] +pub trait GatewayPeerIdentityResolver: Send + Sync + 'static { + async fn resolve(&self, token: &str) -> Result, Status>; +} + +#[derive(Debug)] +struct PeerTokenReviewIdentity { + pod_name: String, + pod_uid: String, +} + +pub struct PeerServiceAccountAuthenticator { + resolver: Arc, +} + +impl std::fmt::Debug for PeerServiceAccountAuthenticator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PeerServiceAccountAuthenticator") + .finish_non_exhaustive() + } +} + +impl PeerServiceAccountAuthenticator { + pub fn new(resolver: Arc) -> Self { + Self { resolver } + } +} + +#[async_trait] +impl Authenticator for PeerServiceAccountAuthenticator { + async fn authenticate( + &self, + headers: &http::HeaderMap, + path: &str, + ) -> Result, Status> { + if path != PEER_RELAY_PATH { + return Ok(None); + } + + let Some(token) = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + else { + return Ok(None); + }; + + let Some(resolved) = self.resolver.resolve(token).await? else { + debug!("K8s gateway peer token did not authenticate; falling through"); + return Ok(None); + }; + + if let Some(claimed_replica) = headers + .get("x-openshell-peer-replica") + .and_then(|v| v.to_str().ok()) + .filter(|v| !v.is_empty()) + && claimed_replica != resolved.pod_name.as_str() + { + warn!( + claimed_replica, + pod_name = %resolved.pod_name, + "gateway peer replica header does not match authenticated pod" + ); + return Err(Status::permission_denied( + "gateway peer replica does not match authenticated pod", + )); + } + + Ok(Some(Principal::Peer(PeerPrincipal { + replica_id: resolved.pod_name, + pod_uid: resolved.pod_uid, + }))) + } +} + +/// Resolver backed by Kubernetes `TokenReview` and a live Pod lookup. +pub struct LiveGatewayPeerResolver { + token_reviews_api: Api, + pods_api: Api, + expected_audience: String, + namespace: String, + expected_service_account: String, + required_pod_labels: Vec<(String, String)>, +} + +impl LiveGatewayPeerResolver { + pub fn new( + client: kube::Client, + namespace: &str, + expected_audience: String, + expected_service_account: String, + required_pod_labels: Vec<(String, String)>, + ) -> Self { + let token_reviews_api: Api = Api::all(client.clone()); + let pods_api: Api = Api::namespaced(client, namespace); + Self { + token_reviews_api, + pods_api, + expected_audience, + namespace: namespace.to_string(), + expected_service_account, + required_pod_labels, + } + } +} + +#[async_trait] +impl GatewayPeerIdentityResolver for LiveGatewayPeerResolver { + async fn resolve(&self, token: &str) -> Result, Status> { + let review = TokenReview { + metadata: ObjectMeta::default(), + spec: TokenReviewSpec { + audiences: Some(vec![self.expected_audience.clone()]), + token: Some(token.to_string()), + }, + status: None, + }; + + let review = self + .token_reviews_api + .create(&PostParams::default(), &review) + .await + .map_err(|err| { + warn!(error = %err, "K8s TokenReview failed for gateway peer"); + Status::internal(format!("peer tokenreview failed: {err}")) + })?; + let status = review + .status + .ok_or_else(|| Status::internal("TokenReview response missing status"))?; + let Some(identity) = peer_token_review_identity( + &status, + &self.expected_audience, + &self.namespace, + &self.expected_service_account, + )? + else { + return Ok(None); + }; + + let pod = self + .pods_api + .get_opt(&identity.pod_name) + .await + .map_err(|err| { + warn!( + pod = %identity.pod_name, + error = %err, + "failed to fetch gateway peer pod" + ); + Status::internal(format!("gateway peer pod GET failed: {err}")) + })?; + let Some(pod) = pod else { + warn!( + pod = %identity.pod_name, + "gateway peer pod referenced by SA token not found" + ); + return Err(Status::not_found("gateway peer pod not found")); + }; + + let actual_uid = pod.metadata.uid.as_deref().unwrap_or_default(); + if actual_uid != identity.pod_uid { + warn!( + pod = %identity.pod_name, + claimed_uid = %identity.pod_uid, + actual_uid, + "gateway peer SA token pod UID does not match live pod" + ); + return Err(Status::permission_denied( + "gateway peer SA token pod UID mismatch", + )); + } + + let actual_service_account = pod + .spec + .as_ref() + .and_then(|spec| spec.service_account_name.as_deref()) + .unwrap_or("default"); + if actual_service_account != self.expected_service_account { + warn!( + pod = %identity.pod_name, + service_account = %actual_service_account, + expected_service_account = %self.expected_service_account, + "gateway peer pod service account does not match TokenReview principal" + ); + return Err(Status::permission_denied( + "gateway peer pod service account mismatch", + )); + } + + validate_required_pod_labels(&pod, &self.required_pod_labels)?; + + info!( + pod_name = %identity.pod_name, + pod_uid = %identity.pod_uid, + service_account = %self.expected_service_account, + "validated gateway peer ServiceAccount token via TokenReview" + ); + + Ok(Some(ResolvedGatewayPeerIdentity { + pod_name: identity.pod_name, + pod_uid: identity.pod_uid, + })) + } +} + +#[allow(clippy::result_large_err)] +fn peer_token_review_identity( + status: &TokenReviewStatus, + expected_audience: &str, + namespace: &str, + expected_service_account: &str, +) -> Result, Status> { + if status.authenticated != Some(true) { + debug!( + error = status.error.as_deref().unwrap_or_default(), + "K8s TokenReview did not authenticate gateway peer token" + ); + return Ok(None); + } + + let audiences = status.audiences.as_deref().unwrap_or_default(); + if !audiences.iter().any(|aud| aud == expected_audience) { + warn!( + expected_audience, + audiences = ?audiences, + "K8s TokenReview authenticated gateway peer token without expected audience" + ); + return Err(Status::unauthenticated( + "gateway peer token audience not accepted", + )); + } + + let user = status + .user + .as_ref() + .ok_or_else(|| Status::permission_denied("TokenReview response missing user info"))?; + let username = user + .username + .as_deref() + .ok_or_else(|| Status::permission_denied("TokenReview response missing username"))?; + let expected_username = format!("system:serviceaccount:{namespace}:{expected_service_account}"); + if username != expected_username { + warn!( + username, + namespace, + service_account = %expected_service_account, + "K8s TokenReview principal is not the configured gateway service account" + ); + return Err(Status::permission_denied( + "gateway peer token is not from the configured service account", + )); + } + + let pod_name = user_extra_one(user, POD_NAME_EXTRA)?; + let pod_uid = user_extra_one(user, POD_UID_EXTRA)?; + Ok(Some(PeerTokenReviewIdentity { pod_name, pod_uid })) +} + +#[allow(clippy::result_large_err)] +fn user_extra_one(user: &UserInfo, key: &str) -> Result { + let Some(values) = user.extra.as_ref().and_then(|extra| extra.get(key)) else { + return Err(Status::permission_denied( + "gateway peer token is not pod-bound", + )); + }; + if values.len() != 1 || values[0].is_empty() { + return Err(Status::permission_denied( + "gateway peer token has invalid pod binding", + )); + } + Ok(values[0].clone()) +} + +#[allow(clippy::result_large_err)] +fn validate_required_pod_labels( + pod: &Pod, + required_labels: &[(String, String)], +) -> Result<(), Status> { + let labels = pod.metadata.labels.as_ref(); + for (key, expected) in required_labels { + let actual = labels + .and_then(|labels| labels.get(key)) + .map(String::as_str) + .unwrap_or_default(); + if actual != expected { + warn!( + pod = %pod.metadata.name.as_deref().unwrap_or_default(), + label = %key, + expected, + actual, + "gateway peer pod missing required label" + ); + return Err(Status::permission_denied( + "gateway peer pod labels do not match", + )); + } + } + Ok(()) +} + +pub fn peer_token_audience_from_env() -> String { + std::env::var(PEER_TOKEN_AUDIENCE_ENV) + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_PEER_TOKEN_AUDIENCE.to_string()) +} + +pub fn peer_service_account_token_file_from_env() -> Option { + if let Ok(path) = std::env::var(PEER_SA_TOKEN_FILE_ENV) + && !path.trim().is_empty() + { + return Some(PathBuf::from(path.trim())); + } + + let default_path = PathBuf::from(DEFAULT_PEER_SA_TOKEN_FILE); + default_path.exists().then_some(default_path) +} + +pub fn load_peer_service_account_token_from_env() -> Result, String> { + let Some(path) = peer_service_account_token_file_from_env() else { + return Ok(None); + }; + + let contents = std::fs::read_to_string(&path) + .map_err(|err| format!("failed to read {}: {err}", path.display()))?; + let token = contents.trim(); + if token.is_empty() { + return Err(format!( + "peer ServiceAccount token file {} is empty", + path.display() + )); + } + + Ok(Some(token.to_string())) +} + +pub fn required_pod_labels_from_env() -> Result, String> { + let raw = std::env::var(PEER_REQUIRED_POD_LABELS_ENV).unwrap_or_default(); + parse_required_pod_labels(&raw) +} + +fn parse_required_pod_labels(raw: &str) -> Result, String> { + let mut labels = Vec::new(); + for entry in raw + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + { + let Some((key, value)) = entry.split_once('=') else { + return Err(format!( + "{PEER_REQUIRED_POD_LABELS_ENV} entry {entry:?} must be key=value" + )); + }; + let key = key.trim(); + let value = value.trim(); + if key.is_empty() || value.is_empty() { + return Err(format!( + "{PEER_REQUIRED_POD_LABELS_ENV} entry {entry:?} must have non-empty key and value" + )); + } + labels.push((key.to_string(), value.to_string())); + } + Ok(labels) +} + +#[cfg(test)] +pub mod test_support { + use super::*; + use std::sync::Mutex; + + pub struct FakeGatewayPeerResolver { + pub outcome: Result, Status>, + pub seen_tokens: Mutex>, + } + + impl FakeGatewayPeerResolver { + pub fn returning(outcome: Result, Status>) -> Self { + Self { + outcome, + seen_tokens: Mutex::new(Vec::new()), + } + } + } + + #[async_trait] + impl GatewayPeerIdentityResolver for FakeGatewayPeerResolver { + async fn resolve( + &self, + token: &str, + ) -> Result, Status> { + self.seen_tokens.lock().unwrap().push(token.to_string()); + match &self.outcome { + Ok(opt) => Ok(opt.clone()), + Err(status) => Err(Status::new(status.code(), status.message())), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::test_support::FakeGatewayPeerResolver; + use super::*; + use std::collections::BTreeMap; + + fn bearer_headers(token: &str) -> http::HeaderMap { + let mut headers = http::HeaderMap::new(); + headers.insert( + "authorization", + http::HeaderValue::from_str(&format!("Bearer {token}")).unwrap(), + ); + headers + } + + fn token_review_status( + authenticated: bool, + audiences: Vec<&str>, + username: &str, + extra: Vec<(&str, &str)>, + ) -> TokenReviewStatus { + TokenReviewStatus { + authenticated: Some(authenticated), + audiences: Some(audiences.into_iter().map(str::to_string).collect()), + error: None, + user: Some(UserInfo { + username: Some(username.to_string()), + uid: Some("sa-uid".to_string()), + groups: Some(vec![ + "system:serviceaccounts".to_string(), + "system:serviceaccounts:openshell".to_string(), + "system:authenticated".to_string(), + ]), + extra: Some( + extra + .into_iter() + .map(|(key, value)| (key.to_string(), vec![value.to_string()])) + .collect(), + ), + }), + } + } + + #[test] + fn peer_token_review_identity_extracts_pod_binding() { + let status = token_review_status( + true, + vec![DEFAULT_PEER_TOKEN_AUDIENCE], + "system:serviceaccount:openshell:openshell", + vec![(POD_NAME_EXTRA, "openshell-0"), (POD_UID_EXTRA, "uid-a")], + ); + + let identity = peer_token_review_identity( + &status, + DEFAULT_PEER_TOKEN_AUDIENCE, + "openshell", + "openshell", + ) + .unwrap() + .expect("authenticated token should resolve"); + + assert_eq!(identity.pod_name, "openshell-0"); + assert_eq!(identity.pod_uid, "uid-a"); + } + + #[test] + fn peer_token_review_identity_rejects_wrong_service_account() { + let status = token_review_status( + true, + vec![DEFAULT_PEER_TOKEN_AUDIENCE], + "system:serviceaccount:openshell:default", + vec![(POD_NAME_EXTRA, "openshell-0"), (POD_UID_EXTRA, "uid-a")], + ); + + let err = peer_token_review_identity( + &status, + DEFAULT_PEER_TOKEN_AUDIENCE, + "openshell", + "openshell", + ) + .expect_err("wrong service account must fail closed"); + assert_eq!(err.code(), tonic::Code::PermissionDenied); + } + + #[test] + fn validate_required_pod_labels_rejects_mismatch() { + let pod = Pod { + metadata: ObjectMeta { + name: Some("openshell-0".to_string()), + labels: Some(BTreeMap::from([( + "app.kubernetes.io/name".to_string(), + "openshell".to_string(), + )])), + ..Default::default() + }, + ..Default::default() + }; + + let err = validate_required_pod_labels( + &pod, + &[( + "app.kubernetes.io/instance".to_string(), + "release-a".to_string(), + )], + ) + .expect_err("missing required label must fail"); + assert_eq!(err.code(), tonic::Code::PermissionDenied); + } + + #[tokio::test] + async fn authenticator_uses_resolved_pod_name_as_replica() { + let resolver = Arc::new(FakeGatewayPeerResolver::returning(Ok(Some( + ResolvedGatewayPeerIdentity { + pod_name: "openshell-0".to_string(), + pod_uid: "uid-a".to_string(), + }, + )))); + let auth = PeerServiceAccountAuthenticator::new(resolver); + + let principal = auth + .authenticate(&bearer_headers("token-a"), PEER_RELAY_PATH) + .await + .unwrap() + .expect("principal"); + + let Principal::Peer(peer) = principal else { + panic!("expected peer principal"); + }; + assert_eq!(peer.replica_id, "openshell-0"); + assert_eq!(peer.pod_uid, "uid-a"); + } + + #[test] + fn parse_required_pod_labels_accepts_comma_list() { + let labels = parse_required_pod_labels( + "app.kubernetes.io/name=openshell,app.kubernetes.io/instance=dev", + ) + .unwrap(); + assert_eq!( + labels, + vec![ + ( + "app.kubernetes.io/name".to_string(), + "openshell".to_string() + ), + ("app.kubernetes.io/instance".to_string(), "dev".to_string()) + ] + ); + } +} diff --git a/crates/openshell-server/src/auth/principal.rs b/crates/openshell-server/src/auth/principal.rs index 9567cc62d2..7d1cc00044 100644 --- a/crates/openshell-server/src/auth/principal.rs +++ b/crates/openshell-server/src/auth/principal.rs @@ -28,6 +28,8 @@ pub enum Principal { /// sandbox UUID. The wrapped `sandbox_id` MUST match any sandbox referenced /// in the request body for sandbox-class methods. Sandbox(#[allow(dead_code)] SandboxPrincipal), + /// Gateway replica authenticated for internal peer RPCs. + Peer(PeerPrincipal), /// Truly unauthenticated caller (health probes, reflection). Sandbox-class /// and user-class methods reject this variant. #[allow(dead_code)] @@ -57,6 +59,15 @@ pub struct SandboxPrincipal { pub trust_domain: Option, } +/// Gateway peer caller. +#[derive(Debug, Clone)] +pub struct PeerPrincipal { + /// Peer replica id supplied by the authenticated caller. + pub replica_id: String, + /// UID of the authenticated Kubernetes pod. + pub pod_uid: String, +} + /// How a [`SandboxPrincipal`] was authenticated. /// /// Variant fields are populated by the producing authenticator and consumed diff --git a/crates/openshell-server/src/auth/sandbox_methods.rs b/crates/openshell-server/src/auth/sandbox_methods.rs index f368298dd3..29e56563cc 100644 --- a/crates/openshell-server/src/auth/sandbox_methods.rs +++ b/crates/openshell-server/src/auth/sandbox_methods.rs @@ -26,6 +26,7 @@ mod tests { "/openshell.v1.OpenShell/ConnectSupervisor" )); assert!(is_sandbox_callable("/openshell.v1.OpenShell/RelayStream")); + assert!(!is_sandbox_callable("/openshell.v1.OpenShell/PeerRelay")); assert!(is_sandbox_callable( "/openshell.v1.OpenShell/GetSandboxConfig" )); diff --git a/crates/openshell-server/src/auth/workspace_authz.rs b/crates/openshell-server/src/auth/workspace_authz.rs index 00b48288fa..0aa38fe0bb 100644 --- a/crates/openshell-server/src/auth/workspace_authz.rs +++ b/crates/openshell-server/src/auth/workspace_authz.rs @@ -190,6 +190,9 @@ pub async fn authorize_workspace( workspace, grant: AuthGrant::Sandbox, }), + Principal::Peer(_) => Err(Status::permission_denied( + "gateway peer principals cannot perform workspace operations", + )), Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } } @@ -223,6 +226,9 @@ pub fn require_platform_admin(admin_role: &str, principal: &Principal) -> Result Principal::Sandbox(_) => Err(Status::permission_denied( "sandbox principals cannot perform cross-workspace operations", )), + Principal::Peer(_) => Err(Status::permission_denied( + "gateway peer principals cannot perform cross-workspace operations", + )), Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 473da2b049..7688884781 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -16,6 +16,7 @@ use crate::persistence::{ }; use crate::sandbox_index::SandboxIndex; use crate::sandbox_watch::SandboxWatchBus; +use crate::supervisor_owner::{OWNER_TTL, SupervisorOwnerIndex}; use crate::supervisor_session::SupervisorSessionRegistry; use crate::tracing_bus::TracingLogBus; use futures::{Stream, StreamExt}; @@ -600,6 +601,13 @@ pub struct ComputeRuntime { rootfs_tar_staging: Arc, } +pub struct SandboxSyncGuard { + // Drop the database guard before the local mutex so another local waiter + // cannot race ahead while this replica still owns the cluster-wide lock. + _distributed: crate::persistence::DistributedMutationGuard, + _local: tokio::sync::OwnedMutexGuard<()>, +} + impl fmt::Debug for ComputeRuntime { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ComputeRuntime").finish_non_exhaustive() @@ -677,13 +685,19 @@ impl ComputeRuntime { } /// Serializes sandbox/provider-profile invariant checks and object writes - /// within this gateway process. + /// across gateway replicas. /// - /// This is a temporary single-gateway guard for cross-object invariants. - /// It is not HA-safe; replace it with DB-backed CAS/resource-version writes - /// tracked by #1255 before enabling multiple gateway writers. - pub(crate) async fn sandbox_sync_guard(&self) -> tokio::sync::OwnedMutexGuard<()> { - self.sync_lock.clone().lock_owned().await + /// The local mutex preserves lock ordering within one process. `PostgreSQL` + /// deployments also hold a session-level advisory lock for the duration. + pub(crate) async fn sandbox_sync_guard( + &self, + ) -> crate::persistence::PersistenceResult { + let local = self.sync_lock.clone().lock_owned().await; + let distributed = self.store.acquire_distributed_mutation_guard().await?; + Ok(SandboxSyncGuard { + _distributed: distributed, + _local: local, + }) } /// Acquires the process-wide lock for code that already holds the @@ -1542,7 +1556,13 @@ impl ComputeRuntime { ) -> Option { let sandbox_id = transition.object_id().to_string(); let expected_resource_version = sandbox_resource_version(transition); - let session_connected = self.supervisor_sessions.has_session(&sandbox_id); + let session_connected = self + .supervisor_session_ready(&sandbox_id) + .await + .unwrap_or_else(|error| { + warn!(sandbox_id, %error, "Failed to resolve supervisor owner during lifecycle reconciliation"); + self.supervisor_sessions.has_session(&sandbox_id) + }); match self .store .update_message_cas::(&sandbox_id, expected_resource_version, |sandbox| { @@ -2021,7 +2041,13 @@ impl ComputeRuntime { match observed { Ok(Some(snapshot)) if snapshot.id == sandbox_id && snapshot.status.is_some() => { - let session_connected = self.supervisor_sessions.has_session(sandbox_id); + let session_connected = self + .supervisor_session_ready(sandbox_id) + .await + .unwrap_or_else(|error| { + warn!(sandbox_id, %error, "Failed to resolve supervisor owner during delete recovery"); + self.supervisor_sessions.has_session(sandbox_id) + }); self.write_delete_recovery_with_retry( sandbox_id, deleting_resource_version, @@ -3233,7 +3259,7 @@ impl ComputeRuntime { expected_resource_version: u64, existing_phase: SandboxPhase, ) -> Result<(), String> { - let session_connected = self.supervisor_sessions.has_session(&incoming.id); + let session_connected = self.supervisor_session_ready(&incoming.id).await?; let sandbox = self .store .update_message_cas::( @@ -3284,8 +3310,77 @@ impl ComputeRuntime { sandbox_id: &str, terminal_delivery_finalized: bool, ) -> Result<(), String> { - self.set_supervisor_session_state(sandbox_id, false, None, terminal_delivery_finalized) + let _guard = self.sync_lock.lock().await; + + // A replacement session may already belong to another gateway. Do not + // let cleanup from this replica overwrite the replacement's Ready state. + if self.supervisor_session_ready(sandbox_id).await? { + return Ok(()); + } + + let existing = self + .store + .get_message::(sandbox_id) .await + .map_err(|error| error.to_string())?; + self.set_supervisor_session_state_from_snapshot( + sandbox_id, + false, + None, + terminal_delivery_finalized, + existing, + ) + .await?; + + // Ownership can move while the CAS above is in flight. Restore Ready + // from the new owner's supervisor instance before releasing the local + // synchronization boundary. + if let Some(instance_id) = self + .supervisor_session_owner_instance_id(sandbox_id) + .await? + { + let existing = self + .store + .get_message::(sandbox_id) + .await + .map_err(|error| error.to_string())?; + self.set_supervisor_session_state_from_snapshot( + sandbox_id, + true, + Some(&instance_id), + false, + existing, + ) + .await?; + } + Ok(()) + } + + async fn supervisor_session_ready(&self, sandbox_id: &str) -> Result { + if self.supervisor_sessions.has_session(sandbox_id) { + return Ok(true); + } + Ok(self + .supervisor_session_owner_instance_id(sandbox_id) + .await? + .is_some()) + } + + async fn supervisor_session_owner_instance_id( + &self, + sandbox_id: &str, + ) -> Result, String> { + let owner_index = SupervisorOwnerIndex::new(self.store.clone(), OWNER_TTL); + let Some(owner) = owner_index + .read(sandbox_id) + .await + .map_err(|error| error.to_string())? + else { + return Ok(None); + }; + let age_ms = openshell_core::time::now_ms() - owner.updated_at_ms; + let ttl_ms = i64::try_from(OWNER_TTL.as_millis()).unwrap_or(i64::MAX); + Ok((age_ms < ttl_ms).then_some(owner.supervisor_instance_id)) } async fn set_supervisor_session_state( @@ -10609,6 +10704,42 @@ mod tests { assert_eq!(ready.message, "Supervisor session disconnected"); } + #[tokio::test] + async fn stale_session_disconnect_preserves_new_remote_owner_ready_state() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready); + sandbox.set_phase(SandboxPhase::Ready as i32); + runtime.store.put_message(&sandbox).await.unwrap(); + + SupervisorOwnerIndex::new(runtime.store.clone(), OWNER_TTL) + .publish( + "sb-1", + "replacement-session", + "supervisor-instance", + 2, + "gateway-b", + "http://gateway-b:8080", + ) + .await + .unwrap(); + + runtime + .supervisor_session_disconnected("sb-1", false) + .await + .unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase()).unwrap(), + SandboxPhase::Ready + ); + } + // --- Composition rule tests --- fn make_ready_driver_status() -> DriverSandboxStatus { diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index 9994208c3e..337b515445 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -49,8 +49,8 @@ use openshell_core::proto::{ ListSandboxProvidersResponse, ListSandboxTemplatesRequest, ListSandboxTemplatesResponse, ListSandboxesRequest, ListSandboxesResponse, ListServicesRequest, ListServicesResponse, ListWorkspaceMembersRequest, ListWorkspaceMembersResponse, ListWorkspacesRequest, - ListWorkspacesResponse, MemoryResourceCapabilities, ProviderProfileResponse, ProviderResponse, - PushSandboxLogsRequest, PushSandboxLogsResponse, RefreshSandboxTokenRequest, + ListWorkspacesResponse, MemoryResourceCapabilities, PeerRelayFrame, ProviderProfileResponse, + ProviderResponse, PushSandboxLogsRequest, PushSandboxLogsResponse, RefreshSandboxTokenRequest, RefreshSandboxTokenResponse, RejectDraftChunkRequest, RejectDraftChunkResponse, RelayFrame, RemoveWorkspaceMemberRequest, RemoveWorkspaceMemberResponse, ReportEndpointStatusRequest, ReportEndpointStatusResponse, ReportMainProcessExitRequest, ReportMainProcessExitResponse, @@ -833,6 +833,16 @@ impl OpenShell for OpenShellService { ) -> Result, Status> { workspace::handle_list_workspace_members(&self.state, request).await } + + type PeerRelayStream = + Pin> + Send + 'static>>; + + async fn peer_relay( + &self, + request: Request>, + ) -> Result, Status> { + crate::supervisor_session::handle_peer_relay(&self.state, request).await + } } fn public_resource_capabilities( diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index c9863813e3..97218d79a6 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -2528,6 +2528,9 @@ async fn resolve_sandbox_by_name_for_principal( Ok(sandbox) } Principal::User(_) => sandbox.ok_or_else(|| Status::not_found("sandbox not found")), + Principal::Peer(_) => Err(Status::permission_denied( + "gateway peer principals may not resolve sandbox configuration", + )), Principal::Anonymous => Err(Status::unauthenticated( "sandbox-scoped methods require an authenticated caller", )), @@ -4613,7 +4616,9 @@ pub(super) async fn handle_report_policy_status( .supersede_older_policies(&req.sandbox_id, version) .await; - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::persistence_error_to_status(error, "acquire policy mutation lock") + })?; let sandbox = state .store .get_message::(&req.sandbox_id) diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status.rs b/crates/openshell-server/src/grpc/policy/endpoint_status.rs index 0081d58aae..82128e1daa 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status.rs @@ -79,7 +79,9 @@ pub(in crate::grpc) async fn handle_report_endpoint_status( // Session validation, configuration derivation, and persistence share the // sandbox mutation boundary. A newly registered supervisor can therefore // invalidate its predecessor before any stale report reaches the CAS. - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::super::persistence_error_to_status(error, "acquire endpoint status mutation lock") + })?; if !state .supervisor_sessions .is_endpoint_status_authority(&req.sandbox_id, &req.supervisor_session_id) @@ -328,7 +330,9 @@ pub async fn reset_endpoint_status_for_supervisor_session( sandbox_id: &str, supervisor_session_id: &str, ) -> Result<(), Status> { - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::super::persistence_error_to_status(error, "acquire endpoint status mutation lock") + })?; if !state .supervisor_sessions .is_current_session(sandbox_id, supervisor_session_id) @@ -373,7 +377,9 @@ pub async fn reset_endpoint_status_after_supervisor_disconnect( state: &Arc, sandbox_id: &str, ) -> Result<(), Status> { - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::super::persistence_error_to_status(error, "acquire endpoint status mutation lock") + })?; if state .supervisor_sessions .current_session_id(sandbox_id) diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs index cd3f863cf3..4c951863a0 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs @@ -408,7 +408,7 @@ async fn global_policy_update_waits_for_endpoint_report_guard() { let before = load_global_settings(state.store.as_ref()) .await .expect("read settings before update"); - let guard = state.compute.sandbox_sync_guard().await; + let guard = state.compute.sandbox_sync_guard().await.unwrap(); let mut pending = Box::pin(handle_update_config(&state, authed_request(update))); // Poll the actual writer while an endpoint report owns the mutation diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index a2563babeb..7cbc2749fa 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -2548,7 +2548,9 @@ pub(super) async fn handle_create_provider( )); } let provider_type = provider.r#type.clone(); - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::persistence_error_to_status(error, "acquire provider mutation lock") + })?; let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -2775,7 +2777,10 @@ pub(super) async fn handle_import_provider_profiles( .ensure_active()?; let (profiles, mut diagnostics) = profiles_from_import_items(&request.profiles); add_empty_profile_set_diagnostic(&profiles, &mut diagnostics); - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = + state.compute.sandbox_sync_guard().await.map_err(|err| { + super::persistence_error_to_status(err, "acquire provider mutation lock") + })?; let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -2867,7 +2872,10 @@ pub(super) async fn handle_update_provider_profiles( let (profiles, mut diagnostics) = profiles_from_import_items(&items); add_empty_profile_set_diagnostic(&profiles, &mut diagnostics); let target_id = normalize_profile_id_request(&request.id)?; - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = + state.compute.sandbox_sync_guard().await.map_err(|err| { + super::persistence_error_to_status(err, "acquire provider mutation lock") + })?; let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -3027,7 +3035,10 @@ pub(super) async fn handle_delete_provider_profile( .name; let id = req.id; let id = normalize_profile_id_request(&id)?; - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = + state.compute.sandbox_sync_guard().await.map_err(|err| { + super::persistence_error_to_status(err, "acquire provider mutation lock") + })?; let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) @@ -3831,7 +3842,9 @@ pub(super) async fn handle_update_provider( // Provider material contributes to the route-report configuration epoch. // Serialize its mutation with route-status validation so a report derived // from the prior revision cannot commit after this update. - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::persistence_error_to_status(error, "acquire provider mutation lock") + })?; let Some(mut provider) = req.provider else { emit_provider_lifecycle( "custom", @@ -4587,7 +4600,10 @@ pub(super) async fn handle_configure_provider_refresh( // configures of providers attached to the same sandbox could each pass // validation before either persisted and both reserve the same key (CWE-362). // This is the same guard sandbox create/attach and profile changes take. - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = + state.compute.sandbox_sync_guard().await.map_err(|err| { + super::persistence_error_to_status(err, "acquire provider mutation lock") + })?; let provider = state .store @@ -5549,7 +5565,7 @@ mod tests { #[tokio::test] async fn import_provider_profile_waits_for_sandbox_sync_guard() { let state = test_server_state().await; - let guard = state.compute.sandbox_sync_guard().await; + let guard = state.compute.sandbox_sync_guard().await.unwrap(); let task_state = state.clone(); let task = tokio::spawn(async move { handle_import_provider_profiles( @@ -8730,7 +8746,7 @@ mod tests { .await .unwrap(); - let guard = state.compute.sandbox_sync_guard().await; + let guard = state.compute.sandbox_sync_guard().await.unwrap(); let task_state = state.clone(); let task = tokio::spawn(async move { handle_delete_provider_profile( @@ -8785,7 +8801,7 @@ mod tests { .await .unwrap(); - let guard = state.compute.sandbox_sync_guard().await; + let guard = state.compute.sandbox_sync_guard().await.unwrap(); let task_state = state.clone(); let task = tokio::spawn(async move { let mut provider = provider_with_values("guarded-provider", "guarded-create"); diff --git a/crates/openshell-server/src/grpc/provider_readiness_tests.rs b/crates/openshell-server/src/grpc/provider_readiness_tests.rs index 1bdf4aa63e..d83bb80ef4 100644 --- a/crates/openshell-server/src/grpc/provider_readiness_tests.rs +++ b/crates/openshell-server/src/grpc/provider_readiness_tests.rs @@ -70,6 +70,7 @@ fn hello() -> SupervisorHello { SupervisorHello { sandbox_id: Uuid::new_v4().to_string(), instance_id: Uuid::new_v4().to_string(), + connection_epoch: 0, supports_provider_readiness: true, } } diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index a12fdadb9a..39edede7ee 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -453,7 +453,9 @@ async fn handle_create_sandbox_inner( let _sandbox_sync_guard = if spec.providers.is_empty() { None } else { - Some(state.compute.sandbox_sync_guard().await) + Some(state.compute.sandbox_sync_guard().await.map_err(|err| { + super::persistence_error_to_status(err, "acquire sandbox mutation lock") + })?) }; // Validate provider names exist (fail fast). @@ -2013,11 +2015,15 @@ pub(super) async fn handle_exec_sandbox( // Open a relay channel through the supervisor session. Use a 15s // session-wait timeout, enough to cover a transient supervisor reconnect // while still failing quickly during normal operation. - let (channel_id, relay_rx) = state - .supervisor_sessions - .open_relay(sandbox.object_id(), std::time::Duration::from_secs(15)) - .await - .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; + let (channel_id, relay_rx) = crate::supervisor_session::open_routed_relay_with_target( + state, + sandbox.object_id(), + relay_open::Target::Ssh(SshRelayTarget {}), + String::new(), + std::time::Duration::from_secs(15), + ) + .await + .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; let command_str = build_remote_exec_command(&req) .map_err(|e| Status::invalid_argument(format!("command construction failed: {e}")))?; @@ -2142,16 +2148,15 @@ pub(super) async fn handle_forward_tcp( } let connection_guard = acquire_forward_connection_guard(state, &init, &sandbox).await?; - let (channel_id, relay_rx) = state - .supervisor_sessions - .open_relay_with_target( - sandbox.object_id(), - target, - init.service_id.clone(), - std::time::Duration::from_secs(15), - ) - .await - .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; + let (channel_id, relay_rx) = crate::supervisor_session::open_routed_relay_with_target( + state, + sandbox.object_id(), + target, + init.service_id.clone(), + std::time::Duration::from_secs(15), + ) + .await + .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; let sandbox_id = sandbox.object_id().to_string(); let (tx, rx) = mpsc::channel::>(256); @@ -2465,11 +2470,15 @@ pub(super) async fn handle_exec_sandbox_interactive( return Err(Status::failed_precondition("sandbox is not ready")); } - let (channel_id, relay_rx) = state - .supervisor_sessions - .open_relay(sandbox.object_id(), std::time::Duration::from_secs(15)) - .await - .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; + let (channel_id, relay_rx) = crate::supervisor_session::open_routed_relay_with_target( + state, + sandbox.object_id(), + relay_open::Target::Ssh(SshRelayTarget {}), + String::new(), + std::time::Duration::from_secs(15), + ) + .await + .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; let command_str = build_remote_exec_command(&req) .map_err(|e| Status::invalid_argument(format!("command construction failed: {e}")))?; @@ -3894,7 +3903,7 @@ mod tests { // Hold the global guard so the handler can resolve the original ID and // acquire its delete gate, but cannot yet revalidate or mutate it. - let global_guard = state.compute.sandbox_sync_guard().await; + let global_guard = state.compute.sandbox_sync_guard().await.unwrap(); let delete_state = state.clone(); let delete = tokio::spawn(async move { handle_delete_sandbox_inner( @@ -5358,7 +5367,7 @@ mod tests { .await .unwrap(); - let guard = state.compute.sandbox_sync_guard().await; + let guard = state.compute.sandbox_sync_guard().await.unwrap(); let task_state = state.clone(); let task = tokio::spawn(async move { handle_create_sandbox( diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index f3a9392693..c8677d890d 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -68,6 +68,9 @@ fn membership_filter_subject<'a>( } } Principal::Sandbox(_) => Ok(None), + Principal::Peer(_) => Err(Status::permission_denied( + "gateway peer principals cannot list workspaces", + )), Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a8cd09acca..79fae6589c 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -38,6 +38,7 @@ mod sandbox_watch; mod service_routing; mod ssh_sessions; mod storage_proto; +mod supervisor_owner; pub mod supervisor_session; mod telemetry; #[cfg(any(test, feature = "test-support"))] @@ -304,6 +305,12 @@ pub struct ServerState { /// distinguish expected transport closes from runtime failures. pub(crate) gateway_shutting_down: AtomicBool, + /// Stable identity for this gateway process. + pub replica_id: String, + + /// Internal endpoint other gateway replicas can dial for peer RPCs. + pub peer_endpoint: Option, + /// Validated built-in and operator-registered supervisor middleware. pub middleware_registry: Arc, @@ -328,6 +335,9 @@ pub struct ServerState { /// bootstrap path. pub compute_driver_authenticator: Option>, + /// Optional K8s `ServiceAccount` authenticator for gateway peer RPCs. + pub peer_authenticator: Option>, + /// Gateway-wide gRPC request rate limiter shared by every multiplex path. pub(crate) grpc_rate_limiter: Option, @@ -404,6 +414,8 @@ impl ServerState { oidc_cache: Option>, credentials: credentials::CredentialRuntime, ) -> Self { + let replica_id = compute::lease::replica_id(); + let peer_endpoint = derive_peer_endpoint(&config); let grpc_rate_limiter = multiplex::GrpcRateLimiter::from_config(&config); let admin_role = config .oidc @@ -423,6 +435,8 @@ impl ServerState { settings_mutex: tokio::sync::Mutex::new(()), supervisor_sessions, gateway_shutting_down: AtomicBool::new(false), + replica_id, + peer_endpoint, extension_mint_limiter: auth::extension_mint_limit::ExtensionMintLimiter::default(), middleware_registry: Arc::new(MiddlewareRegistry::default()), oidc_cache, @@ -430,6 +444,7 @@ impl ServerState { sandbox_session_jwt_authority: None, sandbox_jwt_authenticator: None, compute_driver_authenticator: None, + peer_authenticator: None, grpc_rate_limiter, gateway_interceptors: None, provider_profile_sources: @@ -439,6 +454,34 @@ impl ServerState { } } +fn derive_peer_endpoint(config: &Config) -> Option { + if let Ok(endpoint) = std::env::var("OPENSHELL_PEER_ENDPOINT") + && !endpoint.trim().is_empty() + { + return Some(endpoint.trim().to_string()); + } + + let pod_name = std::env::var("OPENSHELL_POD_NAME").ok()?; + let namespace = std::env::var("OPENSHELL_POD_NAMESPACE").ok()?; + let service = std::env::var("OPENSHELL_PEER_SERVICE_NAME").ok()?; + if pod_name.trim().is_empty() || namespace.trim().is_empty() || service.trim().is_empty() { + return None; + } + + let scheme = if config.tls.is_some() { + "https" + } else { + "http" + }; + Some(format!( + "{scheme}://{pod}.{service}.{namespace}.svc.cluster.local:{port}", + pod = pod_name.trim(), + service = service.trim(), + namespace = namespace.trim(), + port = config.bind_address.port() + )) +} + /// Run the `OpenShell` server. /// /// This starts a multiplexed gRPC/HTTP server on the configured bind address. @@ -700,6 +743,51 @@ pub(crate) async fn run_server( ); } + if std::env::var_os("KUBERNETES_SERVICE_HOST").is_some() { + let namespace = std::env::var("OPENSHELL_POD_NAMESPACE").ok(); + let service_account = std::env::var("OPENSHELL_SERVICE_ACCOUNT_NAME").ok(); + match (namespace, service_account) { + (Some(namespace), Some(service_account)) + if !namespace.trim().is_empty() && !service_account.trim().is_empty() => + { + let required_labels = + auth::peer::required_pod_labels_from_env().map_err(Error::config)?; + match kube::Client::try_default().await { + Ok(client) => { + let audience = auth::peer::peer_token_audience_from_env(); + let resolver = Arc::new(auth::peer::LiveGatewayPeerResolver::new( + client, + namespace.trim(), + audience.clone(), + service_account.trim().to_string(), + required_labels, + )); + let authenticator = + auth::peer::PeerServiceAccountAuthenticator::new(resolver); + state.peer_authenticator = Some(Arc::new(authenticator)); + info!( + namespace = %namespace.trim(), + service_account = %service_account.trim(), + audience, + "gateway peer ServiceAccount TokenReview authentication enabled" + ); + } + Err(err) => warn!( + error = %err, + "in-cluster K8s client construction failed; \ + gateway peer ServiceAccount authentication is disabled" + ), + } + } + _ => { + debug!( + "OPENSHELL_POD_NAMESPACE or OPENSHELL_SERVICE_ACCOUNT_NAME missing; \ + gateway peer ServiceAccount authentication disabled" + ); + } + } + } + let state = Arc::new(state); // Reconcile local-driver running intent before watchers spawn so their @@ -839,6 +927,16 @@ pub(crate) async fn run_server( } startup_tx.send_replace(true); + // The poller observes writes made by other replicas. Single-replica + // backends already see every write through the local update bus. + if !store.is_single_replica() { + sandbox_watch::spawn_store_poller( + store.clone(), + state.sandbox_watch_bus.clone(), + Duration::from_secs(1), + shutdown_rx.clone(), + ); + } ssh_sessions::spawn_session_reaper(store.clone(), Duration::from_hours(1)); supervisor_session::spawn_relay_reaper(state.clone(), Duration::from_secs(30)); provider_refresh::spawn_refresh_worker(state.clone(), Duration::from_mins(1)); diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 2df0ea5403..5dd2cc3b3b 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -675,6 +675,11 @@ fn gateway_principal_fields(principal: &Principal) -> BTreeMap { fields.insert("trust_domain".to_string(), trust_domain.clone()); } } + Principal::Peer(peer) => { + fields.insert("kind".to_string(), "peer".to_string()); + fields.insert("replica_id".to_string(), peer.replica_id.clone()); + fields.insert("pod_uid".to_string(), peer.pod_uid.clone()); + } Principal::Anonymous => { fields.insert("kind".to_string(), "anonymous".to_string()); } @@ -845,12 +850,15 @@ where /// Assemble the authenticator chain for the gateway. /// /// Chain order (first-match-wins): -/// 1. `ComputeDriverAuthenticator` (path-scoped to `IssueSandboxToken`) +/// 1. `PeerServiceAccountAuthenticator` (path-scoped to `PeerRelay`) +/// — validates gateway replica projected `ServiceAccount` tokens with +/// `TokenReview` for internal peer relay calls. No-op on every other path. +/// 2. `ComputeDriverAuthenticator` (path-scoped to `IssueSandboxToken`) /// — delegates a driver-native credential and receives a sandbox identity /// so the handler can mint a gateway JWT. No-op on every other path. -/// 2. `SandboxJwtAuthenticator` — validates gateway-minted JWTs. Recognized +/// 3. `SandboxJwtAuthenticator` — validates gateway-minted JWTs. Recognized /// via a distinctive `kid` so non-matching Bearer tokens fall through. -/// 3. `OidcAuthenticator` — validates user Bearer tokens against the +/// 4. `OidcAuthenticator` — validates user Bearer tokens against the /// configured OIDC issuer. Returns `Unauthenticated` for missing /// Bearer headers so non-OIDC clients can't sneak through. /// @@ -865,6 +873,9 @@ where /// to pass-through unless mTLS or local unauthenticated users are enabled. fn build_authenticator_chain(state: &ServerState) -> Option { let mut authenticators: Vec> = Vec::new(); + if let Some(peer) = state.peer_authenticator.clone() { + authenticators.push(peer); + } if let Some(driver) = state.compute_driver_authenticator.clone() { authenticators.push(driver); } @@ -1040,6 +1051,13 @@ where ))); } } + Principal::Peer(_) => { + if !crate::auth::method_authz::is_peer_callable(&path) { + return Ok(status_response(tonic::Status::permission_denied( + "gateway peer principals may not call this method", + ))); + } + } Principal::Anonymous => { return Ok(status_response(tonic::Status::unauthenticated( "anonymous callers may not call authenticated methods", diff --git a/crates/openshell-server/src/persistence/mod.rs b/crates/openshell-server/src/persistence/mod.rs index 0b365ccd0c..0306bdd312 100644 --- a/crates/openshell-server/src/persistence/mod.rs +++ b/crates/openshell-server/src/persistence/mod.rs @@ -197,6 +197,11 @@ pub enum Store { Sqlite(SqliteStore), } +/// RAII guard for the database-backed cross-object mutation lock. +pub struct DistributedMutationGuard { + _postgres: Option, +} + /// Trait for inferring an object type string from a message type. pub trait ObjectType { fn object_type() -> &'static str; @@ -277,6 +282,23 @@ impl Store { matches!(self, Self::Sqlite(_)) } + /// Serialize mutations whose invariants span multiple persisted objects. + /// + /// `SQLite` deployments are single-replica and use only the caller's local + /// mutex. `PostgreSQL` deployments additionally hold a session-level + /// advisory lock so concurrent gateway replicas cannot validate and write + /// the same cross-object invariant independently. + pub async fn acquire_distributed_mutation_guard( + &self, + ) -> PersistenceResult { + match self { + Self::Postgres(store) => Ok(DistributedMutationGuard { + _postgres: Some(store.acquire_cross_object_lock().await?), + }), + Self::Sqlite(_) => Ok(DistributedMutationGuard { _postgres: None }), + } + } + /// Connect to a persistence store based on the database URL. pub async fn connect(url: &str) -> CoreResult { if url.starts_with("postgres://") || url.starts_with("postgresql://") { diff --git a/crates/openshell-server/src/persistence/postgres.rs b/crates/openshell-server/src/persistence/postgres.rs index c0be94e0c9..ef613a77bd 100644 --- a/crates/openshell-server/src/persistence/postgres.rs +++ b/crates/openshell-server/src/persistence/postgres.rs @@ -14,6 +14,7 @@ use crate::policy_store::{ use openshell_core::SetResourceVersion; use openshell_core::proto::Sandbox; use prost::Message; +use sqlx::pool::PoolConnection; use sqlx::postgres::PgPoolOptions; use sqlx::{Connection, PgPool, Postgres, QueryBuilder, Row}; @@ -34,6 +35,18 @@ pub struct PostgresStore { pool: PgPool, } +// Stable cluster-wide key for serializing sandbox/provider cross-object +// mutations. The bytes spell "OPENSHLL" and stay within PostgreSQL's signed +// 64-bit advisory-lock key space. +const CROSS_OBJECT_ADVISORY_LOCK_KEY: i64 = 0x4f50_454e_5348_4c4c; + +pub(super) struct PostgresAdvisoryLockGuard { + // `close_on_drop` is set before this guard is constructed. Closing the + // dedicated session releases the session-level advisory lock even when a + // request is cancelled or returns early. + _connection: PoolConnection, +} + impl PostgresStore { pub async fn connect(url: &str) -> PersistenceResult { let pool = PgPoolOptions::new() @@ -96,6 +109,21 @@ impl PostgresStore { conn.ping().await.map_err(|e| map_db_error(&e)) } + pub(super) async fn acquire_cross_object_lock( + &self, + ) -> PersistenceResult { + let mut connection = self.pool.acquire().await.map_err(|e| map_db_error(&e))?; + connection.close_on_drop(); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(CROSS_OBJECT_ADVISORY_LOCK_KEY) + .execute(&mut *connection) + .await + .map_err(|e| map_db_error(&e))?; + Ok(PostgresAdvisoryLockGuard { + _connection: connection, + }) + } + /// Test support only: close the underlying connection pool. /// /// Do not call from runtime code; this tears down the active pool. diff --git a/crates/openshell-server/src/provider_refresh.rs b/crates/openshell-server/src/provider_refresh.rs index 6d666fbd56..e8b39cbac4 100644 --- a/crates/openshell-server/src/provider_refresh.rs +++ b/crates/openshell-server/src/provider_refresh.rs @@ -1228,7 +1228,9 @@ async fn apply_minted_credential( // prevents route status from committing against the old provider revision // after the rotation writes, without holding the guard across network I/O. let _sandbox_sync_guard = if let Some(compute) = compute { - Some(compute.sandbox_sync_guard().await) + Some(compute.sandbox_sync_guard().await.map_err(|error| { + Status::internal(format!("acquire provider mutation lock: {error}")) + })?) } else { None }; diff --git a/crates/openshell-server/src/sandbox_watch.rs b/crates/openshell-server/src/sandbox_watch.rs index ac38eba8db..0fbe485f77 100644 --- a/crates/openshell-server/src/sandbox_watch.rs +++ b/crates/openshell-server/src/sandbox_watch.rs @@ -3,12 +3,16 @@ //! In-memory buses to support sandbox watch streaming. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; +use std::time::Duration; -use tokio::sync::broadcast; +use tokio::sync::{broadcast, watch}; use tonic::Status; +use crate::persistence::Store; +use openshell_core::proto::Sandbox; + /// Broadcast bus of sandbox updates keyed by sandbox id. /// /// Producers call [`SandboxWatchBus::notify`] whenever the persisted sandbox record changes. @@ -57,6 +61,71 @@ impl SandboxWatchBus { let mut inner = self.inner.lock().expect("sandbox watch bus lock poisoned"); inner.remove(sandbox_id); } + + fn active_sandbox_ids(&self) -> HashSet { + self.inner + .lock() + .expect("sandbox watch bus lock poisoned") + .iter() + .filter(|(_, sender)| sender.receiver_count() > 0) + .map(|(sandbox_id, _)| sandbox_id.clone()) + .collect() + } +} + +/// Poll persisted sandbox resource versions once per gateway and notify the +/// existing in-memory watch bus when another replica changes a record. +/// +/// The poller performs at most one lookup per actively watched sandbox per +/// interval, regardless of how many clients are watching that sandbox. +pub fn spawn_store_poller( + store: Arc, + bus: SandboxWatchBus, + interval: Duration, + mut shutdown_rx: watch::Receiver, +) { + tokio::spawn(async move { + let mut known_versions: HashMap> = HashMap::new(); + let mut timer = tokio::time::interval(interval); + timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tokio::select! { + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + break; + } + } + _ = timer.tick() => { + let active = bus.active_sandbox_ids(); + known_versions.retain(|sandbox_id, _| active.contains(sandbox_id)); + + for sandbox_id in active { + let current = match store.get_message::(&sandbox_id).await { + Ok(sandbox) => sandbox.map(|sandbox| { + sandbox.metadata.as_ref().map_or(0, |metadata| metadata.resource_version) + }), + Err(err) => { + tracing::warn!( + sandbox_id, + error = %err, + "sandbox watch poller: failed to read persisted sandbox" + ); + continue; + } + }; + + let changed = known_versions + .insert(sandbox_id.clone(), current) + .is_none_or(|previous| previous != current); + if changed { + bus.notify(&sandbox_id); + } + } + } + } + } + }); } /// Helper to translate broadcast lag into a gRPC status. @@ -72,6 +141,7 @@ pub fn broadcast_to_status(err: broadcast::error::RecvError) -> Status { #[cfg(test)] mod tests { use super::*; + use openshell_core::proto::datamodel::v1::ObjectMeta; #[test] fn sandbox_watch_bus_remove_cleans_up() { @@ -114,4 +184,42 @@ mod tests { // Should not panic bus.remove("nonexistent"); } + + #[tokio::test] + async fn shared_store_poller_notifies_remote_resource_version_change() { + let store = Arc::new(crate::persistence::test_store().await); + let bus = SandboxWatchBus::new(); + let sandbox = Sandbox { + metadata: Some(ObjectMeta { + id: "sb-1".to_string(), + name: "sandbox-a".to_string(), + workspace: "default".to_string(), + ..Default::default() + }), + ..Default::default() + }; + store.put_message(&sandbox).await.unwrap(); + + let mut rx = bus.subscribe("sb-1"); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + spawn_store_poller(store.clone(), bus, Duration::from_millis(10), shutdown_rx); + + tokio::time::timeout(Duration::from_secs(1), rx.recv()) + .await + .expect("poller should publish its initial observation") + .unwrap(); + + store + .update_message_cas::("sb-1", 0, |stored| { + stored.set_phase(1); + }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), rx.recv()) + .await + .expect("poller should observe a remote store update") + .unwrap(); + + shutdown_tx.send(true).unwrap(); + } } diff --git a/crates/openshell-server/src/service_routing.rs b/crates/openshell-server/src/service_routing.rs index e2b67e7311..5e9a01f60e 100644 --- a/crates/openshell-server/src/service_routing.rs +++ b/crates/openshell-server/src/service_routing.rs @@ -325,17 +325,16 @@ async fn proxy_to_endpoint( let websocket_upgrade = is_websocket_upgrade(&req); let downstream_upgrade = websocket_upgrade.then(|| hyper::upgrade::on(&mut req)); - let (_channel_id, relay_rx) = state - .supervisor_sessions - .open_relay_with_target( - sandbox.object_id(), - relay_open::Target::Tcp(TcpRelayTarget { - host: RELAY_TARGET_HOST.to_string(), - port: u32::from(target_port), - }), - endpoint.object_id().to_string(), - Duration::from_secs(15), - ) + let (_channel_id, relay_rx) = crate::supervisor_session::open_routed_relay_with_target( + &state, + sandbox.object_id(), + relay_open::Target::Tcp(TcpRelayTarget { + host: RELAY_TARGET_HOST.to_string(), + port: u32::from(target_port), + }), + endpoint.object_id().to_string(), + Duration::from_secs(15), + ) .await .map_err(|err| { warn!(error = %err, sandbox_id = %endpoint.sandbox_id, "sandbox service routing: supervisor relay unavailable"); diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index e89a21bcb5..feb634f056 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -119,7 +119,7 @@ mod tests { const STORAGE_V1_SCHEMA_SHA256: &str = "d68401809d8cea445c35233ef32412bbd041cb2ac5acaf368a0d0bf74d2ddf17"; const PUBLIC_RPC_SCHEMA_SHA256: &str = - "8c1c3677130e3d42314ac7ad8dc76460690560f02a6108bd913d614608156da3"; + "567f060e7b4f5ead636817ee7c41728688b39de16d2ef00396d1b562bdb83045"; const DURABLE_SCHEMA_SHA256: &str = "9eeaa29dfba187bff69fb7bc4f9a13a0f1d7be3f7049a38c8f0e20ce77ec7d8b"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = @@ -518,12 +518,12 @@ mod tests { } assert_eq!( compiled_method_count, - 101 + PROVIDER_READINESS_RPC_SIGNATURES.len(), + 102 + PROVIDER_READINESS_RPC_SIGNATURES.len(), "classify every compiled RPC" ); assert_eq!( methods.len(), - 76 + PROVIDER_READINESS_RPC_SIGNATURES.len(), + 77 + PROVIDER_READINESS_RPC_SIGNATURES.len(), "inventory every public gateway RPC" ); assert_eq!( @@ -531,7 +531,7 @@ mod tests { .iter() .filter(|method| method.starts_with("openshell.v1.OpenShell/")) .count(), - 76 + PROVIDER_READINESS_RPC_SIGNATURES.len() + 77 + PROVIDER_READINESS_RPC_SIGNATURES.len() ); assert!(methods.iter().all(|method| !method.contains(".storage."))); @@ -575,7 +575,7 @@ mod tests { overlap_hash.as_str(), ), ( - (301, 24), + (303, 24), (92, 19), (80, 19), PUBLIC_RPC_SCHEMA_SHA256, diff --git a/crates/openshell-server/src/supervisor_owner.rs b/crates/openshell-server/src/supervisor_owner.rs new file mode 100644 index 0000000000..19d371d722 --- /dev/null +++ b/crates/openshell-server/src/supervisor_owner.rs @@ -0,0 +1,344 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared supervisor-session ownership index for HA gateway replicas. + +use crate::persistence::{PersistenceError, Store, WriteCondition}; +use openshell_core::time::now_ms; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::time::Duration; +use thiserror::Error; + +const OWNER_OBJECT_TYPE: &str = "supervisor_session_owner"; + +pub const OWNER_TTL: Duration = Duration::from_secs(45); + +fn owner_object_id(sandbox_id: &str) -> String { + format!("supervisor-owner:{sandbox_id}") +} + +#[derive(Debug, Error)] +pub enum OwnerError { + #[error("supervisor session is owned by another active gateway replica")] + AlreadyOwned, + #[error("supervisor owner record CAS conflict")] + Conflict, + #[error("persistence error: {0}")] + Store(#[from] PersistenceError), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct OwnerPayload { + sandbox_id: String, + session_id: String, + supervisor_instance_id: String, + connection_epoch: u64, + owner_replica_id: String, + owner_peer_endpoint: String, + connected_at_ms: i64, +} + +#[derive(Debug, Clone)] +pub struct OwnerRecord { + pub session_id: String, + pub supervisor_instance_id: String, + pub connection_epoch: u64, + pub owner_replica_id: String, + pub owner_peer_endpoint: String, + #[allow(dead_code)] + pub connected_at_ms: i64, + pub updated_at_ms: i64, + pub resource_version: u64, +} + +#[derive(Debug, Clone)] +pub struct OwnerGuard { + pub sandbox_id: String, + pub session_id: String, + pub supervisor_instance_id: String, + pub connection_epoch: u64, + pub owner_replica_id: String, + pub owner_peer_endpoint: String, + connected_at_ms: i64, + resource_version: u64, +} + +pub struct SupervisorOwnerIndex { + store: Arc, + ttl: Duration, +} + +impl SupervisorOwnerIndex { + pub fn new(store: Arc, ttl: Duration) -> Self { + Self { store, ttl } + } + + pub async fn publish( + &self, + sandbox_id: &str, + session_id: &str, + supervisor_instance_id: &str, + connection_epoch: u64, + owner_replica_id: &str, + owner_peer_endpoint: &str, + ) -> Result { + let connected_at_ms = now_ms(); + let payload = OwnerPayload { + sandbox_id: sandbox_id.to_string(), + session_id: session_id.to_string(), + supervisor_instance_id: supervisor_instance_id.to_string(), + connection_epoch, + owner_replica_id: owner_replica_id.to_string(), + owner_peer_endpoint: owner_peer_endpoint.to_string(), + connected_at_ms, + }; + + let condition = match self.read(sandbox_id).await? { + None => WriteCondition::MustCreate, + Some(existing) + if can_supersede( + &existing, + supervisor_instance_id, + connection_epoch, + self.ttl, + ) => + { + WriteCondition::MatchResourceVersion(existing.resource_version) + } + Some(_) => return Err(OwnerError::AlreadyOwned), + }; + + let result = self.write_payload(sandbox_id, &payload, condition).await?; + Ok(OwnerGuard { + sandbox_id: sandbox_id.to_string(), + session_id: session_id.to_string(), + supervisor_instance_id: supervisor_instance_id.to_string(), + connection_epoch, + owner_replica_id: owner_replica_id.to_string(), + owner_peer_endpoint: owner_peer_endpoint.to_string(), + connected_at_ms, + resource_version: result.resource_version, + }) + } + + pub async fn renew(&self, guard: &mut OwnerGuard) -> Result<(), OwnerError> { + let payload = OwnerPayload { + sandbox_id: guard.sandbox_id.clone(), + session_id: guard.session_id.clone(), + supervisor_instance_id: guard.supervisor_instance_id.clone(), + connection_epoch: guard.connection_epoch, + owner_replica_id: guard.owner_replica_id.clone(), + owner_peer_endpoint: guard.owner_peer_endpoint.clone(), + connected_at_ms: guard.connected_at_ms, + }; + + match self + .write_payload( + &guard.sandbox_id, + &payload, + WriteCondition::MatchResourceVersion(guard.resource_version), + ) + .await + { + Ok(result) => { + guard.resource_version = result.resource_version; + Ok(()) + } + Err(OwnerError::Store(PersistenceError::Conflict { .. })) => Err(OwnerError::Conflict), + Err(err) => Err(err), + } + } + + pub async fn release_if_current(&self, guard: &OwnerGuard) -> Result<(), OwnerError> { + let Some(record) = self.read(&guard.sandbox_id).await? else { + return Ok(()); + }; + if record.session_id != guard.session_id + || record.owner_replica_id != guard.owner_replica_id + { + return Ok(()); + } + match self + .store + .delete_if( + OWNER_OBJECT_TYPE, + &owner_object_id(&guard.sandbox_id), + record.resource_version, + ) + .await + { + Ok(_) => Ok(()), + Err(PersistenceError::Conflict { .. }) => Err(OwnerError::Conflict), + Err(err) => Err(OwnerError::Store(err)), + } + } + + pub async fn read(&self, sandbox_id: &str) -> Result, OwnerError> { + let Some(record) = self + .store + .get(OWNER_OBJECT_TYPE, &owner_object_id(sandbox_id)) + .await + .map_err(OwnerError::Store)? + else { + return Ok(None); + }; + + let payload: OwnerPayload = serde_json::from_slice(&record.payload) + .map_err(|err| PersistenceError::Decode(err.to_string()))?; + Ok(Some(OwnerRecord { + session_id: payload.session_id, + supervisor_instance_id: payload.supervisor_instance_id, + connection_epoch: payload.connection_epoch, + owner_replica_id: payload.owner_replica_id, + owner_peer_endpoint: payload.owner_peer_endpoint, + connected_at_ms: payload.connected_at_ms, + updated_at_ms: record.updated_at_ms, + resource_version: record.resource_version, + })) + } + + async fn write_payload( + &self, + sandbox_id: &str, + payload: &OwnerPayload, + condition: WriteCondition, + ) -> Result { + let payload_bytes = + serde_json::to_vec(payload).map_err(|err| PersistenceError::Encode(err.to_string())); + let payload_bytes = payload_bytes.map_err(OwnerError::Store)?; + match self + .store + .put_if( + OWNER_OBJECT_TYPE, + &owner_object_id(sandbox_id), + sandbox_id, + "", + &payload_bytes, + None, + condition, + ) + .await + { + Ok(result) => Ok(result), + Err(PersistenceError::UniqueViolation { .. }) => Err(OwnerError::AlreadyOwned), + Err(PersistenceError::Conflict { .. }) => Err(OwnerError::Conflict), + Err(err) => Err(OwnerError::Store(err)), + } + } +} + +fn can_supersede( + existing: &OwnerRecord, + supervisor_instance_id: &str, + connection_epoch: u64, + ttl: Duration, +) -> bool { + let age_ms = now_ms() - existing.updated_at_ms; + let ttl_ms = i64::try_from(ttl.as_millis()).unwrap_or(i64::MAX); + if age_ms >= ttl_ms { + return true; + } + + existing.supervisor_instance_id == supervisor_instance_id + && connection_epoch > existing.connection_epoch +} + +#[cfg(test)] +mod tests { + use super::*; + + async fn test_index(ttl: Duration) -> SupervisorOwnerIndex { + let store = Arc::new(crate::persistence::test_store().await); + SupervisorOwnerIndex::new(store, ttl) + } + + #[tokio::test] + async fn publish_creates_owner() { + let index = test_index(OWNER_TTL).await; + let guard = index + .publish("sbx", "s1", "inst", 1, "gw-1", "http://gw-1") + .await + .unwrap(); + let record = index.read("sbx").await.unwrap().unwrap(); + assert_eq!(record.session_id, guard.session_id); + assert_eq!(record.owner_replica_id, "gw-1"); + } + + #[tokio::test] + async fn publish_does_not_collide_with_sandbox_object_id() { + let index = test_index(OWNER_TTL).await; + index + .store + .put("sandbox", "sbx", "sandbox-a", "default", br"{}", None) + .await + .unwrap(); + + index + .publish("sbx", "s1", "inst", 1, "gw-1", "http://gw-1") + .await + .unwrap(); + + let record = index.read("sbx").await.unwrap().unwrap(); + assert_eq!(record.session_id, "s1"); + assert_eq!(record.owner_replica_id, "gw-1"); + } + + #[tokio::test] + async fn publish_rejects_active_different_instance() { + let index = test_index(OWNER_TTL).await; + index + .publish("sbx", "s1", "inst-a", 1, "gw-1", "http://gw-1") + .await + .unwrap(); + let err = index + .publish("sbx", "s2", "inst-b", 1, "gw-2", "http://gw-2") + .await + .unwrap_err(); + assert!(matches!(err, OwnerError::AlreadyOwned)); + } + + #[tokio::test] + async fn publish_supersedes_same_instance_higher_epoch() { + let index = test_index(OWNER_TTL).await; + index + .publish("sbx", "s1", "inst", 1, "gw-1", "http://gw-1") + .await + .unwrap(); + let guard = index + .publish("sbx", "s2", "inst", 2, "gw-2", "http://gw-2") + .await + .unwrap(); + let record = index.read("sbx").await.unwrap().unwrap(); + assert_eq!(record.session_id, guard.session_id); + assert_eq!(record.owner_replica_id, "gw-2"); + } + + #[tokio::test] + async fn release_if_current_ignores_stale_guard() { + let index = test_index(OWNER_TTL).await; + let old = index + .publish("sbx", "s1", "inst", 1, "gw-1", "http://gw-1") + .await + .unwrap(); + let new = index + .publish("sbx", "s2", "inst", 2, "gw-2", "http://gw-2") + .await + .unwrap(); + index.release_if_current(&old).await.unwrap(); + let record = index.read("sbx").await.unwrap().unwrap(); + assert_eq!(record.session_id, new.session_id); + } + + #[tokio::test] + async fn renew_updates_resource_version() { + let index = test_index(OWNER_TTL).await; + let mut guard = index + .publish("sbx", "s1", "inst", 1, "gw-1", "http://gw-1") + .await + .unwrap(); + let before = guard.resource_version; + index.renew(&mut guard).await.unwrap(); + assert!(guard.resource_version > before); + } +} diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index 1bf961e464..e6ed6afb0f 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -9,15 +9,17 @@ use std::time::{Duration, Instant}; use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; +use tonic::metadata::{Ascii, MetadataValue}; +use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity}; use tonic::{Request, Response, Status}; use tracing::{debug, info, warn}; use uuid::Uuid; use openshell_core::proto::{ - GatewayMessage, ProviderReadinessObservation, RelayFrame, RelayInit, RelayOpen, - ReportMainProcessExitRequest, ReportMainProcessExitResponse, Sandbox, SandboxPhase, - SessionAccepted, SshRelayTarget, SupervisorMessage, gateway_message, relay_open, - supervisor_message, + GatewayMessage, PeerRelayFrame, PeerRelayInit, ProviderReadinessObservation, RelayFrame, + RelayInit, RelayOpen, ReportMainProcessExitRequest, ReportMainProcessExitResponse, Sandbox, + SandboxPhase, SessionAccepted, SshRelayTarget, SupervisorMessage, gateway_message, + open_shell_client, peer_relay_frame, relay_open, supervisor_message, }; use openshell_core::transport_errors::is_expected_transport_close_status; @@ -25,6 +27,7 @@ use crate::ServerState; use crate::auth::principal::Principal; use crate::grpc::provider_readiness::ProviderReadinessEvidence; use crate::persistence::ObjectId; +use crate::supervisor_owner::{OWNER_TTL, OwnerError, OwnerGuard, SupervisorOwnerIndex}; const HEARTBEAT_INTERVAL_SECS: u32 = 15; const RELAY_PENDING_TIMEOUT: Duration = Duration::from_secs(10); @@ -42,6 +45,79 @@ const MAX_PENDING_RELAYS: usize = 256; /// consume the entire global budget. Sits above the SSH-tunnel per-sandbox /// cap (20) so tunnel-specific limits still fire first for that caller. const MAX_PENDING_RELAYS_PER_SANDBOX: usize = 32; +const PEER_TLS_CA_FILE_ENV: &str = "OPENSHELL_PEER_TLS_CA_FILE"; +const PEER_TLS_CERT_FILE_ENV: &str = "OPENSHELL_PEER_TLS_CERT_FILE"; +const PEER_TLS_KEY_FILE_ENV: &str = "OPENSHELL_PEER_TLS_KEY_FILE"; +const PEER_TLS_SERVER_NAME_ENV: &str = "OPENSHELL_PEER_TLS_SERVER_NAME"; + +#[derive(Debug, Default)] +struct PeerTlsClientConfig { + ca_file: Option, + cert_file: Option, + key_file: Option, + server_name: Option, +} + +impl PeerTlsClientConfig { + fn from_env() -> Self { + Self { + ca_file: nonempty_env(PEER_TLS_CA_FILE_ENV).map(Into::into), + cert_file: nonempty_env(PEER_TLS_CERT_FILE_ENV).map(Into::into), + key_file: nonempty_env(PEER_TLS_KEY_FILE_ENV).map(Into::into), + server_name: nonempty_env(PEER_TLS_SERVER_NAME_ENV), + } + } + + fn load(&self) -> Result { + let mut tls = if let Some(path) = self.ca_file.as_deref() { + let pem = std::fs::read(path).map_err(|err| { + Status::failed_precondition(format!( + "failed to read gateway peer TLS CA {}: {err}", + path.display() + )) + })?; + ClientTlsConfig::new().ca_certificate(Certificate::from_pem(pem)) + } else { + ClientTlsConfig::new().with_native_roots() + }; + + match (self.cert_file.as_deref(), self.key_file.as_deref()) { + (Some(cert_path), Some(key_path)) => { + let cert = std::fs::read(cert_path).map_err(|err| { + Status::failed_precondition(format!( + "failed to read gateway peer TLS certificate {}: {err}", + cert_path.display() + )) + })?; + let key = std::fs::read(key_path).map_err(|err| { + Status::failed_precondition(format!( + "failed to read gateway peer TLS key {}: {err}", + key_path.display() + )) + })?; + tls = tls.identity(Identity::from_pem(cert, key)); + } + (None, None) => {} + _ => { + return Err(Status::failed_precondition(format!( + "{PEER_TLS_CERT_FILE_ENV} and {PEER_TLS_KEY_FILE_ENV} must be configured together" + ))); + } + } + + if let Some(server_name) = self.server_name.as_deref() { + tls = tls.domain_name(server_name); + } + Ok(tls) + } +} + +fn nonempty_env(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} // --------------------------------------------------------------------------- // Session registry @@ -512,16 +588,36 @@ impl SupervisorSessionRegistry { ), Status, > { - let tx = self - .wait_for_session(sandbox_id, session_wait_timeout) - .await?; - let channel_id = Uuid::new_v4().to_string(); let relay_open = RelayOpen { channel_id: channel_id.clone(), target: Some(target), service_id, }; + self.open_relay_with_message(sandbox_id, relay_open, session_wait_timeout) + .await + } + + pub async fn open_relay_with_message( + &self, + sandbox_id: &str, + relay_open: RelayOpen, + session_wait_timeout: Duration, + ) -> Result< + ( + String, + oneshot::Receiver>, + ), + Status, + > { + if relay_open.channel_id.is_empty() { + return Err(Status::invalid_argument("relay channel_id is required")); + } + let tx = self + .wait_for_session(sandbox_id, session_wait_timeout) + .await?; + + let channel_id = relay_open.channel_id.clone(); // Register the pending relay before sending RelayOpen to avoid a race. // Both caps are checked and the insert happens under a single lock hold @@ -683,6 +779,18 @@ pub fn spawn_relay_reaper(state: Arc, interval: Duration) { }); } +fn owner_error_to_status(err: OwnerError) -> Status { + match err { + OwnerError::AlreadyOwned => { + Status::unavailable("supervisor session owned by another gateway replica") + } + OwnerError::Conflict => Status::aborted("supervisor owner record changed concurrently"), + OwnerError::Store(err) => { + Status::internal(format!("supervisor owner persistence failed: {err}")) + } + } +} + async fn require_persisted_sandbox( store: &Arc, sandbox_id: &str, @@ -916,6 +1024,427 @@ fn expected_transport_close_during_session_state( ) } +// --------------------------------------------------------------------------- +// PeerRelay gRPC handler and client-side forwarding +// --------------------------------------------------------------------------- + +#[derive(Clone)] +struct PeerAuthInterceptor { + bearer: MetadataValue, + replica_id: MetadataValue, +} + +impl PeerAuthInterceptor { + fn new(token: &str, replica_id: &str) -> Result { + let bearer = MetadataValue::try_from(format!("Bearer {token}")) + .map_err(|_| Status::internal("invalid gateway peer SA token header value"))?; + let replica_id = MetadataValue::try_from(replica_id.to_string()) + .map_err(|_| Status::internal("invalid gateway replica id header value"))?; + Ok(Self { bearer, replica_id }) + } +} + +impl tonic::service::Interceptor for PeerAuthInterceptor { + fn call(&mut self, mut req: Request<()>) -> Result, Status> { + req.metadata_mut() + .insert("authorization", self.bearer.clone()); + req.metadata_mut() + .insert("x-openshell-peer-replica", self.replica_id.clone()); + Ok(req) + } +} + +async fn build_peer_channel(endpoint: &str) -> Result { + let mut ep = Endpoint::from_shared(endpoint.to_string()) + .map_err(|err| Status::internal(format!("invalid gateway peer endpoint: {err}")))? + .connect_timeout(Duration::from_secs(10)) + .http2_keep_alive_interval(Duration::from_secs(10)) + .keep_alive_while_idle(true) + .keep_alive_timeout(Duration::from_secs(20)) + .http2_adaptive_window(true); + + if endpoint.starts_with("https://") { + let peer_tls = PeerTlsClientConfig::from_env().load()?; + ep = ep + .tls_config(peer_tls) + .map_err(|err| Status::internal(format!("failed to configure peer TLS: {err}")))?; + } + + ep.connect() + .await + .map_err(|err| Status::unavailable(format!("gateway peer connection failed: {err}"))) +} + +pub async fn open_routed_relay_with_target( + state: &Arc, + sandbox_id: &str, + target: relay_open::Target, + service_id: String, + session_wait_timeout: Duration, +) -> Result< + ( + String, + oneshot::Receiver>, + ), + Status, +> { + let channel_id = Uuid::new_v4().to_string(); + let relay_open = RelayOpen { + channel_id: channel_id.clone(), + target: Some(target), + service_id, + }; + open_routed_relay_with_message(state, sandbox_id, relay_open, session_wait_timeout).await +} + +pub async fn open_routed_relay_with_message( + state: &Arc, + sandbox_id: &str, + relay_open: RelayOpen, + session_wait_timeout: Duration, +) -> Result< + ( + String, + oneshot::Receiver>, + ), + Status, +> { + let deadline = Instant::now() + session_wait_timeout; + let mut backoff = SESSION_WAIT_INITIAL_BACKOFF; + let owner_index = SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL); + loop { + if state.supervisor_sessions.has_session(sandbox_id) { + match state + .supervisor_sessions + .open_relay_with_message(sandbox_id, relay_open.clone(), Duration::ZERO) + .await + { + Ok(relay) => return Ok(relay), + Err(status) if status.code() == tonic::Code::Unavailable => { + // The session can migrate after `has_session` but before + // RelayOpen reaches its sender. Fall through and reread the + // persisted owner instead of surfacing a handoff race. + warn!( + sandbox_id, + error = %status, + "local supervisor relay disappeared during open; resolving owner again" + ); + } + Err(status) => return Err(status), + } + } + + if let Some(owner) = owner_index + .read(sandbox_id) + .await + .map_err(owner_error_to_status)? + && owner_is_fresh(&owner) + { + if owner.owner_replica_id == state.replica_id { + warn!( + sandbox_id, + owner_replica_id = %owner.owner_replica_id, + "supervisor owner record points at this replica but no local session is registered; retrying" + ); + if Instant::now() + backoff > deadline { + return Err(Status::unavailable("supervisor session not connected")); + } + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(SESSION_WAIT_MAX_BACKOFF); + continue; + } + match open_peer_relay( + state, + owner.owner_peer_endpoint.clone(), + sandbox_id, + relay_open.clone(), + ) + .await + { + Ok(relay) => return Ok(relay), + Err(status) => { + warn!( + sandbox_id, + owner_replica_id = %owner.owner_replica_id, + owner_peer_endpoint = %owner.owner_peer_endpoint, + error = %status, + "gateway peer owner relay open failed; retrying until session wait timeout" + ); + } + } + } + + if Instant::now() + backoff > deadline { + return Err(Status::unavailable("supervisor session not connected")); + } + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(SESSION_WAIT_MAX_BACKOFF); + } +} + +fn owner_is_fresh(owner: &crate::supervisor_owner::OwnerRecord) -> bool { + let age_ms = openshell_core::time::now_ms() - owner.updated_at_ms; + let ttl_ms = i64::try_from(OWNER_TTL.as_millis()).unwrap_or(i64::MAX); + age_ms < ttl_ms +} + +async fn open_peer_relay( + state: &Arc, + owner_peer_endpoint: String, + sandbox_id: &str, + relay_open: RelayOpen, +) -> Result< + ( + String, + oneshot::Receiver>, + ), + Status, +> { + let channel_id = relay_open.channel_id.clone(); + let (relay_tx, relay_rx) = oneshot::channel(); + let stream = connect_peer_relay(state, &owner_peer_endpoint, sandbox_id, relay_open).await?; + let _ = relay_tx.send(Ok(stream)); + Ok((channel_id, relay_rx)) +} + +async fn connect_peer_relay( + state: &Arc, + owner_peer_endpoint: &str, + sandbox_id: &str, + relay_open: RelayOpen, +) -> Result { + let token = crate::auth::peer::load_peer_service_account_token_from_env() + .map_err(|err| { + Status::failed_precondition(format!("gateway peer token load failed: {err}")) + })? + .ok_or_else(|| { + Status::failed_precondition("gateway peer ServiceAccount token is not configured") + })?; + let channel = build_peer_channel(owner_peer_endpoint).await?; + let interceptor = PeerAuthInterceptor::new(&token, &state.replica_id)?; + let mut client = open_shell_client::OpenShellClient::with_interceptor(channel, interceptor); + + let (out_tx, out_rx) = mpsc::channel::(16); + out_tx + .send(PeerRelayFrame { + payload: Some(peer_relay_frame::Payload::Init(PeerRelayInit { + sandbox_id: sandbox_id.to_string(), + relay_open: Some(relay_open), + requester_replica_id: state.replica_id.clone(), + })), + }) + .await + .map_err(|_| Status::internal("failed to initialize peer relay stream"))?; + + let response = client + .peer_relay(ReceiverStream::new(out_rx)) + .await + .map_err(|err| Status::unavailable(format!("gateway peer relay RPC failed: {err}")))?; + let inbound = response.into_inner(); + let (gateway_stream, bridge_stream) = tokio::io::duplex(64 * 1024); + spawn_peer_bridge(bridge_stream, inbound, out_tx, sandbox_id.to_string()); + Ok(gateway_stream) +} + +pub async fn handle_peer_relay( + state: &Arc, + request: Request>, +) -> Result< + Response< + Pin> + Send + 'static>>, + >, + Status, +> { + let peer = match request.extensions().get::() { + Some(Principal::Peer(peer)) => peer.clone(), + _ => { + return Err(Status::permission_denied( + "gateway peer principal is required", + )); + } + }; + let mut inbound = request.into_inner(); + + let first = inbound + .message() + .await? + .ok_or_else(|| Status::invalid_argument("empty PeerRelay stream"))?; + let Some(peer_relay_frame::Payload::Init(init)) = first.payload else { + return Err(Status::invalid_argument( + "first PeerRelayFrame must be init", + )); + }; + if init.sandbox_id.is_empty() { + return Err(Status::invalid_argument("sandbox_id is required")); + } + if init.requester_replica_id != peer.replica_id { + return Err(Status::permission_denied( + "peer relay requester does not match authenticated gateway replica", + )); + } + let relay_open = init + .relay_open + .ok_or_else(|| Status::invalid_argument("relay_open is required"))?; + if relay_open.channel_id.is_empty() { + return Err(Status::invalid_argument("relay channel_id is required")); + } + + info!( + sandbox_id = %init.sandbox_id, + channel_id = %relay_open.channel_id, + requester = %peer.replica_id, + "gateway peer relay: opening local supervisor relay" + ); + + let (channel_id, relay_rx) = state + .supervisor_sessions + .open_relay_with_message(&init.sandbox_id, relay_open, Duration::from_secs(5)) + .await?; + let supervisor_stream = match tokio::time::timeout(Duration::from_secs(10), relay_rx).await { + Ok(Ok(Ok(stream))) => stream, + Ok(Ok(Err(status))) => return Err(status), + Ok(Err(_)) => return Err(Status::unavailable("relay channel dropped")), + Err(_) => return Err(Status::deadline_exceeded("relay open timed out")), + }; + + let (out_tx, out_rx) = mpsc::channel::>(16); + spawn_peer_owner_bridge( + supervisor_stream, + inbound, + out_tx, + init.sandbox_id, + channel_id, + ); + let stream: Pin< + Box> + Send + 'static>, + > = Box::pin(ReceiverStream::new(out_rx)); + Ok(Response::new(stream)) +} + +fn spawn_peer_bridge( + bridge_stream: tokio::io::DuplexStream, + mut inbound: tonic::Streaming, + out_tx: mpsc::Sender, + sandbox_id: String, +) { + let (mut read_half, mut write_half) = tokio::io::split(bridge_stream); + let sandbox_id_in = sandbox_id.clone(); + tokio::spawn(async move { + loop { + match inbound.message().await { + Ok(Some(frame)) => { + let Some(peer_relay_frame::Payload::Data(data)) = frame.payload else { + warn!(sandbox_id = %sandbox_id_in, "gateway peer relay: non-data frame after init"); + break; + }; + if data.is_empty() { + continue; + } + if let Err(err) = + tokio::io::AsyncWriteExt::write_all(&mut write_half, &data).await + { + warn!(sandbox_id = %sandbox_id_in, error = %err, "gateway peer relay: write to duplex failed"); + break; + } + } + Ok(None) => break, + Err(err) => { + warn!(sandbox_id = %sandbox_id_in, error = %err, "gateway peer relay: inbound errored"); + break; + } + } + } + let _ = tokio::io::AsyncWriteExt::shutdown(&mut write_half).await; + }); + + tokio::spawn(async move { + let mut buf = vec![0u8; RELAY_STREAM_CHUNK_SIZE]; + loop { + match tokio::io::AsyncReadExt::read(&mut read_half, &mut buf).await { + Ok(0) => break, + Ok(n) => { + if out_tx + .send(PeerRelayFrame { + payload: Some(peer_relay_frame::Payload::Data(buf[..n].to_vec())), + }) + .await + .is_err() + { + break; + } + } + Err(err) => { + warn!(sandbox_id = %sandbox_id, error = %err, "gateway peer relay: read from duplex failed"); + break; + } + } + } + }); +} + +fn spawn_peer_owner_bridge( + supervisor_stream: tokio::io::DuplexStream, + mut inbound: tonic::Streaming, + out_tx: mpsc::Sender>, + sandbox_id: String, + channel_id: String, +) { + let (mut read_half, mut write_half) = tokio::io::split(supervisor_stream); + let sandbox_id_in = sandbox_id.clone(); + let channel_id_in = channel_id.clone(); + tokio::spawn(async move { + loop { + match inbound.message().await { + Ok(Some(frame)) => { + let Some(peer_relay_frame::Payload::Data(data)) = frame.payload else { + warn!(sandbox_id = %sandbox_id_in, channel_id = %channel_id_in, "gateway peer relay owner: non-data frame after init"); + break; + }; + if data.is_empty() { + continue; + } + if let Err(err) = + tokio::io::AsyncWriteExt::write_all(&mut write_half, &data).await + { + warn!(sandbox_id = %sandbox_id_in, channel_id = %channel_id_in, error = %err, "gateway peer relay owner: write to supervisor relay failed"); + break; + } + } + Ok(None) => break, + Err(err) => { + warn!(sandbox_id = %sandbox_id_in, channel_id = %channel_id_in, error = %err, "gateway peer relay owner: inbound errored"); + break; + } + } + } + let _ = tokio::io::AsyncWriteExt::shutdown(&mut write_half).await; + }); + + tokio::spawn(async move { + let mut buf = vec![0u8; RELAY_STREAM_CHUNK_SIZE]; + loop { + match tokio::io::AsyncReadExt::read(&mut read_half, &mut buf).await { + Ok(0) => break, + Ok(n) => { + if out_tx + .send(Ok(PeerRelayFrame { + payload: Some(peer_relay_frame::Payload::Data(buf[..n].to_vec())), + })) + .await + .is_err() + { + break; + } + } + Err(err) => { + warn!(sandbox_id = %sandbox_id, channel_id = %channel_id, error = %err, "gateway peer relay owner: read from supervisor relay failed"); + break; + } + } + } + }); +} + // --------------------------------------------------------------------------- // ConnectSupervisor gRPC handler // --------------------------------------------------------------------------- @@ -954,10 +1483,35 @@ pub async fn handle_connect_supervisor( let provider_readiness = ProviderReadinessEvidence::from_hello(&hello)?; let session_id = Uuid::new_v4().to_string(); + let owner_peer_endpoint = state.peer_endpoint.clone().unwrap_or_default(); + if !state.store.is_single_replica() && owner_peer_endpoint.is_empty() { + return Err(Status::failed_precondition( + "gateway peer endpoint is required for multi-replica supervisor ownership", + )); + } + let owner_peer_endpoint = if owner_peer_endpoint.is_empty() { + format!("local://{}", state.replica_id) + } else { + owner_peer_endpoint + }; + let owner_index = SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL); + let owner_guard = owner_index + .publish( + &sandbox_id, + &session_id, + &hello.instance_id, + hello.connection_epoch, + &state.replica_id, + &owner_peer_endpoint, + ) + .await + .map_err(owner_error_to_status)?; info!( sandbox_id = %sandbox_id, session_id = %session_id, instance_id = %hello.instance_id, + connection_epoch = hello.connection_epoch, + replica_id = %state.replica_id, "supervisor session: accepted" ); @@ -1028,6 +1582,9 @@ pub async fn handle_connect_supervisor( state .supervisor_sessions .remove_if_current(&sandbox_id, &session_id); + if let Err(err) = owner_index.release_if_current(&owner_guard).await { + warn!(sandbox_id = %sandbox_id, session_id = %session_id, error = %err, "supervisor session: failed to release owner after accept send failure"); + } return Err(Status::internal("failed to send session accepted")); } @@ -1067,6 +1624,7 @@ pub async fn handle_connect_supervisor( let state_clone = Arc::clone(state); let sandbox_id_clone = sandbox_id.clone(); tokio::spawn(async move { + let mut owner_guard = owner_guard; run_session_loop( &state_clone, &sandbox_id_clone, @@ -1074,11 +1632,18 @@ pub async fn handle_connect_supervisor( &tx, &mut inbound, shutdown_rx, + &mut owner_guard, ) .await; let terminal_finalized = state_clone .supervisor_sessions .remove_if_current(&sandbox_id_clone, &session_id); + // Release only this exact ownership record. A newer supervisor session + // may already have published replacement ownership on another replica. + let owner_index = SupervisorOwnerIndex::new(state_clone.store.clone(), OWNER_TTL); + if let Err(err) = owner_index.release_if_current(&owner_guard).await { + warn!(sandbox_id = %sandbox_id_clone, session_id = %session_id, error = %err, "supervisor session: failed to release owner record"); + } if let Some(terminal_finalized) = terminal_finalized { info!(sandbox_id = %sandbox_id_clone, session_id = %session_id, "supervisor session: ended"); state_clone @@ -1179,6 +1744,7 @@ async fn run_session_loop( tx: &mpsc::Sender, inbound: &mut tonic::Streaming, mut shutdown_rx: oneshot::Receiver<()>, + owner_guard: &mut OwnerGuard, ) { let heartbeat_interval = Duration::from_secs(u64::from(HEARTBEAT_INTERVAL_SECS)); let mut heartbeat_timer = tokio::time::interval(heartbeat_interval); @@ -1194,7 +1760,9 @@ async fn run_session_loop( msg = inbound.message() => { match msg { Ok(Some(msg)) => { - handle_supervisor_message(state, sandbox_id, session_id, msg); + if !handle_supervisor_message(state, sandbox_id, session_id, msg, owner_guard).await { + break; + } } Ok(None) => { info!(sandbox_id = %sandbox_id, session_id = %session_id, "supervisor session: stream closed by supervisor"); @@ -1237,15 +1805,25 @@ async fn run_session_loop( } } -fn handle_supervisor_message( +async fn handle_supervisor_message( state: &Arc, sandbox_id: &str, session_id: &str, msg: SupervisorMessage, -) { + owner_guard: &mut OwnerGuard, +) -> bool { match msg.payload { Some(supervisor_message::Payload::Heartbeat(_)) => { - // Heartbeat received — nothing to do for now. + let owner_index = SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL); + if let Err(err) = owner_index.renew(owner_guard).await { + warn!( + sandbox_id = %sandbox_id, + session_id = %session_id, + error = %err, + "supervisor session: owner renewal failed; closing session" + ); + return false; + } } Some(supervisor_message::Payload::RelayOpenResult(result)) => { if result.success { @@ -1286,6 +1864,7 @@ fn handle_supervisor_message( ); } } + true } // --------------------------------------------------------------------------- @@ -1311,6 +1890,42 @@ mod tests { oneshot::channel::<()>().0 } + #[test] + fn peer_tls_client_config_requires_certificate_and_key_together() { + let config = PeerTlsClientConfig { + cert_file: Some("client.crt".into()), + ..Default::default() + }; + + let err = config + .load() + .expect_err("an incomplete peer mTLS identity must fail closed"); + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!(err.message().contains(PEER_TLS_KEY_FILE_ENV)); + } + + #[test] + fn peer_tls_client_config_loads_chart_ca_identity_and_server_name() { + let dir = tempfile::tempdir().unwrap(); + let ca = dir.path().join("ca.crt"); + let cert = dir.path().join("tls.crt"); + let key = dir.path().join("tls.key"); + std::fs::write(&ca, b"test-ca").unwrap(); + std::fs::write(&cert, b"test-cert").unwrap(); + std::fs::write(&key, b"test-key").unwrap(); + + let config = PeerTlsClientConfig { + ca_file: Some(ca), + cert_file: Some(cert), + key_file: Some(key), + server_name: Some("openshell.openshell.svc.cluster.local".to_string()), + }; + + config + .load() + .expect("complete chart peer TLS materials should configure tonic"); + } + fn sandbox_record(id: &str, name: &str) -> Sandbox { Sandbox { metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { diff --git a/crates/openshell-server/tests/common/mod.rs b/crates/openshell-server/tests/common/mod.rs index 64e2762e2a..e34a8bed33 100644 --- a/crates/openshell-server/tests/common/mod.rs +++ b/crates/openshell-server/tests/common/mod.rs @@ -22,10 +22,10 @@ use openshell_core::proto::{ GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, IssueSandboxTokenRequest, IssueSandboxTokenResponse, ListProvidersRequest, - ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, ProviderResponse, - RefreshSandboxTokenRequest, RefreshSandboxTokenResponse, RelayFrame, RevokeSshSessionRequest, - RevokeSshSessionResponse, SandboxResponse, SandboxStreamEvent, ServiceStatus, - SupervisorMessage, TcpForwardFrame, UpdateProviderRequest, WatchSandboxRequest, + ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, PeerRelayFrame, + ProviderResponse, RefreshSandboxTokenRequest, RefreshSandboxTokenResponse, RelayFrame, + RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, SandboxStreamEvent, + ServiceStatus, SupervisorMessage, TcpForwardFrame, UpdateProviderRequest, WatchSandboxRequest, open_shell_client::OpenShellClient, open_shell_server::{OpenShell, OpenShellServer}, }; @@ -584,6 +584,15 @@ impl OpenShell for TestOpenShell { Err(Status::unimplemented("not implemented in test")) } + type PeerRelayStream = ReceiverStream>; + + async fn peer_relay( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + type ForwardTcpStream = std::pin::Pin> + Send>>; diff --git a/crates/openshell-server/tests/supervisor_relay_integration.rs b/crates/openshell-server/tests/supervisor_relay_integration.rs index 31ab9cb1eb..25f74e9847 100644 --- a/crates/openshell-server/tests/supervisor_relay_integration.rs +++ b/crates/openshell-server/tests/supervisor_relay_integration.rs @@ -23,7 +23,7 @@ use hyper_util::{ server::conn::auto::Builder, }; use openshell_core::proto::{ - GatewayMessage, RelayFrame, RelayInit, SupervisorMessage, TcpForwardFrame, + GatewayMessage, PeerRelayFrame, RelayFrame, RelayInit, SupervisorMessage, TcpForwardFrame, open_shell_client::OpenShellClient, open_shell_server::{OpenShell, OpenShellServer}, }; @@ -126,6 +126,16 @@ impl OpenShell for RelayGateway { Err(Status::unimplemented("unused")) } + type PeerRelayStream = + std::pin::Pin> + Send>>; + + async fn peer_relay( + &self, + _: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + type ConnectSupervisorStream = ReceiverStream>; async fn connect_supervisor( &self, diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 1ddfd734e3..3e2e308517 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -342,7 +342,7 @@ async fn run_session_loop(config: SessionConfig) { loop { attempt += 1; - let result = run_single_session(&config).await; + let result = run_single_session(&config, attempt).await; if let Some(updates) = &config.session_id_updates { updates.send_replace(None); } @@ -375,6 +375,7 @@ async fn run_session_loop(config: SessionConfig) { async fn run_single_session( config: &SessionConfig, + connection_epoch: u64, ) -> Result<(), Box> { // Connect to the gateway. The same `Channel` is used for both the // long-lived control stream and all data-plane `RelayStream` calls, so @@ -394,6 +395,7 @@ async fn run_single_session( payload: Some(supervisor_message::Payload::Hello(SupervisorHello { sandbox_id: config.sandbox_id.clone(), instance_id: config.instance_id.clone(), + connection_epoch, supports_provider_readiness: true, })), }) diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 003475df96..f0a96b143d 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -142,6 +142,28 @@ Append these flags to any of the PostgreSQL commands above for OpenShift: --set securityContext.runAsUser=null ``` +### High availability + +Set `replicaCount` above `1` only with `server.externalDbSecret`; the default +SQLite database is per pod and cannot coordinate multiple gateway replicas. +The chart creates a headless peer Service for gateway-to-gateway relay traffic. +StatefulSet pods use stable pod DNS names through that headless Service. +Deployment pods advertise their pod IP with `OPENSHELL_PEER_ENDPOINT`, because +Kubernetes does not assign stable per-pod DNS names to Deployment replicas. + +Gateway peer traffic uses Kubernetes ServiceAccount identity. Each gateway pod +mounts a projected, pod-bound ServiceAccount token with audience +`openshell-gateway-peer`; receiving replicas validate that token with the +Kubernetes TokenReview API, verify the live pod UID and Helm selector labels, +and authorize only the internal `PeerRelay` RPC. The chart does not create or +accept a shared gateway peer Secret. + +With gateway TLS enabled, peer calls use the chart CA and client TLS Secret for +server verification and mTLS. The client verifies the stable gateway Service +DNS name while connecting directly to the owning pod. Custom TLS Secrets must +include that Service DNS name in the server certificate and provide the CA and +client credentials configured by `server.tls`. + ## Secret bootstrap By default, a pre-install/pre-upgrade hook Job runs `openshell-gateway generate-certs` diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 75e651e214..07af6e78b4 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -142,6 +142,28 @@ Append these flags to any of the PostgreSQL commands above for OpenShift: --set securityContext.runAsUser=null ``` +### High availability + +Set `replicaCount` above `1` only with `server.externalDbSecret`; the default +SQLite database is per pod and cannot coordinate multiple gateway replicas. +The chart creates a headless peer Service for gateway-to-gateway relay traffic. +StatefulSet pods use stable pod DNS names through that headless Service. +Deployment pods advertise their pod IP with `OPENSHELL_PEER_ENDPOINT`, because +Kubernetes does not assign stable per-pod DNS names to Deployment replicas. + +Gateway peer traffic uses Kubernetes ServiceAccount identity. Each gateway pod +mounts a projected, pod-bound ServiceAccount token with audience +`openshell-gateway-peer`; receiving replicas validate that token with the +Kubernetes TokenReview API, verify the live pod UID and Helm selector labels, +and authorize only the internal `PeerRelay` RPC. The chart does not create or +accept a shared gateway peer Secret. + +With gateway TLS enabled, peer calls use the chart CA and client TLS Secret for +server verification and mTLS. The client verifies the stable gateway Service +DNS name while connecting directly to the owning pod. Custom TLS Secrets must +include that Service DNS name in the server certificate and provide the CA and +client credentials configured by `server.tls`. + ## Secret bootstrap By default, a pre-install/pre-upgrade hook Job runs `openshell-gateway generate-certs` diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index 63037b075b..baa2e0f69e 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -146,6 +146,25 @@ deploy: sandboxRuntime.image.repository: '{{.IMAGE_REPO_openshell_sandbox}}' sandboxRuntime.image.tag: '{{.IMAGE_TAG_openshell_sandbox}}' profiles: + # Full HA test path: installs Envoy Gateway and layers both HA replicas and + # Gateway API routing values onto the OpenShell release. + - name: high-availability + patches: + - op: add + path: /deploy/helm/releases/0 + value: + name: envoy-gateway + remoteChart: oci://docker.io/envoyproxy/gateway-helm + version: v1.7.2 + namespace: envoy-gateway-system + createNamespace: true + wait: true + - op: add + path: /deploy/helm/releases/1/valuesFiles/- + value: ci/values-high-availability.yaml + - op: add + path: /deploy/helm/releases/1/valuesFiles/- + value: ci/values-gateway.yaml - name: credential-driver-kubernetes-secrets patches: - op: add diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 485fb0241a..e8d4616232 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -52,6 +52,50 @@ spec: - {{ .Values.server.dbUrl | quote }} {{- end }} env: + - name: OPENSHELL_REPLICA_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPENSHELL_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPENSHELL_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if eq (include "openshell.workloadKind" .) "deployment" }} + - name: OPENSHELL_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPENSHELL_PEER_ENDPOINT + value: {{ printf "%s://$(OPENSHELL_POD_IP):%d" (ternary "http" "https" (default false .Values.server.disableTls)) (int .Values.service.port) | quote }} + {{- end }} + - name: OPENSHELL_SERVICE_ACCOUNT_NAME + value: {{ include "openshell.serviceAccountName" . | quote }} + - name: OPENSHELL_PEER_SERVICE_NAME + value: {{ include "openshell.peerServiceName" . | quote }} + - name: OPENSHELL_PEER_TOKEN_AUDIENCE + value: "openshell-gateway-peer" + - name: OPENSHELL_PEER_SERVICE_ACCOUNT_TOKEN_FILE + value: /var/run/secrets/openshell-peer/token + - name: OPENSHELL_PEER_POD_LABELS + value: {{ printf "app.kubernetes.io/name=%s,app.kubernetes.io/instance=%s" (include "openshell.name" .) .Release.Name | quote }} + {{- if not .Values.server.disableTls }} + - name: OPENSHELL_PEER_TLS_SERVER_NAME + value: {{ printf "%s.%s.svc.cluster.local" (include "openshell.fullname" .) .Release.Namespace | quote }} + {{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} + - name: OPENSHELL_PEER_TLS_CA_FILE + value: /etc/openshell-tls/server/ca.crt + {{- end }} + {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} + - name: OPENSHELL_PEER_TLS_CERT_FILE + value: /etc/openshell-tls/peer-client/tls.crt + - name: OPENSHELL_PEER_TLS_KEY_FILE + value: /etc/openshell-tls/peer-client/tls.key + {{- end }} + {{- end }} {{- if not (or .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.vault.enabled) }} - name: {{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . }} valueFrom: @@ -99,6 +143,9 @@ spec: - name: sandbox-jwt mountPath: /etc/openshell-jwt readOnly: true + - name: gateway-peer-token + mountPath: /var/run/secrets/openshell-peer + readOnly: true {{- if not .Values.server.disableTls }} - name: tls-cert mountPath: /etc/openshell-tls/server @@ -109,6 +156,9 @@ spec: readOnly: true {{- end }} {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} + - name: peer-client-tls + mountPath: /etc/openshell-tls/peer-client + readOnly: true - name: tls-client-ca mountPath: /etc/openshell-tls/client-ca readOnly: true @@ -174,6 +224,14 @@ spec: secret: secretName: {{ include "openshell.sandboxJwtSecretName" . }} defaultMode: {{ .Values.server.sandboxJwt.secretDefaultMode | default 0400 }} + - name: gateway-peer-token + projected: + defaultMode: 0400 + sources: + - serviceAccountToken: + path: token + audience: openshell-gateway-peer + expirationSeconds: 3600 {{- if not .Values.server.disableTls }} - name: tls-cert secret: @@ -184,6 +242,9 @@ spec: secretName: {{ include "openshell.fullname" . }}-server-external-tls {{- end }} {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} + - name: peer-client-tls + secret: + secretName: {{ .Values.server.tls.clientTlsSecretName }} - name: tls-client-ca secret: {{- if or (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) }} diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index dd2d103f17..03713b9c26 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -210,6 +210,10 @@ Name of the Secret holding gateway-minted sandbox JWT signing material. {{- .Values.server.sandboxJwt.signingSecretName | default (printf "%s-jwt-keys" (include "openshell.fullname" .)) -}} {{- end }} +{{- define "openshell.peerServiceName" -}} +{{- printf "%s-peer" (include "openshell.fullname" .) -}} +{{- end }} + {{/* gRPC endpoint sandbox pods use to call back into the gateway. An explicit .Values.server.grpcEndpoint is used verbatim. Otherwise it is derived from diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index e48c27cdd2..9265e0a96a 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -9,8 +9,8 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: - # Validate projected sandbox ServiceAccount tokens during the - # IssueSandboxToken bootstrap exchange. + # Validate projected ServiceAccount tokens during sandbox bootstrap and + # internal gateway peer authentication. - apiGroups: - authentication.k8s.io resources: diff --git a/deploy/helm/openshell/templates/peer-role.yaml b/deploy/helm/openshell/templates/peer-role.yaml new file mode 100644 index 0000000000..59ae7479ad --- /dev/null +++ b/deploy/helm/openshell/templates/peer-role.yaml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "openshell.fullname" . }}-peer + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} +rules: + # Gateway peer identity: TokenReview authenticates the projected token, then + # the receiver resolves the returned pod name and UID to the live gateway pod + # in the release namespace. + - apiGroups: + - "" + resources: + - pods + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "openshell.fullname" . }}-peer + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "openshell.fullname" . }}-peer +subjects: + - kind: ServiceAccount + name: {{ include "openshell.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/helm/openshell/templates/peer-service.yaml b/deploy/helm/openshell/templates/peer-service.yaml new file mode 100644 index 0000000000..f5d93af245 --- /dev/null +++ b/deploy/helm/openshell/templates/peer-service.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v1 +kind: Service +metadata: + name: {{ include "openshell.peerServiceName" . }} + labels: + {{- include "openshell.labels" . | nindent 4 }} +spec: + clusterIP: None + publishNotReadyAddresses: true + ports: + - port: {{ .Values.service.port }} + targetPort: grpc + protocol: TCP + name: grpc + appProtocol: grpc + selector: + {{- include "openshell.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/openshell/templates/statefulset.yaml b/deploy/helm/openshell/templates/statefulset.yaml index 30571f80ba..10d0839f60 100644 --- a/deploy/helm/openshell/templates/statefulset.yaml +++ b/deploy/helm/openshell/templates/statefulset.yaml @@ -9,7 +9,7 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} spec: - serviceName: {{ include "openshell.fullname" . }} + serviceName: {{ include "openshell.peerServiceName" . }} replicas: {{ .Values.replicaCount }} selector: matchLabels: diff --git a/deploy/helm/openshell/tests/credential_drivers_test.yaml b/deploy/helm/openshell/tests/credential_drivers_test.yaml index df6de36479..65829712d1 100644 --- a/deploy/helm/openshell/tests/credential_drivers_test.yaml +++ b/deploy/helm/openshell/tests/credential_drivers_test.yaml @@ -46,15 +46,14 @@ tests: - it: injects the default credential storage key-encryption key Secret into the gateway pod by default template: templates/statefulset.yaml asserts: - - equal: - path: spec.template.spec.containers[0].env[0].name - value: OPENSHELL_GATEWAY_CREDENTIAL_KEY_ENCRYPTION_KEY - - matchRegex: - path: spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.name - pattern: 'credential-storage-key-encryption-key$' - - equal: - path: spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.key - value: key-encryption-key + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_GATEWAY_CREDENTIAL_KEY_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: openshell-credential-storage-key-encryption-key + key: key-encryption-key - it: renders Kubernetes Secrets credential driver config template: templates/gateway-config.yaml @@ -178,9 +177,14 @@ tests: - equal: path: kind value: Deployment - - equal: - path: spec.template.spec.containers[0].env[0].name - value: OPENSHELL_GATEWAY_CREDENTIAL_KEY_ENCRYPTION_KEY + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_GATEWAY_CREDENTIAL_KEY_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: openshell-credential-storage-key-encryption-key + key: key-encryption-key - it: allows default credential storage with multiple replicas and an external database template: templates/statefulset.yaml @@ -192,6 +196,11 @@ tests: - equal: path: spec.replicas value: 2 - - equal: - path: spec.template.spec.containers[0].env[0].name - value: OPENSHELL_GATEWAY_CREDENTIAL_KEY_ENCRYPTION_KEY + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_GATEWAY_CREDENTIAL_KEY_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: openshell-credential-storage-key-encryption-key + key: key-encryption-key diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index a9186d3339..4675cd9b6d 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -5,6 +5,8 @@ suite: gateway TOML config shape templates: - templates/gateway-config.yaml - templates/deployment.yaml + - templates/peer-role.yaml + - templates/peer-service.yaml - templates/statefulset.yaml release: name: openshell @@ -153,16 +155,16 @@ tests: server.oidc.caConfigMapName: openshell-oidc-ca asserts: - equal: - path: spec.template.spec.containers[0].volumeMounts[3].name + path: spec.template.spec.containers[0].volumeMounts[4].name value: oidc-ca - equal: - path: spec.template.spec.containers[0].volumeMounts[3].mountPath + path: spec.template.spec.containers[0].volumeMounts[4].mountPath value: /etc/openshell-tls/oidc-ca - equal: - path: spec.template.spec.volumes[2].name + path: spec.template.spec.volumes[3].name value: oidc-ca - equal: - path: spec.template.spec.volumes[2].configMap.name + path: spec.template.spec.volumes[3].configMap.name value: openshell-oidc-ca # Regression for the P1 bug Drew flagged: grpc_endpoint MUST live in the @@ -632,6 +634,128 @@ tests: - failedTemplate: errorMessage: "server.tls.clientCaSecretName cannot be null; omit the key to use the chart default (openshell-server-client-ca), or set to \"\" to disable client certificate verification for HTTPS-only mode" + - it: configures gateway peer identity and projected peer token + template: templates/statefulset.yaml + asserts: + - equal: + path: spec.serviceName + value: openshell-peer + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_PEER_SERVICE_NAME + value: openshell-peer + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_PEER_TOKEN_AUDIENCE + value: openshell-gateway-peer + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_PEER_TLS_SERVER_NAME + value: openshell.my-namespace.svc.cluster.local + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_PEER_TLS_CA_FILE + value: /etc/openshell-tls/server/ca.crt + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_PEER_TLS_CERT_FILE + value: /etc/openshell-tls/peer-client/tls.crt + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_PEER_TLS_KEY_FILE + value: /etc/openshell-tls/peer-client/tls.key + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gateway-peer-token + mountPath: /var/run/secrets/openshell-peer + readOnly: true + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: peer-client-tls + mountPath: /etc/openshell-tls/peer-client + readOnly: true + - equal: + path: spec.template.spec.volumes[2].projected.sources[0].serviceAccountToken.audience + value: openshell-gateway-peer + + - it: configures gateway peer identity and projected peer token for Deployment + template: templates/deployment.yaml + set: + workload.kind: deployment + server.externalDbSecret: my-pg-secret + server.disableTls: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_PEER_ENDPOINT + value: http://$(OPENSHELL_POD_IP):8080 + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_PEER_SERVICE_NAME + value: openshell-peer + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_PEER_TOKEN_AUDIENCE + value: openshell-gateway-peer + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gateway-peer-token + mountPath: /var/run/secrets/openshell-peer + readOnly: true + - equal: + path: spec.template.spec.volumes[2].projected.sources[0].serviceAccountToken.audience + value: openshell-gateway-peer + + - it: renders headless gateway peer service + template: templates/peer-service.yaml + asserts: + - equal: + path: metadata.name + value: openshell-peer + - equal: + path: spec.clusterIP + value: None + - equal: + path: spec.publishNotReadyAddresses + value: true + + - it: grants release-namespace pod lookup for gateway peer identity validation + template: templates/peer-role.yaml + asserts: + - hasDocuments: + count: 2 + - equal: + path: metadata.namespace + value: my-namespace + documentIndex: 0 + - equal: + path: rules[0].resources[0] + value: pods + documentIndex: 0 + - equal: + path: subjects[0].name + value: openshell + documentIndex: 1 + - it: fails when legacy postgres.enabled is set template: templates/statefulset.yaml set: diff --git a/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml b/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml index df6df15693..6b3e320a8d 100644 --- a/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml +++ b/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml @@ -17,13 +17,13 @@ tests: certManager.enabled: false asserts: - equal: - path: spec.template.spec.volumes[3].name + path: spec.template.spec.volumes[5].name value: tls-client-ca - equal: - path: spec.template.spec.volumes[3].secret.secretName + path: spec.template.spec.volumes[5].secret.secretName value: openshell-server-tls - equal: - path: spec.template.spec.volumes[3].secret.items[0].key + path: spec.template.spec.volumes[5].secret.items[0].key value: ca.crt - it: shares the cert-manager server TLS ca.crt when clientCaFromServerTlsSecret is true @@ -33,13 +33,13 @@ tests: certManager.clientCaFromServerTlsSecret: true asserts: - equal: - path: spec.template.spec.volumes[3].name + path: spec.template.spec.volumes[5].name value: tls-client-ca - equal: - path: spec.template.spec.volumes[3].secret.secretName + path: spec.template.spec.volumes[5].secret.secretName value: openshell-server-tls - equal: - path: spec.template.spec.volumes[3].secret.items[0].key + path: spec.template.spec.volumes[5].secret.items[0].key value: ca.crt # Regression: with cert-manager enabled and pkiInitJob left at its default @@ -56,13 +56,13 @@ tests: server.tls.clientCaSecretName: openshell-ca-tls asserts: - equal: - path: spec.template.spec.volumes[3].name + path: spec.template.spec.volumes[5].name value: tls-client-ca - equal: - path: spec.template.spec.volumes[3].secret.secretName + path: spec.template.spec.volumes[5].secret.secretName value: openshell-ca-tls - notExists: - path: spec.template.spec.volumes[3].secret.items + path: spec.template.spec.volumes[5].secret.items - it: omits client CA volume and mount when mTLS is disabled template: templates/statefulset.yaml @@ -96,7 +96,7 @@ tests: asserts: - lengthEqual: path: spec.template.spec.volumes - count: 3 + count: 4 - notContains: path: spec.template.spec.containers[0].volumeMounts content: @@ -115,7 +115,7 @@ tests: asserts: - lengthEqual: path: spec.template.spec.volumes - count: 3 + count: 4 - notContains: path: spec.template.spec.containers[0].volumeMounts content: @@ -132,7 +132,7 @@ tests: asserts: - lengthEqual: path: spec.template.spec.volumes - count: 3 + count: 4 - notContains: path: spec.template.spec.containers[0].volumeMounts content: diff --git a/deploy/kube/manifests/envoy-gateway-openshell.yaml b/deploy/kube/manifests/envoy-gateway-openshell.yaml index 583f2b41ba..68d1ea7d7a 100644 --- a/deploy/kube/manifests/envoy-gateway-openshell.yaml +++ b/deploy/kube/manifests/envoy-gateway-openshell.yaml @@ -15,3 +15,24 @@ metadata: name: eg spec: controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +# OpenShell gRPC streams can remain active across sandbox create, exec, relay, +# and watch operations. Disable Envoy's backend request and stream duration +# timeouts for the OpenShell GRPCRoute so the proxy does not reset long-running +# HTTP/2 streams while gateway pods rotate behind it. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: openshell-grpc-timeouts + namespace: openshell + labels: + app.kubernetes.io/name: openshell +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: GRPCRoute + name: openshell + timeout: + http: + requestTimeout: 0s + maxStreamDuration: 0s diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index c3f4e2416e..43e221b824 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -192,7 +192,7 @@ The most commonly changed values are: | Value | Purpose | |---|---| | `image.repository` / `image.tag` | Gateway container image. Defaults to `ghcr.io/nvidia/openshell/gateway:latest`. | -| `replicaCount` | Number of gateway replicas. Leave at `1` unless you are explicitly testing multi-replica behavior. | +| `replicaCount` | Number of gateway replicas. Values above `1` require shared PostgreSQL through `server.externalDbSecret`. | | `workload.kind` | Gateway workload controller. Use `statefulset` for SQLite or `deployment` with `server.externalDbSecret`. | | `workload.allowMultiReplicaStatefulSet` | Allow `replicaCount > 1` with `workload.kind=statefulset`. Prefer Deployment for external database-backed multi-replica gateways. | | `server.sandboxNamespace` | Namespace where sandbox pods are created. Defaults to the Helm release namespace when left empty. | @@ -210,6 +210,24 @@ The most commonly changed values are: | `supervisor.sandboxRuntime.boundaryPort` | Non-privileged TLS port used between paired supervisor and sandbox Pods. | | `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | +### Run multiple gateway replicas + +Use a Deployment and shared PostgreSQL for high availability: + +```yaml +replicaCount: 2 +workload: + kind: deployment +server: + externalDbSecret: openshell-postgres +``` + +The chart creates the peer Service, projected ServiceAccount identity, and RBAC +that gateway replicas use to route exec, forwarding, file-sync, and service +traffic to the replica holding the external supervisor's live session. Peer +traffic uses the chart TLS materials when gateway TLS is enabled. Do not use +the per-pod SQLite database with more than one replica. + Use a values file for repeatable deployments: ```shell diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 333acd0fec..5bfa0a78f9 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -29,6 +29,7 @@ e2e-docker = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-gpu = ["e2e"] e2e-docker-gpu = ["e2e-docker", "e2e-gpu"] e2e-kubernetes = ["e2e"] +e2e-kubernetes-ha = ["e2e-kubernetes"] e2e-kubernetes-credential-drivers = ["e2e-kubernetes"] e2e-kubernetes-workspace-managed = ["e2e-kubernetes"] e2e-kubernetes-workspace-operator = ["e2e-kubernetes"] @@ -128,9 +129,9 @@ path = "tests/provider_token_exchange.rs" required-features = ["e2e-podman"] [[test]] -name = "readyz_health" -path = "tests/readyz_health.rs" -required-features = ["e2e-kubernetes"] +name = "kubernetes_ha_rebalancing" +path = "tests/kubernetes_ha_rebalancing.rs" +required-features = ["e2e-kubernetes-ha"] [[test]] name = "kubernetes_corporate_proxy" diff --git a/e2e/rust/tests/kubernetes_ha_rebalancing.rs b/e2e/rust/tests/kubernetes_ha_rebalancing.rs new file mode 100644 index 0000000000..ca65f1e3eb --- /dev/null +++ b/e2e/rust/tests/kubernetes_ha_rebalancing.rs @@ -0,0 +1,627 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-ha")] + +use std::fs; +use std::io::Write; +use std::path::Path; +use std::process::Stdio; +use std::time::{Duration, Instant}; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::port::{find_free_port, wait_for_port}; +use openshell_e2e::harness::sandbox::SandboxGuard; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use tokio::process::{Child, Command}; + +static KUBE_HA_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +const HA_SYNC_PAYLOAD_BYTES: usize = 32 * 1024 * 1024; +const HA_SYNC_TIMEOUT: Duration = Duration::from_secs(600); + +#[derive(Clone)] +struct KubeTarget { + context: String, + namespace: String, + release: String, +} + +impl KubeTarget { + fn from_env() -> Self { + Self { + context: required_env("OPENSHELL_E2E_KUBE_CONTEXT"), + namespace: std::env::var("OPENSHELL_E2E_KUBE_NAMESPACE") + .unwrap_or_else(|_| "openshell".to_string()), + release: std::env::var("OPENSHELL_E2E_KUBE_RELEASE") + .unwrap_or_else(|_| "openshell".to_string()), + } + } + + async fn kubectl(&self, args: &[&str]) -> Result { + let output = Command::new("kubectl") + .arg("--context") + .arg(&self.context) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|err| format!("failed to spawn kubectl {args:?}: {err}"))?; + + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + if !output.status.success() { + return Err(format!( + "kubectl {args:?} failed with exit {:?}:\n{combined}", + output.status.code() + )); + } + + Ok(combined) + } + + async fn scale_gateway(&self, replicas: usize) -> Result<(), String> { + let resource = self.gateway_workload_resource().await?; + let replicas_arg = replicas.to_string(); + + self.kubectl(&[ + "-n", + &self.namespace, + "scale", + &resource, + "--replicas", + &replicas_arg, + ]) + .await?; + self.kubectl(&[ + "-n", + &self.namespace, + "rollout", + "status", + &resource, + "--timeout=180s", + ]) + .await?; + Ok(()) + } + + async fn gateway_workload_resource(&self) -> Result { + let deployment = format!("deployment/{}", self.release); + if self + .kubectl(&["-n", &self.namespace, "get", &deployment]) + .await + .is_ok() + { + return Ok(deployment); + } + + let statefulset = format!("statefulset/{}", self.release); + if self + .kubectl(&["-n", &self.namespace, "get", &statefulset]) + .await + .is_ok() + { + return Ok(statefulset); + } + + Err(format!( + "no gateway Deployment or StatefulSet named {} found in namespace {}", + self.release, self.namespace + )) + } + + async fn delete_gateway_pod(&self, pod: &str) -> Result<(), String> { + self.kubectl(&[ + "-n", + &self.namespace, + "delete", + "pod", + pod, + "--wait=true", + "--timeout=90s", + ]) + .await?; + Ok(()) + } + + async fn roll_gateway_pods(&self, pods: Vec, expected: usize) -> Result<(), String> { + for pod in pods { + self.delete_gateway_pod(&pod).await?; + self.wait_for_gateway_pods(expected).await?; + } + Ok(()) + } + + async fn wait_for_gateway_pods(&self, expected: usize) -> Result, String> { + let deadline = Instant::now() + Duration::from_secs(240); + let mut last = String::new(); + + while Instant::now() < deadline { + match self.gateway_pods().await { + Ok(pods) => { + if pods.len() == expected && pods.iter().all(|pod| pod.ready) { + return Ok(pods.into_iter().map(|pod| pod.name).collect()); + } + last = format!( + "pods={:?}", + pods.iter() + .map(|pod| format!("{} ready={}", pod.name, pod.ready)) + .collect::>() + ); + } + Err(err) => last = err, + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + Err(format!( + "gateway pods did not reach expected ready count {expected} within 240s; last={last}" + )) + } + + async fn gateway_pods(&self) -> Result, String> { + let selector = format!("app.kubernetes.io/instance={}", self.release); + let json = self + .kubectl(&[ + "-n", + &self.namespace, + "get", + "pods", + "-l", + &selector, + "-o", + "json", + ]) + .await?; + let value = serde_json::from_str::(&json) + .map_err(|err| format!("failed to parse gateway pod JSON: {err}\n{json}"))?; + let items = value["items"] + .as_array() + .ok_or_else(|| format!("gateway pod JSON missing items array: {value}"))?; + + let mut pods = Vec::new(); + for item in items { + if !item["metadata"]["deletionTimestamp"].is_null() { + continue; + } + let Some(name) = item["metadata"]["name"].as_str() else { + continue; + }; + let ready = item["status"]["conditions"] + .as_array() + .is_some_and(|conditions| { + conditions.iter().any(|condition| { + condition["type"].as_str() == Some("Ready") + && condition["status"].as_str() == Some("True") + }) + }); + pods.push(GatewayPod { + name: name.to_string(), + ready, + }); + } + pods.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(pods) + } +} + +#[derive(Debug)] +struct GatewayPod { + name: String, + ready: bool, +} + +struct PortForward { + port: u16, + child: Child, +} + +impl PortForward { + async fn start(kube: &KubeTarget, pod: &str) -> Result { + let port = find_free_port(); + let mut child = Command::new("kubectl") + .arg("--context") + .arg(&kube.context) + .arg("-n") + .arg(&kube.namespace) + .arg("port-forward") + .arg(format!("pod/{pod}")) + .arg(format!("{port}:8080")) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn() + .map_err(|err| format!("failed to start kubectl port-forward for {pod}: {err}"))?; + + match wait_for_port("127.0.0.1", port, Duration::from_secs(30)).await { + Ok(()) => Ok(Self { port, child }), + Err(err) => { + let status = child.try_wait().ok().flatten(); + let _ = child.kill().await; + Err(format!( + "port-forward to {pod} did not become ready on {port}: {err}; status={status:?}" + )) + } + } + } +} + +impl Drop for PortForward { + fn drop(&mut self) { + let _ = self.child.start_kill(); + } +} + +fn required_env(name: &str) -> String { + std::env::var(name) + .unwrap_or_else(|_| panic!("{name} is not set; run through e2e/rust/e2e-kubernetes.sh")) +} + +async fn exec_through_pod( + kube: &KubeTarget, + pod: &str, + sandbox_name: &str, + marker: &str, +) -> Result<(), String> { + let port_forward = PortForward::start(kube, pod).await?; + let endpoint = format!("http://127.0.0.1:{}", port_forward.port); + + let mut cmd = openshell_cmd(); + cmd.arg("--gateway-endpoint") + .arg(&endpoint) + .args([ + "sandbox", + "exec", + "--name", + sandbox_name, + "--no-tty", + "--", + "printf", + "%s", + marker, + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let output = cmd + .output() + .await + .map_err(|err| format!("failed to spawn openshell exec via {pod}: {err}"))?; + + let combined = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + if !output.status.success() || !combined.contains(marker) { + return Err(format!( + "exec through {pod} ({endpoint}) failed with exit {:?}; expected marker {marker:?}; output:\n{combined}", + output.status.code() + )); + } + + Ok(()) +} + +async fn exec_through_configured_gateway(sandbox_name: &str, marker: &str) -> Result<(), String> { + let mut cmd = openshell_cmd(); + cmd.args([ + "sandbox", + "exec", + "--name", + sandbox_name, + "--no-tty", + "--", + "printf", + "%s", + marker, + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let output = cmd + .output() + .await + .map_err(|err| format!("failed to spawn openshell exec via configured gateway: {err}"))?; + + let combined = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + if !output.status.success() || !combined.contains(marker) { + return Err(format!( + "exec through configured gateway failed with exit {:?}; expected marker {marker:?}; output:\n{combined}", + output.status.code() + )); + } + + Ok(()) +} + +async fn create_sandbox_through_configured_gateway(phase: &str) -> Result { + let marker = format!("ha-create-watch-{phase}"); + let guard = SandboxGuard::create(&["--", "printf", "%s", &marker]).await?; + let output = strip_ansi(&guard.create_output); + + if !output.contains(&marker) { + return Err(format!( + "sandbox create through configured gateway did not include marker {marker:?}; output:\n{output}" + )); + } + + Ok(guard) +} + +async fn assert_exec_through_all_pods( + kube: &KubeTarget, + pods: &[String], + sandbox_name: &str, + phase: &str, +) -> Result<(), String> { + for pod in pods { + let marker = format!("ha-rebalance-{phase}-{pod}"); + exec_through_pod(kube, pod, sandbox_name, &marker).await?; + } + Ok(()) +} + +fn write_deterministic_payload(path: &Path, size: usize) { + let mut file = fs::File::create(path).expect("create HA sync payload"); + let mut offset = 0usize; + let mut remaining = size; + let mut buf = vec![0_u8; 64 * 1024]; + + while remaining > 0 { + let chunk_len = remaining.min(buf.len()); + for (idx, byte) in buf[..chunk_len].iter_mut().enumerate() { + *byte = u8::try_from((offset + idx) % 251).expect("byte value fits"); + } + file.write_all(&buf[..chunk_len]) + .expect("write HA sync payload chunk"); + offset += chunk_len; + remaining -= chunk_len; + } +} + +fn sha256_file(path: &Path) -> String { + let data = fs::read(path).expect("read file for SHA-256"); + let mut hasher = Sha256::new(); + hasher.update(&data); + hex::encode(hasher.finalize()) +} + +fn upload_command(sandbox_name: &str, local_path: &Path, dest: &str) -> Command { + let mut cmd = openshell_cmd(); + cmd.arg("sandbox") + .arg("upload") + .arg(sandbox_name) + .arg(local_path) + .arg(dest) + .arg("--no-git-ignore"); + cmd +} + +fn download_command(sandbox_name: &str, sandbox_path: &str, local_dest: &Path) -> Command { + let mut cmd = openshell_cmd(); + cmd.arg("sandbox") + .arg("download") + .arg(sandbox_name) + .arg(sandbox_path) + .arg(local_dest); + cmd +} + +async fn run_cli_during_gateway_pod_roll( + kube: &KubeTarget, + mut cmd: Command, + operation: &str, +) -> Result { + let pods = kube.wait_for_gateway_pods(2).await?; + + cmd.stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let child = cmd + .spawn() + .map_err(|err| format!("failed to spawn {operation} command: {err}"))?; + + let (roll_result, output_result) = tokio::time::timeout(HA_SYNC_TIMEOUT, async { + let roll = async { + tokio::time::sleep(Duration::from_millis(250)).await; + kube.roll_gateway_pods(pods, 2).await + }; + tokio::join!(roll, child.wait_with_output()) + }) + .await + .map_err(|_| { + format!( + "{operation} command and gateway pod roll did not finish within {HA_SYNC_TIMEOUT:?}" + ) + })?; + + roll_result.map_err(|err| { + format!("gateway pod roll failed while {operation} command was running: {err}") + })?; + + let output = + output_result.map_err(|err| format!("failed to wait for {operation} command: {err}"))?; + let combined = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + if !output.status.success() { + return Err(format!( + "{operation} command failed with exit {:?} during gateway pod roll:\n{combined}", + output.status.code() + )); + } + + Ok(combined) +} + +#[tokio::test] +async fn sandbox_exec_rebalances_across_gateway_scale_and_rollout() { + let _test_lock = KUBE_HA_TEST_LOCK.lock().await; + let kube = KubeTarget::from_env(); + + let mut pods = kube + .wait_for_gateway_pods(2) + .await + .expect("gateway should start with two ready HA replicas"); + + let mut sandbox = create_sandbox_through_configured_gateway("initial") + .await + .expect("sandbox create and readiness watch should succeed through the configured gateway endpoint initially"); + + assert_exec_through_all_pods(&kube, &pods, &sandbox.name, "initial") + .await + .expect("exec should work through every initial gateway pod"); + exec_through_configured_gateway(&sandbox.name, "ha-rebalance-client-initial") + .await + .expect("exec should work through the configured client gateway endpoint initially"); + + kube.scale_gateway(3) + .await + .expect("scale gateway to three replicas"); + pods = kube + .wait_for_gateway_pods(3) + .await + .expect("gateway should scale to three ready replicas"); + assert_exec_through_all_pods(&kube, &pods, &sandbox.name, "scale-up") + .await + .expect("exec should work through every gateway pod after scale-up"); + exec_through_configured_gateway(&sandbox.name, "ha-rebalance-client-scale-up") + .await + .expect("exec should work through the configured client gateway endpoint after scale-up"); + let mut scale_up_sandbox = create_sandbox_through_configured_gateway("scale-up") + .await + .expect( + "sandbox create and readiness watch should succeed through the configured gateway endpoint after scale-up", + ); + scale_up_sandbox.cleanup().await; + + kube.scale_gateway(2) + .await + .expect("scale gateway back to two replicas"); + pods = kube + .wait_for_gateway_pods(2) + .await + .expect("gateway should scale back to two ready replicas"); + assert_exec_through_all_pods(&kube, &pods, &sandbox.name, "scale-down") + .await + .expect("exec should work through every gateway pod after scale-down"); + exec_through_configured_gateway(&sandbox.name, "ha-rebalance-client-scale-down") + .await + .expect("exec should work through the configured client gateway endpoint after scale-down"); + let mut scale_down_sandbox = create_sandbox_through_configured_gateway("scale-down") + .await + .expect( + "sandbox create and readiness watch should succeed through the configured gateway endpoint after scale-down", + ); + scale_down_sandbox.cleanup().await; + + for (idx, pod) in pods.clone().into_iter().enumerate() { + kube.delete_gateway_pod(&pod) + .await + .unwrap_or_else(|err| panic!("delete gateway pod {pod}: {err}")); + pods = kube.wait_for_gateway_pods(2).await.unwrap_or_else(|err| { + panic!("gateway pods should recover after deleting {pod}: {err}") + }); + assert_exec_through_all_pods(&kube, &pods, &sandbox.name, &format!("delete-{pod}")) + .await + .unwrap_or_else(|err| panic!("exec should work after deleting {pod}: {err}")); + exec_through_configured_gateway( + &sandbox.name, + &format!("ha-rebalance-client-delete-{pod}"), + ) + .await + .unwrap_or_else(|err| { + panic!( + "exec should work through the configured client gateway endpoint after deleting {pod}: {err}" + ) + }); + let mut delete_sandbox = + create_sandbox_through_configured_gateway(&format!("delete-{idx}")) + .await + .unwrap_or_else(|err| { + panic!( + "sandbox create and readiness watch should succeed through the configured gateway endpoint after deleting {pod}: {err}" + ) + }); + delete_sandbox.cleanup().await; + } + + sandbox.cleanup().await; +} + +#[tokio::test] +async fn sandbox_file_sync_survives_gateway_pod_rolls() { + let _test_lock = KUBE_HA_TEST_LOCK.lock().await; + let kube = KubeTarget::from_env(); + + kube.scale_gateway(2) + .await + .expect("gateway should run with two HA replicas for sync outage testing"); + kube.wait_for_gateway_pods(2) + .await + .expect("gateway should have two ready replicas before sync outage testing"); + + let mut sandbox = + SandboxGuard::create_keep(&["sh", "-c", "echo Ready && sleep infinity"], "Ready") + .await + .expect("sandbox create --keep for HA sync testing"); + + let tmpdir = tempfile::tempdir().expect("create HA sync tmpdir"); + let upload_dir = tmpdir.path().join("ha-sync-upload"); + fs::create_dir_all(&upload_dir).expect("create HA sync upload dir"); + fs::write(upload_dir.join("marker.txt"), "ha-sync-marker").expect("write HA sync marker"); + + let payload = upload_dir.join("payload.bin"); + write_deterministic_payload(&payload, HA_SYNC_PAYLOAD_BYTES); + let expected_hash = sha256_file(&payload); + + let upload = upload_command(&sandbox.name, &upload_dir, "/sandbox/ha-sync"); + run_cli_during_gateway_pod_roll(&kube, upload, "upload") + .await + .expect("upload should survive rolling gateway pod outages"); + + let remote_payload = "/sandbox/ha-sync/ha-sync-upload/payload.bin"; + let remote_hash_cmd = format!("sha256sum {remote_payload} | awk '{{print $1}}'"); + let remote_hash = sandbox + .exec(&["sh", "-c", &remote_hash_cmd]) + .await + .expect("uploaded payload should be readable in sandbox"); + assert!( + strip_ansi(&remote_hash).contains(&expected_hash), + "uploaded payload SHA-256 mismatch; expected {expected_hash}, got:\n{remote_hash}" + ); + + let download_dir = tmpdir.path().join("ha-sync-download"); + fs::create_dir_all(&download_dir).expect("create HA sync download dir"); + let download = download_command( + &sandbox.name, + "/sandbox/ha-sync/ha-sync-upload", + &download_dir, + ); + run_cli_during_gateway_pod_roll(&kube, download, "download") + .await + .expect("download should survive rolling gateway pod outages"); + + let actual_hash = sha256_file(&download_dir.join("payload.bin")); + assert_eq!( + expected_hash, actual_hash, + "downloaded payload SHA-256 mismatch after gateway pod rolls" + ); + let marker = fs::read_to_string(download_dir.join("marker.txt")) + .expect("read downloaded HA sync marker"); + assert_eq!(marker, "ha-sync-marker", "downloaded marker mismatch"); + + sandbox.cleanup().await; +} diff --git a/e2e/rust/tests/readyz_health.rs b/e2e/rust/tests/readyz_health.rs deleted file mode 100644 index 8f093dabe8..0000000000 --- a/e2e/rust/tests/readyz_health.rs +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-kubernetes")] - -use bytes::Bytes; -use http_body_util::{BodyExt, Empty}; -use hyper::Request; -use hyper_util::rt::TokioIo; -use serde_json::Value; -use std::time::{Duration, Instant}; -use tokio::net::TcpStream; - -fn health_port_from_env() -> u16 { - let raw = std::env::var("OPENSHELL_E2E_HEALTH_PORT").unwrap_or_else(|_| { - panic!( - "OPENSHELL_E2E_HEALTH_PORT is not set. The Kubernetes e2e wrapper \ - (e2e/with-kube-gateway.sh) must export this variable so the \ - /readyz test can reach the gateway health listener." - ) - }); - raw.parse::().unwrap_or_else(|err| { - panic!("OPENSHELL_E2E_HEALTH_PORT=\"{raw}\" is not a valid u16 port: {err}") - }) -} - -async fn http_get_json(port: u16, path: &str) -> Result<(u16, Value), String> { - let stream = TcpStream::connect(("127.0.0.1", port)) - .await - .map_err(|err| format!("connect health endpoint :{port}: {err}"))?; - let (mut sender, conn) = hyper::client::conn::http1::Builder::new() - .handshake(TokioIo::new(stream)) - .await - .map_err(|err| format!("handshake health HTTP/1 client :{port}: {err}"))?; - tokio::spawn(async move { - let _ = conn.await; - }); - - let req = Request::builder() - .method("GET") - .uri(format!("http://127.0.0.1:{port}{path}")) - .body(Empty::::new()) - .map_err(|err| format!("build health request {path}: {err}"))?; - let resp = sender - .send_request(req) - .await - .map_err(|err| format!("send health request {path} to :{port}: {err}"))?; - let status_code = resp.status().as_u16(); - let bytes = resp - .into_body() - .collect() - .await - .map_err(|err| format!("read health response body {path}: {err}"))? - .to_bytes(); - let json = serde_json::from_slice::(&bytes) - .map_err(|err| format!("health endpoint {path} did not return valid JSON: {err}"))?; - - Ok((status_code, json)) -} - -#[tokio::test] -async fn readyz_reports_healthy_database_check() { - let port = health_port_from_env(); - - let deadline = Instant::now() + Duration::from_secs(20); - let timeout_detail = loop { - let observation = match http_get_json(port, "/readyz").await { - Ok((status, payload)) => { - let ready = status == 200 - && payload["status"] == "healthy" - && payload["checks"]["database"]["status"] == "healthy"; - if ready { - assert!( - payload["checks"]["database"]["latency_ms"].is_number(), - "readyz payload should include checks.database.latency_ms: {payload}" - ); - assert!( - payload["checks"]["database"]["error"].is_null(), - "readyz payload should not include checks.database.error when healthy: {payload}" - ); - return; - } - format!("unexpected /readyz response status={status} payload={payload}") - } - Err(err) => err, - }; - - if Instant::now() >= deadline { - break observation; - } - - tokio::time::sleep(Duration::from_secs(1)).await; - }; - panic!("timed out waiting for /readyz healthy response after 20s: {timeout_detail}"); -} diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index c4ed805293..36c321f071 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -101,6 +101,12 @@ EXTERNAL_PG_FIXTURE_SERVICE="openshell-e2e-postgres" EXTERNAL_PG_FIXTURE_USER="openshell" EXTERNAL_PG_FIXTURE_PASSWORD="openshell-e2e-postgres" EXTERNAL_PG_FIXTURE_DATABASE="openshell" +ENVOY_RELEASE_NAME="${OPENSHELL_E2E_ENVOY_RELEASE_NAME:-envoy-gateway}" +ENVOY_NAMESPACE="${OPENSHELL_E2E_ENVOY_NAMESPACE:-envoy-gateway-system}" +ENVOY_CHART_VERSION="${OPENSHELL_E2E_ENVOY_VERSION:-v1.7.2}" +ENVOY_GATEWAY_MANIFEST="${ROOT}/deploy/kube/manifests/envoy-gateway-openshell.yaml" +ENVOY_HELM_INSTALLED=0 +ENVOY_GATEWAY_CONFIG_APPLIED=0 VAULT_FIXTURE_DEPLOYED=0 VAULT_NAMESPACE="${OPENSHELL_E2E_VAULT_NAMESPACE:-openbao}" VAULT_RELEASE_NAME="${OPENSHELL_E2E_VAULT_RELEASE_NAME:-openbao}" @@ -181,6 +187,122 @@ deploy_postgres_fixture() { --from-literal=uri="${pg_uri}" } +use_envoy_gateway() { + case "${OPENSHELL_E2E_KUBE_USE_ENVOY:-0}" in + 1 | true | TRUE | yes | YES) return 0 ;; + *) return 1 ;; + esac +} + +install_envoy_gateway() { + echo "Installing Envoy Gateway (${ENVOY_CHART_VERSION})..." + helmctl upgrade --install "${ENVOY_RELEASE_NAME}" \ + oci://docker.io/envoyproxy/gateway-helm \ + --version "${ENVOY_CHART_VERSION}" \ + --namespace "${ENVOY_NAMESPACE}" --create-namespace \ + --wait --timeout 5m + ENVOY_HELM_INSTALLED=1 + + if ! kctl get namespace "${NAMESPACE}" >/dev/null 2>&1; then + kctl create namespace "${NAMESPACE}" + fi + + kctl apply -f "${ENVOY_GATEWAY_MANIFEST}" + ENVOY_GATEWAY_CONFIG_APPLIED=1 +} + +wait_for_envoy_service() { + local svc_ref="" + local svc_namespace="" + + for _ in $(seq 1 60); do + svc_ref="$(kctl get svc -A \ + -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \ + -o jsonpath='{range .items[0]}{.metadata.namespace}{"/"}{.metadata.name}{end}' \ + 2>/dev/null || true)" + if [ -n "${svc_ref}" ]; then + svc_namespace="${svc_ref%%/*}" + if kctl -n "${svc_namespace}" wait --for=condition=Ready pod \ + -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \ + --timeout=5s >/dev/null 2>&1; then + printf '%s\n' "${svc_ref}" + return 0 + fi + fi + sleep 2 + done + + echo "ERROR: Envoy proxy Service for Gateway ${RELEASE_NAME} was not ready." >&2 + kctl -n "${NAMESPACE}" get gateway,grpcroute -o wide >&2 || true + kctl get svc -A \ + -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \ + -o wide >&2 || true + kctl get pods -A \ + -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \ + -o wide >&2 || true + return 1 +} + +start_gateway_portforward() { + local elapsed=0 + local pf_timeout=30 + local target_port=8080 + local target_namespace="${NAMESPACE}" + local target_service="${RELEASE_NAME}" + local target_service_ref="" + + LOCAL_PORT="$(e2e_pick_port)" + if use_envoy_gateway; then + target_service_ref="$(wait_for_envoy_service)" + target_namespace="${target_service_ref%%/*}" + target_service="${target_service_ref#*/}" + target_port=80 + echo "Starting kubectl port-forward -n ${target_namespace} svc/${target_service} ${LOCAL_PORT}:${target_port} (Envoy Gateway)..." + else + echo "Starting kubectl port-forward svc/${target_service} ${LOCAL_PORT}:${target_port}..." + fi + + kctl -n "${target_namespace}" port-forward "svc/${target_service}" \ + "${LOCAL_PORT}:${target_port}" >"${PORTFORWARD_LOG}" 2>&1 & + PORTFORWARD_PID=$! + + while [ "${elapsed}" -lt "${pf_timeout}" ]; do + if ! kill -0 "${PORTFORWARD_PID}" 2>/dev/null; then + echo "ERROR: kubectl port-forward exited before becoming reachable" >&2 + cat "${PORTFORWARD_LOG}" >&2 || true + return 1 + fi + if curl -s -o /dev/null --connect-timeout 1 "http://127.0.0.1:${LOCAL_PORT}"; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + + echo "ERROR: port-forward did not accept TCP within ${pf_timeout}s" >&2 + cat "${PORTFORWARD_LOG}" >&2 || true + return 1 +} + +stop_gateway_portforward() { + local pid + local pid_var + for pid_var in PORTFORWARD_PID PORTFORWARD_HEALTH_PID; do + pid="${!pid_var}" + [ -n "${pid}" ] || continue + kill "${pid}" >/dev/null 2>&1 || true + for _ in $(seq 1 10); do + if ! kill -0 "${pid}" >/dev/null 2>&1; then + break + fi + sleep 0.5 + done + kill -KILL "${pid}" >/dev/null 2>&1 || true + wait "${pid}" >/dev/null 2>&1 || true + printf -v "${pid_var}" '%s' "" + done +} + cleanup_postgres_fixture() { local secret_name="$1" @@ -320,15 +442,7 @@ cleanup_vault_fixture() { cleanup() { local exit_code=$? - if [ -n "${PORTFORWARD_PID}" ]; then - kill "${PORTFORWARD_PID}" >/dev/null 2>&1 || true - wait "${PORTFORWARD_PID}" >/dev/null 2>&1 || true - fi - - if [ -n "${PORTFORWARD_HEALTH_PID}" ]; then - kill "${PORTFORWARD_HEALTH_PID}" >/dev/null 2>&1 || true - wait "${PORTFORWARD_HEALTH_PID}" >/dev/null 2>&1 || true - fi + stop_gateway_portforward if [ "${exit_code}" -ne 0 ] && [ -n "${KUBE_CONTEXT}" ] && [ -n "${NAMESPACE}" ]; then if command -v kubectl >/dev/null 2>&1 \ @@ -390,6 +504,17 @@ cleanup() { cleanup_vault_fixture fi + if [ "${ENVOY_GATEWAY_CONFIG_APPLIED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then + if command -v kubectl >/dev/null 2>&1; then + kctl -n "${NAMESPACE}" delete backendtrafficpolicy.gateway.envoyproxy.io \ + openshell-grpc-timeouts --ignore-not-found --wait=false \ + >/dev/null 2>&1 || true + kctl delete gatewayclass.gateway.networking.k8s.io eg \ + --ignore-not-found --wait=false >/dev/null 2>&1 || true + fi + ENVOY_GATEWAY_CONFIG_APPLIED=0 + fi + if [ "${CORPORATE_PROXY_FIXTURE_DEPLOYED}" = "1" ]; then kctl -n "${NAMESPACE}" delete secret "${CORPORATE_PROXY_FIXTURE_SECRET}" \ --ignore-not-found >/dev/null 2>&1 || true @@ -438,6 +563,18 @@ cleanup() { fi fi + if [ "${ENVOY_HELM_INSTALLED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then + if command -v helm >/dev/null 2>&1; then + helmctl uninstall "${ENVOY_RELEASE_NAME}" --namespace "${ENVOY_NAMESPACE}" \ + --wait --timeout 60s >/dev/null 2>&1 || true + fi + if command -v kubectl >/dev/null 2>&1; then + kctl delete namespace "${ENVOY_NAMESPACE}" --wait=true --timeout=60s \ + --ignore-not-found >/dev/null 2>&1 || true + fi + ENVOY_HELM_INSTALLED=0 + fi + if [ "${CLUSTER_CREATED_BY_US}" = "1" ] && [ -n "${CLUSTER_NAME}" ]; then if command -v k3d >/dev/null 2>&1 && k3d cluster list "${CLUSTER_NAME}" \ >/dev/null 2>&1; then @@ -453,16 +590,7 @@ trap cleanup EXIT # --- DB-scenario helpers (used only when OPENSHELL_E2E_KUBE_DB_SCENARIOS=1) --- scenario_stop_portforward() { - if [ -n "${PORTFORWARD_PID}" ]; then - kill "${PORTFORWARD_PID}" >/dev/null 2>&1 || true - wait "${PORTFORWARD_PID}" >/dev/null 2>&1 || true - PORTFORWARD_PID="" - fi - if [ -n "${PORTFORWARD_HEALTH_PID}" ]; then - kill "${PORTFORWARD_HEALTH_PID}" >/dev/null 2>&1 || true - wait "${PORTFORWARD_HEALTH_PID}" >/dev/null 2>&1 || true - PORTFORWARD_HEALTH_PID="" - fi + stop_gateway_portforward } scenario_cleanup_release() { @@ -537,7 +665,7 @@ run_scenario() { fi else # Vanilla Kubernetes: reach the gateway in plaintext over port-forward. - if ! start_grpc_portforward; then + if ! start_gateway_portforward; then scenario_record_failure "${scenario_label}" "port-forward failed" return fi @@ -564,6 +692,9 @@ run_scenario() { export CONTAINER_ENGINE="${CONTAINER_ENGINE:-docker}" export OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE="${KUBE_CONTEXT}" export OPENSHELL_E2E_SANDBOX_NAMESPACE="${NAMESPACE}" + export OPENSHELL_E2E_KUBE_CONTEXT="${KUBE_CONTEXT}" + export OPENSHELL_E2E_KUBE_NAMESPACE="${NAMESPACE}" + export OPENSHELL_E2E_KUBE_RELEASE="${RELEASE_NAME}" export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-300}" e2e_import_example_provider_profiles \ @@ -1122,6 +1253,10 @@ if [ -n "${OPENSHELL_E2E_KUBE_EXTRA_VALUES:-}" ]; then helm_values_args+=(--values "${values_file}") done fi +if use_envoy_gateway; then + helm_values_args+=(--values "${ROOT}/deploy/helm/openshell/ci/values-gateway.yaml") + install_envoy_gateway +fi if [ "${OPENSHELL_E2E_KUBE_DB_SCENARIOS:-0}" = "1" ]; then # --- Multi-scenario mode: test all database backends --- @@ -1189,7 +1324,7 @@ else openshift_register_route_gateway || exit 1 else # Vanilla Kubernetes: reach the gateway in plaintext over port-forward. - start_grpc_portforward || exit 1 + start_gateway_portforward || exit 1 GATEWAY_NAME="openshell-e2e-kube-${LOCAL_PORT}" GATEWAY_ENDPOINT="http://127.0.0.1:${LOCAL_PORT}" e2e_register_plaintext_gateway \ @@ -1210,6 +1345,9 @@ else export CONTAINER_ENGINE="${CONTAINER_ENGINE:-docker}" export OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE="${KUBE_CONTEXT}" export OPENSHELL_E2E_SANDBOX_NAMESPACE="${NAMESPACE}" + export OPENSHELL_E2E_KUBE_CONTEXT="${KUBE_CONTEXT}" + export OPENSHELL_E2E_KUBE_NAMESPACE="${NAMESPACE}" + export OPENSHELL_E2E_KUBE_RELEASE="${RELEASE_NAME}" export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-300}" e2e_import_example_provider_profiles \ diff --git a/proto/openshell.proto b/proto/openshell.proto index 8435c73350..16ec2375d2 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -587,6 +587,19 @@ service OpenShell { }; } + // Internal gateway-to-gateway relay forwarding. + // + // A gateway replica that receives a user request for a sandbox whose + // supervisor session is owned by a different replica opens this stream to the + // owner. The first frame carries PeerRelayInit; subsequent frames carry raw + // bytes in either direction. This RPC is authenticated as a gateway peer, not + // as a user or sandbox supervisor. + rpc PeerRelay(stream PeerRelayFrame) returns (stream PeerRelayFrame) { + option (openshell.options.v1.authorization) = { + auth_mode: "peer" + }; + } + // Watch a sandbox and stream updates. // // This stream can include: @@ -2851,6 +2864,9 @@ message SupervisorHello { string sandbox_id = 1; // Supervisor instance ID (e.g. boot id or process epoch). string instance_id = 2; + // Monotonic counter scoped to instance_id. Incremented for each reconnect so + // gateways can distinguish a fresh supervisor connection from stale cleanup. + uint64 connection_epoch = 3; // The supervisor can report credential, policy, and launch-environment installation. bool supports_provider_readiness = 4; } @@ -2944,6 +2960,25 @@ message RelayFrame { } } +// Initial frame for gateway peer relay forwarding. +message PeerRelayInit { + // Stable sandbox UUID whose supervisor relay should be opened. + string sandbox_id = 1; + // Relay target to ask the owning gateway to open on its local supervisor + // session. The channel_id is assigned by the forwarding gateway. + RelayOpen relay_open = 2; + // Gateway replica id that initiated the peer relay. + string requester_replica_id = 3; +} + +// A single frame on the gateway-to-gateway peer relay RPC. +message PeerRelayFrame { + oneof payload { + PeerRelayInit init = 1; + bytes data = 2; + } +} + // Supervisor reports the result of a relay open request. message RelayOpenResult { // Channel identifier from the RelayOpen request. diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 4fafa9c31a..eefce03f82 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -12874,6 +12874,9 @@ type SupervisorHello struct { SandboxId string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` // Supervisor instance ID (e.g. boot id or process epoch). InstanceId string `protobuf:"bytes,2,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` + // Monotonic counter scoped to instance_id. Incremented for each reconnect so + // gateways can distinguish a fresh supervisor connection from stale cleanup. + ConnectionEpoch uint64 `protobuf:"varint,3,opt,name=connection_epoch,json=connectionEpoch,proto3" json:"connection_epoch,omitempty"` // The supervisor can report credential, policy, and launch-environment installation. SupportsProviderReadiness bool `protobuf:"varint,4,opt,name=supports_provider_readiness,json=supportsProviderReadiness,proto3" json:"supports_provider_readiness,omitempty"` unknownFields protoimpl.UnknownFields @@ -12924,6 +12927,13 @@ func (x *SupervisorHello) GetInstanceId() string { return "" } +func (x *SupervisorHello) GetConnectionEpoch() uint64 { + if x != nil { + return x.ConnectionEpoch + } + return 0 +} + func (x *SupervisorHello) GetSupportsProviderReadiness() bool { if x != nil { return x.SupportsProviderReadiness @@ -13628,6 +13638,154 @@ func (*RelayFrame_Init) isRelayFrame_Payload() {} func (*RelayFrame_Data) isRelayFrame_Payload() {} +// Initial frame for gateway peer relay forwarding. +type PeerRelayInit struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Stable sandbox UUID whose supervisor relay should be opened. + SandboxId string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` + // Relay target to ask the owning gateway to open on its local supervisor + // session. The channel_id is assigned by the forwarding gateway. + RelayOpen *RelayOpen `protobuf:"bytes,2,opt,name=relay_open,json=relayOpen,proto3" json:"relay_open,omitempty"` + // Gateway replica id that initiated the peer relay. + RequesterReplicaId string `protobuf:"bytes,3,opt,name=requester_replica_id,json=requesterReplicaId,proto3" json:"requester_replica_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PeerRelayInit) Reset() { + *x = PeerRelayInit{} + mi := &file_openshell_proto_msgTypes[178] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PeerRelayInit) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PeerRelayInit) ProtoMessage() {} + +func (x *PeerRelayInit) ProtoReflect() protoreflect.Message { + mi := &file_openshell_proto_msgTypes[178] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PeerRelayInit.ProtoReflect.Descriptor instead. +func (*PeerRelayInit) Descriptor() ([]byte, []int) { + return file_openshell_proto_rawDescGZIP(), []int{178} +} + +func (x *PeerRelayInit) GetSandboxId() string { + if x != nil { + return x.SandboxId + } + return "" +} + +func (x *PeerRelayInit) GetRelayOpen() *RelayOpen { + if x != nil { + return x.RelayOpen + } + return nil +} + +func (x *PeerRelayInit) GetRequesterReplicaId() string { + if x != nil { + return x.RequesterReplicaId + } + return "" +} + +// A single frame on the gateway-to-gateway peer relay RPC. +type PeerRelayFrame struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Payload: + // + // *PeerRelayFrame_Init + // *PeerRelayFrame_Data + Payload isPeerRelayFrame_Payload `protobuf_oneof:"payload"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PeerRelayFrame) Reset() { + *x = PeerRelayFrame{} + mi := &file_openshell_proto_msgTypes[179] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PeerRelayFrame) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PeerRelayFrame) ProtoMessage() {} + +func (x *PeerRelayFrame) ProtoReflect() protoreflect.Message { + mi := &file_openshell_proto_msgTypes[179] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PeerRelayFrame.ProtoReflect.Descriptor instead. +func (*PeerRelayFrame) Descriptor() ([]byte, []int) { + return file_openshell_proto_rawDescGZIP(), []int{179} +} + +func (x *PeerRelayFrame) GetPayload() isPeerRelayFrame_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *PeerRelayFrame) GetInit() *PeerRelayInit { + if x != nil { + if x, ok := x.Payload.(*PeerRelayFrame_Init); ok { + return x.Init + } + } + return nil +} + +func (x *PeerRelayFrame) GetData() []byte { + if x != nil { + if x, ok := x.Payload.(*PeerRelayFrame_Data); ok { + return x.Data + } + } + return nil +} + +type isPeerRelayFrame_Payload interface { + isPeerRelayFrame_Payload() +} + +type PeerRelayFrame_Init struct { + Init *PeerRelayInit `protobuf:"bytes,1,opt,name=init,proto3,oneof"` +} + +type PeerRelayFrame_Data struct { + Data []byte `protobuf:"bytes,2,opt,name=data,proto3,oneof"` +} + +func (*PeerRelayFrame_Init) isPeerRelayFrame_Payload() {} + +func (*PeerRelayFrame_Data) isPeerRelayFrame_Payload() {} + // Supervisor reports the result of a relay open request. type RelayOpenResult struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -13643,7 +13801,7 @@ type RelayOpenResult struct { func (x *RelayOpenResult) Reset() { *x = RelayOpenResult{} - mi := &file_openshell_proto_msgTypes[178] + mi := &file_openshell_proto_msgTypes[180] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13655,7 +13813,7 @@ func (x *RelayOpenResult) String() string { func (*RelayOpenResult) ProtoMessage() {} func (x *RelayOpenResult) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[178] + mi := &file_openshell_proto_msgTypes[180] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13668,7 +13826,7 @@ func (x *RelayOpenResult) ProtoReflect() protoreflect.Message { // Deprecated: Use RelayOpenResult.ProtoReflect.Descriptor instead. func (*RelayOpenResult) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{178} + return file_openshell_proto_rawDescGZIP(), []int{180} } func (x *RelayOpenResult) GetChannelId() string { @@ -13705,7 +13863,7 @@ type RelayClose struct { func (x *RelayClose) Reset() { *x = RelayClose{} - mi := &file_openshell_proto_msgTypes[179] + mi := &file_openshell_proto_msgTypes[181] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13717,7 +13875,7 @@ func (x *RelayClose) String() string { func (*RelayClose) ProtoMessage() {} func (x *RelayClose) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[179] + mi := &file_openshell_proto_msgTypes[181] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13730,7 +13888,7 @@ func (x *RelayClose) ProtoReflect() protoreflect.Message { // Deprecated: Use RelayClose.ProtoReflect.Descriptor instead. func (*RelayClose) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{179} + return file_openshell_proto_rawDescGZIP(), []int{181} } func (x *RelayClose) GetChannelId() string { @@ -13764,7 +13922,7 @@ type L7RequestSample struct { func (x *L7RequestSample) Reset() { *x = L7RequestSample{} - mi := &file_openshell_proto_msgTypes[180] + mi := &file_openshell_proto_msgTypes[182] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13776,7 +13934,7 @@ func (x *L7RequestSample) String() string { func (*L7RequestSample) ProtoMessage() {} func (x *L7RequestSample) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[180] + mi := &file_openshell_proto_msgTypes[182] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13789,7 +13947,7 @@ func (x *L7RequestSample) ProtoReflect() protoreflect.Message { // Deprecated: Use L7RequestSample.ProtoReflect.Descriptor instead. func (*L7RequestSample) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{180} + return file_openshell_proto_rawDescGZIP(), []int{182} } func (x *L7RequestSample) GetMethod() string { @@ -13863,7 +14021,7 @@ type DenialSummary struct { func (x *DenialSummary) Reset() { *x = DenialSummary{} - mi := &file_openshell_proto_msgTypes[181] + mi := &file_openshell_proto_msgTypes[183] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13875,7 +14033,7 @@ func (x *DenialSummary) String() string { func (*DenialSummary) ProtoMessage() {} func (x *DenialSummary) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[181] + mi := &file_openshell_proto_msgTypes[183] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13888,7 +14046,7 @@ func (x *DenialSummary) ProtoReflect() protoreflect.Message { // Deprecated: Use DenialSummary.ProtoReflect.Descriptor instead. func (*DenialSummary) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{181} + return file_openshell_proto_rawDescGZIP(), []int{183} } func (x *DenialSummary) GetSandboxId() string { @@ -14023,7 +14181,7 @@ type DenialGroupCount struct { func (x *DenialGroupCount) Reset() { *x = DenialGroupCount{} - mi := &file_openshell_proto_msgTypes[182] + mi := &file_openshell_proto_msgTypes[184] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14035,7 +14193,7 @@ func (x *DenialGroupCount) String() string { func (*DenialGroupCount) ProtoMessage() {} func (x *DenialGroupCount) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[182] + mi := &file_openshell_proto_msgTypes[184] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14048,7 +14206,7 @@ func (x *DenialGroupCount) ProtoReflect() protoreflect.Message { // Deprecated: Use DenialGroupCount.ProtoReflect.Descriptor instead. func (*DenialGroupCount) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{182} + return file_openshell_proto_rawDescGZIP(), []int{184} } func (x *DenialGroupCount) GetDenyGroup() string { @@ -14081,7 +14239,7 @@ type NetworkActivitySummary struct { func (x *NetworkActivitySummary) Reset() { *x = NetworkActivitySummary{} - mi := &file_openshell_proto_msgTypes[183] + mi := &file_openshell_proto_msgTypes[185] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14093,7 +14251,7 @@ func (x *NetworkActivitySummary) String() string { func (*NetworkActivitySummary) ProtoMessage() {} func (x *NetworkActivitySummary) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[183] + mi := &file_openshell_proto_msgTypes[185] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14106,7 +14264,7 @@ func (x *NetworkActivitySummary) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkActivitySummary.ProtoReflect.Descriptor instead. func (*NetworkActivitySummary) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{183} + return file_openshell_proto_rawDescGZIP(), []int{185} } func (x *NetworkActivitySummary) GetNetworkActivityCount() uint32 { @@ -14194,7 +14352,7 @@ type PolicyChunk struct { func (x *PolicyChunk) Reset() { *x = PolicyChunk{} - mi := &file_openshell_proto_msgTypes[184] + mi := &file_openshell_proto_msgTypes[186] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14206,7 +14364,7 @@ func (x *PolicyChunk) String() string { func (*PolicyChunk) ProtoMessage() {} func (x *PolicyChunk) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[184] + mi := &file_openshell_proto_msgTypes[186] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14219,7 +14377,7 @@ func (x *PolicyChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use PolicyChunk.ProtoReflect.Descriptor instead. func (*PolicyChunk) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{184} + return file_openshell_proto_rawDescGZIP(), []int{186} } func (x *PolicyChunk) GetId() string { @@ -14407,7 +14565,7 @@ type DraftPolicyUpdate struct { func (x *DraftPolicyUpdate) Reset() { *x = DraftPolicyUpdate{} - mi := &file_openshell_proto_msgTypes[185] + mi := &file_openshell_proto_msgTypes[187] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14419,7 +14577,7 @@ func (x *DraftPolicyUpdate) String() string { func (*DraftPolicyUpdate) ProtoMessage() {} func (x *DraftPolicyUpdate) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[185] + mi := &file_openshell_proto_msgTypes[187] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14432,7 +14590,7 @@ func (x *DraftPolicyUpdate) ProtoReflect() protoreflect.Message { // Deprecated: Use DraftPolicyUpdate.ProtoReflect.Descriptor instead. func (*DraftPolicyUpdate) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{185} + return file_openshell_proto_rawDescGZIP(), []int{187} } func (x *DraftPolicyUpdate) GetDraftVersion() uint64 { @@ -14491,7 +14649,7 @@ type SubmitPolicyAnalysisRequest struct { func (x *SubmitPolicyAnalysisRequest) Reset() { *x = SubmitPolicyAnalysisRequest{} - mi := &file_openshell_proto_msgTypes[186] + mi := &file_openshell_proto_msgTypes[188] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14503,7 +14661,7 @@ func (x *SubmitPolicyAnalysisRequest) String() string { func (*SubmitPolicyAnalysisRequest) ProtoMessage() {} func (x *SubmitPolicyAnalysisRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[186] + mi := &file_openshell_proto_msgTypes[188] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14516,7 +14674,7 @@ func (x *SubmitPolicyAnalysisRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SubmitPolicyAnalysisRequest.ProtoReflect.Descriptor instead. func (*SubmitPolicyAnalysisRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{186} + return file_openshell_proto_rawDescGZIP(), []int{188} } func (x *SubmitPolicyAnalysisRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -14579,7 +14737,7 @@ type SubmitPolicyAnalysisResponse struct { func (x *SubmitPolicyAnalysisResponse) Reset() { *x = SubmitPolicyAnalysisResponse{} - mi := &file_openshell_proto_msgTypes[187] + mi := &file_openshell_proto_msgTypes[189] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14591,7 +14749,7 @@ func (x *SubmitPolicyAnalysisResponse) String() string { func (*SubmitPolicyAnalysisResponse) ProtoMessage() {} func (x *SubmitPolicyAnalysisResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[187] + mi := &file_openshell_proto_msgTypes[189] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14604,7 +14762,7 @@ func (x *SubmitPolicyAnalysisResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SubmitPolicyAnalysisResponse.ProtoReflect.Descriptor instead. func (*SubmitPolicyAnalysisResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{187} + return file_openshell_proto_rawDescGZIP(), []int{189} } func (x *SubmitPolicyAnalysisResponse) GetAcceptedChunks() uint32 { @@ -14649,7 +14807,7 @@ type GetDraftPolicyRequest struct { func (x *GetDraftPolicyRequest) Reset() { *x = GetDraftPolicyRequest{} - mi := &file_openshell_proto_msgTypes[188] + mi := &file_openshell_proto_msgTypes[190] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14661,7 +14819,7 @@ func (x *GetDraftPolicyRequest) String() string { func (*GetDraftPolicyRequest) ProtoMessage() {} func (x *GetDraftPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[188] + mi := &file_openshell_proto_msgTypes[190] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14674,7 +14832,7 @@ func (x *GetDraftPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetDraftPolicyRequest.ProtoReflect.Descriptor instead. func (*GetDraftPolicyRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{188} + return file_openshell_proto_rawDescGZIP(), []int{190} } func (x *GetDraftPolicyRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -14714,7 +14872,7 @@ type GetDraftPolicyResponse struct { func (x *GetDraftPolicyResponse) Reset() { *x = GetDraftPolicyResponse{} - mi := &file_openshell_proto_msgTypes[189] + mi := &file_openshell_proto_msgTypes[191] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14726,7 +14884,7 @@ func (x *GetDraftPolicyResponse) String() string { func (*GetDraftPolicyResponse) ProtoMessage() {} func (x *GetDraftPolicyResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[189] + mi := &file_openshell_proto_msgTypes[191] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14739,7 +14897,7 @@ func (x *GetDraftPolicyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetDraftPolicyResponse.ProtoReflect.Descriptor instead. func (*GetDraftPolicyResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{189} + return file_openshell_proto_rawDescGZIP(), []int{191} } func (x *GetDraftPolicyResponse) GetChunks() []*PolicyChunk { @@ -14790,7 +14948,7 @@ type ApproveDraftChunkRequest struct { func (x *ApproveDraftChunkRequest) Reset() { *x = ApproveDraftChunkRequest{} - mi := &file_openshell_proto_msgTypes[190] + mi := &file_openshell_proto_msgTypes[192] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14802,7 +14960,7 @@ func (x *ApproveDraftChunkRequest) String() string { func (*ApproveDraftChunkRequest) ProtoMessage() {} func (x *ApproveDraftChunkRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[190] + mi := &file_openshell_proto_msgTypes[192] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14815,7 +14973,7 @@ func (x *ApproveDraftChunkRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ApproveDraftChunkRequest.ProtoReflect.Descriptor instead. func (*ApproveDraftChunkRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{190} + return file_openshell_proto_rawDescGZIP(), []int{192} } func (x *ApproveDraftChunkRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -14865,7 +15023,7 @@ type ApproveDraftChunkResponse struct { func (x *ApproveDraftChunkResponse) Reset() { *x = ApproveDraftChunkResponse{} - mi := &file_openshell_proto_msgTypes[191] + mi := &file_openshell_proto_msgTypes[193] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14877,7 +15035,7 @@ func (x *ApproveDraftChunkResponse) String() string { func (*ApproveDraftChunkResponse) ProtoMessage() {} func (x *ApproveDraftChunkResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[191] + mi := &file_openshell_proto_msgTypes[193] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14890,7 +15048,7 @@ func (x *ApproveDraftChunkResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ApproveDraftChunkResponse.ProtoReflect.Descriptor instead. func (*ApproveDraftChunkResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{191} + return file_openshell_proto_rawDescGZIP(), []int{193} } func (x *ApproveDraftChunkResponse) GetPolicyVersion() uint32 { @@ -14926,7 +15084,7 @@ type RejectDraftChunkRequest struct { func (x *RejectDraftChunkRequest) Reset() { *x = RejectDraftChunkRequest{} - mi := &file_openshell_proto_msgTypes[192] + mi := &file_openshell_proto_msgTypes[194] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14938,7 +15096,7 @@ func (x *RejectDraftChunkRequest) String() string { func (*RejectDraftChunkRequest) ProtoMessage() {} func (x *RejectDraftChunkRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[192] + mi := &file_openshell_proto_msgTypes[194] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14951,7 +15109,7 @@ func (x *RejectDraftChunkRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RejectDraftChunkRequest.ProtoReflect.Descriptor instead. func (*RejectDraftChunkRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{192} + return file_openshell_proto_rawDescGZIP(), []int{194} } func (x *RejectDraftChunkRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -14997,7 +15155,7 @@ type RejectDraftChunkResponse struct { func (x *RejectDraftChunkResponse) Reset() { *x = RejectDraftChunkResponse{} - mi := &file_openshell_proto_msgTypes[193] + mi := &file_openshell_proto_msgTypes[195] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15009,7 +15167,7 @@ func (x *RejectDraftChunkResponse) String() string { func (*RejectDraftChunkResponse) ProtoMessage() {} func (x *RejectDraftChunkResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[193] + mi := &file_openshell_proto_msgTypes[195] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15022,7 +15180,7 @@ func (x *RejectDraftChunkResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RejectDraftChunkResponse.ProtoReflect.Descriptor instead. func (*RejectDraftChunkResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{193} + return file_openshell_proto_rawDescGZIP(), []int{195} } // Approve all pending chunks. @@ -15036,7 +15194,7 @@ type DraftChunkApproval struct { func (x *DraftChunkApproval) Reset() { *x = DraftChunkApproval{} - mi := &file_openshell_proto_msgTypes[194] + mi := &file_openshell_proto_msgTypes[196] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15048,7 +15206,7 @@ func (x *DraftChunkApproval) String() string { func (*DraftChunkApproval) ProtoMessage() {} func (x *DraftChunkApproval) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[194] + mi := &file_openshell_proto_msgTypes[196] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15061,7 +15219,7 @@ func (x *DraftChunkApproval) ProtoReflect() protoreflect.Message { // Deprecated: Use DraftChunkApproval.ProtoReflect.Descriptor instead. func (*DraftChunkApproval) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{194} + return file_openshell_proto_rawDescGZIP(), []int{196} } func (x *DraftChunkApproval) GetChunkId() string { @@ -15097,7 +15255,7 @@ type ApproveAllDraftChunksRequest struct { func (x *ApproveAllDraftChunksRequest) Reset() { *x = ApproveAllDraftChunksRequest{} - mi := &file_openshell_proto_msgTypes[195] + mi := &file_openshell_proto_msgTypes[197] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15109,7 +15267,7 @@ func (x *ApproveAllDraftChunksRequest) String() string { func (*ApproveAllDraftChunksRequest) ProtoMessage() {} func (x *ApproveAllDraftChunksRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[195] + mi := &file_openshell_proto_msgTypes[197] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15122,7 +15280,7 @@ func (x *ApproveAllDraftChunksRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ApproveAllDraftChunksRequest.ProtoReflect.Descriptor instead. func (*ApproveAllDraftChunksRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{195} + return file_openshell_proto_rawDescGZIP(), []int{197} } func (x *ApproveAllDraftChunksRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -15177,7 +15335,7 @@ type ApproveAllDraftChunksResponse struct { func (x *ApproveAllDraftChunksResponse) Reset() { *x = ApproveAllDraftChunksResponse{} - mi := &file_openshell_proto_msgTypes[196] + mi := &file_openshell_proto_msgTypes[198] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15189,7 +15347,7 @@ func (x *ApproveAllDraftChunksResponse) String() string { func (*ApproveAllDraftChunksResponse) ProtoMessage() {} func (x *ApproveAllDraftChunksResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[196] + mi := &file_openshell_proto_msgTypes[198] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15202,7 +15360,7 @@ func (x *ApproveAllDraftChunksResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ApproveAllDraftChunksResponse.ProtoReflect.Descriptor instead. func (*ApproveAllDraftChunksResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{196} + return file_openshell_proto_rawDescGZIP(), []int{198} } func (x *ApproveAllDraftChunksResponse) GetPolicyVersion() uint32 { @@ -15252,7 +15410,7 @@ type EditDraftChunkRequest struct { func (x *EditDraftChunkRequest) Reset() { *x = EditDraftChunkRequest{} - mi := &file_openshell_proto_msgTypes[197] + mi := &file_openshell_proto_msgTypes[199] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15264,7 +15422,7 @@ func (x *EditDraftChunkRequest) String() string { func (*EditDraftChunkRequest) ProtoMessage() {} func (x *EditDraftChunkRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[197] + mi := &file_openshell_proto_msgTypes[199] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15277,7 +15435,7 @@ func (x *EditDraftChunkRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use EditDraftChunkRequest.ProtoReflect.Descriptor instead. func (*EditDraftChunkRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{197} + return file_openshell_proto_rawDescGZIP(), []int{199} } func (x *EditDraftChunkRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -15323,7 +15481,7 @@ type EditDraftChunkResponse struct { func (x *EditDraftChunkResponse) Reset() { *x = EditDraftChunkResponse{} - mi := &file_openshell_proto_msgTypes[198] + mi := &file_openshell_proto_msgTypes[200] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15335,7 +15493,7 @@ func (x *EditDraftChunkResponse) String() string { func (*EditDraftChunkResponse) ProtoMessage() {} func (x *EditDraftChunkResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[198] + mi := &file_openshell_proto_msgTypes[200] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15348,7 +15506,7 @@ func (x *EditDraftChunkResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use EditDraftChunkResponse.ProtoReflect.Descriptor instead. func (*EditDraftChunkResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{198} + return file_openshell_proto_rawDescGZIP(), []int{200} } // Reverse an approval (remove merged rule from active policy). @@ -15368,7 +15526,7 @@ type UndoDraftChunkRequest struct { func (x *UndoDraftChunkRequest) Reset() { *x = UndoDraftChunkRequest{} - mi := &file_openshell_proto_msgTypes[199] + mi := &file_openshell_proto_msgTypes[201] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15380,7 +15538,7 @@ func (x *UndoDraftChunkRequest) String() string { func (*UndoDraftChunkRequest) ProtoMessage() {} func (x *UndoDraftChunkRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[199] + mi := &file_openshell_proto_msgTypes[201] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15393,7 +15551,7 @@ func (x *UndoDraftChunkRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UndoDraftChunkRequest.ProtoReflect.Descriptor instead. func (*UndoDraftChunkRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{199} + return file_openshell_proto_rawDescGZIP(), []int{201} } func (x *UndoDraftChunkRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -15436,7 +15594,7 @@ type UndoDraftChunkResponse struct { func (x *UndoDraftChunkResponse) Reset() { *x = UndoDraftChunkResponse{} - mi := &file_openshell_proto_msgTypes[200] + mi := &file_openshell_proto_msgTypes[202] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15448,7 +15606,7 @@ func (x *UndoDraftChunkResponse) String() string { func (*UndoDraftChunkResponse) ProtoMessage() {} func (x *UndoDraftChunkResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[200] + mi := &file_openshell_proto_msgTypes[202] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15461,7 +15619,7 @@ func (x *UndoDraftChunkResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UndoDraftChunkResponse.ProtoReflect.Descriptor instead. func (*UndoDraftChunkResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{200} + return file_openshell_proto_rawDescGZIP(), []int{202} } func (x *UndoDraftChunkResponse) GetPolicyVersion() uint32 { @@ -15493,7 +15651,7 @@ type ClearDraftChunksRequest struct { func (x *ClearDraftChunksRequest) Reset() { *x = ClearDraftChunksRequest{} - mi := &file_openshell_proto_msgTypes[201] + mi := &file_openshell_proto_msgTypes[203] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15505,7 +15663,7 @@ func (x *ClearDraftChunksRequest) String() string { func (*ClearDraftChunksRequest) ProtoMessage() {} func (x *ClearDraftChunksRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[201] + mi := &file_openshell_proto_msgTypes[203] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15518,7 +15676,7 @@ func (x *ClearDraftChunksRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ClearDraftChunksRequest.ProtoReflect.Descriptor instead. func (*ClearDraftChunksRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{201} + return file_openshell_proto_rawDescGZIP(), []int{203} } func (x *ClearDraftChunksRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -15552,7 +15710,7 @@ type ClearDraftChunksResponse struct { func (x *ClearDraftChunksResponse) Reset() { *x = ClearDraftChunksResponse{} - mi := &file_openshell_proto_msgTypes[202] + mi := &file_openshell_proto_msgTypes[204] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15564,7 +15722,7 @@ func (x *ClearDraftChunksResponse) String() string { func (*ClearDraftChunksResponse) ProtoMessage() {} func (x *ClearDraftChunksResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[202] + mi := &file_openshell_proto_msgTypes[204] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15577,7 +15735,7 @@ func (x *ClearDraftChunksResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ClearDraftChunksResponse.ProtoReflect.Descriptor instead. func (*ClearDraftChunksResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{202} + return file_openshell_proto_rawDescGZIP(), []int{204} } func (x *ClearDraftChunksResponse) GetChunksCleared() uint32 { @@ -15599,7 +15757,7 @@ type GetDraftHistoryRequest struct { func (x *GetDraftHistoryRequest) Reset() { *x = GetDraftHistoryRequest{} - mi := &file_openshell_proto_msgTypes[203] + mi := &file_openshell_proto_msgTypes[205] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15611,7 +15769,7 @@ func (x *GetDraftHistoryRequest) String() string { func (*GetDraftHistoryRequest) ProtoMessage() {} func (x *GetDraftHistoryRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[203] + mi := &file_openshell_proto_msgTypes[205] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15624,7 +15782,7 @@ func (x *GetDraftHistoryRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetDraftHistoryRequest.ProtoReflect.Descriptor instead. func (*GetDraftHistoryRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{203} + return file_openshell_proto_rawDescGZIP(), []int{205} } func (x *GetDraftHistoryRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -15658,7 +15816,7 @@ type DraftHistoryEntry struct { func (x *DraftHistoryEntry) Reset() { *x = DraftHistoryEntry{} - mi := &file_openshell_proto_msgTypes[204] + mi := &file_openshell_proto_msgTypes[206] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15670,7 +15828,7 @@ func (x *DraftHistoryEntry) String() string { func (*DraftHistoryEntry) ProtoMessage() {} func (x *DraftHistoryEntry) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[204] + mi := &file_openshell_proto_msgTypes[206] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15683,7 +15841,7 @@ func (x *DraftHistoryEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use DraftHistoryEntry.ProtoReflect.Descriptor instead. func (*DraftHistoryEntry) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{204} + return file_openshell_proto_rawDescGZIP(), []int{206} } func (x *DraftHistoryEntry) GetEventTime() *timestamppb.Timestamp { @@ -15724,7 +15882,7 @@ type GetDraftHistoryResponse struct { func (x *GetDraftHistoryResponse) Reset() { *x = GetDraftHistoryResponse{} - mi := &file_openshell_proto_msgTypes[205] + mi := &file_openshell_proto_msgTypes[207] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15736,7 +15894,7 @@ func (x *GetDraftHistoryResponse) String() string { func (*GetDraftHistoryResponse) ProtoMessage() {} func (x *GetDraftHistoryResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[205] + mi := &file_openshell_proto_msgTypes[207] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15749,7 +15907,7 @@ func (x *GetDraftHistoryResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetDraftHistoryResponse.ProtoReflect.Descriptor instead. func (*GetDraftHistoryResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{205} + return file_openshell_proto_rawDescGZIP(), []int{207} } func (x *GetDraftHistoryResponse) GetEntries() []*DraftHistoryEntry { @@ -15774,7 +15932,7 @@ type CreateWorkspaceRequest struct { func (x *CreateWorkspaceRequest) Reset() { *x = CreateWorkspaceRequest{} - mi := &file_openshell_proto_msgTypes[206] + mi := &file_openshell_proto_msgTypes[208] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15786,7 +15944,7 @@ func (x *CreateWorkspaceRequest) String() string { func (*CreateWorkspaceRequest) ProtoMessage() {} func (x *CreateWorkspaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[206] + mi := &file_openshell_proto_msgTypes[208] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15799,7 +15957,7 @@ func (x *CreateWorkspaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateWorkspaceRequest.ProtoReflect.Descriptor instead. func (*CreateWorkspaceRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{206} + return file_openshell_proto_rawDescGZIP(), []int{208} } func (x *CreateWorkspaceRequest) GetName() string { @@ -15833,7 +15991,7 @@ type CreateWorkspaceResponse struct { func (x *CreateWorkspaceResponse) Reset() { *x = CreateWorkspaceResponse{} - mi := &file_openshell_proto_msgTypes[207] + mi := &file_openshell_proto_msgTypes[209] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15845,7 +16003,7 @@ func (x *CreateWorkspaceResponse) String() string { func (*CreateWorkspaceResponse) ProtoMessage() {} func (x *CreateWorkspaceResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[207] + mi := &file_openshell_proto_msgTypes[209] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15858,7 +16016,7 @@ func (x *CreateWorkspaceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateWorkspaceResponse.ProtoReflect.Descriptor instead. func (*CreateWorkspaceResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{207} + return file_openshell_proto_rawDescGZIP(), []int{209} } func (x *CreateWorkspaceResponse) GetWorkspace() *datamodelv1.Workspace { @@ -15879,7 +16037,7 @@ type GetWorkspaceRequest struct { func (x *GetWorkspaceRequest) Reset() { *x = GetWorkspaceRequest{} - mi := &file_openshell_proto_msgTypes[208] + mi := &file_openshell_proto_msgTypes[210] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15891,7 +16049,7 @@ func (x *GetWorkspaceRequest) String() string { func (*GetWorkspaceRequest) ProtoMessage() {} func (x *GetWorkspaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[208] + mi := &file_openshell_proto_msgTypes[210] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15904,7 +16062,7 @@ func (x *GetWorkspaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkspaceRequest.ProtoReflect.Descriptor instead. func (*GetWorkspaceRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{208} + return file_openshell_proto_rawDescGZIP(), []int{210} } func (x *GetWorkspaceRequest) GetName() string { @@ -15924,7 +16082,7 @@ type GetWorkspaceResponse struct { func (x *GetWorkspaceResponse) Reset() { *x = GetWorkspaceResponse{} - mi := &file_openshell_proto_msgTypes[209] + mi := &file_openshell_proto_msgTypes[211] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15936,7 +16094,7 @@ func (x *GetWorkspaceResponse) String() string { func (*GetWorkspaceResponse) ProtoMessage() {} func (x *GetWorkspaceResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[209] + mi := &file_openshell_proto_msgTypes[211] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15949,7 +16107,7 @@ func (x *GetWorkspaceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkspaceResponse.ProtoReflect.Descriptor instead. func (*GetWorkspaceResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{209} + return file_openshell_proto_rawDescGZIP(), []int{211} } func (x *GetWorkspaceResponse) GetWorkspace() *datamodelv1.Workspace { @@ -15976,7 +16134,7 @@ type ListWorkspacesRequest struct { func (x *ListWorkspacesRequest) Reset() { *x = ListWorkspacesRequest{} - mi := &file_openshell_proto_msgTypes[210] + mi := &file_openshell_proto_msgTypes[212] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15988,7 +16146,7 @@ func (x *ListWorkspacesRequest) String() string { func (*ListWorkspacesRequest) ProtoMessage() {} func (x *ListWorkspacesRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[210] + mi := &file_openshell_proto_msgTypes[212] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16001,7 +16159,7 @@ func (x *ListWorkspacesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkspacesRequest.ProtoReflect.Descriptor instead. func (*ListWorkspacesRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{210} + return file_openshell_proto_rawDescGZIP(), []int{212} } func (x *ListWorkspacesRequest) GetPageSize() int32 { @@ -16037,7 +16195,7 @@ type ListWorkspacesResponse struct { func (x *ListWorkspacesResponse) Reset() { *x = ListWorkspacesResponse{} - mi := &file_openshell_proto_msgTypes[211] + mi := &file_openshell_proto_msgTypes[213] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16049,7 +16207,7 @@ func (x *ListWorkspacesResponse) String() string { func (*ListWorkspacesResponse) ProtoMessage() {} func (x *ListWorkspacesResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[211] + mi := &file_openshell_proto_msgTypes[213] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16062,7 +16220,7 @@ func (x *ListWorkspacesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkspacesResponse.ProtoReflect.Descriptor instead. func (*ListWorkspacesResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{211} + return file_openshell_proto_rawDescGZIP(), []int{213} } func (x *ListWorkspacesResponse) GetWorkspaces() []*datamodelv1.Workspace { @@ -16093,7 +16251,7 @@ type DeleteWorkspaceRequest struct { func (x *DeleteWorkspaceRequest) Reset() { *x = DeleteWorkspaceRequest{} - mi := &file_openshell_proto_msgTypes[212] + mi := &file_openshell_proto_msgTypes[214] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16105,7 +16263,7 @@ func (x *DeleteWorkspaceRequest) String() string { func (*DeleteWorkspaceRequest) ProtoMessage() {} func (x *DeleteWorkspaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[212] + mi := &file_openshell_proto_msgTypes[214] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16118,7 +16276,7 @@ func (x *DeleteWorkspaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteWorkspaceRequest.ProtoReflect.Descriptor instead. func (*DeleteWorkspaceRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{212} + return file_openshell_proto_rawDescGZIP(), []int{214} } func (x *DeleteWorkspaceRequest) GetName() string { @@ -16152,7 +16310,7 @@ type DeleteWorkspaceResponse struct { func (x *DeleteWorkspaceResponse) Reset() { *x = DeleteWorkspaceResponse{} - mi := &file_openshell_proto_msgTypes[213] + mi := &file_openshell_proto_msgTypes[215] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16164,7 +16322,7 @@ func (x *DeleteWorkspaceResponse) String() string { func (*DeleteWorkspaceResponse) ProtoMessage() {} func (x *DeleteWorkspaceResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[213] + mi := &file_openshell_proto_msgTypes[215] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16177,7 +16335,7 @@ func (x *DeleteWorkspaceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteWorkspaceResponse.ProtoReflect.Descriptor instead. func (*DeleteWorkspaceResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{213} + return file_openshell_proto_rawDescGZIP(), []int{215} } func (x *DeleteWorkspaceResponse) GetOutcome() DeletionOutcome { @@ -16201,7 +16359,7 @@ type WorkspaceMember struct { func (x *WorkspaceMember) Reset() { *x = WorkspaceMember{} - mi := &file_openshell_proto_msgTypes[214] + mi := &file_openshell_proto_msgTypes[216] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16213,7 +16371,7 @@ func (x *WorkspaceMember) String() string { func (*WorkspaceMember) ProtoMessage() {} func (x *WorkspaceMember) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[214] + mi := &file_openshell_proto_msgTypes[216] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16226,7 +16384,7 @@ func (x *WorkspaceMember) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkspaceMember.ProtoReflect.Descriptor instead. func (*WorkspaceMember) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{214} + return file_openshell_proto_rawDescGZIP(), []int{216} } func (x *WorkspaceMember) GetMetadata() *datamodelv1.ObjectMeta { @@ -16267,7 +16425,7 @@ type AddWorkspaceMemberRequest struct { func (x *AddWorkspaceMemberRequest) Reset() { *x = AddWorkspaceMemberRequest{} - mi := &file_openshell_proto_msgTypes[215] + mi := &file_openshell_proto_msgTypes[217] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16279,7 +16437,7 @@ func (x *AddWorkspaceMemberRequest) String() string { func (*AddWorkspaceMemberRequest) ProtoMessage() {} func (x *AddWorkspaceMemberRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[215] + mi := &file_openshell_proto_msgTypes[217] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16292,7 +16450,7 @@ func (x *AddWorkspaceMemberRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AddWorkspaceMemberRequest.ProtoReflect.Descriptor instead. func (*AddWorkspaceMemberRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{215} + return file_openshell_proto_rawDescGZIP(), []int{217} } func (x *AddWorkspaceMemberRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -16333,7 +16491,7 @@ type AddWorkspaceMemberResponse struct { func (x *AddWorkspaceMemberResponse) Reset() { *x = AddWorkspaceMemberResponse{} - mi := &file_openshell_proto_msgTypes[216] + mi := &file_openshell_proto_msgTypes[218] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16345,7 +16503,7 @@ func (x *AddWorkspaceMemberResponse) String() string { func (*AddWorkspaceMemberResponse) ProtoMessage() {} func (x *AddWorkspaceMemberResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[216] + mi := &file_openshell_proto_msgTypes[218] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16358,7 +16516,7 @@ func (x *AddWorkspaceMemberResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AddWorkspaceMemberResponse.ProtoReflect.Descriptor instead. func (*AddWorkspaceMemberResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{216} + return file_openshell_proto_rawDescGZIP(), []int{218} } func (x *AddWorkspaceMemberResponse) GetMember() *WorkspaceMember { @@ -16384,7 +16542,7 @@ type RemoveWorkspaceMemberRequest struct { func (x *RemoveWorkspaceMemberRequest) Reset() { *x = RemoveWorkspaceMemberRequest{} - mi := &file_openshell_proto_msgTypes[217] + mi := &file_openshell_proto_msgTypes[219] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16396,7 +16554,7 @@ func (x *RemoveWorkspaceMemberRequest) String() string { func (*RemoveWorkspaceMemberRequest) ProtoMessage() {} func (x *RemoveWorkspaceMemberRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[217] + mi := &file_openshell_proto_msgTypes[219] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16409,7 +16567,7 @@ func (x *RemoveWorkspaceMemberRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWorkspaceMemberRequest.ProtoReflect.Descriptor instead. func (*RemoveWorkspaceMemberRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{217} + return file_openshell_proto_rawDescGZIP(), []int{219} } func (x *RemoveWorkspaceMemberRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -16450,7 +16608,7 @@ type RemoveWorkspaceMemberResponse struct { func (x *RemoveWorkspaceMemberResponse) Reset() { *x = RemoveWorkspaceMemberResponse{} - mi := &file_openshell_proto_msgTypes[218] + mi := &file_openshell_proto_msgTypes[220] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16462,7 +16620,7 @@ func (x *RemoveWorkspaceMemberResponse) String() string { func (*RemoveWorkspaceMemberResponse) ProtoMessage() {} func (x *RemoveWorkspaceMemberResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[218] + mi := &file_openshell_proto_msgTypes[220] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16475,7 +16633,7 @@ func (x *RemoveWorkspaceMemberResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWorkspaceMemberResponse.ProtoReflect.Descriptor instead. func (*RemoveWorkspaceMemberResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{218} + return file_openshell_proto_rawDescGZIP(), []int{220} } func (x *RemoveWorkspaceMemberResponse) GetOutcome() DeletionOutcome { @@ -16502,7 +16660,7 @@ type ListWorkspaceMembersRequest struct { func (x *ListWorkspaceMembersRequest) Reset() { *x = ListWorkspaceMembersRequest{} - mi := &file_openshell_proto_msgTypes[219] + mi := &file_openshell_proto_msgTypes[221] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16514,7 +16672,7 @@ func (x *ListWorkspaceMembersRequest) String() string { func (*ListWorkspaceMembersRequest) ProtoMessage() {} func (x *ListWorkspaceMembersRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[219] + mi := &file_openshell_proto_msgTypes[221] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16527,7 +16685,7 @@ func (x *ListWorkspaceMembersRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkspaceMembersRequest.ProtoReflect.Descriptor instead. func (*ListWorkspaceMembersRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{219} + return file_openshell_proto_rawDescGZIP(), []int{221} } func (x *ListWorkspaceMembersRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSelector { @@ -16563,7 +16721,7 @@ type ListWorkspaceMembersResponse struct { func (x *ListWorkspaceMembersResponse) Reset() { *x = ListWorkspaceMembersResponse{} - mi := &file_openshell_proto_msgTypes[220] + mi := &file_openshell_proto_msgTypes[222] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16575,7 +16733,7 @@ func (x *ListWorkspaceMembersResponse) String() string { func (*ListWorkspaceMembersResponse) ProtoMessage() {} func (x *ListWorkspaceMembersResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[220] + mi := &file_openshell_proto_msgTypes[222] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16588,7 +16746,7 @@ func (x *ListWorkspaceMembersResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkspaceMembersResponse.ProtoReflect.Descriptor instead. func (*ListWorkspaceMembersResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{220} + return file_openshell_proto_rawDescGZIP(), []int{222} } func (x *ListWorkspaceMembersResponse) GetMembers() []*WorkspaceMember { @@ -16623,7 +16781,7 @@ type ExtensionServiceCredential struct { func (x *ExtensionServiceCredential) Reset() { *x = ExtensionServiceCredential{} - mi := &file_openshell_proto_msgTypes[221] + mi := &file_openshell_proto_msgTypes[223] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16635,7 +16793,7 @@ func (x *ExtensionServiceCredential) String() string { func (*ExtensionServiceCredential) ProtoMessage() {} func (x *ExtensionServiceCredential) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[221] + mi := &file_openshell_proto_msgTypes[223] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16648,7 +16806,7 @@ func (x *ExtensionServiceCredential) ProtoReflect() protoreflect.Message { // Deprecated: Use ExtensionServiceCredential.ProtoReflect.Descriptor instead. func (*ExtensionServiceCredential) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{221} + return file_openshell_proto_rawDescGZIP(), []int{223} } func (x *ExtensionServiceCredential) GetServiceName() string { @@ -16685,7 +16843,7 @@ type EndpointObservation struct { func (x *EndpointObservation) Reset() { *x = EndpointObservation{} - mi := &file_openshell_proto_msgTypes[222] + mi := &file_openshell_proto_msgTypes[224] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16697,7 +16855,7 @@ func (x *EndpointObservation) String() string { func (*EndpointObservation) ProtoMessage() {} func (x *EndpointObservation) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[222] + mi := &file_openshell_proto_msgTypes[224] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16710,7 +16868,7 @@ func (x *EndpointObservation) ProtoReflect() protoreflect.Message { // Deprecated: Use EndpointObservation.ProtoReflect.Descriptor instead. func (*EndpointObservation) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{222} + return file_openshell_proto_rawDescGZIP(), []int{224} } func (x *EndpointObservation) GetEndpointId() string { @@ -16753,7 +16911,7 @@ type ReportEndpointStatusRequest struct { func (x *ReportEndpointStatusRequest) Reset() { *x = ReportEndpointStatusRequest{} - mi := &file_openshell_proto_msgTypes[223] + mi := &file_openshell_proto_msgTypes[225] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16765,7 +16923,7 @@ func (x *ReportEndpointStatusRequest) String() string { func (*ReportEndpointStatusRequest) ProtoMessage() {} func (x *ReportEndpointStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[223] + mi := &file_openshell_proto_msgTypes[225] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16778,7 +16936,7 @@ func (x *ReportEndpointStatusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportEndpointStatusRequest.ProtoReflect.Descriptor instead. func (*ReportEndpointStatusRequest) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{223} + return file_openshell_proto_rawDescGZIP(), []int{225} } func (x *ReportEndpointStatusRequest) GetSandboxId() string { @@ -16839,7 +16997,7 @@ type ReportEndpointStatusResponse struct { func (x *ReportEndpointStatusResponse) Reset() { *x = ReportEndpointStatusResponse{} - mi := &file_openshell_proto_msgTypes[224] + mi := &file_openshell_proto_msgTypes[226] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16851,7 +17009,7 @@ func (x *ReportEndpointStatusResponse) String() string { func (*ReportEndpointStatusResponse) ProtoMessage() {} func (x *ReportEndpointStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[224] + mi := &file_openshell_proto_msgTypes[226] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16864,7 +17022,7 @@ func (x *ReportEndpointStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportEndpointStatusResponse.ProtoReflect.Descriptor instead. func (*ReportEndpointStatusResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{224} + return file_openshell_proto_rawDescGZIP(), []int{226} } // A configured endpoint and its last accepted network result in one record. @@ -16893,7 +17051,7 @@ type EndpointStatus struct { func (x *EndpointStatus) Reset() { *x = EndpointStatus{} - mi := &file_openshell_proto_msgTypes[225] + mi := &file_openshell_proto_msgTypes[227] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16905,7 +17063,7 @@ func (x *EndpointStatus) String() string { func (*EndpointStatus) ProtoMessage() {} func (x *EndpointStatus) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[225] + mi := &file_openshell_proto_msgTypes[227] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16918,7 +17076,7 @@ func (x *EndpointStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use EndpointStatus.ProtoReflect.Descriptor instead. func (*EndpointStatus) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{225} + return file_openshell_proto_rawDescGZIP(), []int{227} } func (x *EndpointStatus) GetEndpointId() string { @@ -16988,7 +17146,7 @@ type SandboxProvisioning struct { func (x *SandboxProvisioning) Reset() { *x = SandboxProvisioning{} - mi := &file_openshell_proto_msgTypes[226] + mi := &file_openshell_proto_msgTypes[228] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -17000,7 +17158,7 @@ func (x *SandboxProvisioning) String() string { func (*SandboxProvisioning) ProtoMessage() {} func (x *SandboxProvisioning) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[226] + mi := &file_openshell_proto_msgTypes[228] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -17013,7 +17171,7 @@ func (x *SandboxProvisioning) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxProvisioning.ProtoReflect.Descriptor instead. func (*SandboxProvisioning) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{226} + return file_openshell_proto_rawDescGZIP(), []int{228} } func (x *SandboxProvisioning) GetAttemptId() string { @@ -17106,7 +17264,7 @@ type SandboxServiceExposure struct { func (x *SandboxServiceExposure) Reset() { *x = SandboxServiceExposure{} - mi := &file_openshell_proto_msgTypes[227] + mi := &file_openshell_proto_msgTypes[229] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -17118,7 +17276,7 @@ func (x *SandboxServiceExposure) String() string { func (*SandboxServiceExposure) ProtoMessage() {} func (x *SandboxServiceExposure) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[227] + mi := &file_openshell_proto_msgTypes[229] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -17131,7 +17289,7 @@ func (x *SandboxServiceExposure) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxServiceExposure.ProtoReflect.Descriptor instead. func (*SandboxServiceExposure) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{227} + return file_openshell_proto_rawDescGZIP(), []int{229} } func (x *SandboxServiceExposure) GetService() string { @@ -18109,12 +18267,13 @@ const file_openshell_proto_rawDesc = "" + "relay_open\x18\x04 \x01(\v2\x17.openshell.v1.RelayOpenH\x00R\trelayOpen\x12;\n" + "\vrelay_close\x18\x05 \x01(\v2\x18.openshell.v1.RelayCloseH\x00R\n" + "relayCloseB\t\n" + - "\apayload\"\x91\x01\n" + + "\apayload\"\xbc\x01\n" + "\x0fSupervisorHello\x12\x1d\n" + "\n" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12\x1f\n" + "\vinstance_id\x18\x02 \x01(\tR\n" + - "instanceId\x12>\n" + + "instanceId\x12)\n" + + "\x10connection_epoch\x18\x03 \x01(\x04R\x0fconnectionEpoch\x12>\n" + "\x1bsupports_provider_readiness\x18\x04 \x01(\bR\x19supportsProviderReadiness\"\x99\x01\n" + "\x0fSessionAccepted\x12\x1d\n" + "\n" + @@ -18156,6 +18315,16 @@ const file_openshell_proto_rawDesc = "" + "RelayFrame\x12-\n" + "\x04init\x18\x01 \x01(\v2\x17.openshell.v1.RelayInitH\x00R\x04init\x12\x14\n" + "\x04data\x18\x02 \x01(\fH\x00R\x04dataB\t\n" + + "\apayload\"\x98\x01\n" + + "\rPeerRelayInit\x12\x1d\n" + + "\n" + + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x126\n" + + "\n" + + "relay_open\x18\x02 \x01(\v2\x17.openshell.v1.RelayOpenR\trelayOpen\x120\n" + + "\x14requester_replica_id\x18\x03 \x01(\tR\x12requesterReplicaId\"d\n" + + "\x0ePeerRelayFrame\x121\n" + + "\x04init\x18\x01 \x01(\v2\x1b.openshell.v1.PeerRelayInitH\x00R\x04init\x12\x14\n" + + "\x04data\x18\x02 \x01(\fH\x00R\x04dataB\t\n" + "\apayload\"`\n" + "\x0fRelayOpenResult\x12\x1d\n" + "\n" + @@ -18563,7 +18732,7 @@ const file_openshell_proto_rawDesc = "" + "&ENDPOINT_RESULT_CREDENTIAL_UNAVAILABLE\x10\x04\x12\x1e\n" + "\x1aENDPOINT_RESULT_TLS_FAILED\x10\x05\x12$\n" + " ENDPOINT_RESULT_TRANSPORT_FAILED\x10\x06\x12%\n" + - "!ENDPOINT_RESULT_UPSTREAM_REJECTED\x10\a2\xc2Q\n" + + "!ENDPOINT_RESULT_UPSTREAM_REJECTED\x10\a2\x9bR\n" + "\tOpenShell\x12Z\n" + "\x06Health\x12\x1b.openshell.v1.HealthRequest\x1a\x1c.openshell.v1.HealthResponse\"\x15\x82\xb5\x18\x11\n" + "\x0funauthenticated\x12i\n" + @@ -18685,7 +18854,10 @@ const file_openshell_proto_rawDesc = "" + "\x17FinalizeMainProcessExit\x12,.openshell.v1.FinalizeMainProcessExitRequest\x1a-.openshell.v1.FinalizeMainProcessExitResponse\"\r\x82\xb5\x18\t\n" + "\asandbox\x12T\n" + "\vRelayStream\x12\x18.openshell.v1.RelayFrame\x1a\x18.openshell.v1.RelayFrame\"\r\x82\xb5\x18\t\n" + - "\asandbox(\x010\x01\x12w\n" + + "\asandbox(\x010\x01\x12W\n" + + "\tPeerRelay\x12\x1c.openshell.v1.PeerRelayFrame\x1a\x1c.openshell.v1.PeerRelayFrame\"\n" + + "\x82\xb5\x18\x06\n" + + "\x04peer(\x010\x01\x12w\n" + "\fWatchSandbox\x12!.openshell.v1.WatchSandboxRequest\x1a .openshell.v1.SandboxStreamEvent\" \x82\xb5\x18\x1c\n" + "\x06bearer\x12\x04user\"\fsandbox:read0\x01\x12|\n" + "\x14SubmitPolicyAnalysis\x12).openshell.v1.SubmitPolicyAnalysisRequest\x1a*.openshell.v1.SubmitPolicyAnalysisResponse\"\r\x82\xb5\x18\t\n" + @@ -18738,7 +18910,7 @@ func file_openshell_proto_rawDescGZIP() []byte { } var file_openshell_proto_enumTypes = make([]protoimpl.EnumInfo, 17) -var file_openshell_proto_msgTypes = make([]protoimpl.MessageInfo, 250) +var file_openshell_proto_msgTypes = make([]protoimpl.MessageInfo, 252) var file_openshell_proto_goTypes = []any{ (SandboxPhase)(0), // 0: openshell.v1.SandboxPhase (ProviderMutationKind)(0), // 1: openshell.v1.ProviderMutationKind @@ -18935,103 +19107,105 @@ var file_openshell_proto_goTypes = []any{ (*TcpRelayTarget)(nil), // 192: openshell.v1.TcpRelayTarget (*RelayInit)(nil), // 193: openshell.v1.RelayInit (*RelayFrame)(nil), // 194: openshell.v1.RelayFrame - (*RelayOpenResult)(nil), // 195: openshell.v1.RelayOpenResult - (*RelayClose)(nil), // 196: openshell.v1.RelayClose - (*L7RequestSample)(nil), // 197: openshell.v1.L7RequestSample - (*DenialSummary)(nil), // 198: openshell.v1.DenialSummary - (*DenialGroupCount)(nil), // 199: openshell.v1.DenialGroupCount - (*NetworkActivitySummary)(nil), // 200: openshell.v1.NetworkActivitySummary - (*PolicyChunk)(nil), // 201: openshell.v1.PolicyChunk - (*DraftPolicyUpdate)(nil), // 202: openshell.v1.DraftPolicyUpdate - (*SubmitPolicyAnalysisRequest)(nil), // 203: openshell.v1.SubmitPolicyAnalysisRequest - (*SubmitPolicyAnalysisResponse)(nil), // 204: openshell.v1.SubmitPolicyAnalysisResponse - (*GetDraftPolicyRequest)(nil), // 205: openshell.v1.GetDraftPolicyRequest - (*GetDraftPolicyResponse)(nil), // 206: openshell.v1.GetDraftPolicyResponse - (*ApproveDraftChunkRequest)(nil), // 207: openshell.v1.ApproveDraftChunkRequest - (*ApproveDraftChunkResponse)(nil), // 208: openshell.v1.ApproveDraftChunkResponse - (*RejectDraftChunkRequest)(nil), // 209: openshell.v1.RejectDraftChunkRequest - (*RejectDraftChunkResponse)(nil), // 210: openshell.v1.RejectDraftChunkResponse - (*DraftChunkApproval)(nil), // 211: openshell.v1.DraftChunkApproval - (*ApproveAllDraftChunksRequest)(nil), // 212: openshell.v1.ApproveAllDraftChunksRequest - (*ApproveAllDraftChunksResponse)(nil), // 213: openshell.v1.ApproveAllDraftChunksResponse - (*EditDraftChunkRequest)(nil), // 214: openshell.v1.EditDraftChunkRequest - (*EditDraftChunkResponse)(nil), // 215: openshell.v1.EditDraftChunkResponse - (*UndoDraftChunkRequest)(nil), // 216: openshell.v1.UndoDraftChunkRequest - (*UndoDraftChunkResponse)(nil), // 217: openshell.v1.UndoDraftChunkResponse - (*ClearDraftChunksRequest)(nil), // 218: openshell.v1.ClearDraftChunksRequest - (*ClearDraftChunksResponse)(nil), // 219: openshell.v1.ClearDraftChunksResponse - (*GetDraftHistoryRequest)(nil), // 220: openshell.v1.GetDraftHistoryRequest - (*DraftHistoryEntry)(nil), // 221: openshell.v1.DraftHistoryEntry - (*GetDraftHistoryResponse)(nil), // 222: openshell.v1.GetDraftHistoryResponse - (*CreateWorkspaceRequest)(nil), // 223: openshell.v1.CreateWorkspaceRequest - (*CreateWorkspaceResponse)(nil), // 224: openshell.v1.CreateWorkspaceResponse - (*GetWorkspaceRequest)(nil), // 225: openshell.v1.GetWorkspaceRequest - (*GetWorkspaceResponse)(nil), // 226: openshell.v1.GetWorkspaceResponse - (*ListWorkspacesRequest)(nil), // 227: openshell.v1.ListWorkspacesRequest - (*ListWorkspacesResponse)(nil), // 228: openshell.v1.ListWorkspacesResponse - (*DeleteWorkspaceRequest)(nil), // 229: openshell.v1.DeleteWorkspaceRequest - (*DeleteWorkspaceResponse)(nil), // 230: openshell.v1.DeleteWorkspaceResponse - (*WorkspaceMember)(nil), // 231: openshell.v1.WorkspaceMember - (*AddWorkspaceMemberRequest)(nil), // 232: openshell.v1.AddWorkspaceMemberRequest - (*AddWorkspaceMemberResponse)(nil), // 233: openshell.v1.AddWorkspaceMemberResponse - (*RemoveWorkspaceMemberRequest)(nil), // 234: openshell.v1.RemoveWorkspaceMemberRequest - (*RemoveWorkspaceMemberResponse)(nil), // 235: openshell.v1.RemoveWorkspaceMemberResponse - (*ListWorkspaceMembersRequest)(nil), // 236: openshell.v1.ListWorkspaceMembersRequest - (*ListWorkspaceMembersResponse)(nil), // 237: openshell.v1.ListWorkspaceMembersResponse - (*ExtensionServiceCredential)(nil), // 238: openshell.v1.ExtensionServiceCredential - (*EndpointObservation)(nil), // 239: openshell.v1.EndpointObservation - (*ReportEndpointStatusRequest)(nil), // 240: openshell.v1.ReportEndpointStatusRequest - (*ReportEndpointStatusResponse)(nil), // 241: openshell.v1.ReportEndpointStatusResponse - (*EndpointStatus)(nil), // 242: openshell.v1.EndpointStatus - (*SandboxProvisioning)(nil), // 243: openshell.v1.SandboxProvisioning - (*SandboxServiceExposure)(nil), // 244: openshell.v1.SandboxServiceExposure - nil, // 245: openshell.v1.SandboxSpec.EnvironmentEntry - nil, // 246: openshell.v1.SandboxTemplate.LabelsEntry - nil, // 247: openshell.v1.SandboxTemplate.AnnotationsEntry - nil, // 248: openshell.v1.SandboxTemplate.EnvironmentEntry - nil, // 249: openshell.v1.SandboxWorkloadConfig.EnvironmentEntry - nil, // 250: openshell.v1.PlatformEvent.MetadataEntry - nil, // 251: openshell.v1.CreateSandboxRequest.LabelsEntry - nil, // 252: openshell.v1.CreateSandboxRequest.AnnotationsEntry - nil, // 253: openshell.v1.SandboxResponse.ServiceUrlsEntry - nil, // 254: openshell.v1.ExecSandboxRequest.EnvironmentEntry - nil, // 255: openshell.v1.SandboxLogLine.FieldsEntry - nil, // 256: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry - nil, // 257: openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry - nil, // 258: openshell.v1.ProviderProfile.AnnotationsEntry - nil, // 259: openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry - nil, // 260: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry - nil, // 261: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry - nil, // 262: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry - nil, // 263: openshell.v1.UpdateConfigRequest.AnnotationsEntry - nil, // 264: openshell.v1.UpdateConfigResponse.AnnotationsEntry - nil, // 265: openshell.v1.SandboxPolicyRevision.ProvenanceEntry - nil, // 266: openshell.v1.CreateWorkspaceRequest.LabelsEntry - (*timestamppb.Timestamp)(nil), // 267: google.protobuf.Timestamp - (*datamodelv1.ObjectMeta)(nil), // 268: openshell.datamodel.v1.ObjectMeta - (*sandboxv1.SandboxPolicy)(nil), // 269: openshell.sandbox.v1.SandboxPolicy - (*structpb.Struct)(nil), // 270: google.protobuf.Struct - (*durationpb.Duration)(nil), // 271: google.protobuf.Duration - (*datamodelv1.WorkspaceSelector)(nil), // 272: openshell.datamodel.v1.WorkspaceSelector - (*datamodelv1.Provider)(nil), // 273: openshell.datamodel.v1.Provider - (sandboxv1.PolicySource)(0), // 274: openshell.sandbox.v1.PolicySource - (*sandboxv1.NetworkEndpoint)(nil), // 275: openshell.sandbox.v1.NetworkEndpoint - (*sandboxv1.NetworkBinary)(nil), // 276: openshell.sandbox.v1.NetworkBinary - (*sandboxv1.SettingValue)(nil), // 277: openshell.sandbox.v1.SettingValue - (*sandboxv1.NetworkPolicyRule)(nil), // 278: openshell.sandbox.v1.NetworkPolicyRule - (*sandboxv1.L7DenyRule)(nil), // 279: openshell.sandbox.v1.L7DenyRule - (*sandboxv1.L7Rule)(nil), // 280: openshell.sandbox.v1.L7Rule - (*datamodelv1.Workspace)(nil), // 281: openshell.datamodel.v1.Workspace - (*sandboxv1.GetSandboxConfigRequest)(nil), // 282: openshell.sandbox.v1.GetSandboxConfigRequest - (*sandboxv1.GetGatewayConfigRequest)(nil), // 283: openshell.sandbox.v1.GetGatewayConfigRequest - (*sandboxv1.GetSandboxConfigResponse)(nil), // 284: openshell.sandbox.v1.GetSandboxConfigResponse - (*sandboxv1.GetGatewayConfigResponse)(nil), // 285: openshell.sandbox.v1.GetGatewayConfigResponse + (*PeerRelayInit)(nil), // 195: openshell.v1.PeerRelayInit + (*PeerRelayFrame)(nil), // 196: openshell.v1.PeerRelayFrame + (*RelayOpenResult)(nil), // 197: openshell.v1.RelayOpenResult + (*RelayClose)(nil), // 198: openshell.v1.RelayClose + (*L7RequestSample)(nil), // 199: openshell.v1.L7RequestSample + (*DenialSummary)(nil), // 200: openshell.v1.DenialSummary + (*DenialGroupCount)(nil), // 201: openshell.v1.DenialGroupCount + (*NetworkActivitySummary)(nil), // 202: openshell.v1.NetworkActivitySummary + (*PolicyChunk)(nil), // 203: openshell.v1.PolicyChunk + (*DraftPolicyUpdate)(nil), // 204: openshell.v1.DraftPolicyUpdate + (*SubmitPolicyAnalysisRequest)(nil), // 205: openshell.v1.SubmitPolicyAnalysisRequest + (*SubmitPolicyAnalysisResponse)(nil), // 206: openshell.v1.SubmitPolicyAnalysisResponse + (*GetDraftPolicyRequest)(nil), // 207: openshell.v1.GetDraftPolicyRequest + (*GetDraftPolicyResponse)(nil), // 208: openshell.v1.GetDraftPolicyResponse + (*ApproveDraftChunkRequest)(nil), // 209: openshell.v1.ApproveDraftChunkRequest + (*ApproveDraftChunkResponse)(nil), // 210: openshell.v1.ApproveDraftChunkResponse + (*RejectDraftChunkRequest)(nil), // 211: openshell.v1.RejectDraftChunkRequest + (*RejectDraftChunkResponse)(nil), // 212: openshell.v1.RejectDraftChunkResponse + (*DraftChunkApproval)(nil), // 213: openshell.v1.DraftChunkApproval + (*ApproveAllDraftChunksRequest)(nil), // 214: openshell.v1.ApproveAllDraftChunksRequest + (*ApproveAllDraftChunksResponse)(nil), // 215: openshell.v1.ApproveAllDraftChunksResponse + (*EditDraftChunkRequest)(nil), // 216: openshell.v1.EditDraftChunkRequest + (*EditDraftChunkResponse)(nil), // 217: openshell.v1.EditDraftChunkResponse + (*UndoDraftChunkRequest)(nil), // 218: openshell.v1.UndoDraftChunkRequest + (*UndoDraftChunkResponse)(nil), // 219: openshell.v1.UndoDraftChunkResponse + (*ClearDraftChunksRequest)(nil), // 220: openshell.v1.ClearDraftChunksRequest + (*ClearDraftChunksResponse)(nil), // 221: openshell.v1.ClearDraftChunksResponse + (*GetDraftHistoryRequest)(nil), // 222: openshell.v1.GetDraftHistoryRequest + (*DraftHistoryEntry)(nil), // 223: openshell.v1.DraftHistoryEntry + (*GetDraftHistoryResponse)(nil), // 224: openshell.v1.GetDraftHistoryResponse + (*CreateWorkspaceRequest)(nil), // 225: openshell.v1.CreateWorkspaceRequest + (*CreateWorkspaceResponse)(nil), // 226: openshell.v1.CreateWorkspaceResponse + (*GetWorkspaceRequest)(nil), // 227: openshell.v1.GetWorkspaceRequest + (*GetWorkspaceResponse)(nil), // 228: openshell.v1.GetWorkspaceResponse + (*ListWorkspacesRequest)(nil), // 229: openshell.v1.ListWorkspacesRequest + (*ListWorkspacesResponse)(nil), // 230: openshell.v1.ListWorkspacesResponse + (*DeleteWorkspaceRequest)(nil), // 231: openshell.v1.DeleteWorkspaceRequest + (*DeleteWorkspaceResponse)(nil), // 232: openshell.v1.DeleteWorkspaceResponse + (*WorkspaceMember)(nil), // 233: openshell.v1.WorkspaceMember + (*AddWorkspaceMemberRequest)(nil), // 234: openshell.v1.AddWorkspaceMemberRequest + (*AddWorkspaceMemberResponse)(nil), // 235: openshell.v1.AddWorkspaceMemberResponse + (*RemoveWorkspaceMemberRequest)(nil), // 236: openshell.v1.RemoveWorkspaceMemberRequest + (*RemoveWorkspaceMemberResponse)(nil), // 237: openshell.v1.RemoveWorkspaceMemberResponse + (*ListWorkspaceMembersRequest)(nil), // 238: openshell.v1.ListWorkspaceMembersRequest + (*ListWorkspaceMembersResponse)(nil), // 239: openshell.v1.ListWorkspaceMembersResponse + (*ExtensionServiceCredential)(nil), // 240: openshell.v1.ExtensionServiceCredential + (*EndpointObservation)(nil), // 241: openshell.v1.EndpointObservation + (*ReportEndpointStatusRequest)(nil), // 242: openshell.v1.ReportEndpointStatusRequest + (*ReportEndpointStatusResponse)(nil), // 243: openshell.v1.ReportEndpointStatusResponse + (*EndpointStatus)(nil), // 244: openshell.v1.EndpointStatus + (*SandboxProvisioning)(nil), // 245: openshell.v1.SandboxProvisioning + (*SandboxServiceExposure)(nil), // 246: openshell.v1.SandboxServiceExposure + nil, // 247: openshell.v1.SandboxSpec.EnvironmentEntry + nil, // 248: openshell.v1.SandboxTemplate.LabelsEntry + nil, // 249: openshell.v1.SandboxTemplate.AnnotationsEntry + nil, // 250: openshell.v1.SandboxTemplate.EnvironmentEntry + nil, // 251: openshell.v1.SandboxWorkloadConfig.EnvironmentEntry + nil, // 252: openshell.v1.PlatformEvent.MetadataEntry + nil, // 253: openshell.v1.CreateSandboxRequest.LabelsEntry + nil, // 254: openshell.v1.CreateSandboxRequest.AnnotationsEntry + nil, // 255: openshell.v1.SandboxResponse.ServiceUrlsEntry + nil, // 256: openshell.v1.ExecSandboxRequest.EnvironmentEntry + nil, // 257: openshell.v1.SandboxLogLine.FieldsEntry + nil, // 258: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry + nil, // 259: openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry + nil, // 260: openshell.v1.ProviderProfile.AnnotationsEntry + nil, // 261: openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry + nil, // 262: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry + nil, // 263: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry + nil, // 264: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry + nil, // 265: openshell.v1.UpdateConfigRequest.AnnotationsEntry + nil, // 266: openshell.v1.UpdateConfigResponse.AnnotationsEntry + nil, // 267: openshell.v1.SandboxPolicyRevision.ProvenanceEntry + nil, // 268: openshell.v1.CreateWorkspaceRequest.LabelsEntry + (*timestamppb.Timestamp)(nil), // 269: google.protobuf.Timestamp + (*datamodelv1.ObjectMeta)(nil), // 270: openshell.datamodel.v1.ObjectMeta + (*sandboxv1.SandboxPolicy)(nil), // 271: openshell.sandbox.v1.SandboxPolicy + (*structpb.Struct)(nil), // 272: google.protobuf.Struct + (*durationpb.Duration)(nil), // 273: google.protobuf.Duration + (*datamodelv1.WorkspaceSelector)(nil), // 274: openshell.datamodel.v1.WorkspaceSelector + (*datamodelv1.Provider)(nil), // 275: openshell.datamodel.v1.Provider + (sandboxv1.PolicySource)(0), // 276: openshell.sandbox.v1.PolicySource + (*sandboxv1.NetworkEndpoint)(nil), // 277: openshell.sandbox.v1.NetworkEndpoint + (*sandboxv1.NetworkBinary)(nil), // 278: openshell.sandbox.v1.NetworkBinary + (*sandboxv1.SettingValue)(nil), // 279: openshell.sandbox.v1.SettingValue + (*sandboxv1.NetworkPolicyRule)(nil), // 280: openshell.sandbox.v1.NetworkPolicyRule + (*sandboxv1.L7DenyRule)(nil), // 281: openshell.sandbox.v1.L7DenyRule + (*sandboxv1.L7Rule)(nil), // 282: openshell.sandbox.v1.L7Rule + (*datamodelv1.Workspace)(nil), // 283: openshell.datamodel.v1.Workspace + (*sandboxv1.GetSandboxConfigRequest)(nil), // 284: openshell.sandbox.v1.GetSandboxConfigRequest + (*sandboxv1.GetGatewayConfigRequest)(nil), // 285: openshell.sandbox.v1.GetGatewayConfigRequest + (*sandboxv1.GetSandboxConfigResponse)(nil), // 286: openshell.sandbox.v1.GetSandboxConfigResponse + (*sandboxv1.GetGatewayConfigResponse)(nil), // 287: openshell.sandbox.v1.GetGatewayConfigResponse } var file_openshell_proto_depIdxs = []int32{ - 267, // 0: openshell.v1.IssueSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp - 267, // 1: openshell.v1.RefreshSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp - 238, // 2: openshell.v1.RefreshSandboxTokenResponse.extension_credentials:type_name -> openshell.v1.ExtensionServiceCredential - 267, // 3: openshell.v1.RefreshSandboxTokenResponse.sandbox_expiration_time:type_name -> google.protobuf.Timestamp + 269, // 0: openshell.v1.IssueSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp + 269, // 1: openshell.v1.RefreshSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp + 240, // 2: openshell.v1.RefreshSandboxTokenResponse.extension_credentials:type_name -> openshell.v1.ExtensionServiceCredential + 269, // 3: openshell.v1.RefreshSandboxTokenResponse.sandbox_expiration_time:type_name -> google.protobuf.Timestamp 12, // 4: openshell.v1.HealthResponse.status:type_name -> openshell.v1.ServiceStatus 12, // 5: openshell.v1.GetGatewayInfoResponse.status:type_name -> openshell.v1.ServiceStatus 27, // 6: openshell.v1.GetGatewayInfoResponse.compute_drivers:type_name -> openshell.v1.ComputeDriverInfo @@ -19040,110 +19214,110 @@ var file_openshell_proto_depIdxs = []int32{ 30, // 9: openshell.v1.ResourceCapabilities.cpu:type_name -> openshell.v1.CpuResourceCapabilities 31, // 10: openshell.v1.ResourceCapabilities.memory:type_name -> openshell.v1.MemoryResourceCapabilities 32, // 11: openshell.v1.ResourceCapabilities.gpu:type_name -> openshell.v1.GpuResourceCapabilities - 268, // 12: openshell.v1.Sandbox.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 270, // 12: openshell.v1.Sandbox.metadata:type_name -> openshell.datamodel.v1.ObjectMeta 34, // 13: openshell.v1.Sandbox.spec:type_name -> openshell.v1.SandboxSpec 45, // 14: openshell.v1.Sandbox.status:type_name -> openshell.v1.SandboxStatus 44, // 15: openshell.v1.Sandbox.created_from_workload_template:type_name -> openshell.v1.SandboxWorkloadTemplateProvenance - 245, // 16: openshell.v1.SandboxSpec.environment:type_name -> openshell.v1.SandboxSpec.EnvironmentEntry + 247, // 16: openshell.v1.SandboxSpec.environment:type_name -> openshell.v1.SandboxSpec.EnvironmentEntry 37, // 17: openshell.v1.SandboxSpec.template:type_name -> openshell.v1.SandboxTemplate - 269, // 18: openshell.v1.SandboxSpec.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 271, // 18: openshell.v1.SandboxSpec.policy:type_name -> openshell.sandbox.v1.SandboxPolicy 35, // 19: openshell.v1.SandboxSpec.resource_requirements:type_name -> openshell.v1.ResourceRequirements 36, // 20: openshell.v1.ResourceRequirements.gpu:type_name -> openshell.v1.GpuResourceRequirements - 246, // 21: openshell.v1.SandboxTemplate.labels:type_name -> openshell.v1.SandboxTemplate.LabelsEntry - 247, // 22: openshell.v1.SandboxTemplate.annotations:type_name -> openshell.v1.SandboxTemplate.AnnotationsEntry - 248, // 23: openshell.v1.SandboxTemplate.environment:type_name -> openshell.v1.SandboxTemplate.EnvironmentEntry - 270, // 24: openshell.v1.SandboxTemplate.resources:type_name -> google.protobuf.Struct - 270, // 25: openshell.v1.SandboxTemplate.driver_config:type_name -> google.protobuf.Struct - 268, // 26: openshell.v1.SandboxWorkloadTemplate.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 248, // 21: openshell.v1.SandboxTemplate.labels:type_name -> openshell.v1.SandboxTemplate.LabelsEntry + 249, // 22: openshell.v1.SandboxTemplate.annotations:type_name -> openshell.v1.SandboxTemplate.AnnotationsEntry + 250, // 23: openshell.v1.SandboxTemplate.environment:type_name -> openshell.v1.SandboxTemplate.EnvironmentEntry + 272, // 24: openshell.v1.SandboxTemplate.resources:type_name -> google.protobuf.Struct + 272, // 25: openshell.v1.SandboxTemplate.driver_config:type_name -> google.protobuf.Struct + 270, // 26: openshell.v1.SandboxWorkloadTemplate.metadata:type_name -> openshell.datamodel.v1.ObjectMeta 39, // 27: openshell.v1.SandboxWorkloadTemplate.spec:type_name -> openshell.v1.SandboxWorkloadTemplateSpec 40, // 28: openshell.v1.SandboxWorkloadTemplateSpec.workload:type_name -> openshell.v1.SandboxWorkloadConfig - 270, // 29: openshell.v1.SandboxWorkloadTemplateSpec.driver_config:type_name -> google.protobuf.Struct + 272, // 29: openshell.v1.SandboxWorkloadTemplateSpec.driver_config:type_name -> google.protobuf.Struct 42, // 30: openshell.v1.SandboxWorkloadTemplateSpec.desired_service_level:type_name -> openshell.v1.SandboxServiceLevel - 249, // 31: openshell.v1.SandboxWorkloadConfig.environment:type_name -> openshell.v1.SandboxWorkloadConfig.EnvironmentEntry + 251, // 31: openshell.v1.SandboxWorkloadConfig.environment:type_name -> openshell.v1.SandboxWorkloadConfig.EnvironmentEntry 41, // 32: openshell.v1.SandboxWorkloadConfig.resources:type_name -> openshell.v1.SandboxResources 36, // 33: openshell.v1.SandboxResources.gpu:type_name -> openshell.v1.GpuResourceRequirements 43, // 34: openshell.v1.SandboxServiceLevel.startup:type_name -> openshell.v1.SandboxStartup - 271, // 35: openshell.v1.SandboxStartup.ready_within:type_name -> google.protobuf.Duration + 273, // 35: openshell.v1.SandboxStartup.ready_within:type_name -> google.protobuf.Duration 46, // 36: openshell.v1.SandboxStatus.conditions:type_name -> openshell.v1.SandboxCondition 0, // 37: openshell.v1.SandboxStatus.phase:type_name -> openshell.v1.SandboxPhase - 242, // 38: openshell.v1.SandboxStatus.endpoint_statuses:type_name -> openshell.v1.EndpointStatus + 244, // 38: openshell.v1.SandboxStatus.endpoint_statuses:type_name -> openshell.v1.EndpointStatus 171, // 39: openshell.v1.SandboxStatus.configuration_admission:type_name -> openshell.v1.SandboxConfigurationAdmission - 243, // 40: openshell.v1.SandboxStatus.provisioning:type_name -> openshell.v1.SandboxProvisioning - 267, // 41: openshell.v1.SandboxCondition.transition_time:type_name -> google.protobuf.Timestamp - 267, // 42: openshell.v1.PlatformEvent.event_time:type_name -> google.protobuf.Timestamp - 250, // 43: openshell.v1.PlatformEvent.metadata:type_name -> openshell.v1.PlatformEvent.MetadataEntry - 272, // 44: openshell.v1.CreateSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 245, // 40: openshell.v1.SandboxStatus.provisioning:type_name -> openshell.v1.SandboxProvisioning + 269, // 41: openshell.v1.SandboxCondition.transition_time:type_name -> google.protobuf.Timestamp + 269, // 42: openshell.v1.PlatformEvent.event_time:type_name -> google.protobuf.Timestamp + 252, // 43: openshell.v1.PlatformEvent.metadata:type_name -> openshell.v1.PlatformEvent.MetadataEntry + 274, // 44: openshell.v1.CreateSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 34, // 45: openshell.v1.CreateSandboxRequest.spec:type_name -> openshell.v1.SandboxSpec - 251, // 46: openshell.v1.CreateSandboxRequest.labels:type_name -> openshell.v1.CreateSandboxRequest.LabelsEntry - 252, // 47: openshell.v1.CreateSandboxRequest.annotations:type_name -> openshell.v1.CreateSandboxRequest.AnnotationsEntry - 244, // 48: openshell.v1.CreateSandboxRequest.service_exposures:type_name -> openshell.v1.SandboxServiceExposure - 272, // 49: openshell.v1.CreateSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 253, // 46: openshell.v1.CreateSandboxRequest.labels:type_name -> openshell.v1.CreateSandboxRequest.LabelsEntry + 254, // 47: openshell.v1.CreateSandboxRequest.annotations:type_name -> openshell.v1.CreateSandboxRequest.AnnotationsEntry + 246, // 48: openshell.v1.CreateSandboxRequest.service_exposures:type_name -> openshell.v1.SandboxServiceExposure + 274, // 49: openshell.v1.CreateSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 38, // 50: openshell.v1.CreateSandboxTemplateRequest.template:type_name -> openshell.v1.SandboxWorkloadTemplate - 272, // 51: openshell.v1.GetSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 52: openshell.v1.ListSandboxTemplatesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 53: openshell.v1.DeleteSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 51: openshell.v1.GetSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 52: openshell.v1.ListSandboxTemplatesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 53: openshell.v1.DeleteSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 38, // 54: openshell.v1.SandboxTemplateResponse.template:type_name -> openshell.v1.SandboxWorkloadTemplate 38, // 55: openshell.v1.ListSandboxTemplatesResponse.templates:type_name -> openshell.v1.SandboxWorkloadTemplate 15, // 56: openshell.v1.DeleteSandboxTemplateResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 272, // 57: openshell.v1.BeginRootfsTarStagingRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 267, // 58: openshell.v1.BeginRootfsTarStagingResponse.expiration_time:type_name -> google.protobuf.Timestamp - 272, // 59: openshell.v1.GetSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 60: openshell.v1.ListSandboxesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 61: openshell.v1.ListSandboxProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 62: openshell.v1.AttachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 63: openshell.v1.DetachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 64: openshell.v1.DeleteSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 65: openshell.v1.StopSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 66: openshell.v1.StartSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 57: openshell.v1.BeginRootfsTarStagingRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 269, // 58: openshell.v1.BeginRootfsTarStagingResponse.expiration_time:type_name -> google.protobuf.Timestamp + 274, // 59: openshell.v1.GetSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 60: openshell.v1.ListSandboxesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 61: openshell.v1.ListSandboxProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 62: openshell.v1.AttachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 63: openshell.v1.DetachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 64: openshell.v1.DeleteSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 65: openshell.v1.StopSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 66: openshell.v1.StartSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 33, // 67: openshell.v1.SandboxResponse.sandbox:type_name -> openshell.v1.Sandbox - 253, // 68: openshell.v1.SandboxResponse.service_urls:type_name -> openshell.v1.SandboxResponse.ServiceUrlsEntry + 255, // 68: openshell.v1.SandboxResponse.service_urls:type_name -> openshell.v1.SandboxResponse.ServiceUrlsEntry 33, // 69: openshell.v1.ListSandboxesResponse.sandboxes:type_name -> openshell.v1.Sandbox - 273, // 70: openshell.v1.ListSandboxProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider + 275, // 70: openshell.v1.ListSandboxProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider 33, // 71: openshell.v1.AttachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox 75, // 72: openshell.v1.AttachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt 33, // 73: openshell.v1.DetachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox 75, // 74: openshell.v1.DetachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt 73, // 75: openshell.v1.ConfigSnapshotRevision.sandbox_config:type_name -> openshell.v1.SandboxConfigRevision 71, // 76: openshell.v1.ConfigSnapshotRevision.provider_target:type_name -> openshell.v1.ProviderDesiredIdentity - 274, // 77: openshell.v1.SandboxConfigRevision.policy_source:type_name -> openshell.sandbox.v1.PolicySource + 276, // 77: openshell.v1.SandboxConfigRevision.policy_source:type_name -> openshell.sandbox.v1.PolicySource 4, // 78: openshell.v1.ConfigUpdateOperation.component:type_name -> openshell.v1.ConfigComponent 72, // 79: openshell.v1.ConfigUpdateOperation.target_revision:type_name -> openshell.v1.ConfigSnapshotRevision 6, // 80: openshell.v1.ConfigUpdateOperation.state:type_name -> openshell.v1.ConfigUpdateOperationState 5, // 81: openshell.v1.ConfigUpdateOperation.outcome:type_name -> openshell.v1.ConfigApplyOutcome - 267, // 82: openshell.v1.ConfigUpdateOperation.created_time:type_name -> google.protobuf.Timestamp - 267, // 83: openshell.v1.ConfigUpdateOperation.updated_time:type_name -> google.protobuf.Timestamp - 267, // 84: openshell.v1.ConfigUpdateOperation.completed_time:type_name -> google.protobuf.Timestamp + 269, // 82: openshell.v1.ConfigUpdateOperation.created_time:type_name -> google.protobuf.Timestamp + 269, // 83: openshell.v1.ConfigUpdateOperation.updated_time:type_name -> google.protobuf.Timestamp + 269, // 84: openshell.v1.ConfigUpdateOperation.completed_time:type_name -> google.protobuf.Timestamp 1, // 85: openshell.v1.ProviderMutationReceipt.kind:type_name -> openshell.v1.ProviderMutationKind 71, // 86: openshell.v1.ProviderMutationReceipt.desired:type_name -> openshell.v1.ProviderDesiredIdentity - 267, // 87: openshell.v1.ProviderMutationReceipt.persisted_time:type_name -> google.protobuf.Timestamp + 269, // 87: openshell.v1.ProviderMutationReceipt.persisted_time:type_name -> google.protobuf.Timestamp 3, // 88: openshell.v1.ProviderReadinessObservation.reason:type_name -> openshell.v1.ProviderReadinessReason 75, // 89: openshell.v1.ProviderReadinessStatus.receipt:type_name -> openshell.v1.ProviderMutationReceipt 2, // 90: openshell.v1.ProviderReadinessStatus.state:type_name -> openshell.v1.ProviderReadinessState 3, // 91: openshell.v1.ProviderReadinessStatus.reason:type_name -> openshell.v1.ProviderReadinessReason 76, // 92: openshell.v1.ProviderReadinessStatus.observed:type_name -> openshell.v1.ProviderReadinessObservation - 267, // 93: openshell.v1.ProviderReadinessStatus.observed_time:type_name -> google.protobuf.Timestamp - 267, // 94: openshell.v1.ProviderReadinessStatus.evaluated_time:type_name -> google.protobuf.Timestamp + 269, // 93: openshell.v1.ProviderReadinessStatus.observed_time:type_name -> google.protobuf.Timestamp + 269, // 94: openshell.v1.ProviderReadinessStatus.evaluated_time:type_name -> google.protobuf.Timestamp 74, // 95: openshell.v1.ProviderReadinessStatus.operation:type_name -> openshell.v1.ConfigUpdateOperation - 272, // 96: openshell.v1.GetSandboxProviderStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 96: openshell.v1.GetSandboxProviderStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 77, // 97: openshell.v1.GetSandboxProviderStatusResponse.status:type_name -> openshell.v1.ProviderReadinessStatus 76, // 98: openshell.v1.ReportProviderReadinessRequest.observation:type_name -> openshell.v1.ProviderReadinessObservation - 271, // 99: openshell.v1.ReportProviderReadinessResponse.report_interval:type_name -> google.protobuf.Duration - 271, // 100: openshell.v1.ReportProviderReadinessResponse.observation_ttl:type_name -> google.protobuf.Duration + 273, // 99: openshell.v1.ReportProviderReadinessResponse.report_interval:type_name -> google.protobuf.Duration + 273, // 100: openshell.v1.ReportProviderReadinessResponse.observation_ttl:type_name -> google.protobuf.Duration 15, // 101: openshell.v1.DeleteSandboxResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 272, // 102: openshell.v1.CreateSshSessionRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 267, // 103: openshell.v1.CreateSshSessionResponse.expiration_time:type_name -> google.protobuf.Timestamp - 272, // 104: openshell.v1.ExposeServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 105: openshell.v1.GetServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 106: openshell.v1.ListServicesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 102: openshell.v1.CreateSshSessionRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 269, // 103: openshell.v1.CreateSshSessionResponse.expiration_time:type_name -> google.protobuf.Timestamp + 274, // 104: openshell.v1.ExposeServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 105: openshell.v1.GetServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 106: openshell.v1.ListServicesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 92, // 107: openshell.v1.ListServicesResponse.services:type_name -> openshell.v1.ServiceEndpointResponse - 272, // 108: openshell.v1.DeleteServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 108: openshell.v1.DeleteServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 15, // 109: openshell.v1.DeleteServiceResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 268, // 110: openshell.v1.ServiceEndpoint.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 270, // 110: openshell.v1.ServiceEndpoint.metadata:type_name -> openshell.datamodel.v1.ObjectMeta 91, // 111: openshell.v1.ServiceEndpointResponse.endpoint:type_name -> openshell.v1.ServiceEndpoint 15, // 112: openshell.v1.RevokeSshSessionResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 272, // 113: openshell.v1.ExecSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 254, // 114: openshell.v1.ExecSandboxRequest.environment:type_name -> openshell.v1.ExecSandboxRequest.EnvironmentEntry - 271, // 115: openshell.v1.ExecSandboxRequest.execution_timeout:type_name -> google.protobuf.Duration + 274, // 113: openshell.v1.ExecSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 256, // 114: openshell.v1.ExecSandboxRequest.environment:type_name -> openshell.v1.ExecSandboxRequest.EnvironmentEntry + 273, // 115: openshell.v1.ExecSandboxRequest.execution_timeout:type_name -> google.protobuf.Duration 96, // 116: openshell.v1.ExecSandboxEvent.stdout:type_name -> openshell.v1.ExecSandboxStdout 97, // 117: openshell.v1.ExecSandboxEvent.stderr:type_name -> openshell.v1.ExecSandboxStderr 98, // 118: openshell.v1.ExecSandboxEvent.exit:type_name -> openshell.v1.ExecSandboxExit @@ -19152,355 +19326,359 @@ var file_openshell_proto_depIdxs = []int32{ 100, // 121: openshell.v1.TcpForwardFrame.init:type_name -> openshell.v1.TcpForwardInit 95, // 122: openshell.v1.ExecSandboxInput.start:type_name -> openshell.v1.ExecSandboxRequest 103, // 123: openshell.v1.ExecSandboxInput.resize:type_name -> openshell.v1.ExecSandboxWindowResize - 268, // 124: openshell.v1.SshSession.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 267, // 125: openshell.v1.SshSession.expiration_time:type_name -> google.protobuf.Timestamp - 272, // 126: openshell.v1.WatchSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 267, // 127: openshell.v1.WatchSandboxRequest.since_time:type_name -> google.protobuf.Timestamp + 270, // 124: openshell.v1.SshSession.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 269, // 125: openshell.v1.SshSession.expiration_time:type_name -> google.protobuf.Timestamp + 274, // 126: openshell.v1.WatchSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 269, // 127: openshell.v1.WatchSandboxRequest.since_time:type_name -> google.protobuf.Timestamp 33, // 128: openshell.v1.SandboxStreamEvent.sandbox:type_name -> openshell.v1.Sandbox 107, // 129: openshell.v1.SandboxStreamEvent.log:type_name -> openshell.v1.SandboxLogLine 47, // 130: openshell.v1.SandboxStreamEvent.event:type_name -> openshell.v1.PlatformEvent 108, // 131: openshell.v1.SandboxStreamEvent.warning:type_name -> openshell.v1.SandboxStreamWarning - 202, // 132: openshell.v1.SandboxStreamEvent.draft_policy_update:type_name -> openshell.v1.DraftPolicyUpdate - 267, // 133: openshell.v1.SandboxLogLine.event_time:type_name -> google.protobuf.Timestamp - 255, // 134: openshell.v1.SandboxLogLine.fields:type_name -> openshell.v1.SandboxLogLine.FieldsEntry - 272, // 135: openshell.v1.CreateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 273, // 136: openshell.v1.CreateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider - 272, // 137: openshell.v1.GetProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 138: openshell.v1.ListProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 139: openshell.v1.UpdateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 273, // 140: openshell.v1.UpdateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider - 256, // 141: openshell.v1.UpdateProviderRequest.credential_expiration_times:type_name -> openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry - 272, // 142: openshell.v1.DeleteProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 273, // 143: openshell.v1.ProviderResponse.provider:type_name -> openshell.datamodel.v1.Provider + 204, // 132: openshell.v1.SandboxStreamEvent.draft_policy_update:type_name -> openshell.v1.DraftPolicyUpdate + 269, // 133: openshell.v1.SandboxLogLine.event_time:type_name -> google.protobuf.Timestamp + 257, // 134: openshell.v1.SandboxLogLine.fields:type_name -> openshell.v1.SandboxLogLine.FieldsEntry + 274, // 135: openshell.v1.CreateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 275, // 136: openshell.v1.CreateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider + 274, // 137: openshell.v1.GetProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 138: openshell.v1.ListProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 139: openshell.v1.UpdateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 275, // 140: openshell.v1.UpdateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider + 258, // 141: openshell.v1.UpdateProviderRequest.credential_expiration_times:type_name -> openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry + 274, // 142: openshell.v1.DeleteProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 275, // 143: openshell.v1.ProviderResponse.provider:type_name -> openshell.datamodel.v1.Provider 75, // 144: openshell.v1.ProviderResponse.target_receipts:type_name -> openshell.v1.ProviderMutationReceipt - 273, // 145: openshell.v1.ListProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider - 272, // 146: openshell.v1.ListProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 147: openshell.v1.GetProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 275, // 145: openshell.v1.ListProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider + 274, // 146: openshell.v1.ListProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 147: openshell.v1.GetProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 137, // 148: openshell.v1.ProviderProfileImportItem.profile:type_name -> openshell.v1.ProviderProfile - 271, // 149: openshell.v1.ProviderCredentialTokenGrant.cache_ttl:type_name -> google.protobuf.Duration + 273, // 149: openshell.v1.ProviderCredentialTokenGrant.cache_ttl:type_name -> google.protobuf.Duration 120, // 150: openshell.v1.ProviderCredentialTokenGrant.audience_overrides:type_name -> openshell.v1.ProviderCredentialTokenGrantAudienceOverride 7, // 151: openshell.v1.ProviderCredentialTokenGrant.grant_type:type_name -> openshell.v1.ProviderCredentialTokenGrantType 121, // 152: openshell.v1.ProviderCredentialTokenGrant.subject_token:type_name -> openshell.v1.ProviderCredentialTokenGrantSubjectToken 126, // 153: openshell.v1.ProviderProfileCredential.refresh:type_name -> openshell.v1.ProviderCredentialRefresh 122, // 154: openshell.v1.ProviderProfileCredential.token_grant:type_name -> openshell.v1.ProviderCredentialTokenGrant 8, // 155: openshell.v1.ProviderCredentialRefresh.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy - 271, // 156: openshell.v1.ProviderCredentialRefresh.refresh_before:type_name -> google.protobuf.Duration - 271, // 157: openshell.v1.ProviderCredentialRefresh.max_lifetime:type_name -> google.protobuf.Duration + 273, // 156: openshell.v1.ProviderCredentialRefresh.refresh_before:type_name -> google.protobuf.Duration + 273, // 157: openshell.v1.ProviderCredentialRefresh.max_lifetime:type_name -> google.protobuf.Duration 124, // 158: openshell.v1.ProviderCredentialRefresh.material:type_name -> openshell.v1.ProviderCredentialRefreshMaterial 125, // 159: openshell.v1.ProviderCredentialRefresh.additional_outputs:type_name -> openshell.v1.ProviderCredentialRefreshOutput 8, // 160: openshell.v1.ProviderCredentialRefreshStatus.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy - 267, // 161: openshell.v1.ProviderCredentialRefreshStatus.expiration_time:type_name -> google.protobuf.Timestamp - 267, // 162: openshell.v1.ProviderCredentialRefreshStatus.next_refresh_time:type_name -> google.protobuf.Timestamp - 267, // 163: openshell.v1.ProviderCredentialRefreshStatus.last_refresh_time:type_name -> google.protobuf.Timestamp + 269, // 161: openshell.v1.ProviderCredentialRefreshStatus.expiration_time:type_name -> google.protobuf.Timestamp + 269, // 162: openshell.v1.ProviderCredentialRefreshStatus.next_refresh_time:type_name -> google.protobuf.Timestamp + 269, // 163: openshell.v1.ProviderCredentialRefreshStatus.last_refresh_time:type_name -> google.protobuf.Timestamp 14, // 164: openshell.v1.ProviderCredentialRefreshStatus.recovery_action:type_name -> openshell.v1.ProviderCredentialRefreshRecoveryAction - 267, // 165: openshell.v1.ProviderCredentialRefreshStatus.last_error_time:type_name -> google.protobuf.Timestamp - 272, // 166: openshell.v1.GetProviderRefreshStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 269, // 165: openshell.v1.ProviderCredentialRefreshStatus.last_error_time:type_name -> google.protobuf.Timestamp + 274, // 166: openshell.v1.GetProviderRefreshStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 127, // 167: openshell.v1.GetProviderRefreshStatusResponse.credentials:type_name -> openshell.v1.ProviderCredentialRefreshStatus - 272, // 168: openshell.v1.ConfigureProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 168: openshell.v1.ConfigureProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 8, // 169: openshell.v1.ConfigureProviderRefreshRequest.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy - 257, // 170: openshell.v1.ConfigureProviderRefreshRequest.material:type_name -> openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry - 267, // 171: openshell.v1.ConfigureProviderRefreshRequest.expiration_time:type_name -> google.protobuf.Timestamp + 259, // 170: openshell.v1.ConfigureProviderRefreshRequest.material:type_name -> openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry + 269, // 171: openshell.v1.ConfigureProviderRefreshRequest.expiration_time:type_name -> google.protobuf.Timestamp 127, // 172: openshell.v1.ConfigureProviderRefreshResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus - 272, // 173: openshell.v1.RotateProviderCredentialRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 173: openshell.v1.RotateProviderCredentialRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 127, // 174: openshell.v1.RotateProviderCredentialResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus - 272, // 175: openshell.v1.DeleteProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 175: openshell.v1.DeleteProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 15, // 176: openshell.v1.DeleteProviderRefreshResponse.outcome:type_name -> openshell.v1.DeletionOutcome 9, // 177: openshell.v1.ProviderProfile.category:type_name -> openshell.v1.ProviderProfileCategory 123, // 178: openshell.v1.ProviderProfile.credentials:type_name -> openshell.v1.ProviderProfileCredential - 275, // 179: openshell.v1.ProviderProfile.endpoints:type_name -> openshell.sandbox.v1.NetworkEndpoint - 276, // 180: openshell.v1.ProviderProfile.binaries:type_name -> openshell.sandbox.v1.NetworkBinary + 277, // 179: openshell.v1.ProviderProfile.endpoints:type_name -> openshell.sandbox.v1.NetworkEndpoint + 278, // 180: openshell.v1.ProviderProfile.binaries:type_name -> openshell.sandbox.v1.NetworkBinary 128, // 181: openshell.v1.ProviderProfile.discovery:type_name -> openshell.v1.ProviderProfileDiscovery - 258, // 182: openshell.v1.ProviderProfile.annotations:type_name -> openshell.v1.ProviderProfile.AnnotationsEntry + 260, // 182: openshell.v1.ProviderProfile.annotations:type_name -> openshell.v1.ProviderProfile.AnnotationsEntry 137, // 183: openshell.v1.ProviderProfileResponse.profile:type_name -> openshell.v1.ProviderProfile 137, // 184: openshell.v1.ListProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile - 272, // 185: openshell.v1.ImportProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 185: openshell.v1.ImportProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 118, // 186: openshell.v1.ImportProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem 119, // 187: openshell.v1.ImportProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic 137, // 188: openshell.v1.ImportProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile - 272, // 189: openshell.v1.UpdateProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 189: openshell.v1.UpdateProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 118, // 190: openshell.v1.UpdateProviderProfilesRequest.profile:type_name -> openshell.v1.ProviderProfileImportItem 119, // 191: openshell.v1.UpdateProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic 137, // 192: openshell.v1.UpdateProviderProfilesResponse.profile:type_name -> openshell.v1.ProviderProfile - 272, // 193: openshell.v1.LintProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 193: openshell.v1.LintProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 118, // 194: openshell.v1.LintProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem 119, // 195: openshell.v1.LintProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic 15, // 196: openshell.v1.DeleteProviderResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 272, // 197: openshell.v1.DeleteProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 197: openshell.v1.DeleteProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 15, // 198: openshell.v1.DeleteProviderProfileResponse.outcome:type_name -> openshell.v1.DeletionOutcome 150, // 199: openshell.v1.StaticCredentialBinding.endpoints:type_name -> openshell.v1.StaticCredentialEndpointBinding - 259, // 200: openshell.v1.GetSandboxProviderEnvironmentResponse.environment:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry - 260, // 201: openshell.v1.GetSandboxProviderEnvironmentResponse.credential_expiration_times:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry - 261, // 202: openshell.v1.GetSandboxProviderEnvironmentResponse.dynamic_credentials:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry - 262, // 203: openshell.v1.GetSandboxProviderEnvironmentResponse.static_credential_bindings:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry + 261, // 200: openshell.v1.GetSandboxProviderEnvironmentResponse.environment:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry + 262, // 201: openshell.v1.GetSandboxProviderEnvironmentResponse.credential_expiration_times:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry + 263, // 202: openshell.v1.GetSandboxProviderEnvironmentResponse.dynamic_credentials:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry + 264, // 203: openshell.v1.GetSandboxProviderEnvironmentResponse.static_credential_bindings:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry 3, // 204: openshell.v1.GetSandboxProviderEnvironmentResponse.readiness_reason:type_name -> openshell.v1.ProviderReadinessReason - 271, // 205: openshell.v1.ExchangeProviderSubjectTokenResponse.expires_after:type_name -> google.protobuf.Duration - 272, // 206: openshell.v1.UpdateConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 269, // 207: openshell.v1.UpdateConfigRequest.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 277, // 208: openshell.v1.UpdateConfigRequest.setting_value:type_name -> openshell.sandbox.v1.SettingValue + 273, // 205: openshell.v1.ExchangeProviderSubjectTokenResponse.expires_after:type_name -> google.protobuf.Duration + 274, // 206: openshell.v1.UpdateConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 271, // 207: openshell.v1.UpdateConfigRequest.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 279, // 208: openshell.v1.UpdateConfigRequest.setting_value:type_name -> openshell.sandbox.v1.SettingValue 156, // 209: openshell.v1.UpdateConfigRequest.merge_operations:type_name -> openshell.v1.PolicyMergeOperation - 263, // 210: openshell.v1.UpdateConfigRequest.annotations:type_name -> openshell.v1.UpdateConfigRequest.AnnotationsEntry + 265, // 210: openshell.v1.UpdateConfigRequest.annotations:type_name -> openshell.v1.UpdateConfigRequest.AnnotationsEntry 157, // 211: openshell.v1.PolicyMergeOperation.add_rule:type_name -> openshell.v1.AddNetworkRule 158, // 212: openshell.v1.PolicyMergeOperation.remove_endpoint:type_name -> openshell.v1.RemoveNetworkEndpoint 159, // 213: openshell.v1.PolicyMergeOperation.remove_rule:type_name -> openshell.v1.RemoveNetworkRule 161, // 214: openshell.v1.PolicyMergeOperation.add_deny_rules:type_name -> openshell.v1.AddDenyRules 162, // 215: openshell.v1.PolicyMergeOperation.add_allow_rules:type_name -> openshell.v1.AddAllowRules 163, // 216: openshell.v1.PolicyMergeOperation.remove_binary:type_name -> openshell.v1.RemoveNetworkBinary - 278, // 217: openshell.v1.AddNetworkRule.rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 276, // 218: openshell.v1.L7RuleTarget.binaries:type_name -> openshell.sandbox.v1.NetworkBinary - 279, // 219: openshell.v1.AddDenyRules.deny_rules:type_name -> openshell.sandbox.v1.L7DenyRule + 280, // 217: openshell.v1.AddNetworkRule.rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 278, // 218: openshell.v1.L7RuleTarget.binaries:type_name -> openshell.sandbox.v1.NetworkBinary + 281, // 219: openshell.v1.AddDenyRules.deny_rules:type_name -> openshell.sandbox.v1.L7DenyRule 160, // 220: openshell.v1.AddDenyRules.target:type_name -> openshell.v1.L7RuleTarget - 280, // 221: openshell.v1.AddAllowRules.rules:type_name -> openshell.sandbox.v1.L7Rule + 282, // 221: openshell.v1.AddAllowRules.rules:type_name -> openshell.sandbox.v1.L7Rule 160, // 222: openshell.v1.AddAllowRules.target:type_name -> openshell.v1.L7RuleTarget - 264, // 223: openshell.v1.UpdateConfigResponse.annotations:type_name -> openshell.v1.UpdateConfigResponse.AnnotationsEntry - 272, // 224: openshell.v1.GetSandboxPolicyStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 266, // 223: openshell.v1.UpdateConfigResponse.annotations:type_name -> openshell.v1.UpdateConfigResponse.AnnotationsEntry + 274, // 224: openshell.v1.GetSandboxPolicyStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 174, // 225: openshell.v1.GetSandboxPolicyStatusResponse.revision:type_name -> openshell.v1.SandboxPolicyRevision - 272, // 226: openshell.v1.ListSandboxPoliciesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 226: openshell.v1.ListSandboxPoliciesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector 174, // 227: openshell.v1.ListSandboxPoliciesResponse.revisions:type_name -> openshell.v1.SandboxPolicyRevision 11, // 228: openshell.v1.ReportPolicyStatusRequest.status:type_name -> openshell.v1.PolicyStatus 10, // 229: openshell.v1.SandboxConfigurationAdmission.state:type_name -> openshell.v1.ConfigurationAdmissionState 171, // 230: openshell.v1.ReportSandboxConfigurationRequest.admission:type_name -> openshell.v1.SandboxConfigurationAdmission 11, // 231: openshell.v1.SandboxPolicyRevision.status:type_name -> openshell.v1.PolicyStatus - 267, // 232: openshell.v1.SandboxPolicyRevision.created_time:type_name -> google.protobuf.Timestamp - 267, // 233: openshell.v1.SandboxPolicyRevision.loaded_time:type_name -> google.protobuf.Timestamp - 269, // 234: openshell.v1.SandboxPolicyRevision.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 265, // 235: openshell.v1.SandboxPolicyRevision.provenance:type_name -> openshell.v1.SandboxPolicyRevision.ProvenanceEntry - 272, // 236: openshell.v1.GetSandboxLogsRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 267, // 237: openshell.v1.GetSandboxLogsRequest.since_time:type_name -> google.protobuf.Timestamp + 269, // 232: openshell.v1.SandboxPolicyRevision.created_time:type_name -> google.protobuf.Timestamp + 269, // 233: openshell.v1.SandboxPolicyRevision.loaded_time:type_name -> google.protobuf.Timestamp + 271, // 234: openshell.v1.SandboxPolicyRevision.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 267, // 235: openshell.v1.SandboxPolicyRevision.provenance:type_name -> openshell.v1.SandboxPolicyRevision.ProvenanceEntry + 274, // 236: openshell.v1.GetSandboxLogsRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 269, // 237: openshell.v1.GetSandboxLogsRequest.since_time:type_name -> google.protobuf.Timestamp 107, // 238: openshell.v1.PushSandboxLogsRequest.logs:type_name -> openshell.v1.SandboxLogLine 107, // 239: openshell.v1.GetSandboxLogsResponse.logs:type_name -> openshell.v1.SandboxLogLine 181, // 240: openshell.v1.SupervisorMessage.hello:type_name -> openshell.v1.SupervisorHello 184, // 241: openshell.v1.SupervisorMessage.heartbeat:type_name -> openshell.v1.SupervisorHeartbeat - 195, // 242: openshell.v1.SupervisorMessage.relay_open_result:type_name -> openshell.v1.RelayOpenResult - 196, // 243: openshell.v1.SupervisorMessage.relay_close:type_name -> openshell.v1.RelayClose + 197, // 242: openshell.v1.SupervisorMessage.relay_open_result:type_name -> openshell.v1.RelayOpenResult + 198, // 243: openshell.v1.SupervisorMessage.relay_close:type_name -> openshell.v1.RelayClose 182, // 244: openshell.v1.GatewayMessage.session_accepted:type_name -> openshell.v1.SessionAccepted 183, // 245: openshell.v1.GatewayMessage.session_rejected:type_name -> openshell.v1.SessionRejected 185, // 246: openshell.v1.GatewayMessage.heartbeat:type_name -> openshell.v1.GatewayHeartbeat 190, // 247: openshell.v1.GatewayMessage.relay_open:type_name -> openshell.v1.RelayOpen - 196, // 248: openshell.v1.GatewayMessage.relay_close:type_name -> openshell.v1.RelayClose - 271, // 249: openshell.v1.SessionAccepted.heartbeat_interval:type_name -> google.protobuf.Duration + 198, // 248: openshell.v1.GatewayMessage.relay_close:type_name -> openshell.v1.RelayClose + 273, // 249: openshell.v1.SessionAccepted.heartbeat_interval:type_name -> google.protobuf.Duration 191, // 250: openshell.v1.RelayOpen.ssh:type_name -> openshell.v1.SshRelayTarget 192, // 251: openshell.v1.RelayOpen.tcp:type_name -> openshell.v1.TcpRelayTarget 193, // 252: openshell.v1.RelayFrame.init:type_name -> openshell.v1.RelayInit - 267, // 253: openshell.v1.DenialSummary.first_seen_time:type_name -> google.protobuf.Timestamp - 267, // 254: openshell.v1.DenialSummary.last_seen_time:type_name -> google.protobuf.Timestamp - 197, // 255: openshell.v1.DenialSummary.l7_request_samples:type_name -> openshell.v1.L7RequestSample - 199, // 256: openshell.v1.NetworkActivitySummary.denials_by_group:type_name -> openshell.v1.DenialGroupCount - 278, // 257: openshell.v1.PolicyChunk.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 267, // 258: openshell.v1.PolicyChunk.created_time:type_name -> google.protobuf.Timestamp - 267, // 259: openshell.v1.PolicyChunk.decided_time:type_name -> google.protobuf.Timestamp - 267, // 260: openshell.v1.PolicyChunk.first_seen_time:type_name -> google.protobuf.Timestamp - 267, // 261: openshell.v1.PolicyChunk.last_seen_time:type_name -> google.protobuf.Timestamp - 269, // 262: openshell.v1.PolicyChunk.current_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 269, // 263: openshell.v1.PolicyChunk.candidate_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 272, // 264: openshell.v1.SubmitPolicyAnalysisRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 198, // 265: openshell.v1.SubmitPolicyAnalysisRequest.summaries:type_name -> openshell.v1.DenialSummary - 201, // 266: openshell.v1.SubmitPolicyAnalysisRequest.proposed_chunks:type_name -> openshell.v1.PolicyChunk - 200, // 267: openshell.v1.SubmitPolicyAnalysisRequest.network_activity_summaries:type_name -> openshell.v1.NetworkActivitySummary - 272, // 268: openshell.v1.GetDraftPolicyRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 201, // 269: openshell.v1.GetDraftPolicyResponse.chunks:type_name -> openshell.v1.PolicyChunk - 267, // 270: openshell.v1.GetDraftPolicyResponse.last_analyzed_time:type_name -> google.protobuf.Timestamp - 272, // 271: openshell.v1.ApproveDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 272: openshell.v1.RejectDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 273: openshell.v1.ApproveAllDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 211, // 274: openshell.v1.ApproveAllDraftChunksRequest.approvals:type_name -> openshell.v1.DraftChunkApproval - 272, // 275: openshell.v1.EditDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 278, // 276: openshell.v1.EditDraftChunkRequest.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 272, // 277: openshell.v1.UndoDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 278: openshell.v1.ClearDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 272, // 279: openshell.v1.GetDraftHistoryRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 267, // 280: openshell.v1.DraftHistoryEntry.event_time:type_name -> google.protobuf.Timestamp - 221, // 281: openshell.v1.GetDraftHistoryResponse.entries:type_name -> openshell.v1.DraftHistoryEntry - 266, // 282: openshell.v1.CreateWorkspaceRequest.labels:type_name -> openshell.v1.CreateWorkspaceRequest.LabelsEntry - 281, // 283: openshell.v1.CreateWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace - 281, // 284: openshell.v1.GetWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace - 281, // 285: openshell.v1.ListWorkspacesResponse.workspaces:type_name -> openshell.datamodel.v1.Workspace - 15, // 286: openshell.v1.DeleteWorkspaceResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 268, // 287: openshell.v1.WorkspaceMember.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 13, // 288: openshell.v1.WorkspaceMember.role:type_name -> openshell.v1.WorkspaceRole - 272, // 289: openshell.v1.AddWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 13, // 290: openshell.v1.AddWorkspaceMemberRequest.role:type_name -> openshell.v1.WorkspaceRole - 231, // 291: openshell.v1.AddWorkspaceMemberResponse.member:type_name -> openshell.v1.WorkspaceMember - 272, // 292: openshell.v1.RemoveWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 15, // 293: openshell.v1.RemoveWorkspaceMemberResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 272, // 294: openshell.v1.ListWorkspaceMembersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 231, // 295: openshell.v1.ListWorkspaceMembersResponse.members:type_name -> openshell.v1.WorkspaceMember - 267, // 296: openshell.v1.ExtensionServiceCredential.expiration_time:type_name -> google.protobuf.Timestamp - 16, // 297: openshell.v1.EndpointObservation.result:type_name -> openshell.v1.EndpointResult - 239, // 298: openshell.v1.ReportEndpointStatusRequest.observations:type_name -> openshell.v1.EndpointObservation - 16, // 299: openshell.v1.EndpointStatus.last_result:type_name -> openshell.v1.EndpointResult - 267, // 300: openshell.v1.EndpointStatus.last_reported_time:type_name -> google.protobuf.Timestamp - 267, // 301: openshell.v1.SandboxProvisioning.configuration_change_time:type_name -> google.protobuf.Timestamp - 267, // 302: openshell.v1.SandboxProvisioning.first_rejection_time:type_name -> google.protobuf.Timestamp - 267, // 303: openshell.v1.SandboxProvisioning.deadline:type_name -> google.protobuf.Timestamp - 267, // 304: openshell.v1.SandboxProvisioning.timeout_time:type_name -> google.protobuf.Timestamp - 267, // 305: openshell.v1.SandboxProvisioning.cleanup_completed_time:type_name -> google.protobuf.Timestamp - 267, // 306: openshell.v1.SandboxProvisioning.cleanup_retry_time:type_name -> google.protobuf.Timestamp - 267, // 307: openshell.v1.SandboxProvisioning.attachment_change_time:type_name -> google.protobuf.Timestamp - 267, // 308: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp - 267, // 309: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp - 123, // 310: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry.value:type_name -> openshell.v1.ProviderProfileCredential - 151, // 311: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry.value:type_name -> openshell.v1.StaticCredentialBinding - 21, // 312: openshell.v1.OpenShell.Health:input_type -> openshell.v1.HealthRequest - 23, // 313: openshell.v1.OpenShell.GetCurrentUser:input_type -> openshell.v1.GetCurrentUserRequest - 25, // 314: openshell.v1.OpenShell.GetGatewayInfo:input_type -> openshell.v1.GetGatewayInfoRequest - 48, // 315: openshell.v1.OpenShell.CreateSandbox:input_type -> openshell.v1.CreateSandboxRequest - 56, // 316: openshell.v1.OpenShell.BeginRootfsTarStaging:input_type -> openshell.v1.BeginRootfsTarStagingRequest - 58, // 317: openshell.v1.OpenShell.GetSandbox:input_type -> openshell.v1.GetSandboxRequest - 59, // 318: openshell.v1.OpenShell.ListSandboxes:input_type -> openshell.v1.ListSandboxesRequest - 49, // 319: openshell.v1.OpenShell.CreateSandboxTemplate:input_type -> openshell.v1.CreateSandboxTemplateRequest - 50, // 320: openshell.v1.OpenShell.GetSandboxTemplate:input_type -> openshell.v1.GetSandboxTemplateRequest - 51, // 321: openshell.v1.OpenShell.ListSandboxTemplates:input_type -> openshell.v1.ListSandboxTemplatesRequest - 52, // 322: openshell.v1.OpenShell.DeleteSandboxTemplate:input_type -> openshell.v1.DeleteSandboxTemplateRequest - 60, // 323: openshell.v1.OpenShell.ListSandboxProviders:input_type -> openshell.v1.ListSandboxProvidersRequest - 61, // 324: openshell.v1.OpenShell.AttachSandboxProvider:input_type -> openshell.v1.AttachSandboxProviderRequest - 62, // 325: openshell.v1.OpenShell.DetachSandboxProvider:input_type -> openshell.v1.DetachSandboxProviderRequest - 78, // 326: openshell.v1.OpenShell.GetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest - 63, // 327: openshell.v1.OpenShell.DeleteSandbox:input_type -> openshell.v1.DeleteSandboxRequest - 64, // 328: openshell.v1.OpenShell.StopSandbox:input_type -> openshell.v1.StopSandboxRequest - 65, // 329: openshell.v1.OpenShell.StartSandbox:input_type -> openshell.v1.StartSandboxRequest - 83, // 330: openshell.v1.OpenShell.CreateSshSession:input_type -> openshell.v1.CreateSshSessionRequest - 85, // 331: openshell.v1.OpenShell.ExposeService:input_type -> openshell.v1.ExposeServiceRequest - 86, // 332: openshell.v1.OpenShell.GetService:input_type -> openshell.v1.GetServiceRequest - 87, // 333: openshell.v1.OpenShell.ListServices:input_type -> openshell.v1.ListServicesRequest - 89, // 334: openshell.v1.OpenShell.DeleteService:input_type -> openshell.v1.DeleteServiceRequest - 93, // 335: openshell.v1.OpenShell.RevokeSshSession:input_type -> openshell.v1.RevokeSshSessionRequest - 95, // 336: openshell.v1.OpenShell.ExecSandbox:input_type -> openshell.v1.ExecSandboxRequest - 101, // 337: openshell.v1.OpenShell.ForwardTcp:input_type -> openshell.v1.TcpForwardFrame - 102, // 338: openshell.v1.OpenShell.ExecSandboxInteractive:input_type -> openshell.v1.ExecSandboxInput - 109, // 339: openshell.v1.OpenShell.CreateProvider:input_type -> openshell.v1.CreateProviderRequest - 110, // 340: openshell.v1.OpenShell.GetProvider:input_type -> openshell.v1.GetProviderRequest - 111, // 341: openshell.v1.OpenShell.ListProviders:input_type -> openshell.v1.ListProvidersRequest - 116, // 342: openshell.v1.OpenShell.ListProviderProfiles:input_type -> openshell.v1.ListProviderProfilesRequest - 117, // 343: openshell.v1.OpenShell.GetProviderProfile:input_type -> openshell.v1.GetProviderProfileRequest - 140, // 344: openshell.v1.OpenShell.ImportProviderProfiles:input_type -> openshell.v1.ImportProviderProfilesRequest - 142, // 345: openshell.v1.OpenShell.UpdateProviderProfiles:input_type -> openshell.v1.UpdateProviderProfilesRequest - 144, // 346: openshell.v1.OpenShell.LintProviderProfiles:input_type -> openshell.v1.LintProviderProfilesRequest - 112, // 347: openshell.v1.OpenShell.UpdateProvider:input_type -> openshell.v1.UpdateProviderRequest - 129, // 348: openshell.v1.OpenShell.GetProviderRefreshStatus:input_type -> openshell.v1.GetProviderRefreshStatusRequest - 131, // 349: openshell.v1.OpenShell.ConfigureProviderRefresh:input_type -> openshell.v1.ConfigureProviderRefreshRequest - 133, // 350: openshell.v1.OpenShell.RotateProviderCredential:input_type -> openshell.v1.RotateProviderCredentialRequest - 135, // 351: openshell.v1.OpenShell.DeleteProviderRefresh:input_type -> openshell.v1.DeleteProviderRefreshRequest - 113, // 352: openshell.v1.OpenShell.DeleteProvider:input_type -> openshell.v1.DeleteProviderRequest - 147, // 353: openshell.v1.OpenShell.DeleteProviderProfile:input_type -> openshell.v1.DeleteProviderProfileRequest - 282, // 354: openshell.v1.OpenShell.GetSandboxConfig:input_type -> openshell.sandbox.v1.GetSandboxConfigRequest - 283, // 355: openshell.v1.OpenShell.GetGatewayConfig:input_type -> openshell.sandbox.v1.GetGatewayConfigRequest - 155, // 356: openshell.v1.OpenShell.UpdateConfig:input_type -> openshell.v1.UpdateConfigRequest - 165, // 357: openshell.v1.OpenShell.GetSandboxPolicyStatus:input_type -> openshell.v1.GetSandboxPolicyStatusRequest - 167, // 358: openshell.v1.OpenShell.ListSandboxPolicies:input_type -> openshell.v1.ListSandboxPoliciesRequest - 169, // 359: openshell.v1.OpenShell.ReportPolicyStatus:input_type -> openshell.v1.ReportPolicyStatusRequest - 240, // 360: openshell.v1.OpenShell.ReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest - 80, // 361: openshell.v1.OpenShell.ReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest - 172, // 362: openshell.v1.OpenShell.ReportSandboxConfiguration:input_type -> openshell.v1.ReportSandboxConfigurationRequest - 149, // 363: openshell.v1.OpenShell.GetSandboxProviderEnvironment:input_type -> openshell.v1.GetSandboxProviderEnvironmentRequest - 153, // 364: openshell.v1.OpenShell.ExchangeProviderSubjectToken:input_type -> openshell.v1.ExchangeProviderSubjectTokenRequest - 175, // 365: openshell.v1.OpenShell.GetSandboxLogs:input_type -> openshell.v1.GetSandboxLogsRequest - 176, // 366: openshell.v1.OpenShell.PushSandboxLogs:input_type -> openshell.v1.PushSandboxLogsRequest - 179, // 367: openshell.v1.OpenShell.ConnectSupervisor:input_type -> openshell.v1.SupervisorMessage - 186, // 368: openshell.v1.OpenShell.ReportMainProcessExit:input_type -> openshell.v1.ReportMainProcessExitRequest - 188, // 369: openshell.v1.OpenShell.FinalizeMainProcessExit:input_type -> openshell.v1.FinalizeMainProcessExitRequest - 194, // 370: openshell.v1.OpenShell.RelayStream:input_type -> openshell.v1.RelayFrame - 105, // 371: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest - 203, // 372: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest - 205, // 373: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest - 207, // 374: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest - 209, // 375: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest - 212, // 376: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest - 214, // 377: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest - 216, // 378: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest - 218, // 379: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest - 220, // 380: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest - 17, // 381: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest - 19, // 382: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest - 223, // 383: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest - 225, // 384: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest - 227, // 385: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest - 229, // 386: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest - 232, // 387: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest - 234, // 388: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest - 236, // 389: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest - 22, // 390: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse - 24, // 391: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse - 26, // 392: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse - 66, // 393: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse - 57, // 394: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse - 66, // 395: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse - 67, // 396: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse - 53, // 397: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 53, // 398: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 54, // 399: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse - 55, // 400: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse - 68, // 401: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse - 69, // 402: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse - 70, // 403: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse - 79, // 404: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse - 82, // 405: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse - 66, // 406: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse - 66, // 407: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse - 84, // 408: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse - 92, // 409: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse - 92, // 410: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse - 88, // 411: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse - 90, // 412: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse - 94, // 413: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse - 99, // 414: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent - 101, // 415: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame - 99, // 416: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent - 114, // 417: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse - 114, // 418: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse - 115, // 419: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse - 139, // 420: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse - 138, // 421: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse - 141, // 422: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse - 143, // 423: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse - 145, // 424: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse - 114, // 425: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse - 130, // 426: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse - 132, // 427: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse - 134, // 428: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse - 136, // 429: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse - 146, // 430: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse - 148, // 431: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse - 284, // 432: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse - 285, // 433: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse - 164, // 434: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse - 166, // 435: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse - 168, // 436: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse - 170, // 437: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse - 241, // 438: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse - 81, // 439: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse - 173, // 440: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse - 152, // 441: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse - 154, // 442: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse - 178, // 443: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse - 177, // 444: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse - 180, // 445: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage - 187, // 446: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse - 189, // 447: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse - 194, // 448: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame - 106, // 449: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent - 204, // 450: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse - 206, // 451: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse - 208, // 452: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse - 210, // 453: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse - 213, // 454: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse - 215, // 455: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse - 217, // 456: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse - 219, // 457: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse - 222, // 458: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse - 18, // 459: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse - 20, // 460: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse - 224, // 461: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse - 226, // 462: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse - 228, // 463: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse - 230, // 464: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse - 233, // 465: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse - 235, // 466: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse - 237, // 467: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse - 390, // [390:468] is the sub-list for method output_type - 312, // [312:390] is the sub-list for method input_type - 312, // [312:312] is the sub-list for extension type_name - 312, // [312:312] is the sub-list for extension extendee - 0, // [0:312] is the sub-list for field type_name + 190, // 253: openshell.v1.PeerRelayInit.relay_open:type_name -> openshell.v1.RelayOpen + 195, // 254: openshell.v1.PeerRelayFrame.init:type_name -> openshell.v1.PeerRelayInit + 269, // 255: openshell.v1.DenialSummary.first_seen_time:type_name -> google.protobuf.Timestamp + 269, // 256: openshell.v1.DenialSummary.last_seen_time:type_name -> google.protobuf.Timestamp + 199, // 257: openshell.v1.DenialSummary.l7_request_samples:type_name -> openshell.v1.L7RequestSample + 201, // 258: openshell.v1.NetworkActivitySummary.denials_by_group:type_name -> openshell.v1.DenialGroupCount + 280, // 259: openshell.v1.PolicyChunk.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 269, // 260: openshell.v1.PolicyChunk.created_time:type_name -> google.protobuf.Timestamp + 269, // 261: openshell.v1.PolicyChunk.decided_time:type_name -> google.protobuf.Timestamp + 269, // 262: openshell.v1.PolicyChunk.first_seen_time:type_name -> google.protobuf.Timestamp + 269, // 263: openshell.v1.PolicyChunk.last_seen_time:type_name -> google.protobuf.Timestamp + 271, // 264: openshell.v1.PolicyChunk.current_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 271, // 265: openshell.v1.PolicyChunk.candidate_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 274, // 266: openshell.v1.SubmitPolicyAnalysisRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 200, // 267: openshell.v1.SubmitPolicyAnalysisRequest.summaries:type_name -> openshell.v1.DenialSummary + 203, // 268: openshell.v1.SubmitPolicyAnalysisRequest.proposed_chunks:type_name -> openshell.v1.PolicyChunk + 202, // 269: openshell.v1.SubmitPolicyAnalysisRequest.network_activity_summaries:type_name -> openshell.v1.NetworkActivitySummary + 274, // 270: openshell.v1.GetDraftPolicyRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 203, // 271: openshell.v1.GetDraftPolicyResponse.chunks:type_name -> openshell.v1.PolicyChunk + 269, // 272: openshell.v1.GetDraftPolicyResponse.last_analyzed_time:type_name -> google.protobuf.Timestamp + 274, // 273: openshell.v1.ApproveDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 274: openshell.v1.RejectDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 275: openshell.v1.ApproveAllDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 213, // 276: openshell.v1.ApproveAllDraftChunksRequest.approvals:type_name -> openshell.v1.DraftChunkApproval + 274, // 277: openshell.v1.EditDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 280, // 278: openshell.v1.EditDraftChunkRequest.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 274, // 279: openshell.v1.UndoDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 280: openshell.v1.ClearDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 274, // 281: openshell.v1.GetDraftHistoryRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 269, // 282: openshell.v1.DraftHistoryEntry.event_time:type_name -> google.protobuf.Timestamp + 223, // 283: openshell.v1.GetDraftHistoryResponse.entries:type_name -> openshell.v1.DraftHistoryEntry + 268, // 284: openshell.v1.CreateWorkspaceRequest.labels:type_name -> openshell.v1.CreateWorkspaceRequest.LabelsEntry + 283, // 285: openshell.v1.CreateWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace + 283, // 286: openshell.v1.GetWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace + 283, // 287: openshell.v1.ListWorkspacesResponse.workspaces:type_name -> openshell.datamodel.v1.Workspace + 15, // 288: openshell.v1.DeleteWorkspaceResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 270, // 289: openshell.v1.WorkspaceMember.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 13, // 290: openshell.v1.WorkspaceMember.role:type_name -> openshell.v1.WorkspaceRole + 274, // 291: openshell.v1.AddWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 13, // 292: openshell.v1.AddWorkspaceMemberRequest.role:type_name -> openshell.v1.WorkspaceRole + 233, // 293: openshell.v1.AddWorkspaceMemberResponse.member:type_name -> openshell.v1.WorkspaceMember + 274, // 294: openshell.v1.RemoveWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 15, // 295: openshell.v1.RemoveWorkspaceMemberResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 274, // 296: openshell.v1.ListWorkspaceMembersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 233, // 297: openshell.v1.ListWorkspaceMembersResponse.members:type_name -> openshell.v1.WorkspaceMember + 269, // 298: openshell.v1.ExtensionServiceCredential.expiration_time:type_name -> google.protobuf.Timestamp + 16, // 299: openshell.v1.EndpointObservation.result:type_name -> openshell.v1.EndpointResult + 241, // 300: openshell.v1.ReportEndpointStatusRequest.observations:type_name -> openshell.v1.EndpointObservation + 16, // 301: openshell.v1.EndpointStatus.last_result:type_name -> openshell.v1.EndpointResult + 269, // 302: openshell.v1.EndpointStatus.last_reported_time:type_name -> google.protobuf.Timestamp + 269, // 303: openshell.v1.SandboxProvisioning.configuration_change_time:type_name -> google.protobuf.Timestamp + 269, // 304: openshell.v1.SandboxProvisioning.first_rejection_time:type_name -> google.protobuf.Timestamp + 269, // 305: openshell.v1.SandboxProvisioning.deadline:type_name -> google.protobuf.Timestamp + 269, // 306: openshell.v1.SandboxProvisioning.timeout_time:type_name -> google.protobuf.Timestamp + 269, // 307: openshell.v1.SandboxProvisioning.cleanup_completed_time:type_name -> google.protobuf.Timestamp + 269, // 308: openshell.v1.SandboxProvisioning.cleanup_retry_time:type_name -> google.protobuf.Timestamp + 269, // 309: openshell.v1.SandboxProvisioning.attachment_change_time:type_name -> google.protobuf.Timestamp + 269, // 310: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp + 269, // 311: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp + 123, // 312: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry.value:type_name -> openshell.v1.ProviderProfileCredential + 151, // 313: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry.value:type_name -> openshell.v1.StaticCredentialBinding + 21, // 314: openshell.v1.OpenShell.Health:input_type -> openshell.v1.HealthRequest + 23, // 315: openshell.v1.OpenShell.GetCurrentUser:input_type -> openshell.v1.GetCurrentUserRequest + 25, // 316: openshell.v1.OpenShell.GetGatewayInfo:input_type -> openshell.v1.GetGatewayInfoRequest + 48, // 317: openshell.v1.OpenShell.CreateSandbox:input_type -> openshell.v1.CreateSandboxRequest + 56, // 318: openshell.v1.OpenShell.BeginRootfsTarStaging:input_type -> openshell.v1.BeginRootfsTarStagingRequest + 58, // 319: openshell.v1.OpenShell.GetSandbox:input_type -> openshell.v1.GetSandboxRequest + 59, // 320: openshell.v1.OpenShell.ListSandboxes:input_type -> openshell.v1.ListSandboxesRequest + 49, // 321: openshell.v1.OpenShell.CreateSandboxTemplate:input_type -> openshell.v1.CreateSandboxTemplateRequest + 50, // 322: openshell.v1.OpenShell.GetSandboxTemplate:input_type -> openshell.v1.GetSandboxTemplateRequest + 51, // 323: openshell.v1.OpenShell.ListSandboxTemplates:input_type -> openshell.v1.ListSandboxTemplatesRequest + 52, // 324: openshell.v1.OpenShell.DeleteSandboxTemplate:input_type -> openshell.v1.DeleteSandboxTemplateRequest + 60, // 325: openshell.v1.OpenShell.ListSandboxProviders:input_type -> openshell.v1.ListSandboxProvidersRequest + 61, // 326: openshell.v1.OpenShell.AttachSandboxProvider:input_type -> openshell.v1.AttachSandboxProviderRequest + 62, // 327: openshell.v1.OpenShell.DetachSandboxProvider:input_type -> openshell.v1.DetachSandboxProviderRequest + 78, // 328: openshell.v1.OpenShell.GetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest + 63, // 329: openshell.v1.OpenShell.DeleteSandbox:input_type -> openshell.v1.DeleteSandboxRequest + 64, // 330: openshell.v1.OpenShell.StopSandbox:input_type -> openshell.v1.StopSandboxRequest + 65, // 331: openshell.v1.OpenShell.StartSandbox:input_type -> openshell.v1.StartSandboxRequest + 83, // 332: openshell.v1.OpenShell.CreateSshSession:input_type -> openshell.v1.CreateSshSessionRequest + 85, // 333: openshell.v1.OpenShell.ExposeService:input_type -> openshell.v1.ExposeServiceRequest + 86, // 334: openshell.v1.OpenShell.GetService:input_type -> openshell.v1.GetServiceRequest + 87, // 335: openshell.v1.OpenShell.ListServices:input_type -> openshell.v1.ListServicesRequest + 89, // 336: openshell.v1.OpenShell.DeleteService:input_type -> openshell.v1.DeleteServiceRequest + 93, // 337: openshell.v1.OpenShell.RevokeSshSession:input_type -> openshell.v1.RevokeSshSessionRequest + 95, // 338: openshell.v1.OpenShell.ExecSandbox:input_type -> openshell.v1.ExecSandboxRequest + 101, // 339: openshell.v1.OpenShell.ForwardTcp:input_type -> openshell.v1.TcpForwardFrame + 102, // 340: openshell.v1.OpenShell.ExecSandboxInteractive:input_type -> openshell.v1.ExecSandboxInput + 109, // 341: openshell.v1.OpenShell.CreateProvider:input_type -> openshell.v1.CreateProviderRequest + 110, // 342: openshell.v1.OpenShell.GetProvider:input_type -> openshell.v1.GetProviderRequest + 111, // 343: openshell.v1.OpenShell.ListProviders:input_type -> openshell.v1.ListProvidersRequest + 116, // 344: openshell.v1.OpenShell.ListProviderProfiles:input_type -> openshell.v1.ListProviderProfilesRequest + 117, // 345: openshell.v1.OpenShell.GetProviderProfile:input_type -> openshell.v1.GetProviderProfileRequest + 140, // 346: openshell.v1.OpenShell.ImportProviderProfiles:input_type -> openshell.v1.ImportProviderProfilesRequest + 142, // 347: openshell.v1.OpenShell.UpdateProviderProfiles:input_type -> openshell.v1.UpdateProviderProfilesRequest + 144, // 348: openshell.v1.OpenShell.LintProviderProfiles:input_type -> openshell.v1.LintProviderProfilesRequest + 112, // 349: openshell.v1.OpenShell.UpdateProvider:input_type -> openshell.v1.UpdateProviderRequest + 129, // 350: openshell.v1.OpenShell.GetProviderRefreshStatus:input_type -> openshell.v1.GetProviderRefreshStatusRequest + 131, // 351: openshell.v1.OpenShell.ConfigureProviderRefresh:input_type -> openshell.v1.ConfigureProviderRefreshRequest + 133, // 352: openshell.v1.OpenShell.RotateProviderCredential:input_type -> openshell.v1.RotateProviderCredentialRequest + 135, // 353: openshell.v1.OpenShell.DeleteProviderRefresh:input_type -> openshell.v1.DeleteProviderRefreshRequest + 113, // 354: openshell.v1.OpenShell.DeleteProvider:input_type -> openshell.v1.DeleteProviderRequest + 147, // 355: openshell.v1.OpenShell.DeleteProviderProfile:input_type -> openshell.v1.DeleteProviderProfileRequest + 284, // 356: openshell.v1.OpenShell.GetSandboxConfig:input_type -> openshell.sandbox.v1.GetSandboxConfigRequest + 285, // 357: openshell.v1.OpenShell.GetGatewayConfig:input_type -> openshell.sandbox.v1.GetGatewayConfigRequest + 155, // 358: openshell.v1.OpenShell.UpdateConfig:input_type -> openshell.v1.UpdateConfigRequest + 165, // 359: openshell.v1.OpenShell.GetSandboxPolicyStatus:input_type -> openshell.v1.GetSandboxPolicyStatusRequest + 167, // 360: openshell.v1.OpenShell.ListSandboxPolicies:input_type -> openshell.v1.ListSandboxPoliciesRequest + 169, // 361: openshell.v1.OpenShell.ReportPolicyStatus:input_type -> openshell.v1.ReportPolicyStatusRequest + 242, // 362: openshell.v1.OpenShell.ReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest + 80, // 363: openshell.v1.OpenShell.ReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest + 172, // 364: openshell.v1.OpenShell.ReportSandboxConfiguration:input_type -> openshell.v1.ReportSandboxConfigurationRequest + 149, // 365: openshell.v1.OpenShell.GetSandboxProviderEnvironment:input_type -> openshell.v1.GetSandboxProviderEnvironmentRequest + 153, // 366: openshell.v1.OpenShell.ExchangeProviderSubjectToken:input_type -> openshell.v1.ExchangeProviderSubjectTokenRequest + 175, // 367: openshell.v1.OpenShell.GetSandboxLogs:input_type -> openshell.v1.GetSandboxLogsRequest + 176, // 368: openshell.v1.OpenShell.PushSandboxLogs:input_type -> openshell.v1.PushSandboxLogsRequest + 179, // 369: openshell.v1.OpenShell.ConnectSupervisor:input_type -> openshell.v1.SupervisorMessage + 186, // 370: openshell.v1.OpenShell.ReportMainProcessExit:input_type -> openshell.v1.ReportMainProcessExitRequest + 188, // 371: openshell.v1.OpenShell.FinalizeMainProcessExit:input_type -> openshell.v1.FinalizeMainProcessExitRequest + 194, // 372: openshell.v1.OpenShell.RelayStream:input_type -> openshell.v1.RelayFrame + 196, // 373: openshell.v1.OpenShell.PeerRelay:input_type -> openshell.v1.PeerRelayFrame + 105, // 374: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest + 205, // 375: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest + 207, // 376: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest + 209, // 377: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest + 211, // 378: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest + 214, // 379: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest + 216, // 380: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest + 218, // 381: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest + 220, // 382: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest + 222, // 383: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest + 17, // 384: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest + 19, // 385: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest + 225, // 386: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest + 227, // 387: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest + 229, // 388: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest + 231, // 389: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest + 234, // 390: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest + 236, // 391: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest + 238, // 392: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest + 22, // 393: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse + 24, // 394: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse + 26, // 395: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse + 66, // 396: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse + 57, // 397: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse + 66, // 398: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse + 67, // 399: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse + 53, // 400: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 53, // 401: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 54, // 402: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse + 55, // 403: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse + 68, // 404: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse + 69, // 405: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse + 70, // 406: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse + 79, // 407: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse + 82, // 408: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse + 66, // 409: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse + 66, // 410: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse + 84, // 411: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse + 92, // 412: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse + 92, // 413: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse + 88, // 414: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse + 90, // 415: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse + 94, // 416: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse + 99, // 417: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent + 101, // 418: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame + 99, // 419: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent + 114, // 420: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse + 114, // 421: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse + 115, // 422: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse + 139, // 423: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse + 138, // 424: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse + 141, // 425: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse + 143, // 426: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse + 145, // 427: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse + 114, // 428: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse + 130, // 429: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse + 132, // 430: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse + 134, // 431: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse + 136, // 432: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse + 146, // 433: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse + 148, // 434: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse + 286, // 435: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse + 287, // 436: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse + 164, // 437: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse + 166, // 438: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse + 168, // 439: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse + 170, // 440: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse + 243, // 441: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse + 81, // 442: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse + 173, // 443: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse + 152, // 444: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse + 154, // 445: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse + 178, // 446: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse + 177, // 447: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse + 180, // 448: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage + 187, // 449: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse + 189, // 450: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse + 194, // 451: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame + 196, // 452: openshell.v1.OpenShell.PeerRelay:output_type -> openshell.v1.PeerRelayFrame + 106, // 453: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent + 206, // 454: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse + 208, // 455: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse + 210, // 456: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse + 212, // 457: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse + 215, // 458: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse + 217, // 459: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse + 219, // 460: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse + 221, // 461: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse + 224, // 462: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse + 18, // 463: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse + 20, // 464: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse + 226, // 465: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse + 228, // 466: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse + 230, // 467: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse + 232, // 468: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse + 235, // 469: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse + 237, // 470: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse + 239, // 471: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse + 393, // [393:472] is the sub-list for method output_type + 314, // [314:393] is the sub-list for method input_type + 314, // [314:314] is the sub-list for extension type_name + 314, // [314:314] is the sub-list for extension extendee + 0, // [0:314] is the sub-list for field type_name } func init() { file_openshell_proto_init() } @@ -19571,13 +19749,17 @@ func file_openshell_proto_init() { (*RelayFrame_Init)(nil), (*RelayFrame_Data)(nil), } + file_openshell_proto_msgTypes[179].OneofWrappers = []any{ + (*PeerRelayFrame_Init)(nil), + (*PeerRelayFrame_Data)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_openshell_proto_rawDesc), len(file_openshell_proto_rawDesc)), NumEnums: 17, - NumMessages: 250, + NumMessages: 252, NumExtensions: 0, NumServices: 1, }, diff --git a/sdk/go/proto/openshellv1/openshell_grpc.pb.go b/sdk/go/proto/openshellv1/openshell_grpc.pb.go index 0609450c89..56aa770ae0 100644 --- a/sdk/go/proto/openshellv1/openshell_grpc.pb.go +++ b/sdk/go/proto/openshellv1/openshell_grpc.pb.go @@ -82,6 +82,7 @@ const ( OpenShell_ReportMainProcessExit_FullMethodName = "/openshell.v1.OpenShell/ReportMainProcessExit" OpenShell_FinalizeMainProcessExit_FullMethodName = "/openshell.v1.OpenShell/FinalizeMainProcessExit" OpenShell_RelayStream_FullMethodName = "/openshell.v1.OpenShell/RelayStream" + OpenShell_PeerRelay_FullMethodName = "/openshell.v1.OpenShell/PeerRelay" OpenShell_WatchSandbox_FullMethodName = "/openshell.v1.OpenShell/WatchSandbox" OpenShell_SubmitPolicyAnalysis_FullMethodName = "/openshell.v1.OpenShell/SubmitPolicyAnalysis" OpenShell_GetDraftPolicy_FullMethodName = "/openshell.v1.OpenShell/GetDraftPolicy" @@ -267,6 +268,14 @@ type OpenShellClient interface { // This rides the same TCP+TLS+HTTP/2 connection as ConnectSupervisor — // no new TLS handshake, no reverse HTTP CONNECT. RelayStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[RelayFrame, RelayFrame], error) + // Internal gateway-to-gateway relay forwarding. + // + // A gateway replica that receives a user request for a sandbox whose + // supervisor session is owned by a different replica opens this stream to the + // owner. The first frame carries PeerRelayInit; subsequent frames carry raw + // bytes in either direction. This RPC is authenticated as a gateway peer, not + // as a user or sandbox supervisor. + PeerRelay(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PeerRelayFrame, PeerRelayFrame], error) // Watch a sandbox and stream updates. // // This stream can include: @@ -943,9 +952,22 @@ func (c *openShellClient) RelayStream(ctx context.Context, opts ...grpc.CallOpti // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type OpenShell_RelayStreamClient = grpc.BidiStreamingClient[RelayFrame, RelayFrame] +func (c *openShellClient) PeerRelay(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PeerRelayFrame, PeerRelayFrame], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &OpenShell_ServiceDesc.Streams[6], OpenShell_PeerRelay_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[PeerRelayFrame, PeerRelayFrame]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type OpenShell_PeerRelayClient = grpc.BidiStreamingClient[PeerRelayFrame, PeerRelayFrame] + func (c *openShellClient) WatchSandbox(ctx context.Context, in *WatchSandboxRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SandboxStreamEvent], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &OpenShell_ServiceDesc.Streams[6], OpenShell_WatchSandbox_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &OpenShell_ServiceDesc.Streams[7], OpenShell_WatchSandbox_FullMethodName, cOpts...) if err != nil { return nil, err } @@ -1306,6 +1328,14 @@ type OpenShellServer interface { // This rides the same TCP+TLS+HTTP/2 connection as ConnectSupervisor — // no new TLS handshake, no reverse HTTP CONNECT. RelayStream(grpc.BidiStreamingServer[RelayFrame, RelayFrame]) error + // Internal gateway-to-gateway relay forwarding. + // + // A gateway replica that receives a user request for a sandbox whose + // supervisor session is owned by a different replica opens this stream to the + // owner. The first frame carries PeerRelayInit; subsequent frames carry raw + // bytes in either direction. This RPC is authenticated as a gateway peer, not + // as a user or sandbox supervisor. + PeerRelay(grpc.BidiStreamingServer[PeerRelayFrame, PeerRelayFrame]) error // Watch a sandbox and stream updates. // // This stream can include: @@ -1545,6 +1575,9 @@ func (UnimplementedOpenShellServer) FinalizeMainProcessExit(context.Context, *Fi func (UnimplementedOpenShellServer) RelayStream(grpc.BidiStreamingServer[RelayFrame, RelayFrame]) error { return status.Error(codes.Unimplemented, "method RelayStream not implemented") } +func (UnimplementedOpenShellServer) PeerRelay(grpc.BidiStreamingServer[PeerRelayFrame, PeerRelayFrame]) error { + return status.Error(codes.Unimplemented, "method PeerRelay not implemented") +} func (UnimplementedOpenShellServer) WatchSandbox(*WatchSandboxRequest, grpc.ServerStreamingServer[SandboxStreamEvent]) error { return status.Error(codes.Unimplemented, "method WatchSandbox not implemented") } @@ -2623,6 +2656,13 @@ func _OpenShell_RelayStream_Handler(srv interface{}, stream grpc.ServerStream) e // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type OpenShell_RelayStreamServer = grpc.BidiStreamingServer[RelayFrame, RelayFrame] +func _OpenShell_PeerRelay_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(OpenShellServer).PeerRelay(&grpc.GenericServerStream[PeerRelayFrame, PeerRelayFrame]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type OpenShell_PeerRelayServer = grpc.BidiStreamingServer[PeerRelayFrame, PeerRelayFrame] + func _OpenShell_WatchSandbox_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(WatchSandboxRequest) if err := stream.RecvMsg(m); err != nil { @@ -3285,6 +3325,12 @@ var OpenShell_ServiceDesc = grpc.ServiceDesc{ ServerStreams: true, ClientStreams: true, }, + { + StreamName: "PeerRelay", + Handler: _OpenShell_PeerRelay_Handler, + ServerStreams: true, + ClientStreams: true, + }, { StreamName: "WatchSandbox", Handler: _OpenShell_WatchSandbox_Handler, diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 8fccf3779d..a41d6f0b9c 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -331,14 +331,13 @@ When `userns` is configured (e.g. `userns = "auto"` or `userns = "keep-id"`): helm -n openshell status openshell helm -n openshell get values openshell kubectl -n openshell get deployment,statefulset,pod,svc,pvc -kubectl -n openshell logs deployment/openshell -c openshell-gateway --tail=200 -kubectl -n openshell logs statefulset/openshell -c openshell-gateway --tail=200 -kubectl -n openshell rollout status deployment/openshell -kubectl -n openshell rollout status statefulset/openshell +GATEWAY_DEPLOYMENT="$(kubectl -n openshell get deployment openshell >/dev/null 2>&1 && echo deployment/openshell || echo statefulset/openshell)" +kubectl -n openshell logs "${GATEWAY_DEPLOYMENT}" -c openshell-gateway --tail=200 +kubectl -n openshell rollout status "${GATEWAY_DEPLOYMENT}" ``` -Use the log and rollout commands for the workload kind that exists in the -release. Look for failed installs, unexpected values, missing namespace, wrong +Use the log and rollout commands for the gateway resource kind that exists in +the release. Look for failed installs, unexpected values, missing namespace, wrong image tag, TLS settings that do not match the registered endpoint, and scheduling failures. @@ -378,6 +377,54 @@ kubectl -n get deployment,service,pod -l app.kubernetes.io/name= logs deployment/ --tail=200 ``` +Multi-replica gateways serialize cross-object sandbox and provider mutations +with a PostgreSQL advisory lock. If those RPCs stall while ordinary reads and +health checks remain responsive, inspect long-running database sessions and +advisory-lock waiters. Do not print the database URI or Secret contents into +logs: + +```sql +SELECT pid, granted, waitstart +FROM pg_locks +WHERE locktype = 'advisory'; +``` + +For multi-replica gateway installs, supervisor and client session traffic may +be served by a non-owner gateway replica and relayed to the current supervisor +owner over the internal `PeerRelay` RPC. Check the headless peer Service, +projected peer ServiceAccount token volume, and TokenReview RBAC: + +```bash +kubectl -n openshell get svc openshell-peer -o wide +kubectl -n openshell get endpoints openshell-peer +kubectl -n openshell get pod -l app.kubernetes.io/instance=openshell \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{.spec.volumes[?(@.name=="gateway-peer-token")]}{"\n"}{.spec.volumes[?(@.name=="peer-client-tls")]}{"\n"}{.spec.containers[0].env[?(@.name=="OPENSHELL_PEER_SERVICE_ACCOUNT_TOKEN_FILE")]}{"\n"}{.spec.containers[0].env[?(@.name=="OPENSHELL_PEER_ENDPOINT")]}{"\n"}{.spec.containers[0].env[?(@.name=="OPENSHELL_PEER_TLS_SERVER_NAME")]}{"\n"}{end}' +kubectl auth can-i create tokenreviews.authentication.k8s.io \ + --as=system:serviceaccount:openshell:openshell +kubectl auth can-i get pods -n openshell \ + --as=system:serviceaccount:openshell:openshell +kubectl -n openshell logs "${GATEWAY_DEPLOYMENT}" --tail=200 | grep -E 'gateway peer|PeerRelay|supervisor owner|owner relay' +``` + +Expected gateway startup logs include +`gateway peer ServiceAccount TokenReview authentication enabled`. If peer relay +calls fail with `Unauthenticated`, verify the `gateway-peer-token` projected +volume has audience `openshell-gateway-peer` and that the receiving gateway can +create TokenReviews. If they fail with `PermissionDenied`, verify the gateway +ServiceAccount name, release namespace, pod UID, and Helm selector labels match +the live gateway pods. Deployment-backed gateway pods should also publish +`OPENSHELL_PEER_ENDPOINT` from their pod IP. The +`OPENSHELL_PEER_SERVICE_ACCOUNT_TOKEN_FILE` name follows the existing +token-file convention used by `OPENSHELL_SANDBOX_TOKEN_FILE` and +`OPENSHELL_K8S_SA_TOKEN_FILE`. + +For TLS-enabled gateways, peer clients verify the stable gateway Service DNS +name and load the chart CA plus client identity from +`OPENSHELL_PEER_TLS_CA_FILE`, `OPENSHELL_PEER_TLS_CERT_FILE`, and +`OPENSHELL_PEER_TLS_KEY_FILE`. If peer calls fail during TLS negotiation, verify +the `peer-client-tls` volume exists, those files are readable, and the server +certificate includes the name in `OPENSHELL_PEER_TLS_SERVER_NAME`. + Check required Helm deployment secrets: ```bash @@ -508,8 +555,8 @@ certificate CA — verify both are from the same issuer. Check the image references currently used by the gateway deployment: ```bash -kubectl -n openshell get deployment openshell -o jsonpath="{.spec.template.spec.containers[*].image}{\"\n\"}{.spec.template.spec.containers[*].env[?(@.name==\"OPENSHELL_SUPERVISOR_IMAGE\")].value}{\"\n\"}" -kubectl -n openshell get statefulset openshell -o jsonpath="{.spec.template.spec.containers[*].image}{\"\n\"}{.spec.template.spec.containers[*].env[?(@.name==\"OPENSHELL_SUPERVISOR_IMAGE\")].value}{\"\n\"}" +GATEWAY_DEPLOYMENT="$(kubectl -n openshell get deployment openshell >/dev/null 2>&1 && echo deployment/openshell || echo statefulset/openshell)" +kubectl -n openshell get "${GATEWAY_DEPLOYMENT}" -o jsonpath="{.spec.template.spec.containers[*].image}{\"\n\"}{.spec.template.spec.containers[*].env[?(@.name==\"OPENSHELL_SUPERVISOR_IMAGE\")].value}{\"\n\"}" helm -n openshell get values openshell | grep -E 'repository|tag|supervisorImage|workload' ``` @@ -563,8 +610,8 @@ If the gateway is healthy but sandbox creation fails: ```bash kubectl -n openshell get pods kubectl -n openshell get events --sort-by=.lastTimestamp | tail -n 50 -kubectl -n openshell logs deployment/openshell -c openshell-gateway --tail=200 -kubectl -n openshell logs statefulset/openshell -c openshell-gateway --tail=200 +GATEWAY_DEPLOYMENT="$(kubectl -n openshell get deployment openshell >/dev/null 2>&1 && echo deployment/openshell || echo statefulset/openshell)" +kubectl -n openshell logs "${GATEWAY_DEPLOYMENT}" -c openshell-gateway --tail=200 ``` Check the configured sandbox namespace: diff --git a/tasks/test.toml b/tasks/test.toml index 30f348bebd..317f676c19 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -196,6 +196,12 @@ env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:ha-rebalancing"] +description = "Run the Kubernetes HA rebalancing suite through Envoy against two gateway replicas and external PostgreSQL" +env = { OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET = "openshell-ha-pg", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-high-availability.yaml", OPENSHELL_E2E_KUBE_TEST = "kubernetes_ha_rebalancing", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha", OPENSHELL_E2E_KUBE_USE_ENVOY = "1" } +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" + ["e2e:kubernetes:credential-drivers"] description = "Run Kubernetes e2e for provider credential storage backed by Kubernetes Secrets and Vault" env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_KUBE_TEST = "credential_drivers", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-credential-drivers" } From 0246e4ea48b8ea2a252e8ebd6059f9b25f560daf Mon Sep 17 00:00:00 2001 From: divesh Date: Mon, 14 Sep 2026 13:41:49 -0700 Subject: [PATCH 2/5] perf(server): cache peer connections, tokens, and owner lookups Every forwarded relay rebuilt its setup from scratch: an owner lookup, a blocking read of the peer token, a TLS connect to the owning replica, and a TokenReview plus Pod GET on the receiving side. Sandbox service routing does this per HTTP request, so the apiserver calls scaled with traffic. Cache all of it on ServerState: - peer channels pooled per endpoint, so relays multiplex over one connection instead of redialing - peer tokens keyed by SHA-256, expiring at min(ttl, token exp) so a hit cannot accept an expired token - owner records for 3s against a 45s ownership TTL, still freshness checked before use Entries are evicted when a relay fails. Also raise HTTP/2 max_concurrent_streams to 1024, since pooling funnels every relay between two replicas onto one connection and hyper's default of 200 sits below the 256 pending-relay budget. Signed-off-by: divesh --- crates/openshell-server/src/auth/peer.rs | 236 ++++++++++++++++- crates/openshell-server/src/lib.rs | 10 + crates/openshell-server/src/multiplex.rs | 10 + .../src/supervisor_session.rs | 244 +++++++++++++++++- 4 files changed, 486 insertions(+), 14 deletions(-) diff --git a/crates/openshell-server/src/auth/peer.rs b/crates/openshell-server/src/auth/peer.rs index b92c77bd31..79dbda132a 100644 --- a/crates/openshell-server/src/auth/peer.rs +++ b/crates/openshell-server/src/auth/peer.rs @@ -18,8 +18,11 @@ use k8s_openapi::api::{ }; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::api::{Api, PostParams}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use tonic::Status; use tracing::{debug, info, warn}; @@ -37,6 +40,10 @@ pub const DEFAULT_PEER_SA_TOKEN_FILE: &str = "/var/run/secrets/openshell-peer/to /// Environment variable with comma-separated `key=value` pod labels required /// on authenticated gateway peer pods. pub const PEER_REQUIRED_POD_LABELS_ENV: &str = "OPENSHELL_PEER_POD_LABELS"; +/// Environment variable overriding how long a verified peer token is cached. +pub const PEER_TOKEN_CACHE_TTL_SECS_ENV: &str = "OPENSHELL_PEER_TOKEN_CACHE_TTL_SECS"; +/// Default lifetime of a cached peer token verification. +pub const DEFAULT_PEER_TOKEN_CACHE_TTL: Duration = Duration::from_mins(1); const POD_NAME_EXTRA: &str = "authentication.kubernetes.io/pod-name"; const POD_UID_EXTRA: &str = "authentication.kubernetes.io/pod-uid"; @@ -251,6 +258,120 @@ impl GatewayPeerIdentityResolver for LiveGatewayPeerResolver { } } +struct CachedPeerIdentity { + identity: ResolvedGatewayPeerIdentity, + expires_at: Instant, +} + +/// Caches verified peer identities so a burst of relays from the same replica +/// costs one `TokenReview` plus one Pod GET instead of one of each per relay. +/// +/// Only successful verifications are cached. An entry lives for `ttl` or until +/// the token itself expires, whichever comes first. +pub struct CachingGatewayPeerResolver { + inner: Arc, + ttl: Duration, + entries: Mutex>, +} + +impl CachingGatewayPeerResolver { + pub fn new(inner: Arc, ttl: Duration) -> Self { + Self { + inner, + ttl, + entries: Mutex::new(HashMap::new()), + } + } + + fn cached(&self, key: &str) -> Option { + let now = Instant::now(); + let mut entries = self.entries.lock().unwrap(); + let entry = entries.get(key)?; + if entry.expires_at <= now { + entries.remove(key); + return None; + } + Some(entry.identity.clone()) + } + + fn store(&self, key: &str, identity: &ResolvedGatewayPeerIdentity, token: &str) { + let Some(lifetime) = self.entry_lifetime(token) else { + return; + }; + let now = Instant::now(); + let mut entries = self.entries.lock().unwrap(); + entries.retain(|_, entry| entry.expires_at > now); + entries.insert( + key.to_string(), + CachedPeerIdentity { + identity: identity.clone(), + expires_at: now + lifetime, + }, + ); + } + + /// Never outlive the token. A token expiring in 5s is cached for 5s even + /// when `ttl` is longer, so a cache hit can't accept an expired token. + fn entry_lifetime(&self, token: &str) -> Option { + let Some(expires_in) = jwt_expires_in(token) else { + return Some(self.ttl); + }; + if expires_in.is_zero() { + return None; + } + Some(self.ttl.min(expires_in)) + } +} + +#[async_trait] +impl GatewayPeerIdentityResolver for CachingGatewayPeerResolver { + async fn resolve(&self, token: &str) -> Result, Status> { + let key = token_cache_key(token); + if let Some(identity) = self.cached(&key) { + return Ok(Some(identity)); + } + + let resolved = self.inner.resolve(token).await?; + if let Some(identity) = resolved.as_ref() { + self.store(&key, identity, token); + } + Ok(resolved) + } +} + +/// Hashed so long-lived map keys never hold raw credentials. +fn token_cache_key(token: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(token.as_bytes()); + hex::encode(hasher.finalize()) +} + +/// Reads `exp` from the JWT payload without verifying the signature — +/// `TokenReview` already did that, this only bounds the cache entry. +fn jwt_expires_in(token: &str) -> Option { + use base64::Engine as _; + + let mut parts = token.split('.'); + let _header = parts.next()?; + let payload = parts.next()?; + let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(payload) + .ok()?; + let claims: serde_json::Value = serde_json::from_slice(&decoded).ok()?; + let exp = claims.get("exp")?.as_i64()?; + let now_secs = openshell_core::time::now_ms() / 1_000; + Some(Duration::from_secs( + u64::try_from(exp.saturating_sub(now_secs)).unwrap_or(0), + )) +} + +pub fn peer_token_cache_ttl_from_env() -> Duration { + std::env::var(PEER_TOKEN_CACHE_TTL_SECS_ENV) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .map_or(DEFAULT_PEER_TOKEN_CACHE_TTL, Duration::from_secs) +} + #[allow(clippy::result_large_err)] fn peer_token_review_identity( status: &TokenReviewStatus, @@ -577,6 +698,119 @@ mod tests { assert_eq!(peer.pod_uid, "uid-a"); } + fn identity() -> ResolvedGatewayPeerIdentity { + ResolvedGatewayPeerIdentity { + pod_name: "openshell-0".to_string(), + pod_uid: "uid-a".to_string(), + } + } + + /// Builds an unsigned JWT whose payload carries only `exp`. The cache reads + /// `exp` without verifying the signature, so the other segments are filler. + fn token_expiring_in(secs: i64) -> String { + use base64::Engine as _; + let exp = (openshell_core::time::now_ms() / 1_000) + secs; + let payload = + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(format!(r#"{{"exp":{exp}}}"#)); + format!("header.{payload}.signature") + } + + #[tokio::test] + async fn caching_resolver_reuses_a_verified_token() { + let inner = Arc::new(FakeGatewayPeerResolver::returning(Ok(Some(identity())))); + let resolver = CachingGatewayPeerResolver::new(inner.clone(), Duration::from_mins(1)); + let token = token_expiring_in(3600); + + for _ in 0..5 { + let peer = resolver.resolve(&token).await.unwrap(); + assert_eq!(peer.unwrap().pod_name, "openshell-0"); + } + + assert_eq!( + inner.seen_tokens.lock().unwrap().len(), + 1, + "five relays with the same token must cost one TokenReview" + ); + } + + #[tokio::test] + async fn caching_resolver_verifies_each_distinct_token() { + let inner = Arc::new(FakeGatewayPeerResolver::returning(Ok(Some(identity())))); + let resolver = CachingGatewayPeerResolver::new(inner.clone(), Duration::from_mins(1)); + + resolver.resolve(&token_expiring_in(3600)).await.unwrap(); + resolver.resolve(&token_expiring_in(1800)).await.unwrap(); + + assert_eq!(inner.seen_tokens.lock().unwrap().len(), 2); + } + + #[tokio::test] + async fn caching_resolver_does_not_cache_a_rejected_token() { + let inner = Arc::new(FakeGatewayPeerResolver::returning(Ok(None))); + let resolver = CachingGatewayPeerResolver::new(inner.clone(), Duration::from_mins(1)); + let token = token_expiring_in(3600); + + assert!(resolver.resolve(&token).await.unwrap().is_none()); + assert!(resolver.resolve(&token).await.unwrap().is_none()); + + assert_eq!( + inner.seen_tokens.lock().unwrap().len(), + 2, + "a rejection must be revalidated, never served from cache" + ); + } + + #[tokio::test] + async fn caching_resolver_does_not_cache_an_expired_token() { + let inner = Arc::new(FakeGatewayPeerResolver::returning(Ok(Some(identity())))); + let resolver = CachingGatewayPeerResolver::new(inner.clone(), Duration::from_mins(1)); + let token = token_expiring_in(-1); + + resolver.resolve(&token).await.unwrap(); + resolver.resolve(&token).await.unwrap(); + + assert_eq!(inner.seen_tokens.lock().unwrap().len(), 2); + } + + #[test] + fn entry_lifetime_never_outlives_the_token() { + let inner = Arc::new(FakeGatewayPeerResolver::returning(Ok(Some(identity())))); + let resolver = CachingGatewayPeerResolver::new(inner, Duration::from_mins(1)); + + let short = resolver + .entry_lifetime(&token_expiring_in(5)) + .expect("a live token should be cacheable"); + assert!(short <= Duration::from_secs(5)); + + let capped = resolver + .entry_lifetime(&token_expiring_in(3600)) + .expect("a long-lived token should be cacheable"); + assert_eq!(capped, Duration::from_mins(1)); + + assert!(resolver.entry_lifetime(&token_expiring_in(-1)).is_none()); + } + + #[test] + fn opaque_tokens_fall_back_to_the_configured_ttl() { + let inner = Arc::new(FakeGatewayPeerResolver::returning(Ok(Some(identity())))); + let resolver = CachingGatewayPeerResolver::new(inner, Duration::from_secs(30)); + + assert_eq!( + resolver.entry_lifetime("not-a-jwt"), + Some(Duration::from_secs(30)) + ); + } + + #[test] + fn token_cache_key_does_not_contain_the_token() { + let key = token_cache_key("super-secret-token"); + + assert!(!key.contains("super-secret-token")); + assert_eq!(key.len(), 64); + assert_eq!(key, token_cache_key("super-secret-token")); + assert_ne!(key, token_cache_key("another-token")); + } + #[test] fn parse_required_pod_labels_accepts_comma_list() { let labels = parse_required_pod_labels( diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 79fae6589c..9a23a08c3a 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -311,6 +311,10 @@ pub struct ServerState { /// Internal endpoint other gateway replicas can dial for peer RPCs. pub peer_endpoint: Option, + /// Reused peer connections, peer token, and owner lookups for relay + /// forwarding. Keeps per-relay cost off the connection and auth paths. + pub peer_routes: Arc, + /// Validated built-in and operator-registered supervisor middleware. pub middleware_registry: Arc, @@ -437,6 +441,7 @@ impl ServerState { gateway_shutting_down: AtomicBool::new(false), replica_id, peer_endpoint, + peer_routes: Arc::new(supervisor_session::PeerRouteCache::default()), extension_mint_limiter: auth::extension_mint_limit::ExtensionMintLimiter::default(), middleware_registry: Arc::new(MiddlewareRegistry::default()), oidc_cache, @@ -762,6 +767,10 @@ pub(crate) async fn run_server( service_account.trim().to_string(), required_labels, )); + let cache_ttl = auth::peer::peer_token_cache_ttl_from_env(); + let resolver = Arc::new(auth::peer::CachingGatewayPeerResolver::new( + resolver, cache_ttl, + )); let authenticator = auth::peer::PeerServiceAccountAuthenticator::new(resolver); state.peer_authenticator = Some(Arc::new(authenticator)); @@ -769,6 +778,7 @@ pub(crate) async fn run_server( namespace = %namespace.trim(), service_account = %service_account.trim(), audience, + token_cache_ttl_secs = cache_ttl.as_secs(), "gateway peer ServiceAccount TokenReview authentication enabled" ); } diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 5dd2cc3b3b..3309cad01f 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -200,6 +200,11 @@ macro_rules! request_id_middleware { const MAX_GRPC_DECODE_SIZE: usize = 1_048_576; const MAX_INTERCEPTED_GRPC_BODY_SIZE: usize = MAX_GRPC_DECODE_SIZE + 5; +/// Concurrent HTTP/2 streams allowed per connection. Sits above the +/// per-replica pending relay budget so pooled peer connections are bounded by +/// the relay caps rather than by the transport. +const MAX_HTTP2_CONCURRENT_STREAMS: u32 = 1024; + /// Multiplexed gRPC/HTTP service. #[derive(Clone)] pub struct MultiplexService { @@ -271,10 +276,15 @@ impl MultiplexService { // it the gateway never PINGs them, so idle/half-dead connections linger and orphan // in-flight relay execs. The timer is required — hyper panics on the keepalive // interval without one. + // + // Peer relays from one replica now share a single pooled connection, so every + // forwarded session for every sandbox counts against this one limit. hyper's + // default of 200 would cap the whole replica pair below MAX_PENDING_RELAYS. builder .http2() .timer(TokioTimer::new()) .adaptive_window(true) + .max_concurrent_streams(MAX_HTTP2_CONCURRENT_STREAMS) .keep_alive_interval(Some(Duration::from_secs(20))) .keep_alive_timeout(Duration::from_secs(10)); diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index e6ed6afb0f..e5c658fff1 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -49,6 +49,12 @@ const PEER_TLS_CA_FILE_ENV: &str = "OPENSHELL_PEER_TLS_CA_FILE"; const PEER_TLS_CERT_FILE_ENV: &str = "OPENSHELL_PEER_TLS_CERT_FILE"; const PEER_TLS_KEY_FILE_ENV: &str = "OPENSHELL_PEER_TLS_KEY_FILE"; const PEER_TLS_SERVER_NAME_ENV: &str = "OPENSHELL_PEER_TLS_SERVER_NAME"; +/// How long a resolved owner record is reused before rereading the store. +/// Well below `OWNER_TTL` so a cache hit can never outlive the record itself. +const OWNER_CACHE_TTL: Duration = Duration::from_secs(3); +/// How long the projected peer `ServiceAccount` token is held in memory. +/// The kubelet rotates the file hourly, so this only bounds staleness. +const PEER_TOKEN_CACHE_TTL: Duration = Duration::from_mins(5); #[derive(Debug, Default)] struct PeerTlsClientConfig { @@ -119,6 +125,120 @@ fn nonempty_env(name: &str) -> Option { .filter(|value| !value.is_empty()) } +/// Reusable peer state: one HTTP/2 channel per peer endpoint, the projected +/// `ServiceAccount` token, and recently resolved owner records. +/// +/// Without this every forwarded relay paid a TLS handshake, a blocking token +/// file read, and a store read. +#[derive(Default)] +pub struct PeerRouteCache { + channels: Mutex>, + token: Mutex>, + owners: Mutex>, +} + +/// Hand-written so the cached `ServiceAccount` token is never formatted. +impl std::fmt::Debug for PeerRouteCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PeerRouteCache").finish_non_exhaustive() + } +} + +struct CachedPeerToken { + token: String, + refresh_at: Instant, +} + +struct CachedOwner { + record: crate::supervisor_owner::OwnerRecord, + expires_at: Instant, +} + +impl PeerRouteCache { + /// Cloning a `Channel` shares the existing connection, so concurrent relays + /// to the same peer multiplex as HTTP/2 streams instead of dialing again. + async fn channel(&self, endpoint: &str) -> Result { + let cached = self.channels.lock().unwrap().get(endpoint).cloned(); + if let Some(channel) = cached { + return Ok(channel); + } + + let connected = build_peer_channel(endpoint).await?; + let mut channels = self.channels.lock().unwrap(); + // Two relays can miss together; keep whichever landed first so both + // end up on one connection and the loser's channel is dropped. + Ok(channels + .entry(endpoint.to_string()) + .or_insert(connected) + .clone()) + } + + /// Drops a peer connection so the next relay redials. Needed when a pod is + /// replaced and its endpoint now points at a dead or recycled address. + fn evict_channel(&self, endpoint: &str) { + self.channels.lock().unwrap().remove(endpoint); + } + + async fn peer_token(&self) -> Result { + let cached = self + .token + .lock() + .unwrap() + .as_ref() + .filter(|cached| cached.refresh_at > Instant::now()) + .map(|cached| cached.token.clone()); + if let Some(token) = cached { + return Ok(token); + } + + let loaded = tokio::task::spawn_blocking( + crate::auth::peer::load_peer_service_account_token_from_env, + ) + .await + .map_err(|_| Status::internal("gateway peer token read task failed"))? + .map_err(|err| { + Status::failed_precondition(format!("gateway peer token load failed: {err}")) + })? + .ok_or_else(|| { + Status::failed_precondition("gateway peer ServiceAccount token is not configured") + })?; + + *self.token.lock().unwrap() = Some(CachedPeerToken { + token: loaded.clone(), + refresh_at: Instant::now() + PEER_TOKEN_CACHE_TTL, + }); + Ok(loaded) + } + + fn cached_owner(&self, sandbox_id: &str) -> Option { + let now = Instant::now(); + let mut owners = self.owners.lock().unwrap(); + let entry = owners.get(sandbox_id)?; + if entry.expires_at <= now { + owners.remove(sandbox_id); + return None; + } + Some(entry.record.clone()) + } + + fn store_owner(&self, sandbox_id: &str, record: &crate::supervisor_owner::OwnerRecord) { + let now = Instant::now(); + let mut owners = self.owners.lock().unwrap(); + owners.retain(|_, entry| entry.expires_at > now); + owners.insert( + sandbox_id.to_string(), + CachedOwner { + record: record.clone(), + expires_at: now + OWNER_CACHE_TTL, + }, + ); + } + + fn evict_owner(&self, sandbox_id: &str) { + self.owners.lock().unwrap().remove(sandbox_id); + } +} + // --------------------------------------------------------------------------- // Session registry // --------------------------------------------------------------------------- @@ -1134,10 +1254,7 @@ pub async fn open_routed_relay_with_message( } } - if let Some(owner) = owner_index - .read(sandbox_id) - .await - .map_err(owner_error_to_status)? + if let Some(owner) = resolve_owner(state, &owner_index, sandbox_id).await? && owner_is_fresh(&owner) { if owner.owner_replica_id == state.replica_id { @@ -1146,6 +1263,7 @@ pub async fn open_routed_relay_with_message( owner_replica_id = %owner.owner_replica_id, "supervisor owner record points at this replica but no local session is registered; retrying" ); + state.peer_routes.evict_owner(sandbox_id); if Instant::now() + backoff > deadline { return Err(Status::unavailable("supervisor session not connected")); } @@ -1170,6 +1288,9 @@ pub async fn open_routed_relay_with_message( error = %status, "gateway peer owner relay open failed; retrying until session wait timeout" ); + // The record may name a replaced pod, so retry against a + // fresh read rather than the cached endpoint. + state.peer_routes.evict_owner(sandbox_id); } } } @@ -1182,6 +1303,26 @@ pub async fn open_routed_relay_with_message( } } +/// Reads the owning replica, reusing a recent result when one is cached. +async fn resolve_owner( + state: &Arc, + owner_index: &SupervisorOwnerIndex, + sandbox_id: &str, +) -> Result, Status> { + if let Some(record) = state.peer_routes.cached_owner(sandbox_id) { + return Ok(Some(record)); + } + + let record = owner_index + .read(sandbox_id) + .await + .map_err(owner_error_to_status)?; + if let Some(record) = record.as_ref() { + state.peer_routes.store_owner(sandbox_id, record); + } + Ok(record) +} + fn owner_is_fresh(owner: &crate::supervisor_owner::OwnerRecord) -> bool { let age_ms = openshell_core::time::now_ms() - owner.updated_at_ms; let ttl_ms = i64::try_from(OWNER_TTL.as_millis()).unwrap_or(i64::MAX); @@ -1213,14 +1354,8 @@ async fn connect_peer_relay( sandbox_id: &str, relay_open: RelayOpen, ) -> Result { - let token = crate::auth::peer::load_peer_service_account_token_from_env() - .map_err(|err| { - Status::failed_precondition(format!("gateway peer token load failed: {err}")) - })? - .ok_or_else(|| { - Status::failed_precondition("gateway peer ServiceAccount token is not configured") - })?; - let channel = build_peer_channel(owner_peer_endpoint).await?; + let token = state.peer_routes.peer_token().await?; + let channel = state.peer_routes.channel(owner_peer_endpoint).await?; let interceptor = PeerAuthInterceptor::new(&token, &state.replica_id)?; let mut client = open_shell_client::OpenShellClient::with_interceptor(channel, interceptor); @@ -1239,7 +1374,10 @@ async fn connect_peer_relay( let response = client .peer_relay(ReceiverStream::new(out_rx)) .await - .map_err(|err| Status::unavailable(format!("gateway peer relay RPC failed: {err}")))?; + .map_err(|err| { + state.peer_routes.evict_channel(owner_peer_endpoint); + Status::unavailable(format!("gateway peer relay RPC failed: {err}")) + })?; let inbound = response.into_inner(); let (gateway_stream, bridge_stream) = tokio::io::duplex(64 * 1024); spawn_peer_bridge(bridge_stream, inbound, out_tx, sandbox_id.to_string()); @@ -2710,4 +2848,84 @@ mod tests { .contains_key("ch-fresh") ); } + + fn owner_record(replica: &str) -> crate::supervisor_owner::OwnerRecord { + crate::supervisor_owner::OwnerRecord { + session_id: "session-a".to_string(), + supervisor_instance_id: "instance-a".to_string(), + connection_epoch: 1, + owner_replica_id: replica.to_string(), + owner_peer_endpoint: format!("http://{replica}:8080"), + connected_at_ms: 0, + updated_at_ms: openshell_core::time::now_ms(), + resource_version: 1, + } + } + + #[test] + fn owner_cache_returns_stored_record() { + let cache = PeerRouteCache::default(); + cache.store_owner("sbx-a", &owner_record("replica-a")); + + let cached = cache + .cached_owner("sbx-a") + .expect("record should be cached"); + assert_eq!(cached.owner_replica_id, "replica-a"); + } + + #[test] + fn owner_cache_misses_for_unknown_sandbox() { + let cache = PeerRouteCache::default(); + cache.store_owner("sbx-a", &owner_record("replica-a")); + + assert!(cache.cached_owner("sbx-b").is_none()); + } + + #[test] + fn evict_owner_forces_a_fresh_read() { + let cache = PeerRouteCache::default(); + cache.store_owner("sbx-a", &owner_record("replica-a")); + cache.evict_owner("sbx-a"); + + assert!(cache.cached_owner("sbx-a").is_none()); + } + + #[test] + fn owner_cache_drops_entries_past_their_ttl() { + let cache = PeerRouteCache::default(); + cache.owners.lock().unwrap().insert( + "sbx-a".to_string(), + CachedOwner { + record: owner_record("replica-a"), + expires_at: Instant::now().checked_sub(Duration::from_secs(1)).unwrap(), + }, + ); + + assert!(cache.cached_owner("sbx-a").is_none()); + assert!(!cache.owners.lock().unwrap().contains_key("sbx-a")); + } + + #[test] + fn owner_cache_ttl_stays_below_owner_record_ttl() { + // A cache hit must never extend the window in which a stale owner + // looks routable; `owner_is_fresh` is what enforces the real TTL. + assert!(OWNER_CACHE_TTL < OWNER_TTL); + } + + #[tokio::test] + async fn evict_channel_removes_only_the_named_peer() { + let cache = PeerRouteCache::default(); + let channel = Endpoint::from_static("http://10.0.0.1:8080").connect_lazy(); + { + let mut channels = cache.channels.lock().unwrap(); + channels.insert("http://10.0.0.1:8080".to_string(), channel.clone()); + channels.insert("http://10.0.0.2:8080".to_string(), channel); + } + + cache.evict_channel("http://10.0.0.1:8080"); + + let channels = cache.channels.lock().unwrap(); + assert!(!channels.contains_key("http://10.0.0.1:8080")); + assert!(channels.contains_key("http://10.0.0.2:8080")); + } } From f36ae35e61e1fc2020cb6daca17d9ff9a678c1f4 Mon Sep 17 00:00:00 2001 From: divesh Date: Mon, 14 Sep 2026 13:42:36 -0700 Subject: [PATCH 3/5] perf(server): pool upstream connections for sandbox services Each HTTP request to a sandbox service opened its own supervisor relay, paying a new TCP connection and HTTP/1 handshake every time. Worse, it counted against the 32 in-flight relay cap, so a service handling more than 32 concurrent requests failed outright. Pool idle upstreams per endpoint and port, up to 8 each for 15s. Reuse is safe because the pool only returns a connection hyper reports as ready, and HTTP/1 cannot start a request until the previous body has drained. Upgrades are never pooled since they take the connection over, and a failed send evicts that endpoint. Pruning is bounded per key, with the full sweep limited to once per 30s. Signed-off-by: divesh --- crates/openshell-server/src/lib.rs | 5 + .../openshell-server/src/service_routing.rs | 373 +++++++++++++++--- 2 files changed, 320 insertions(+), 58 deletions(-) diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 9a23a08c3a..48e11c1dbf 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -315,6 +315,10 @@ pub struct ServerState { /// forwarding. Keeps per-relay cost off the connection and auth paths. pub peer_routes: Arc, + /// Idle HTTP/1 upstreams to sandbox services, so routed requests reuse a + /// relay instead of opening one per request. + pub service_upstreams: Arc, + /// Validated built-in and operator-registered supervisor middleware. pub middleware_registry: Arc, @@ -442,6 +446,7 @@ impl ServerState { replica_id, peer_endpoint, peer_routes: Arc::new(supervisor_session::PeerRouteCache::default()), + service_upstreams: Arc::new(service_routing::ServiceUpstreamPool::default()), extension_mint_limiter: auth::extension_mint_limit::ExtensionMintLimiter::default(), middleware_registry: Arc::new(MiddlewareRegistry::default()), oidc_cache, diff --git a/crates/openshell-server/src/service_routing.rs b/crates/openshell-server/src/service_routing.rs index 5e9a01f60e..66bac3e2ab 100644 --- a/crates/openshell-server/src/service_routing.rs +++ b/crates/openshell-server/src/service_routing.rs @@ -17,9 +17,10 @@ use openshell_ocsf::{ HttpActivityBuilder, HttpRequest, HttpResponse as OcsfHttpResponse, NetworkActivityBuilder, OCSF_TARGET, OcsfEvent, SeverityId, StateId, StatusId, Url as OcsfUrl, }; +use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr}; -use std::sync::Arc; -use std::time::Duration; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use tokio::io::AsyncWriteExt; use tracing::{info, warn}; @@ -31,6 +32,107 @@ const ROUTING_RULE_NAME: &str = "sandbox_service_routing"; const ROUTING_RULE_TYPE: &str = "gateway"; const RELAY_RULE_NAME: &str = "sandbox_service_relay"; const RELAY_TARGET_HOST: &str = "127.0.0.1"; +/// How long an idle upstream is kept. Deliberately short: a sandbox app can +/// close its side at any time, and a connection we hand out after it died +/// fails the request. +const UPSTREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(15); +/// Idle upstreams kept per endpoint. Concurrent requests need one each, +/// because HTTP/1 serves a single request at a time per connection. +const UPSTREAM_MAX_IDLE_PER_ENDPOINT: usize = 8; +/// Minimum gap between full sweeps of the pool. +const UPSTREAM_SWEEP_INTERVAL: Duration = Duration::from_secs(30); + +type UpstreamSender = hyper::client::conn::http1::SendRequest; + +struct PooledUpstream { + sender: UpstreamSender, + idle_since: Instant, +} + +#[derive(Default)] +struct UpstreamPoolInner { + idle: HashMap>, + last_sweep: Option, +} + +impl UpstreamPoolInner { + /// Full sweeps are rate-limited so a large pool can't turn every insert + /// into a scan of every endpoint. + fn sweep_if_due(&mut self, now: Instant) { + if let Some(last) = self.last_sweep + && now.duration_since(last) < UPSTREAM_SWEEP_INTERVAL + { + return; + } + self.last_sweep = Some(now); + self.idle.retain(|_, entries| { + entries.retain(|entry| is_reusable(entry, now)); + !entries.is_empty() + }); + } +} + +/// Idle upstream connections to sandbox services, keyed by endpoint. +/// +/// Without this every HTTP request opened its own supervisor relay, which +/// meant a new TCP connection inside the sandbox and a new HTTP/1 handshake +/// per request, and counted against the 32 in-flight relay cap per sandbox. +#[derive(Default)] +pub struct ServiceUpstreamPool { + inner: Mutex, +} + +impl std::fmt::Debug for ServiceUpstreamPool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ServiceUpstreamPool") + .finish_non_exhaustive() + } +} + +impl ServiceUpstreamPool { + /// Takes an upstream that is connected and free. + /// + /// `is_ready` is what makes reuse safe: HTTP/1 can't start a request until + /// the previous response body has been read, and hyper only reports ready + /// once that has happened. + fn take(&self, key: &str) -> Option { + let now = Instant::now(); + let mut inner = self.inner.lock().unwrap(); + let entries = inner.idle.get_mut(key)?; + entries.retain(|entry| is_reusable(entry, now)); + let index = entries.iter().position(|entry| entry.sender.is_ready())?; + Some(entries.swap_remove(index).sender) + } + + fn put(&self, key: &str, sender: UpstreamSender) { + if sender.is_closed() { + return; + } + let now = Instant::now(); + let mut inner = self.inner.lock().unwrap(); + inner.sweep_if_due(now); + let entries = inner.idle.entry(key.to_string()).or_default(); + if entries.len() >= UPSTREAM_MAX_IDLE_PER_ENDPOINT { + return; + } + entries.push(PooledUpstream { + sender, + idle_since: now, + }); + } + + fn evict(&self, key: &str) { + self.inner.lock().unwrap().idle.remove(key); + } +} + +fn is_reusable(entry: &PooledUpstream, now: Instant) -> bool { + !entry.sender.is_closed() && now.duration_since(entry.idle_since) <= UPSTREAM_IDLE_TIMEOUT +} + +fn upstream_pool_key(endpoint_id: &str, target_port: u16) -> String { + format!("{endpoint_id}|{target_port}") +} impl ObjectType for ServiceEndpoint { fn object_type() -> &'static str { @@ -325,8 +427,74 @@ async fn proxy_to_endpoint( let websocket_upgrade = is_websocket_upgrade(&req); let downstream_upgrade = websocket_upgrade.then(|| hyper::upgrade::on(&mut req)); + // An upgrade takes the connection over, so it is never pooled. + let pool_key = upstream_pool_key(endpoint.object_id(), target_port); + let pooled = if websocket_upgrade { + None + } else { + state.service_upstreams.take(&pool_key) + }; + + let mut sender = match pooled { + Some(sender) => sender, + None => open_upstream(&state, &sandbox, &endpoint, target_port, websocket_upgrade).await?, + }; + + let upstream = build_upstream_request(req, target_port, websocket_upgrade)?; + let mut response = sender.send_request(upstream).await.map_err(|err| { + warn!(error = %err, "sandbox service routing: upstream HTTP request failed"); + state.service_upstreams.evict(&pool_key); + let route_err = ServiceRouteError::service_unreachable(); + emit_service_relay_failure(&endpoint, target_port, route_err.reason); + route_err + })?; + + if !websocket_upgrade { + state.service_upstreams.put(&pool_key, sender); + } + + if websocket_upgrade && response.status() == StatusCode::SWITCHING_PROTOCOLS { + let upstream_upgrade = hyper::upgrade::on(&mut response); + let downstream_upgrade = downstream_upgrade.ok_or_else(|| { + let err = ServiceRouteError::service_unreachable(); + emit_service_relay_failure(&endpoint, target_port, "websocket upgrade unavailable"); + err + })?; + tokio::spawn(async move { + match (downstream_upgrade.await, upstream_upgrade.await) { + (Ok(downstream), Ok(upstream)) => { + let mut downstream = TokioIo::new(downstream); + let mut upstream = TokioIo::new(upstream); + let _ = tokio::io::copy_bidirectional(&mut downstream, &mut upstream).await; + let _ = downstream.shutdown().await; + let _ = upstream.shutdown().await; + } + (Err(err), _) => { + warn!(error = %err, "sandbox service routing: downstream WebSocket upgrade failed"); + } + (_, Err(err)) => { + warn!(error = %err, "sandbox service routing: upstream WebSocket upgrade failed"); + } + } + }); + + let (parts, _) = response.into_parts(); + return Ok(Response::from_parts(parts, Body::empty())); + } + + let (parts, body) = response.into_parts(); + Ok(Response::from_parts(parts, Body::new(body))) +} + +async fn open_upstream( + state: &Arc, + sandbox: &Sandbox, + endpoint: &ServiceEndpoint, + target_port: u16, + websocket_upgrade: bool, +) -> Result { let (_channel_id, relay_rx) = crate::supervisor_session::open_routed_relay_with_target( - &state, + state, sandbox.object_id(), relay_open::Target::Tcp(TcpRelayTarget { host: RELAY_TARGET_HOST.to_string(), @@ -335,40 +503,40 @@ async fn proxy_to_endpoint( endpoint.object_id().to_string(), Duration::from_secs(15), ) - .await - .map_err(|err| { - warn!(error = %err, sandbox_id = %endpoint.sandbox_id, "sandbox service routing: supervisor relay unavailable"); - let route_err = ServiceRouteError::service_unreachable(); - emit_service_relay_failure(&endpoint, target_port, route_err.reason); - route_err - })?; + .await + .map_err(|err| { + warn!(error = %err, sandbox_id = %endpoint.sandbox_id, "sandbox service routing: supervisor relay unavailable"); + let route_err = ServiceRouteError::service_unreachable(); + emit_service_relay_failure(endpoint, target_port, route_err.reason); + route_err + })?; let relay = tokio::time::timeout(Duration::from_secs(10), relay_rx) .await .map_err(|_| { let err = ServiceRouteError::service_unreachable(); - emit_service_relay_failure(&endpoint, target_port, "relay claim timed out"); + emit_service_relay_failure(endpoint, target_port, "relay claim timed out"); err })? .map_err(|_| { let err = ServiceRouteError::service_unreachable(); - emit_service_relay_failure(&endpoint, target_port, "relay claim canceled"); + emit_service_relay_failure(endpoint, target_port, "relay claim canceled"); err })? .map_err(|err| { warn!(error = %err, "sandbox service routing: relay target open failed"); let route_err = ServiceRouteError::service_unreachable(); - emit_service_relay_failure(&endpoint, target_port, route_err.reason); + emit_service_relay_failure(endpoint, target_port, route_err.reason); route_err })?; - let (mut sender, conn) = hyper::client::conn::http1::Builder::new() + let (sender, conn) = hyper::client::conn::http1::Builder::new() .handshake(TokioIo::new(relay)) .await .map_err(|err| { warn!(error = %err, "sandbox service routing: failed to start upstream HTTP client"); let route_err = ServiceRouteError::service_unreachable(); - emit_service_relay_failure(&endpoint, target_port, route_err.reason); + emit_service_relay_failure(endpoint, target_port, route_err.reason); route_err })?; @@ -386,45 +554,7 @@ async fn proxy_to_endpoint( }); } - let upstream = build_upstream_request(req, target_port, websocket_upgrade)?; - let mut response = sender.send_request(upstream).await.map_err(|err| { - warn!(error = %err, "sandbox service routing: upstream HTTP request failed"); - let route_err = ServiceRouteError::service_unreachable(); - emit_service_relay_failure(&endpoint, target_port, route_err.reason); - route_err - })?; - - if websocket_upgrade && response.status() == StatusCode::SWITCHING_PROTOCOLS { - let upstream_upgrade = hyper::upgrade::on(&mut response); - let downstream_upgrade = downstream_upgrade.ok_or_else(|| { - let err = ServiceRouteError::service_unreachable(); - emit_service_relay_failure(&endpoint, target_port, "websocket upgrade unavailable"); - err - })?; - tokio::spawn(async move { - match (downstream_upgrade.await, upstream_upgrade.await) { - (Ok(downstream), Ok(upstream)) => { - let mut downstream = TokioIo::new(downstream); - let mut upstream = TokioIo::new(upstream); - let _ = tokio::io::copy_bidirectional(&mut downstream, &mut upstream).await; - let _ = downstream.shutdown().await; - let _ = upstream.shutdown().await; - } - (Err(err), _) => { - warn!(error = %err, "sandbox service routing: downstream WebSocket upgrade failed"); - } - (_, Err(err)) => { - warn!(error = %err, "sandbox service routing: upstream WebSocket upgrade failed"); - } - } - }); - - let (parts, _) = response.into_parts(); - return Ok(Response::from_parts(parts, Body::empty())); - } - - let (parts, body) = response.into_parts(); - Ok(Response::from_parts(parts, Body::new(body))) + Ok(sender) } async fn load_endpoint( @@ -819,9 +949,9 @@ mod tests { id: "endpoint-id".to_string(), name: "my-sandbox--web".to_string(), created_time: openshell_core::time::timestamp_from_millis(1_700_000_000_000).ok(), - labels: std::collections::HashMap::default(), + labels: HashMap::default(), resource_version: 0, - annotations: std::collections::HashMap::new(), + annotations: HashMap::new(), workspace: "default".to_string(), deletion_time: None, }), @@ -1204,9 +1334,9 @@ mod tests { id: "ep-1".to_string(), name: "my-sandbox--web".to_string(), created_time: openshell_core::time::timestamp_from_millis(1_700_000_000_000).ok(), - labels: std::collections::HashMap::default(), + labels: HashMap::default(), resource_version: 0, - annotations: std::collections::HashMap::new(), + annotations: HashMap::new(), workspace: "default".to_string(), deletion_time: None, }), @@ -1227,4 +1357,131 @@ mod tests { "should not find endpoint in wrong workspace" ); } + + /// Returns a live upstream plus the sandbox end of the connection. Hold + /// the returned half: dropping it closes the upstream. + async fn test_upstream() -> (UpstreamSender, tokio::io::DuplexStream) { + let (client_io, server_io) = tokio::io::duplex(1024); + let (sender, conn) = hyper::client::conn::http1::Builder::new() + .handshake(TokioIo::new(client_io)) + .await + .unwrap(); + tokio::spawn(async move { + let _ = conn.await; + }); + tokio::time::sleep(Duration::from_millis(20)).await; + (sender, server_io) + } + + #[test] + fn upstream_pool_key_separates_endpoints_and_ports() { + assert_eq!(upstream_pool_key("ep-a", 8080), "ep-a|8080"); + assert_ne!( + upstream_pool_key("ep-a", 8080), + upstream_pool_key("ep-b", 8080) + ); + assert_ne!( + upstream_pool_key("ep-a", 8080), + upstream_pool_key("ep-a", 9090) + ); + } + + #[tokio::test] + async fn pool_hands_back_a_ready_upstream() { + let pool = ServiceUpstreamPool::default(); + let (sender, _sandbox) = test_upstream().await; + + pool.put("ep-a|8080", sender); + + assert!( + pool.take("ep-a|8080").is_some(), + "put upstream should be reusable" + ); + assert!( + pool.take("ep-a|8080").is_none(), + "an upstream in use must not be handed out twice" + ); + } + + #[tokio::test] + async fn pool_does_not_hand_out_another_endpoints_upstream() { + let pool = ServiceUpstreamPool::default(); + let (sender, _sandbox) = test_upstream().await; + + pool.put("ep-a|8080", sender); + + assert!(pool.take("ep-b|8080").is_none()); + } + + #[tokio::test] + async fn pool_drops_an_upstream_the_sandbox_closed() { + let pool = ServiceUpstreamPool::default(); + let (sender, sandbox) = test_upstream().await; + pool.put("ep-a|8080", sender); + + drop(sandbox); + tokio::time::sleep(Duration::from_millis(50)).await; + + assert!( + pool.take("ep-a|8080").is_none(), + "a closed upstream must never be reused" + ); + } + + #[tokio::test] + async fn pool_drops_upstreams_past_the_idle_timeout() { + let pool = ServiceUpstreamPool::default(); + let (sender, _sandbox) = test_upstream().await; + pool.inner.lock().unwrap().idle.insert( + "ep-a|8080".to_string(), + vec![PooledUpstream { + sender, + idle_since: Instant::now() + .checked_sub(UPSTREAM_IDLE_TIMEOUT + Duration::from_secs(1)) + .unwrap(), + }], + ); + + assert!(pool.take("ep-a|8080").is_none()); + } + + #[tokio::test] + async fn pool_caps_idle_upstreams_per_endpoint() { + let pool = ServiceUpstreamPool::default(); + let mut sandboxes = Vec::new(); + for _ in 0..(UPSTREAM_MAX_IDLE_PER_ENDPOINT + 4) { + let (sender, sandbox) = test_upstream().await; + sandboxes.push(sandbox); + pool.put("ep-a|8080", sender); + } + + let held = pool + .inner + .lock() + .unwrap() + .idle + .get("ep-a|8080") + .map_or(0, Vec::len); + assert_eq!(held, UPSTREAM_MAX_IDLE_PER_ENDPOINT); + // Every connection stayed open, so the cap dropped the surplus rather + // than the pool losing entries to closure. + assert_eq!(sandboxes.len(), UPSTREAM_MAX_IDLE_PER_ENDPOINT + 4); + } + + #[tokio::test] + async fn evict_drops_every_upstream_for_an_endpoint() { + let pool = ServiceUpstreamPool::default(); + let (first, _sandbox_a) = test_upstream().await; + let (second, _sandbox_b) = test_upstream().await; + pool.put("ep-a|8080", first); + pool.put("ep-b|8080", second); + + pool.evict("ep-a|8080"); + + assert!(pool.take("ep-a|8080").is_none()); + assert!( + pool.take("ep-b|8080").is_some(), + "eviction must be scoped to one endpoint" + ); + } } From 7085db9246ead9d28ff5adfb22d66587caf9248d Mon Sep 17 00:00:00 2001 From: Divesh Chowdary <47188680+FrostGod@users.noreply.github.com> Date: Fri, 18 Sep 2026 00:25:23 -0700 Subject: [PATCH 4/5] fix(server): address HA gateway review findings (#3449) - Let a gateway own supervisor sessions without a peer endpoint. Requiring one whenever the store is PostgreSQL broke every single-instance PostgreSQL deployment, because no sandbox supervisor could connect. A cross-replica request to an owner that advertises no endpoint now fails immediately naming the cause, instead of retrying until the wait timeout. - Close a supervisor session on heartbeat only when another replica owns it, or after renewals fail for the ownership TTL. A database error no longer drops every session heartbeating during an outage. - Clamp owner record ages at zero so a skewed or corrupt stored timestamp cannot produce a negative age. - Bound the cross-object advisory lock with a lock timeout, so a stuck holder fails instead of blocking every mutation in the fleet. - Refuse to start when a peer endpoint is configured on a multi-replica backend but peer authentication is unavailable, and warn when a multi-replica backend has no peer endpoint at all. - Reject a plaintext peer endpoint when the gateway serves TLS. - Skip the sandbox watch poller on single-replica backends, where the local update bus already sees every write. - Rate-limit the peer owner cache sweep so an insert no longer scans the whole map under the lock. - Retry GET and HEAD on a pooled upstream the sandbox closed, instead of returning 502, and drop an emptied endpoint from the pool right away. - Document the gateway peer environment variables and the post-rollout ownership skew operators should expect. Signed-off-by: divesh --- architecture/gateway.md | 15 ++- crates/openshell-server/src/compute/mod.rs | 11 +- crates/openshell-server/src/lib.rs | 98 +++++++++++++- .../src/persistence/postgres.rs | 10 ++ crates/openshell-server/src/sandbox_watch.rs | 4 + .../openshell-server/src/service_routing.rs | 94 ++++++++++++-- .../openshell-server/src/supervisor_owner.rs | 91 ++++++++++++- .../src/supervisor_session.rs | 120 ++++++++++++++---- docs/reference/gateway-config.mdx | 20 +++ 9 files changed, 410 insertions(+), 53 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index dfa49d85ad..f95674fd61 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -379,6 +379,11 @@ deadline. Each retry re-reads the owner record, so a supervisor reconnect or heartbeat can surface a new owner; if no fresh reachable owner appears before the deadline, the client operation fails rather than electing an owner itself. +Nothing redistributes established sessions, so after a rolling restart the last +surviving replica holds most sessions and a new replica serves none until +sandboxes reconnect. That skew decays only as sandboxes churn. Client traffic +stays correct throughout because a non-owner relays to the owner. + File upload and download use tar-over-SSH through the same relay path. A gateway pod termination drops the active SSH proxy byte stream, so the CLI retries the whole sync operation with a fresh SSH session instead of attempting mid-stream @@ -393,10 +398,12 @@ also trust the chart CA, present the chart-generated client certificate for mTLS, and verify the stable gateway Service DNS name even when connecting to a Deployment pod IP. -`WatchSandbox` uses the local update bus for same-replica writes. One shared -poller per gateway observes resource-version changes made by other replicas and -feeds that bus for all local watchers, avoiding a database poll per client -stream. +`WatchSandbox` uses the local update bus for same-replica writes. On +multi-replica backends one shared poller per gateway observes resource-version +changes made by other replicas and feeds that bus for all local watchers, +avoiding a database poll per client stream. SQLite deployments do not run the +poller because they are single-replica and the local bus already sees every +write. Mutations whose invariants span sandbox, provider-profile, policy, or provider records take a process-local mutex and a shared PostgreSQL advisory lock. The diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 7688884781..dc0b5a79f9 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -3378,9 +3378,9 @@ impl ComputeRuntime { else { return Ok(None); }; - let age_ms = openshell_core::time::now_ms() - owner.updated_at_ms; - let ttl_ms = i64::try_from(OWNER_TTL.as_millis()).unwrap_or(i64::MAX); - Ok((age_ms < ttl_ms).then_some(owner.supervisor_instance_id)) + Ok(owner + .is_fresh(OWNER_TTL) + .then_some(owner.supervisor_instance_id)) } async fn set_supervisor_session_state( @@ -4028,8 +4028,9 @@ impl ComputeRuntime { } let sandbox = decode_sandbox_record(¤t_record)?; - let age_ms = - openshell_core::time::now_ms().saturating_sub(current_record.created_at_ms); + let age_ms = openshell_core::time::now_ms() + .saturating_sub(current_record.created_at_ms) + .max(0); if age_ms < grace_ms { return Ok(()); } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 48e11c1dbf..ee3ef1768e 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -492,6 +492,27 @@ fn derive_peer_endpoint(config: &Config) -> Option { )) } +/// Reject a plaintext peer endpoint on a gateway that serves TLS. +/// +/// Peer relay traffic carries whole supervisor sessions between replicas. The +/// chart renders a plaintext peer endpoint only when the gateway itself serves +fn validate_peer_endpoint_scheme(config: &Config, peer_endpoint: &str) -> Result<()> { + if peer_endpoint.starts_with("https://") { + return Ok(()); + } + if config.tls.is_some() { + return Err(Error::config(format!( + "gateway peer endpoint {peer_endpoint} is plaintext but this gateway serves TLS; \ + set an https:// OPENSHELL_PEER_ENDPOINT so peer relay traffic is not downgraded" + ))); + } + warn!( + peer_endpoint, + "gateway peer relay traffic is plaintext because this gateway does not serve TLS" + ); + Ok(()) +} + /// Run the `OpenShell` server. /// /// This starts a multiplexed gRPC/HTTP server on the configured bind address. @@ -753,6 +774,20 @@ pub(crate) async fn run_server( ); } + let peer_routing_expected = state.peer_endpoint.is_some() && !state.store.is_single_replica(); + if let Some(peer_endpoint) = state.peer_endpoint.as_deref() + && peer_routing_expected + { + validate_peer_endpoint_scheme(&state.config, peer_endpoint)?; + } + if state.peer_endpoint.is_none() && !state.store.is_single_replica() { + warn!( + "no gateway peer endpoint configured; this replica owns its supervisor sessions but \ + peers cannot reach it. Single-gateway deployments are unaffected; set \ + OPENSHELL_PEER_ENDPOINT on every replica when running more than one." + ); + } + if std::env::var_os("KUBERNETES_SERVICE_HOST").is_some() { let namespace = std::env::var("OPENSHELL_POD_NAMESPACE").ok(); let service_account = std::env::var("OPENSHELL_SERVICE_ACCOUNT_NAME").ok(); @@ -787,6 +822,13 @@ pub(crate) async fn run_server( "gateway peer ServiceAccount TokenReview authentication enabled" ); } + Err(err) if peer_routing_expected => { + return Err(Error::config(format!( + "in-cluster K8s client construction failed ({err}); \ + gateway peer authentication is required because \ + OPENSHELL_PEER_ENDPOINT is configured" + ))); + } Err(err) => warn!( error = %err, "in-cluster K8s client construction failed; \ @@ -794,6 +836,14 @@ pub(crate) async fn run_server( ), } } + _ if peer_routing_expected => { + return Err(Error::config( + "OPENSHELL_POD_NAMESPACE or OPENSHELL_SERVICE_ACCOUNT_NAME missing; \ + both are required for gateway peer authentication because \ + OPENSHELL_PEER_ENDPOINT is configured" + .to_string(), + )); + } _ => { debug!( "OPENSHELL_POD_NAMESPACE or OPENSHELL_SERVICE_ACCOUNT_NAME missing; \ @@ -801,6 +851,12 @@ pub(crate) async fn run_server( ); } } + } else if peer_routing_expected { + return Err(Error::config( + "OPENSHELL_PEER_ENDPOINT is configured but the gateway is not running in a \ + Kubernetes cluster, so gateway peer authentication is unavailable" + .to_string(), + )); } let state = Arc::new(state); @@ -942,13 +998,13 @@ pub(crate) async fn run_server( } startup_tx.send_replace(true); - // The poller observes writes made by other replicas. Single-replica - // backends already see every write through the local update bus. + // The poller exists to observe writes made by other replicas. Single- + // replica backends have none, so it would only add load. if !store.is_single_replica() { sandbox_watch::spawn_store_poller( store.clone(), state.sandbox_watch_bus.clone(), - Duration::from_secs(1), + sandbox_watch::DEFAULT_STORE_POLL_INTERVAL, shutdown_rx.clone(), ); } @@ -1862,7 +1918,7 @@ mod tests { MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, bind_gateway_listener, classify_initial_bytes, configured_compute_driver, extension_token_ttl, is_benign_tls_handshake_failure, mint_gateway_extension_credential, - serve_gateway_listener, + serve_gateway_listener, validate_peer_endpoint_scheme, }; use openshell_core::{ Config, @@ -1882,6 +1938,40 @@ mod tests { use crate::tls_test_utils::generate_test_certs_with_ca; + fn tls_enabled_config() -> Config { + Config::new(Some(openshell_core::TlsConfig { + cert_path: "/tmp/cert.pem".into(), + key_path: "/tmp/key.pem".into(), + client_ca_path: None, + require_client_auth: false, + external_cert_path: None, + external_key_path: None, + external_server_names: Vec::new(), + })) + } + + #[test] + fn plaintext_peer_endpoint_is_rejected_on_a_tls_gateway() { + let error = validate_peer_endpoint_scheme(&tls_enabled_config(), "http://10.0.0.1:8080") + .expect_err("plaintext peer endpoint must not be accepted alongside gateway TLS"); + assert!( + error.to_string().contains("plaintext"), + "error should name the downgrade: {error}" + ); + } + + #[test] + fn https_peer_endpoint_is_accepted_on_a_tls_gateway() { + validate_peer_endpoint_scheme(&tls_enabled_config(), "https://10.0.0.1:8080").unwrap(); + } + + #[test] + fn plaintext_peer_endpoint_is_allowed_on_a_plaintext_gateway() { + let config = Config::new(None); + assert!(config.tls.is_none()); + validate_peer_endpoint_scheme(&config, "http://10.0.0.1:8080").unwrap(); + } + static DETECTION_PROBE_ORDER: LazyLock>> = LazyLock::new(|| Mutex::new(Vec::new())); diff --git a/crates/openshell-server/src/persistence/postgres.rs b/crates/openshell-server/src/persistence/postgres.rs index ef613a77bd..d79335b5f5 100644 --- a/crates/openshell-server/src/persistence/postgres.rs +++ b/crates/openshell-server/src/persistence/postgres.rs @@ -40,6 +40,11 @@ pub struct PostgresStore { // 64-bit advisory-lock key space. const CROSS_OBJECT_ADVISORY_LOCK_KEY: i64 = 0x4f50_454e_5348_4c4c; +// Bounds the wait for the cross-object lock. The holder only validates and +// writes, so a wait this long means a stuck replica; failing beats blocking +// every sandbox and provider mutation in the fleet indefinitely. +const CROSS_OBJECT_ADVISORY_LOCK_TIMEOUT: &str = "10s"; + pub(super) struct PostgresAdvisoryLockGuard { // `close_on_drop` is set before this guard is constructed. Closing the // dedicated session releases the session-level advisory lock even when a @@ -114,6 +119,11 @@ impl PostgresStore { ) -> PersistenceResult { let mut connection = self.pool.acquire().await.map_err(|e| map_db_error(&e))?; connection.close_on_drop(); + sqlx::query("SELECT set_config('lock_timeout', $1, false)") + .bind(CROSS_OBJECT_ADVISORY_LOCK_TIMEOUT) + .execute(&mut *connection) + .await + .map_err(|e| map_db_error(&e))?; sqlx::query("SELECT pg_advisory_lock($1)") .bind(CROSS_OBJECT_ADVISORY_LOCK_KEY) .execute(&mut *connection) diff --git a/crates/openshell-server/src/sandbox_watch.rs b/crates/openshell-server/src/sandbox_watch.rs index 0fbe485f77..3a62fce398 100644 --- a/crates/openshell-server/src/sandbox_watch.rs +++ b/crates/openshell-server/src/sandbox_watch.rs @@ -13,6 +13,10 @@ use tonic::Status; use crate::persistence::Store; use openshell_core::proto::Sandbox; +/// How often [`spawn_store_poller`] rechecks watched sandboxes for writes made +/// by other gateway replicas. +pub const DEFAULT_STORE_POLL_INTERVAL: Duration = Duration::from_secs(1); + /// Broadcast bus of sandbox updates keyed by sandbox id. /// /// Producers call [`SandboxWatchBus::notify`] whenever the persisted sandbox record changes. diff --git a/crates/openshell-server/src/service_routing.rs b/crates/openshell-server/src/service_routing.rs index 66bac3e2ab..43078c8138 100644 --- a/crates/openshell-server/src/service_routing.rs +++ b/crates/openshell-server/src/service_routing.rs @@ -100,8 +100,14 @@ impl ServiceUpstreamPool { let mut inner = self.inner.lock().unwrap(); let entries = inner.idle.get_mut(key)?; entries.retain(|entry| is_reusable(entry, now)); - let index = entries.iter().position(|entry| entry.sender.is_ready())?; - Some(entries.swap_remove(index).sender) + let taken = entries + .iter() + .position(|entry| entry.sender.is_ready()) + .map(|index| entries.swap_remove(index).sender); + if entries.is_empty() { + inner.idle.remove(key); + } + taken } fn put(&self, key: &str, sender: UpstreamSender) { @@ -435,19 +441,34 @@ async fn proxy_to_endpoint( state.service_upstreams.take(&pool_key) }; + let reused = pooled.is_some(); let mut sender = match pooled { Some(sender) => sender, None => open_upstream(&state, &sandbox, &endpoint, target_port, websocket_upgrade).await?, }; let upstream = build_upstream_request(req, target_port, websocket_upgrade)?; - let mut response = sender.send_request(upstream).await.map_err(|err| { - warn!(error = %err, "sandbox service routing: upstream HTTP request failed"); - state.service_upstreams.evict(&pool_key); - let route_err = ServiceRouteError::service_unreachable(); - emit_service_relay_failure(&endpoint, target_port, route_err.reason); - route_err - })?; + let replay = reused.then(|| replayable_request(&upstream)).flatten(); + let mut response = match sender.send_request(upstream).await { + Ok(response) => response, + Err(err) => { + warn!(error = %err, "sandbox service routing: upstream HTTP request failed"); + state.service_upstreams.evict(&pool_key); + let Some(replay) = replay else { + let route_err = ServiceRouteError::service_unreachable(); + emit_service_relay_failure(&endpoint, target_port, route_err.reason); + return Err(route_err); + }; + sender = + open_upstream(&state, &sandbox, &endpoint, target_port, websocket_upgrade).await?; + sender.send_request(replay).await.map_err(|err| { + warn!(error = %err, "sandbox service routing: upstream HTTP retry failed"); + let route_err = ServiceRouteError::service_unreachable(); + emit_service_relay_failure(&endpoint, target_port, route_err.reason); + route_err + })? + } + }; if !websocket_upgrade { state.service_upstreams.put(&pool_key, sender); @@ -574,6 +595,23 @@ async fn load_endpoint( .ok_or_else(ServiceRouteError::endpoint_not_found) } +/// Copies a request that can be sent again on a fresh connection. +/// +/// A pooled connection can be closed by the sandbox between the liveness check +/// and the send. Only bodyless methods are replayable, because the original +/// body is consumed by the failed attempt. +fn replayable_request(request: &Request) -> Option> { + if !matches!(*request.method(), Method::GET | Method::HEAD) { + return None; + } + let mut replay = Request::new(Body::empty()); + *replay.method_mut() = request.method().clone(); + *replay.uri_mut() = request.uri().clone(); + *replay.version_mut() = request.version(); + *replay.headers_mut() = request.headers().clone(); + Some(replay) +} + fn build_upstream_request( req: Request, target_port: u16, @@ -1403,6 +1441,44 @@ mod tests { ); } + #[tokio::test] + async fn take_removes_an_endpoint_left_with_no_upstreams() { + let pool = ServiceUpstreamPool::default(); + let (sender, _sandbox) = test_upstream().await; + pool.put("ep-a|8080", sender); + + assert!(pool.take("ep-a|8080").is_some()); + assert!( + !pool.inner.lock().unwrap().idle.contains_key("ep-a|8080"), + "an emptied endpoint must not linger until the next sweep" + ); + } + + #[test] + fn only_bodyless_requests_are_replayable() { + for method in [Method::GET, Method::HEAD] { + let mut request = Request::new(Body::empty()); + *request.method_mut() = method.clone(); + *request.uri_mut() = "/health".parse().unwrap(); + request + .headers_mut() + .insert(header::HOST, HeaderValue::from_static("svc")); + + let replay = + replayable_request(&request).expect("bodyless method should be replayable"); + assert_eq!(*replay.method(), method); + assert_eq!(replay.uri().path(), "/health"); + assert_eq!(replay.headers().get(header::HOST).unwrap(), "svc"); + } + + let mut post = Request::new(Body::empty()); + *post.method_mut() = Method::POST; + assert!( + replayable_request(&post).is_none(), + "a request with a body cannot be replayed" + ); + } + #[tokio::test] async fn pool_does_not_hand_out_another_endpoints_upstream() { let pool = ServiceUpstreamPool::default(); diff --git a/crates/openshell-server/src/supervisor_owner.rs b/crates/openshell-server/src/supervisor_owner.rs index 19d371d722..a517bce0fa 100644 --- a/crates/openshell-server/src/supervisor_owner.rs +++ b/crates/openshell-server/src/supervisor_owner.rs @@ -7,7 +7,7 @@ use crate::persistence::{PersistenceError, Store, WriteCondition}; use openshell_core::time::now_ms; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use thiserror::Error; const OWNER_OBJECT_TYPE: &str = "supervisor_session_owner"; @@ -28,6 +28,14 @@ pub enum OwnerError { Store(#[from] PersistenceError), } +impl OwnerError { + /// True when another replica holds ownership, as opposed to the store + /// being unreachable. + pub fn is_ownership_lost(&self) -> bool { + matches!(self, Self::AlreadyOwned | Self::Conflict) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] struct OwnerPayload { sandbox_id: String, @@ -52,6 +60,14 @@ pub struct OwnerRecord { pub resource_version: u64, } +impl OwnerRecord { + /// True while the record's last update is within `ttl`. + pub fn is_fresh(&self, ttl: Duration) -> bool { + let ttl_ms = i64::try_from(ttl.as_millis()).unwrap_or(i64::MAX); + now_ms().saturating_sub(self.updated_at_ms).max(0) < ttl_ms + } +} + #[derive(Debug, Clone)] pub struct OwnerGuard { pub sandbox_id: String, @@ -62,6 +78,15 @@ pub struct OwnerGuard { pub owner_peer_endpoint: String, connected_at_ms: i64, resource_version: u64, + last_renewed_at: Instant, +} + +impl OwnerGuard { + /// True once renewals have failed for long enough that another replica can + /// supersede this claim, making it unsafe to keep serving the session. + pub fn claim_expired(&self, ttl: Duration) -> bool { + self.last_renewed_at.elapsed() >= ttl + } } pub struct SupervisorOwnerIndex { @@ -119,6 +144,7 @@ impl SupervisorOwnerIndex { owner_peer_endpoint: owner_peer_endpoint.to_string(), connected_at_ms, resource_version: result.resource_version, + last_renewed_at: Instant::now(), }) } @@ -143,6 +169,7 @@ impl SupervisorOwnerIndex { { Ok(result) => { guard.resource_version = result.resource_version; + guard.last_renewed_at = Instant::now(); Ok(()) } Err(OwnerError::Store(PersistenceError::Conflict { .. })) => Err(OwnerError::Conflict), @@ -234,9 +261,7 @@ fn can_supersede( connection_epoch: u64, ttl: Duration, ) -> bool { - let age_ms = now_ms() - existing.updated_at_ms; - let ttl_ms = i64::try_from(ttl.as_millis()).unwrap_or(i64::MAX); - if age_ms >= ttl_ms { + if !existing.is_fresh(ttl) { return true; } @@ -253,6 +278,64 @@ mod tests { SupervisorOwnerIndex::new(store, ttl) } + fn record_updated_at(updated_at_ms: i64) -> OwnerRecord { + OwnerRecord { + session_id: "s1".to_string(), + supervisor_instance_id: "inst".to_string(), + connection_epoch: 1, + owner_replica_id: "gw-1".to_string(), + owner_peer_endpoint: "https://gw-1".to_string(), + connected_at_ms: 0, + updated_at_ms, + resource_version: 1, + } + } + + fn owner_ttl_ms() -> i64 { + i64::try_from(OWNER_TTL.as_millis()).unwrap() + } + + #[test] + fn freshness_clamps_a_future_timestamp_instead_of_going_negative() { + let skewed = record_updated_at(now_ms() + owner_ttl_ms() * 10); + assert!(skewed.is_fresh(OWNER_TTL)); + assert!(!can_supersede(&skewed, "other-inst", 99, OWNER_TTL)); + } + + #[test] + fn only_ownership_conflicts_count_as_lost_ownership() { + assert!(OwnerError::AlreadyOwned.is_ownership_lost()); + assert!(OwnerError::Conflict.is_ownership_lost()); + assert!( + !OwnerError::Store(PersistenceError::Database("db unreachable".to_string())) + .is_ownership_lost() + ); + } + + #[tokio::test] + async fn a_claim_expires_once_renewals_stop_for_the_ttl() { + let index = test_index(OWNER_TTL).await; + let guard = index + .publish("sbx", "s1", "inst", 1, "gw-1", "http://gw-1") + .await + .unwrap(); + assert!(!guard.claim_expired(OWNER_TTL)); + assert!(guard.claim_expired(Duration::ZERO)); + } + + #[test] + fn freshness_survives_a_corrupt_timestamp() { + let corrupt = record_updated_at(i64::MIN); + assert!(!corrupt.is_fresh(OWNER_TTL)); + } + + #[test] + fn freshness_expires_past_the_ttl() { + let stale = record_updated_at(now_ms() - owner_ttl_ms() - 1); + assert!(!stale.is_fresh(OWNER_TTL)); + assert!(can_supersede(&stale, "other-inst", 1, OWNER_TTL)); + } + #[tokio::test] async fn publish_creates_owner() { let index = test_index(OWNER_TTL).await; diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index e5c658fff1..e88dda3ee8 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -51,7 +51,13 @@ const PEER_TLS_KEY_FILE_ENV: &str = "OPENSHELL_PEER_TLS_KEY_FILE"; const PEER_TLS_SERVER_NAME_ENV: &str = "OPENSHELL_PEER_TLS_SERVER_NAME"; /// How long a resolved owner record is reused before rereading the store. /// Well below `OWNER_TTL` so a cache hit can never outlive the record itself. +/// Marks an owner record written by a gateway that advertises no peer endpoint. +/// Only that gateway can serve such a session, so no peer should dial it. +const LOCAL_OWNER_ENDPOINT_SCHEME: &str = "local://"; const OWNER_CACHE_TTL: Duration = Duration::from_secs(3); +/// How often the owner cache reclaims expired entries. Rate-limited so an +/// insert never scans the whole map. +const OWNER_CACHE_SWEEP_INTERVAL: Duration = Duration::from_secs(30); /// How long the projected peer `ServiceAccount` token is held in memory. /// The kubelet rotates the file hourly, so this only bounds staleness. const PEER_TOKEN_CACHE_TTL: Duration = Duration::from_mins(5); @@ -134,7 +140,13 @@ fn nonempty_env(name: &str) -> Option { pub struct PeerRouteCache { channels: Mutex>, token: Mutex>, - owners: Mutex>, + owners: Mutex, +} + +#[derive(Default)] +struct OwnerCache { + entries: HashMap, + last_sweep: Option, } /// Hand-written so the cached `ServiceAccount` token is never formatted. @@ -213,9 +225,9 @@ impl PeerRouteCache { fn cached_owner(&self, sandbox_id: &str) -> Option { let now = Instant::now(); let mut owners = self.owners.lock().unwrap(); - let entry = owners.get(sandbox_id)?; + let entry = owners.entries.get(sandbox_id)?; if entry.expires_at <= now { - owners.remove(sandbox_id); + owners.entries.remove(sandbox_id); return None; } Some(entry.record.clone()) @@ -224,8 +236,16 @@ impl PeerRouteCache { fn store_owner(&self, sandbox_id: &str, record: &crate::supervisor_owner::OwnerRecord) { let now = Instant::now(); let mut owners = self.owners.lock().unwrap(); - owners.retain(|_, entry| entry.expires_at > now); - owners.insert( + // Expiry is enforced per entry on read, so the full scan only needs to + // reclaim memory. Rate-limit it to keep inserts off an O(n) path. + if owners + .last_sweep + .is_none_or(|last| now.duration_since(last) >= OWNER_CACHE_SWEEP_INTERVAL) + { + owners.last_sweep = Some(now); + owners.entries.retain(|_, entry| entry.expires_at > now); + } + owners.entries.insert( sandbox_id.to_string(), CachedOwner { record: record.clone(), @@ -235,7 +255,7 @@ impl PeerRouteCache { } fn evict_owner(&self, sandbox_id: &str) { - self.owners.lock().unwrap().remove(sandbox_id); + self.owners.lock().unwrap().entries.remove(sandbox_id); } } @@ -1271,6 +1291,13 @@ pub async fn open_routed_relay_with_message( backoff = (backoff * 2).min(SESSION_WAIT_MAX_BACKOFF); continue; } + if owner_endpoint_is_local_only(&owner.owner_peer_endpoint) { + return Err(Status::failed_precondition(format!( + "sandbox is owned by gateway replica {} which advertises no peer endpoint; \ + set OPENSHELL_PEER_ENDPOINT on every replica to route across replicas", + owner.owner_replica_id + ))); + } match open_peer_relay( state, owner.owner_peer_endpoint.clone(), @@ -1324,9 +1351,17 @@ async fn resolve_owner( } fn owner_is_fresh(owner: &crate::supervisor_owner::OwnerRecord) -> bool { - let age_ms = openshell_core::time::now_ms() - owner.updated_at_ms; - let ttl_ms = i64::try_from(OWNER_TTL.as_millis()).unwrap_or(i64::MAX); - age_ms < ttl_ms + owner.is_fresh(OWNER_TTL) +} + +/// Endpoint recorded when this replica advertises none. +fn local_owner_endpoint(replica_id: &str) -> String { + format!("{LOCAL_OWNER_ENDPOINT_SCHEME}{replica_id}") +} + +/// True when an owner record names a gateway that no peer can dial. +fn owner_endpoint_is_local_only(endpoint: &str) -> bool { + endpoint.starts_with(LOCAL_OWNER_ENDPOINT_SCHEME) } async fn open_peer_relay( @@ -1621,17 +1656,10 @@ pub async fn handle_connect_supervisor( let provider_readiness = ProviderReadinessEvidence::from_hello(&hello)?; let session_id = Uuid::new_v4().to_string(); - let owner_peer_endpoint = state.peer_endpoint.clone().unwrap_or_default(); - if !state.store.is_single_replica() && owner_peer_endpoint.is_empty() { - return Err(Status::failed_precondition( - "gateway peer endpoint is required for multi-replica supervisor ownership", - )); - } - let owner_peer_endpoint = if owner_peer_endpoint.is_empty() { - format!("local://{}", state.replica_id) - } else { - owner_peer_endpoint - }; + let owner_peer_endpoint = state.peer_endpoint.as_deref().map_or_else( + || local_owner_endpoint(&state.replica_id), + ToString::to_string, + ); let owner_index = SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL); let owner_guard = owner_index .publish( @@ -1953,14 +1981,39 @@ async fn handle_supervisor_message( match msg.payload { Some(supervisor_message::Payload::Heartbeat(_)) => { let owner_index = SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL); - if let Err(err) = owner_index.renew(owner_guard).await { - warn!( + match owner_index.renew(owner_guard).await { + Ok(()) => {} + // Only a real ownership change ends the session. A store error + // means the database did not answer, and closing on that would + // drop every session heartbeating during the outage. + Err(err) if err.is_ownership_lost() => { + warn!( + sandbox_id = %sandbox_id, + session_id = %session_id, + error = %err, + "supervisor session: ownership lost; closing session" + ); + return false; + } + // Past the TTL our record is stale, so another replica may + // already have superseded it. Close rather than serve a + // session we can no longer claim. + Err(err) if owner_guard.claim_expired(OWNER_TTL) => { + warn!( + sandbox_id = %sandbox_id, + session_id = %session_id, + error = %err, + "supervisor session: owner renewal failed past the ownership TTL; \ + closing session" + ); + return false; + } + Err(err) => warn!( sandbox_id = %sandbox_id, session_id = %session_id, error = %err, - "supervisor session: owner renewal failed; closing session" - ); - return false; + "supervisor session: owner renewal failed; retrying on next heartbeat" + ), } } Some(supervisor_message::Payload::RelayOpenResult(result)) => { @@ -2862,6 +2915,19 @@ mod tests { } } + #[test] + fn a_gateway_without_a_peer_endpoint_still_records_ownership() { + let endpoint = local_owner_endpoint("gw-0"); + assert_eq!(endpoint, "local://gw-0"); + assert!(owner_endpoint_is_local_only(&endpoint)); + } + + #[test] + fn a_dialable_owner_endpoint_is_not_local_only() { + assert!(!owner_endpoint_is_local_only("https://10.0.0.1:8080")); + assert!(!owner_endpoint_is_local_only("http://10.0.0.1:8080")); + } + #[test] fn owner_cache_returns_stored_record() { let cache = PeerRouteCache::default(); @@ -2893,7 +2959,7 @@ mod tests { #[test] fn owner_cache_drops_entries_past_their_ttl() { let cache = PeerRouteCache::default(); - cache.owners.lock().unwrap().insert( + cache.owners.lock().unwrap().entries.insert( "sbx-a".to_string(), CachedOwner { record: owner_record("replica-a"), @@ -2902,7 +2968,7 @@ mod tests { ); assert!(cache.cached_owner("sbx-a").is_none()); - assert!(!cache.owners.lock().unwrap().contains_key("sbx-a")); + assert!(!cache.owners.lock().unwrap().entries.contains_key("sbx-a")); } #[test] diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index d614b5637e..583e3ec016 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -505,6 +505,26 @@ args = [ ] ``` +## Gateway Peer Routing + +Multi-replica gateways relay session-bound traffic to the replica that owns a sandbox supervisor. The Helm chart sets these variables; set them manually only outside the chart. + +| Variable | Purpose | +|---|---| +| `OPENSHELL_PEER_ENDPOINT` | Address other replicas use to reach this one. Derived from the pod DNS name when unset. | +| `OPENSHELL_PEER_SERVICE_NAME` | Peer Service used to derive the endpoint. | +| `OPENSHELL_POD_NAME` / `OPENSHELL_POD_NAMESPACE` | Pod identity used for endpoint derivation and peer verification. | +| `OPENSHELL_SERVICE_ACCOUNT_NAME` | ServiceAccount that peer tokens must present. | +| `OPENSHELL_PEER_SERVICE_ACCOUNT_TOKEN_FILE` | Projected peer token path. Defaults to `/var/run/secrets/openshell-peer/token`. | +| `OPENSHELL_PEER_TOKEN_AUDIENCE` | Required token audience. Defaults to `openshell-gateway-peer`. | +| `OPENSHELL_PEER_POD_LABELS` | Comma-separated labels a calling pod must carry. | +| `OPENSHELL_PEER_TOKEN_CACHE_TTL_SECS` | How long a verified peer identity is cached. Defaults to `60`. | +| `OPENSHELL_PEER_TLS_CA_FILE` | CA bundle for peer connections. Falls back to the platform trust store. | +| `OPENSHELL_PEER_TLS_CERT_FILE` / `OPENSHELL_PEER_TLS_KEY_FILE` | Client certificate and key for peer mTLS. Set both or neither. | +| `OPENSHELL_PEER_TLS_SERVER_NAME` | Name verified on peer certificates. | + +The gateway refuses to start when a peer endpoint is configured on a multi-replica backend and peer authentication is unavailable, because the replica could not serve relays from its peers. That happens when the pod identity variables are missing, when the in-cluster Kubernetes client cannot be built, or when the gateway is not running in a cluster. A plaintext `http://` peer endpoint is also rejected when the gateway serves TLS; use it only when the gateway itself serves plaintext. + ## Driver References Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. From 9dacd76628dc70350c4e302a8caa92fd816b4109 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 21 Sep 2026 12:15:54 -0700 Subject: [PATCH 5/5] fix(server): harden HA supervisor ownership Signed-off-by: Drew Newberry --- architecture/gateway.md | 15 +- .../tests/ensure_providers_integration.rs | 21 ++ .../openshell-cli/tests/mtls_integration.rs | 21 ++ .../tests/provider_commands_integration.rs | 21 ++ .../sandbox_create_lifecycle_integration.rs | 21 ++ .../sandbox_name_fallback_integration.rs | 21 ++ crates/openshell-sdk/tests/client_mock.rs | 21 ++ .../openshell-server/src/auth/method_authz.rs | 11 +- crates/openshell-server/src/auth/peer.rs | 34 ++- crates/openshell-server/src/grpc/mod.rs | 21 ++ crates/openshell-server/src/grpc/policy.rs | 20 +- .../src/grpc/policy/endpoint_status.rs | 59 ++++- .../src/grpc/provider_readiness.rs | 86 ++++++- crates/openshell-server/src/grpc/sandbox.rs | 16 +- crates/openshell-server/src/multiplex.rs | 2 +- .../openshell-server/src/service_routing.rs | 40 +++- crates/openshell-server/src/storage_proto.rs | 19 +- .../src/supervisor_session.rs | 165 +++++++++++-- crates/openshell-server/tests/common/mod.rs | 21 ++ .../tests/supervisor_relay_integration.rs | 21 ++ proto/openshell.proto | 22 ++ sdk/go/proto/openshellv1/openshell.pb.go | 219 ++++++++++-------- sdk/go/proto/openshellv1/openshell_grpc.pb.go | 116 ++++++++++ 23 files changed, 873 insertions(+), 140 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index f95674fd61..8b9e70dfa1 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -378,6 +378,11 @@ the serving gateway retries ownership lookup until the normal relay wait deadline. Each retry re-reads the owner record, so a supervisor reconnect or heartbeat can surface a new owner; if no fresh reachable owner appears before the deadline, the client operation fails rather than electing an owner itself. +Provider-readiness reports, endpoint-status reports, and provider-status reads +also follow the durable owner record through unary peer RPCs. The owning replica +validates the current supervisor session and keeps the in-memory evidence; a +non-owner never accepts evidence from a stale local session or projects a +remote session as disconnected. Nothing redistributes established sessions, so after a rolling restart the last surviving replica holds most sessions and a new replica serves none until @@ -393,7 +398,7 @@ Gateway peer RPCs authenticate with Kubernetes ServiceAccount identity rather than a shared secret. Helm mounts a projected, pod-bound token with audience `openshell-gateway-peer`; the receiving gateway validates it through TokenReview, checks the live pod UID and chart selector labels, and authorizes -only the internal peer relay method. When gateway TLS is enabled, peer clients +only the internal peer RPC methods. When gateway TLS is enabled, peer clients also trust the chart CA, present the chart-generated client certificate for mTLS, and verify the stable gateway Service DNS name even when connecting to a Deployment pod IP. @@ -867,7 +872,13 @@ Provider receipts, installation status, and common operations represent absolute Provider installation reports belong to the existing `ConnectSupervisor` session. Each report names that session, has an increasing sequence, and expires unless the supervisor reports again. Reconnection or disconnect invalidates prior observations; stored change records survive a gateway restart, but runtime evidence does not. Replaying an identical report cannot extend its lifetime. -Reports and status also compare the supervisor instance with the sandbox's persisted current instance. A different supervisor becoming current invalidates an older connection, including one retained by another gateway replica. Observations stay local to the gateway holding the supervisor session; a status request reaching a replica without that session returns pending. Multi-replica deployments therefore retain the existing supervisor-session routing requirement. +Reports and status also compare the supervisor instance with the sandbox's +persisted current instance. A different supervisor becoming current invalidates +an older connection, including one retained by another gateway replica. +Observations stay local to the gateway holding the supervisor session. A status +request or report reaching another replica follows the shared owner record to +that gateway, which remains the sole authority for accepting and projecting the +session's evidence. The supervisor reports success only after it installs the matching credentials, activates the effective policy, and receives an acknowledgment from the authenticated workload boundary that it installed the environment for future processes. Environment synchronization shares the process-launch lock, and its acknowledgment identifies the exact publication, including retries at the same provider revision. Failed policy installation cannot reuse evidence for a different installed policy. Ready and revoked statuses also recheck the requested sandbox, provider, attachment and configuration identities; revision fingerprints are compared only for equality. Revocation applies to future credential resolution and future processes. Requests already forwarded upstream can still finish. diff --git a/crates/openshell-cli/tests/ensure_providers_integration.rs b/crates/openshell-cli/tests/ensure_providers_integration.rs index 374301c961..e7221e0079 100644 --- a/crates/openshell-cli/tests/ensure_providers_integration.rs +++ b/crates/openshell-cli/tests/ensure_providers_integration.rs @@ -79,6 +79,27 @@ impl TestOpenShell { #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn peer_report_provider_readiness( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_report_endpoint_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_get_sandbox_provider_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + async fn report_endpoint_status( &self, _request: tonic::Request, diff --git a/crates/openshell-cli/tests/mtls_integration.rs b/crates/openshell-cli/tests/mtls_integration.rs index cff6a46f5c..ca969dde13 100644 --- a/crates/openshell-cli/tests/mtls_integration.rs +++ b/crates/openshell-cli/tests/mtls_integration.rs @@ -32,6 +32,27 @@ struct TestOpenShell; #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn peer_report_provider_readiness( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_report_endpoint_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_get_sandbox_provider_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + async fn report_endpoint_status( &self, _request: tonic::Request, diff --git a/crates/openshell-cli/tests/provider_commands_integration.rs b/crates/openshell-cli/tests/provider_commands_integration.rs index ace3827d2e..7d5a047acd 100644 --- a/crates/openshell-cli/tests/provider_commands_integration.rs +++ b/crates/openshell-cli/tests/provider_commands_integration.rs @@ -221,6 +221,27 @@ impl TestOpenShell { #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn peer_report_provider_readiness( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_report_endpoint_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_get_sandbox_provider_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + async fn report_endpoint_status( &self, _request: tonic::Request, diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index e46cd4345a..f1408493bc 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -92,6 +92,27 @@ struct TestOpenShell { #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn peer_report_provider_readiness( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_report_endpoint_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_get_sandbox_provider_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + async fn begin_rootfs_tar_staging( &self, _request: tonic::Request, diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs index 0f446d8f37..7a5780d7f3 100644 --- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs +++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs @@ -47,6 +47,27 @@ struct TestOpenShell { #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn peer_report_provider_readiness( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_report_endpoint_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_get_sandbox_provider_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + async fn report_endpoint_status( &self, _request: tonic::Request, diff --git a/crates/openshell-sdk/tests/client_mock.rs b/crates/openshell-sdk/tests/client_mock.rs index 8e4ea7df32..2bf67ddc60 100644 --- a/crates/openshell-sdk/tests/client_mock.rs +++ b/crates/openshell-sdk/tests/client_mock.rs @@ -161,6 +161,27 @@ fn workload_template_proto(name: &str, workspace: &str) -> proto::SandboxWorkloa #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn peer_report_provider_readiness( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_report_endpoint_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_get_sandbox_provider_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + async fn report_endpoint_status( &self, _request: tonic::Request, diff --git a/crates/openshell-server/src/auth/method_authz.rs b/crates/openshell-server/src/auth/method_authz.rs index 4d91408644..06f6784cc8 100644 --- a/crates/openshell-server/src/auth/method_authz.rs +++ b/crates/openshell-server/src/auth/method_authz.rs @@ -150,8 +150,15 @@ mod tests { "/openshell.v1.OpenShell/ConnectSupervisor" )); assert!(!is_user_callable("/openshell.v1.OpenShell/RelayStream")); - assert!(!is_user_callable("/openshell.v1.OpenShell/PeerRelay")); - assert!(is_peer_callable("/openshell.v1.OpenShell/PeerRelay")); + for path in [ + "/openshell.v1.OpenShell/PeerRelay", + "/openshell.v1.OpenShell/PeerReportProviderReadiness", + "/openshell.v1.OpenShell/PeerReportEndpointStatus", + "/openshell.v1.OpenShell/PeerGetSandboxProviderStatus", + ] { + assert!(!is_user_callable(path)); + assert!(is_peer_callable(path)); + } // Unauthenticated methods are not "user callable" — they're // intercepted before principal evaluation. assert!(!is_user_callable("/openshell.v1.OpenShell/Health")); diff --git a/crates/openshell-server/src/auth/peer.rs b/crates/openshell-server/src/auth/peer.rs index 79dbda132a..e9053649a8 100644 --- a/crates/openshell-server/src/auth/peer.rs +++ b/crates/openshell-server/src/auth/peer.rs @@ -27,6 +27,7 @@ use tonic::Status; use tracing::{debug, info, warn}; /// gRPC path for internal gateway relay forwarding. +#[cfg(test)] pub const PEER_RELAY_PATH: &str = "/openshell.v1.OpenShell/PeerRelay"; /// Audience used for gateway-to-gateway projected `ServiceAccount` tokens. pub const DEFAULT_PEER_TOKEN_AUDIENCE: &str = "openshell-gateway-peer"; @@ -88,7 +89,7 @@ impl Authenticator for PeerServiceAccountAuthenticator { headers: &http::HeaderMap, path: &str, ) -> Result, Status> { - if path != PEER_RELAY_PATH { + if !crate::auth::method_authz::is_peer_callable(path) { return Ok(None); } @@ -698,6 +699,37 @@ mod tests { assert_eq!(peer.pod_uid, "uid-a"); } + #[tokio::test] + async fn authenticator_accepts_each_peer_rpc_and_ignores_non_peer_rpcs() { + let resolver = Arc::new(FakeGatewayPeerResolver::returning(Ok(Some( + ResolvedGatewayPeerIdentity { + pod_name: "openshell-0".to_string(), + pod_uid: "uid-a".to_string(), + }, + )))); + let auth = PeerServiceAccountAuthenticator::new(resolver.clone()); + let headers = bearer_headers("token-a"); + + for path in [ + PEER_RELAY_PATH, + "/openshell.v1.OpenShell/PeerReportProviderReadiness", + "/openshell.v1.OpenShell/PeerReportEndpointStatus", + "/openshell.v1.OpenShell/PeerGetSandboxProviderStatus", + ] { + assert!(matches!( + auth.authenticate(&headers, path).await.unwrap(), + Some(Principal::Peer(_)) + )); + } + assert!( + auth.authenticate(&headers, "/openshell.v1.OpenShell/ListSandboxes") + .await + .unwrap() + .is_none() + ); + assert_eq!(resolver.seen_tokens.lock().unwrap().len(), 4); + } + fn identity() -> ResolvedGatewayPeerIdentity { ResolvedGatewayPeerIdentity { pod_name: "openshell-0".to_string(), diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index 337b515445..fcca8e2bd1 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -843,6 +843,27 @@ impl OpenShell for OpenShellService { ) -> Result, Status> { crate::supervisor_session::handle_peer_relay(&self.state, request).await } + + async fn peer_report_provider_readiness( + &self, + request: Request, + ) -> Result, Status> { + provider_readiness::handle_peer_report_provider_readiness(&self.state, request).await + } + + async fn peer_report_endpoint_status( + &self, + request: Request, + ) -> Result, Status> { + policy::handle_peer_report_endpoint_status(&self.state, request).await + } + + async fn peer_get_sandbox_provider_status( + &self, + request: Request, + ) -> Result, Status> { + provider_readiness::handle_peer_get_sandbox_provider_status(&self.state, request).await + } } fn public_resource_capabilities( diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 97218d79a6..3324813ad6 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -14,7 +14,9 @@ mod endpoint_status; mod provisioning_clock; pub use provisioning_clock::configuration_change; -pub(super) use endpoint_status::handle_report_endpoint_status; +pub(super) use endpoint_status::{ + handle_peer_report_endpoint_status, handle_report_endpoint_status, +}; pub use endpoint_status::{ invalidate_endpoint_status_on_startup, reset_endpoint_status_for_supervisor_session, retry_endpoint_status_after_supervisor_disconnect, @@ -3601,7 +3603,9 @@ async fn handle_update_config_inner( )); } let _settings_guard = state.settings_mutex.lock().await; - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::persistence_error_to_status(error, "acquire policy mutation lock") + })?; if has_merge_ops { return Err(Status::invalid_argument( @@ -3790,7 +3794,9 @@ async fn handle_update_config_inner( if has_setting { let _settings_guard = state.settings_mutex.lock().await; - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::persistence_error_to_status(error, "acquire policy mutation lock") + })?; if key == POLICY_SETTING_KEY { return Err(Status::invalid_argument( @@ -3885,7 +3891,9 @@ async fn handle_update_config_inner( )); } - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::persistence_error_to_status(error, "acquire policy mutation lock") + })?; if has_merge_ops { let global_settings = load_global_settings(state.store.as_ref()).await?; if global_settings.settings.contains_key(POLICY_SETTING_KEY) { @@ -4416,7 +4424,9 @@ pub(super) async fn handle_report_sandbox_configuration( if reported == ConfigurationAdmissionState::Unspecified { return Err(Status::invalid_argument("admission state is required")); } - let _guard = state.compute.sandbox_sync_guard().await; + let _guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::persistence_error_to_status(error, "acquire configuration admission lock") + })?; let mut sandbox = state .store .get_message::(&sandbox_id) diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status.rs b/crates/openshell-server/src/grpc/policy/endpoint_status.rs index 82128e1daa..462099198b 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status.rs @@ -13,6 +13,7 @@ use crate::ServerState; use crate::persistence::{ObjectId, ObjectWorkspace}; use crate::policy_store::PolicyStoreExt; use crate::provider_profile_sources::EffectiveProviderProfileCatalog; +use crate::supervisor_owner::{OWNER_TTL, SupervisorOwnerIndex}; use crate::supervisor_session::EndpointReportCursor; use openshell_core::GetResourceVersion; use openshell_core::endpoint_status::initial_endpoint_status; @@ -61,6 +62,37 @@ pub(in crate::grpc) async fn handle_report_endpoint_status( let sandbox_id = request.get_ref().sandbox_id.clone(); crate::auth::guard::enforce_sandbox_scope(&request, &sandbox_id)?; let req = request.into_inner(); + if let Some(owner) = + crate::supervisor_session::remote_supervisor_owner(state, &sandbox_id).await? + { + let response = + crate::supervisor_session::forward_endpoint_status_to_owner(state, &owner, req).await?; + return Ok(Response::new(response)); + } + handle_report_endpoint_status_inner(state, req).await +} + +pub(in crate::grpc) async fn handle_peer_report_endpoint_status( + state: &Arc, + request: Request, +) -> Result, Status> { + if !matches!( + request + .extensions() + .get::(), + Some(crate::auth::principal::Principal::Peer(_)) + ) { + return Err(Status::permission_denied( + "gateway peer principal is required", + )); + } + handle_report_endpoint_status_inner(state, request.into_inner()).await +} + +async fn handle_report_endpoint_status_inner( + state: &Arc, + req: ReportEndpointStatusRequest, +) -> Result, Status> { if req.sandbox_id.is_empty() { return Err(Status::invalid_argument("sandbox_id is required")); } @@ -387,6 +419,9 @@ pub async fn reset_endpoint_status_after_supervisor_disconnect( { return Ok(()); } + if has_fresh_shared_owner(state, sandbox_id).await? { + return Ok(()); + } let sandbox = state .store .get_message::(sandbox_id) @@ -447,9 +482,15 @@ pub async fn retry_endpoint_status_after_supervisor_disconnect( /// Invalidate endpoint results left by sessions from an earlier gateway process. /// /// Supervisor sessions are intentionally process-local. This reconciliation -/// runs before gateway listeners are bound, so persisted success can never be -/// served without a session in the current process that owns the observation. +/// runs before gateway listeners are bound. A fresh shared owner preserves its +/// evidence; records without one are reset so stale success is never served. pub async fn invalidate_endpoint_status_on_startup(state: &Arc) -> Result<(), Status> { + let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { + super::super::persistence_error_to_status( + error, + "acquire endpoint status startup reconciliation lock", + ) + })?; let mut offset = 0; loop { let sandboxes = state @@ -474,6 +515,9 @@ pub async fn invalidate_endpoint_status_on_startup(state: &Arc) -> continue; } let sandbox_id = sandbox.object_id(); + if has_fresh_shared_owner(state, sandbox_id).await? { + continue; + } let expected_resource_version = sandbox.get_resource_version(); let updated = state .store @@ -504,6 +548,17 @@ pub async fn invalidate_endpoint_status_on_startup(state: &Arc) -> } } +async fn has_fresh_shared_owner( + state: &Arc, + sandbox_id: &str, +) -> Result { + SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL) + .read(sandbox_id) + .await + .map(|owner| owner.is_some_and(|owner| owner.is_fresh(OWNER_TTL))) + .map_err(|error| Status::unavailable(format!("resolve supervisor owner failed: {error}"))) +} + fn invalidate_endpoint_status_without_session(sandbox: &mut Sandbox) { let Some(status) = sandbox.status.as_mut() else { return; diff --git a/crates/openshell-server/src/grpc/provider_readiness.rs b/crates/openshell-server/src/grpc/provider_readiness.rs index 881975a867..3de9148b27 100644 --- a/crates/openshell-server/src/grpc/provider_readiness.rs +++ b/crates/openshell-server/src/grpc/provider_readiness.rs @@ -307,6 +307,26 @@ pub(super) async fn handle_get_sandbox_provider_status( MinWorkspaceRole::User, ) .await?; + if let Some(owner) = + crate::supervisor_session::remote_supervisor_owner(state, sandbox.object_id()).await? + { + let response = crate::supervisor_session::forward_provider_status_query_to_owner( + state, + &owner, + sandbox.object_id(), + request, + ) + .await?; + return Ok(Response::new(response)); + } + handle_get_sandbox_provider_status_resolved(state, request, sandbox).await +} + +async fn handle_get_sandbox_provider_status_resolved( + state: &Arc, + request: GetSandboxProviderStatusRequest, + sandbox: Sandbox, +) -> Result, Status> { let workspace = sandbox.object_workspace().to_string(); // Validate the selector before it can contribute to a durable observation. if request.provider.len() > super::MAX_NAME_LEN { @@ -408,6 +428,31 @@ pub(super) async fn handle_get_sandbox_provider_status( })) } +pub(super) async fn handle_peer_get_sandbox_provider_status( + state: &Arc, + request: Request, +) -> Result, Status> { + ensure_peer_request(&request)?; + let request = request.into_inner(); + let workspace = + crate::auth::workspace_authz::selected_workspace_name(request.workspace_scope.as_ref())?; + if request.sandbox.is_empty() { + return Err(Status::invalid_argument("sandbox is required")); + } + let sandbox = state + .store + .get_message_by_name::(workspace, &request.sandbox) + .await + .map_err(|error| Status::internal(format!("fetch sandbox failed: {error}")))? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + if !state.supervisor_sessions.has_session(sandbox.object_id()) { + return Err(Status::failed_precondition( + "gateway peer does not own the sandbox supervisor session", + )); + } + handle_get_sandbox_provider_status_resolved(state, request, sandbox).await +} + fn observation_matches( observation: &ProviderReadinessObservation, desired: &ProviderDesiredIdentity, @@ -568,9 +613,33 @@ pub(super) async fn handle_report_provider_readiness( ) -> Result, Status> { let sandbox_id = request.get_ref().sandbox_id.clone(); authorize_provider_readiness(&request, &sandbox_id)?; + let request = request.into_inner(); + if let Some(owner) = + crate::supervisor_session::remote_supervisor_owner(state, &sandbox_id).await? + { + let response = + crate::supervisor_session::forward_provider_readiness_to_owner(state, &owner, request) + .await?; + return Ok(Response::new(response)); + } + handle_report_provider_readiness_inner(state, request).await +} + +pub(super) async fn handle_peer_report_provider_readiness( + state: &Arc, + request: Request, +) -> Result, Status> { + ensure_peer_request(&request)?; + handle_report_provider_readiness_inner(state, request.into_inner()).await +} + +async fn handle_report_provider_readiness_inner( + state: &Arc, + request: ReportProviderReadinessRequest, +) -> Result, Status> { + let sandbox_id = request.sandbox_id.clone(); canonical_uuid(&sandbox_id)?; let observation = request - .into_inner() .observation .ok_or_else(|| Status::invalid_argument("provider readiness observation is required"))?; canonical_uuid(&observation.session_id)?; @@ -615,6 +684,21 @@ pub(super) async fn handle_report_provider_readiness( })) } +fn ensure_peer_request(request: &Request) -> Result<(), Status> { + if matches!( + request + .extensions() + .get::(), + Some(crate::auth::principal::Principal::Peer(_)) + ) { + Ok(()) + } else { + Err(Status::permission_denied( + "gateway peer principal is required", + )) + } +} + #[cfg(test)] #[path = "provider_readiness_tests.rs"] mod tests; diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 39edede7ee..bffdc6ebbf 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -1235,7 +1235,10 @@ pub(super) async fn handle_attach_sandbox_provider( if let Some(probe) = attach_wait_probe { probe.notify_one(); } - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = + state.compute.sandbox_sync_guard().await.map_err(|err| { + super::persistence_error_to_status(err, "acquire sandbox mutation lock") + })?; let provider_record = get_provider_record(state.store.as_ref(), &workspace, &request.provider) .await .map_err(|err| { @@ -1405,7 +1408,10 @@ pub(super) async fn handle_detach_sandbox_provider( ))); } - let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await; + let _sandbox_sync_guard = + state.compute.sandbox_sync_guard().await.map_err(|err| { + super::persistence_error_to_status(err, "acquire sandbox mutation lock") + })?; let sandbox_name = sandbox.object_name().to_string(); let sandbox_id = sandbox .metadata @@ -1637,9 +1643,13 @@ async fn handle_start_sandbox_inner( state.sandbox_session_jwt_authority.as_deref(), ) .await?; + let remote_authority = + crate::supervisor_session::remote_supervisor_owner(state, sandbox.object_id()) + .await? + .is_some(); state .supervisor_sessions - .project_endpoint_status(&mut sandbox); + .project_endpoint_status(&mut sandbox, remote_authority); info!(sandbox_name = %name, "StartSandbox request completed successfully"); Ok(Response::new(SandboxResponse { sandbox: Some(sandbox), diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 3309cad01f..713054b7fe 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -860,7 +860,7 @@ where /// Assemble the authenticator chain for the gateway. /// /// Chain order (first-match-wins): -/// 1. `PeerServiceAccountAuthenticator` (path-scoped to `PeerRelay`) +/// 1. `PeerServiceAccountAuthenticator` (path-scoped to peer RPCs) /// — validates gateway replica projected `ServiceAccount` tokens with /// `TokenReview` for internal peer relay calls. No-op on every other path. /// 2. `ComputeDriverAuthenticator` (path-scoped to `IssueSandboxToken`) diff --git a/crates/openshell-server/src/service_routing.rs b/crates/openshell-server/src/service_routing.rs index 43078c8138..3d49d74d70 100644 --- a/crates/openshell-server/src/service_routing.rs +++ b/crates/openshell-server/src/service_routing.rs @@ -449,19 +449,30 @@ async fn proxy_to_endpoint( let upstream = build_upstream_request(req, target_port, websocket_upgrade)?; let replay = reused.then(|| replayable_request(&upstream)).flatten(); - let mut response = match sender.send_request(upstream).await { + let first_attempt = if reused { + sender.try_send_request(upstream).await.map_err(|mut err| { + let request = err.take_message(); + (err.into_error(), request) + }) + } else { + sender + .send_request(upstream) + .await + .map_err(|err| (err, None)) + }; + let mut response = match first_attempt { Ok(response) => response, - Err(err) => { + Err((err, recovered)) => { warn!(error = %err, "sandbox service routing: upstream HTTP request failed"); state.service_upstreams.evict(&pool_key); - let Some(replay) = replay else { + let Some(retry_request) = recovered.or(replay) else { let route_err = ServiceRouteError::service_unreachable(); emit_service_relay_failure(&endpoint, target_port, route_err.reason); return Err(route_err); }; sender = open_upstream(&state, &sandbox, &endpoint, target_port, websocket_upgrade).await?; - sender.send_request(replay).await.map_err(|err| { + sender.send_request(retry_request).await.map_err(|err| { warn!(error = %err, "sandbox service routing: upstream HTTP retry failed"); let route_err = ServiceRouteError::service_unreachable(); emit_service_relay_failure(&endpoint, target_port, route_err.reason); @@ -1504,6 +1515,27 @@ mod tests { ); } + #[tokio::test] + async fn closed_upstream_recovers_an_unsent_post_for_retry() { + let (mut sender, sandbox) = test_upstream().await; + drop(sandbox); + tokio::time::sleep(Duration::from_millis(50)).await; + + let mut request = Request::new(Body::from("payload")); + *request.method_mut() = Method::POST; + *request.uri_mut() = "/write".parse().unwrap(); + let mut error = sender + .try_send_request(request) + .await + .expect_err("a closed connection must reject the request"); + let recovered = error + .take_message() + .expect("an unsent request must be recoverable for a fresh connection"); + + assert_eq!(recovered.method(), Method::POST); + assert_eq!(recovered.uri().path(), "/write"); + } + #[tokio::test] async fn pool_drops_upstreams_past_the_idle_timeout() { let pool = ServiceUpstreamPool::default(); diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index feb634f056..b768b2c216 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -119,7 +119,7 @@ mod tests { const STORAGE_V1_SCHEMA_SHA256: &str = "d68401809d8cea445c35233ef32412bbd041cb2ac5acaf368a0d0bf74d2ddf17"; const PUBLIC_RPC_SCHEMA_SHA256: &str = - "567f060e7b4f5ead636817ee7c41728688b39de16d2ef00396d1b562bdb83045"; + "60ae9faddbc4e8ebb7e22d8aa61cdb8362d4cb7ac8602e2bbc178080fdf566ec"; const DURABLE_SCHEMA_SHA256: &str = "9eeaa29dfba187bff69fb7bc4f9a13a0f1d7be3f7049a38c8f0e20ce77ec7d8b"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = @@ -158,6 +158,11 @@ mod tests { "openshell.v1.OpenShell/GetSandboxProviderStatus|.openshell.v1.GetSandboxProviderStatusRequest|.openshell.v1.GetSandboxProviderStatusResponse|false|false", "openshell.v1.OpenShell/ReportProviderReadiness|.openshell.v1.ReportProviderReadinessRequest|.openshell.v1.ReportProviderReadinessResponse|false|false", ]; + const PEER_OWNER_RPC_SIGNATURES: [&str; 3] = [ + "openshell.v1.OpenShell/PeerGetSandboxProviderStatus|.openshell.v1.GetSandboxProviderStatusRequest|.openshell.v1.GetSandboxProviderStatusResponse|false|false", + "openshell.v1.OpenShell/PeerReportEndpointStatus|.openshell.v1.ReportEndpointStatusRequest|.openshell.v1.ReportEndpointStatusResponse|false|false", + "openshell.v1.OpenShell/PeerReportProviderReadiness|.openshell.v1.ReportProviderReadinessRequest|.openshell.v1.ReportProviderReadinessResponse|false|false", + ]; // Synthetic SandboxSpec bytes with log level, provider, and command fields, // emitted before the gateway-owned attachment epoch field was introduced. const PRE_READINESS_SANDBOX_SPEC: &str = @@ -516,14 +521,20 @@ mod tests { "provider readiness RPC is missing or changed: {signature}" ); } + for signature in PEER_OWNER_RPC_SIGNATURES { + assert!( + methods.iter().any(|method| method == signature), + "peer owner RPC is missing or changed: {signature}" + ); + } assert_eq!( compiled_method_count, - 102 + PROVIDER_READINESS_RPC_SIGNATURES.len(), + 102 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len(), "classify every compiled RPC" ); assert_eq!( methods.len(), - 77 + PROVIDER_READINESS_RPC_SIGNATURES.len(), + 77 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len(), "inventory every public gateway RPC" ); assert_eq!( @@ -531,7 +542,7 @@ mod tests { .iter() .filter(|method| method.starts_with("openshell.v1.OpenShell/")) .count(), - 77 + PROVIDER_READINESS_RPC_SIGNATURES.len() + 77 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len() ); assert!(methods.iter().all(|method| !method.contains(".storage."))); diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index e88dda3ee8..a1140e9fd3 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -16,9 +16,11 @@ use tracing::{debug, info, warn}; use uuid::Uuid; use openshell_core::proto::{ - GatewayMessage, PeerRelayFrame, PeerRelayInit, ProviderReadinessObservation, RelayFrame, - RelayInit, RelayOpen, ReportMainProcessExitRequest, ReportMainProcessExitResponse, Sandbox, - SandboxPhase, SessionAccepted, SshRelayTarget, SupervisorMessage, gateway_message, + GatewayMessage, GetSandboxProviderStatusRequest, GetSandboxProviderStatusResponse, + PeerRelayFrame, PeerRelayInit, ProviderReadinessObservation, RelayFrame, RelayInit, RelayOpen, + ReportEndpointStatusRequest, ReportEndpointStatusResponse, ReportMainProcessExitRequest, + ReportMainProcessExitResponse, ReportProviderReadinessRequest, ReportProviderReadinessResponse, + Sandbox, SandboxPhase, SessionAccepted, SshRelayTarget, SupervisorMessage, gateway_message, open_shell_client, peer_relay_frame, relay_open, supervisor_message, }; use openshell_core::transport_errors::is_expected_transport_close_status; @@ -30,6 +32,7 @@ use crate::persistence::ObjectId; use crate::supervisor_owner::{OWNER_TTL, OwnerError, OwnerGuard, SupervisorOwnerIndex}; const HEARTBEAT_INTERVAL_SECS: u32 = 15; +const OWNER_RENEW_TIMEOUT: Duration = Duration::from_secs(5); const RELAY_PENDING_TIMEOUT: Duration = Duration::from_secs(10); /// Initial backoff between session-availability polls in `wait_for_session`. const SESSION_WAIT_INITIAL_BACKOFF: Duration = Duration::from_millis(100); @@ -594,7 +597,7 @@ impl SupervisorSessionRegistry { /// Fail closed when projecting persisted endpoint status without a live, /// initialized observation authority in the current gateway process. - pub(crate) fn project_endpoint_status(&self, sandbox: &mut Sandbox) { + pub(crate) fn project_endpoint_status(&self, sandbox: &mut Sandbox, remote_authority: bool) { let sandbox_id = sandbox.object_id(); let has_authority = self .sessions @@ -602,7 +605,7 @@ impl SupervisorSessionRegistry { .unwrap() .get(sandbox_id) .is_some_and(|session| session.endpoint_status_initialized); - if has_authority { + if has_authority || remote_authority { return; } let Some(status) = sandbox.status.as_mut() else { @@ -1215,6 +1218,109 @@ async fn build_peer_channel(endpoint: &str) -> Result { .map_err(|err| Status::unavailable(format!("gateway peer connection failed: {err}"))) } +async fn peer_rpc_client( + state: &Arc, + endpoint: &str, +) -> Result< + open_shell_client::OpenShellClient< + tonic::service::interceptor::InterceptedService, + >, + Status, +> { + let token = state.peer_routes.peer_token().await?; + let channel = state.peer_routes.channel(endpoint).await?; + let interceptor = PeerAuthInterceptor::new(&token, &state.replica_id)?; + Ok(open_shell_client::OpenShellClient::with_interceptor( + channel, + interceptor, + )) +} + +pub(crate) async fn remote_supervisor_owner( + state: &Arc, + sandbox_id: &str, +) -> Result, Status> { + let owner_index = SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL); + // Supervisor-bound state must follow the durable owner record. A cached + // local owner can remain valid for relay discovery while ownership has + // already moved, but it must never authorize a stale local session to + // publish readiness or endpoint observations. + let Some(owner) = owner_index + .read(sandbox_id) + .await + .map_err(owner_error_to_status)? + else { + state.peer_routes.evict_owner(sandbox_id); + return Ok(None); + }; + if !owner_is_fresh(&owner) { + state.peer_routes.evict_owner(sandbox_id); + return Ok(None); + } + state.peer_routes.store_owner(sandbox_id, &owner); + if owner.owner_replica_id == state.replica_id { + return Ok(None); + } + if owner_endpoint_is_local_only(&owner.owner_peer_endpoint) { + return Err(Status::failed_precondition(format!( + "sandbox is owned by gateway replica {} which advertises no peer endpoint", + owner.owner_replica_id + ))); + } + Ok(Some(owner)) +} + +pub(crate) async fn forward_provider_readiness_to_owner( + state: &Arc, + owner: &crate::supervisor_owner::OwnerRecord, + request: ReportProviderReadinessRequest, +) -> Result { + let sandbox_id = request.sandbox_id.clone(); + let mut client = peer_rpc_client(state, &owner.owner_peer_endpoint).await?; + client + .peer_report_provider_readiness(request) + .await + .map(Response::into_inner) + .inspect_err(|_| { + state.peer_routes.evict_channel(&owner.owner_peer_endpoint); + state.peer_routes.evict_owner(&sandbox_id); + }) +} + +pub(crate) async fn forward_endpoint_status_to_owner( + state: &Arc, + owner: &crate::supervisor_owner::OwnerRecord, + request: ReportEndpointStatusRequest, +) -> Result { + let sandbox_id = request.sandbox_id.clone(); + let mut client = peer_rpc_client(state, &owner.owner_peer_endpoint).await?; + client + .peer_report_endpoint_status(request) + .await + .map(Response::into_inner) + .inspect_err(|_| { + state.peer_routes.evict_channel(&owner.owner_peer_endpoint); + state.peer_routes.evict_owner(&sandbox_id); + }) +} + +pub(crate) async fn forward_provider_status_query_to_owner( + state: &Arc, + owner: &crate::supervisor_owner::OwnerRecord, + sandbox_id: &str, + request: GetSandboxProviderStatusRequest, +) -> Result { + let mut client = peer_rpc_client(state, &owner.owner_peer_endpoint).await?; + client + .peer_get_sandbox_provider_status(request) + .await + .map(Response::into_inner) + .inspect_err(|_| { + state.peer_routes.evict_channel(&owner.owner_peer_endpoint); + state.peer_routes.evict_owner(sandbox_id); + }) +} + pub async fn open_routed_relay_with_target( state: &Arc, sandbox_id: &str, @@ -1711,12 +1817,18 @@ pub async fn handle_connect_supervisor( state .supervisor_sessions .remove_if_current(&sandbox_id, &session_id); + if let Err(err) = owner_index.release_if_current(&owner_guard).await { + warn!(sandbox_id, session_id, error = %err, "supervisor session: failed to release owner after endpoint status initialization failure"); + } return Err(error); } if !state .supervisor_sessions .initialize_endpoint_status_authority(&sandbox_id, &session_id) { + if let Err(err) = owner_index.release_if_current(&owner_guard).await { + warn!(sandbox_id, session_id, error = %err, "supervisor session: failed to release superseded owner during endpoint status initialization"); + } return Err(Status::failed_precondition( "supervisor session was replaced during endpoint status initialization", )); @@ -1729,6 +1841,9 @@ pub async fn handle_connect_supervisor( state .supervisor_sessions .remove_if_current(&sandbox_id, &session_id); + if let Err(err) = owner_index.release_if_current(&owner_guard).await { + warn!(sandbox_id, session_id, error = %err, "supervisor session: failed to release owner after provider readiness initialization failure"); + } return Err(error); } @@ -1767,6 +1882,9 @@ pub async fn handle_connect_supervisor( state .supervisor_sessions .remove_if_current(&sandbox_id, &session_id); + if let Err(release_error) = owner_index.release_if_current(&owner_guard).await { + warn!(sandbox_id, session_id, error = %release_error, "supervisor session: failed to release owner after lifecycle persistence failure"); + } warn!( sandbox_id = %sandbox_id, session_id = %session_id, @@ -1981,12 +2099,12 @@ async fn handle_supervisor_message( match msg.payload { Some(supervisor_message::Payload::Heartbeat(_)) => { let owner_index = SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL); - match owner_index.renew(owner_guard).await { - Ok(()) => {} + match tokio::time::timeout(OWNER_RENEW_TIMEOUT, owner_index.renew(owner_guard)).await { + Ok(Ok(())) => {} // Only a real ownership change ends the session. A store error // means the database did not answer, and closing on that would // drop every session heartbeating during the outage. - Err(err) if err.is_ownership_lost() => { + Ok(Err(err)) if err.is_ownership_lost() => { warn!( sandbox_id = %sandbox_id, session_id = %session_id, @@ -1998,7 +2116,7 @@ async fn handle_supervisor_message( // Past the TTL our record is stale, so another replica may // already have superseded it. Close rather than serve a // session we can no longer claim. - Err(err) if owner_guard.claim_expired(OWNER_TTL) => { + Ok(Err(err)) if owner_guard.claim_expired(OWNER_TTL) => { warn!( sandbox_id = %sandbox_id, session_id = %session_id, @@ -2008,12 +2126,27 @@ async fn handle_supervisor_message( ); return false; } - Err(err) => warn!( + Ok(Err(err)) => warn!( sandbox_id = %sandbox_id, session_id = %session_id, error = %err, "supervisor session: owner renewal failed; retrying on next heartbeat" ), + Err(_) if owner_guard.claim_expired(OWNER_TTL) => { + warn!( + sandbox_id = %sandbox_id, + session_id = %session_id, + timeout_ms = OWNER_RENEW_TIMEOUT.as_millis(), + "supervisor session: owner renewal timed out past the ownership TTL; closing session" + ); + return false; + } + Err(_) => warn!( + sandbox_id = %sandbox_id, + session_id = %session_id, + timeout_ms = OWNER_RENEW_TIMEOUT.as_millis(), + "supervisor session: owner renewal timed out; retrying on next heartbeat" + ), } } Some(supervisor_message::Payload::RelayOpenResult(result)) => { @@ -2183,11 +2316,17 @@ mod tests { }; let mut without_session = sandbox.clone(); - registry.project_endpoint_status(&mut without_session); + registry.project_endpoint_status(&mut without_session, false); let projected_status = without_session.status.expect("status"); assert_eq!(projected_status.endpoint_statuses, vec![unknown.clone()]); assert_eq!(projected_status.conditions, vec![ready.clone()]); + let mut remotely_owned = sandbox.clone(); + registry.project_endpoint_status(&mut remotely_owned, true); + let remote_status = remotely_owned.status.expect("status"); + assert_eq!(remote_status.endpoint_statuses, vec![endpoint.clone()]); + assert_eq!(remote_status.conditions, vec![ready.clone()]); + let (session_tx, _session_rx) = mpsc::channel(1); registry.register( "sandbox-1".to_string(), @@ -2196,13 +2335,13 @@ mod tests { make_shutdown(), ); let mut before_initialization = sandbox.clone(); - registry.project_endpoint_status(&mut before_initialization); + registry.project_endpoint_status(&mut before_initialization, false); let uninitialized_status = before_initialization.status.expect("status"); assert_eq!(uninitialized_status.endpoint_statuses, vec![unknown]); assert_eq!(uninitialized_status.conditions, vec![ready.clone()]); assert!(registry.initialize_endpoint_status_authority("sandbox-1", "session-1")); - registry.project_endpoint_status(&mut sandbox); + registry.project_endpoint_status(&mut sandbox, false); let initialized_status = sandbox.status.expect("status"); assert_eq!(initialized_status.endpoint_statuses, vec![endpoint]); assert_eq!(initialized_status.conditions, vec![ready]); diff --git a/crates/openshell-server/tests/common/mod.rs b/crates/openshell-server/tests/common/mod.rs index e34a8bed33..414251734d 100644 --- a/crates/openshell-server/tests/common/mod.rs +++ b/crates/openshell-server/tests/common/mod.rs @@ -53,6 +53,27 @@ pub struct TestOpenShell; #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn peer_report_provider_readiness( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_report_endpoint_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_get_sandbox_provider_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + async fn report_endpoint_status( &self, _request: tonic::Request, diff --git a/crates/openshell-server/tests/supervisor_relay_integration.rs b/crates/openshell-server/tests/supervisor_relay_integration.rs index 25f74e9847..5856acbbda 100644 --- a/crates/openshell-server/tests/supervisor_relay_integration.rs +++ b/crates/openshell-server/tests/supervisor_relay_integration.rs @@ -48,6 +48,27 @@ struct RelayGateway { #[tonic::async_trait] impl OpenShell for RelayGateway { + async fn peer_report_provider_readiness( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_report_endpoint_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + + async fn peer_get_sandbox_provider_status( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + async fn report_endpoint_status( &self, _request: tonic::Request, diff --git a/proto/openshell.proto b/proto/openshell.proto index 16ec2375d2..34d7d0b5c8 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -600,6 +600,28 @@ service OpenShell { }; } + // Internal gateway-to-owner forwarding for supervisor-bound state. + rpc PeerReportProviderReadiness(ReportProviderReadinessRequest) + returns (ReportProviderReadinessResponse) { + option (openshell.options.v1.authorization) = { + auth_mode: "peer" + }; + } + + rpc PeerReportEndpointStatus(ReportEndpointStatusRequest) + returns (ReportEndpointStatusResponse) { + option (openshell.options.v1.authorization) = { + auth_mode: "peer" + }; + } + + rpc PeerGetSandboxProviderStatus(GetSandboxProviderStatusRequest) + returns (GetSandboxProviderStatusResponse) { + option (openshell.options.v1.authorization) = { + auth_mode: "peer" + }; + } + // Watch a sandbox and stream updates. // // This stream can include: diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index eefce03f82..44f9b61efa 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -18732,7 +18732,7 @@ const file_openshell_proto_rawDesc = "" + "&ENDPOINT_RESULT_CREDENTIAL_UNAVAILABLE\x10\x04\x12\x1e\n" + "\x1aENDPOINT_RESULT_TLS_FAILED\x10\x05\x12$\n" + " ENDPOINT_RESULT_TRANSPORT_FAILED\x10\x06\x12%\n" + - "!ENDPOINT_RESULT_UPSTREAM_REJECTED\x10\a2\x9bR\n" + + "!ENDPOINT_RESULT_UPSTREAM_REJECTED\x10\a2\xafU\n" + "\tOpenShell\x12Z\n" + "\x06Health\x12\x1b.openshell.v1.HealthRequest\x1a\x1c.openshell.v1.HealthResponse\"\x15\x82\xb5\x18\x11\n" + "\x0funauthenticated\x12i\n" + @@ -18857,7 +18857,16 @@ const file_openshell_proto_rawDesc = "" + "\asandbox(\x010\x01\x12W\n" + "\tPeerRelay\x12\x1c.openshell.v1.PeerRelayFrame\x1a\x1c.openshell.v1.PeerRelayFrame\"\n" + "\x82\xb5\x18\x06\n" + - "\x04peer(\x010\x01\x12w\n" + + "\x04peer(\x010\x01\x12\x86\x01\n" + + "\x1bPeerReportProviderReadiness\x12,.openshell.v1.ReportProviderReadinessRequest\x1a-.openshell.v1.ReportProviderReadinessResponse\"\n" + + "\x82\xb5\x18\x06\n" + + "\x04peer\x12}\n" + + "\x18PeerReportEndpointStatus\x12).openshell.v1.ReportEndpointStatusRequest\x1a*.openshell.v1.ReportEndpointStatusResponse\"\n" + + "\x82\xb5\x18\x06\n" + + "\x04peer\x12\x89\x01\n" + + "\x1cPeerGetSandboxProviderStatus\x12-.openshell.v1.GetSandboxProviderStatusRequest\x1a..openshell.v1.GetSandboxProviderStatusResponse\"\n" + + "\x82\xb5\x18\x06\n" + + "\x04peer\x12w\n" + "\fWatchSandbox\x12!.openshell.v1.WatchSandboxRequest\x1a .openshell.v1.SandboxStreamEvent\" \x82\xb5\x18\x1c\n" + "\x06bearer\x12\x04user\"\fsandbox:read0\x01\x12|\n" + "\x14SubmitPolicyAnalysis\x12).openshell.v1.SubmitPolicyAnalysisRequest\x1a*.openshell.v1.SubmitPolicyAnalysisResponse\"\r\x82\xb5\x18\t\n" + @@ -19576,106 +19585,112 @@ var file_openshell_proto_depIdxs = []int32{ 188, // 371: openshell.v1.OpenShell.FinalizeMainProcessExit:input_type -> openshell.v1.FinalizeMainProcessExitRequest 194, // 372: openshell.v1.OpenShell.RelayStream:input_type -> openshell.v1.RelayFrame 196, // 373: openshell.v1.OpenShell.PeerRelay:input_type -> openshell.v1.PeerRelayFrame - 105, // 374: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest - 205, // 375: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest - 207, // 376: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest - 209, // 377: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest - 211, // 378: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest - 214, // 379: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest - 216, // 380: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest - 218, // 381: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest - 220, // 382: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest - 222, // 383: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest - 17, // 384: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest - 19, // 385: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest - 225, // 386: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest - 227, // 387: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest - 229, // 388: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest - 231, // 389: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest - 234, // 390: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest - 236, // 391: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest - 238, // 392: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest - 22, // 393: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse - 24, // 394: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse - 26, // 395: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse - 66, // 396: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse - 57, // 397: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse - 66, // 398: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse - 67, // 399: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse - 53, // 400: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 53, // 401: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 54, // 402: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse - 55, // 403: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse - 68, // 404: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse - 69, // 405: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse - 70, // 406: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse - 79, // 407: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse - 82, // 408: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse - 66, // 409: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse - 66, // 410: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse - 84, // 411: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse - 92, // 412: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse - 92, // 413: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse - 88, // 414: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse - 90, // 415: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse - 94, // 416: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse - 99, // 417: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent - 101, // 418: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame - 99, // 419: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent - 114, // 420: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse - 114, // 421: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse - 115, // 422: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse - 139, // 423: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse - 138, // 424: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse - 141, // 425: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse - 143, // 426: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse - 145, // 427: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse - 114, // 428: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse - 130, // 429: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse - 132, // 430: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse - 134, // 431: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse - 136, // 432: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse - 146, // 433: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse - 148, // 434: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse - 286, // 435: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse - 287, // 436: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse - 164, // 437: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse - 166, // 438: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse - 168, // 439: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse - 170, // 440: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse - 243, // 441: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse - 81, // 442: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse - 173, // 443: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse - 152, // 444: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse - 154, // 445: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse - 178, // 446: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse - 177, // 447: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse - 180, // 448: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage - 187, // 449: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse - 189, // 450: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse - 194, // 451: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame - 196, // 452: openshell.v1.OpenShell.PeerRelay:output_type -> openshell.v1.PeerRelayFrame - 106, // 453: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent - 206, // 454: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse - 208, // 455: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse - 210, // 456: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse - 212, // 457: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse - 215, // 458: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse - 217, // 459: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse - 219, // 460: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse - 221, // 461: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse - 224, // 462: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse - 18, // 463: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse - 20, // 464: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse - 226, // 465: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse - 228, // 466: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse - 230, // 467: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse - 232, // 468: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse - 235, // 469: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse - 237, // 470: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse - 239, // 471: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse - 393, // [393:472] is the sub-list for method output_type - 314, // [314:393] is the sub-list for method input_type + 80, // 374: openshell.v1.OpenShell.PeerReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest + 242, // 375: openshell.v1.OpenShell.PeerReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest + 78, // 376: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest + 105, // 377: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest + 205, // 378: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest + 207, // 379: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest + 209, // 380: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest + 211, // 381: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest + 214, // 382: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest + 216, // 383: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest + 218, // 384: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest + 220, // 385: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest + 222, // 386: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest + 17, // 387: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest + 19, // 388: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest + 225, // 389: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest + 227, // 390: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest + 229, // 391: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest + 231, // 392: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest + 234, // 393: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest + 236, // 394: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest + 238, // 395: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest + 22, // 396: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse + 24, // 397: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse + 26, // 398: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse + 66, // 399: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse + 57, // 400: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse + 66, // 401: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse + 67, // 402: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse + 53, // 403: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 53, // 404: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 54, // 405: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse + 55, // 406: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse + 68, // 407: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse + 69, // 408: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse + 70, // 409: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse + 79, // 410: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse + 82, // 411: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse + 66, // 412: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse + 66, // 413: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse + 84, // 414: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse + 92, // 415: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse + 92, // 416: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse + 88, // 417: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse + 90, // 418: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse + 94, // 419: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse + 99, // 420: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent + 101, // 421: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame + 99, // 422: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent + 114, // 423: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse + 114, // 424: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse + 115, // 425: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse + 139, // 426: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse + 138, // 427: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse + 141, // 428: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse + 143, // 429: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse + 145, // 430: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse + 114, // 431: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse + 130, // 432: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse + 132, // 433: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse + 134, // 434: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse + 136, // 435: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse + 146, // 436: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse + 148, // 437: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse + 286, // 438: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse + 287, // 439: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse + 164, // 440: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse + 166, // 441: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse + 168, // 442: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse + 170, // 443: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse + 243, // 444: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse + 81, // 445: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse + 173, // 446: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse + 152, // 447: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse + 154, // 448: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse + 178, // 449: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse + 177, // 450: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse + 180, // 451: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage + 187, // 452: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse + 189, // 453: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse + 194, // 454: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame + 196, // 455: openshell.v1.OpenShell.PeerRelay:output_type -> openshell.v1.PeerRelayFrame + 81, // 456: openshell.v1.OpenShell.PeerReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse + 243, // 457: openshell.v1.OpenShell.PeerReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse + 79, // 458: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse + 106, // 459: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent + 206, // 460: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse + 208, // 461: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse + 210, // 462: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse + 212, // 463: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse + 215, // 464: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse + 217, // 465: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse + 219, // 466: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse + 221, // 467: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse + 224, // 468: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse + 18, // 469: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse + 20, // 470: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse + 226, // 471: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse + 228, // 472: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse + 230, // 473: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse + 232, // 474: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse + 235, // 475: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse + 237, // 476: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse + 239, // 477: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse + 396, // [396:478] is the sub-list for method output_type + 314, // [314:396] is the sub-list for method input_type 314, // [314:314] is the sub-list for extension type_name 314, // [314:314] is the sub-list for extension extendee 0, // [0:314] is the sub-list for field type_name diff --git a/sdk/go/proto/openshellv1/openshell_grpc.pb.go b/sdk/go/proto/openshellv1/openshell_grpc.pb.go index 56aa770ae0..d185e24283 100644 --- a/sdk/go/proto/openshellv1/openshell_grpc.pb.go +++ b/sdk/go/proto/openshellv1/openshell_grpc.pb.go @@ -83,6 +83,9 @@ const ( OpenShell_FinalizeMainProcessExit_FullMethodName = "/openshell.v1.OpenShell/FinalizeMainProcessExit" OpenShell_RelayStream_FullMethodName = "/openshell.v1.OpenShell/RelayStream" OpenShell_PeerRelay_FullMethodName = "/openshell.v1.OpenShell/PeerRelay" + OpenShell_PeerReportProviderReadiness_FullMethodName = "/openshell.v1.OpenShell/PeerReportProviderReadiness" + OpenShell_PeerReportEndpointStatus_FullMethodName = "/openshell.v1.OpenShell/PeerReportEndpointStatus" + OpenShell_PeerGetSandboxProviderStatus_FullMethodName = "/openshell.v1.OpenShell/PeerGetSandboxProviderStatus" OpenShell_WatchSandbox_FullMethodName = "/openshell.v1.OpenShell/WatchSandbox" OpenShell_SubmitPolicyAnalysis_FullMethodName = "/openshell.v1.OpenShell/SubmitPolicyAnalysis" OpenShell_GetDraftPolicy_FullMethodName = "/openshell.v1.OpenShell/GetDraftPolicy" @@ -276,6 +279,10 @@ type OpenShellClient interface { // bytes in either direction. This RPC is authenticated as a gateway peer, not // as a user or sandbox supervisor. PeerRelay(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PeerRelayFrame, PeerRelayFrame], error) + // Internal gateway-to-owner forwarding for supervisor-bound state. + PeerReportProviderReadiness(ctx context.Context, in *ReportProviderReadinessRequest, opts ...grpc.CallOption) (*ReportProviderReadinessResponse, error) + PeerReportEndpointStatus(ctx context.Context, in *ReportEndpointStatusRequest, opts ...grpc.CallOption) (*ReportEndpointStatusResponse, error) + PeerGetSandboxProviderStatus(ctx context.Context, in *GetSandboxProviderStatusRequest, opts ...grpc.CallOption) (*GetSandboxProviderStatusResponse, error) // Watch a sandbox and stream updates. // // This stream can include: @@ -965,6 +972,36 @@ func (c *openShellClient) PeerRelay(ctx context.Context, opts ...grpc.CallOption // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type OpenShell_PeerRelayClient = grpc.BidiStreamingClient[PeerRelayFrame, PeerRelayFrame] +func (c *openShellClient) PeerReportProviderReadiness(ctx context.Context, in *ReportProviderReadinessRequest, opts ...grpc.CallOption) (*ReportProviderReadinessResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReportProviderReadinessResponse) + err := c.cc.Invoke(ctx, OpenShell_PeerReportProviderReadiness_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *openShellClient) PeerReportEndpointStatus(ctx context.Context, in *ReportEndpointStatusRequest, opts ...grpc.CallOption) (*ReportEndpointStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReportEndpointStatusResponse) + err := c.cc.Invoke(ctx, OpenShell_PeerReportEndpointStatus_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *openShellClient) PeerGetSandboxProviderStatus(ctx context.Context, in *GetSandboxProviderStatusRequest, opts ...grpc.CallOption) (*GetSandboxProviderStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetSandboxProviderStatusResponse) + err := c.cc.Invoke(ctx, OpenShell_PeerGetSandboxProviderStatus_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *openShellClient) WatchSandbox(ctx context.Context, in *WatchSandboxRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SandboxStreamEvent], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &OpenShell_ServiceDesc.Streams[7], OpenShell_WatchSandbox_FullMethodName, cOpts...) @@ -1336,6 +1373,10 @@ type OpenShellServer interface { // bytes in either direction. This RPC is authenticated as a gateway peer, not // as a user or sandbox supervisor. PeerRelay(grpc.BidiStreamingServer[PeerRelayFrame, PeerRelayFrame]) error + // Internal gateway-to-owner forwarding for supervisor-bound state. + PeerReportProviderReadiness(context.Context, *ReportProviderReadinessRequest) (*ReportProviderReadinessResponse, error) + PeerReportEndpointStatus(context.Context, *ReportEndpointStatusRequest) (*ReportEndpointStatusResponse, error) + PeerGetSandboxProviderStatus(context.Context, *GetSandboxProviderStatusRequest) (*GetSandboxProviderStatusResponse, error) // Watch a sandbox and stream updates. // // This stream can include: @@ -1578,6 +1619,15 @@ func (UnimplementedOpenShellServer) RelayStream(grpc.BidiStreamingServer[RelayFr func (UnimplementedOpenShellServer) PeerRelay(grpc.BidiStreamingServer[PeerRelayFrame, PeerRelayFrame]) error { return status.Error(codes.Unimplemented, "method PeerRelay not implemented") } +func (UnimplementedOpenShellServer) PeerReportProviderReadiness(context.Context, *ReportProviderReadinessRequest) (*ReportProviderReadinessResponse, error) { + return nil, status.Error(codes.Unimplemented, "method PeerReportProviderReadiness not implemented") +} +func (UnimplementedOpenShellServer) PeerReportEndpointStatus(context.Context, *ReportEndpointStatusRequest) (*ReportEndpointStatusResponse, error) { + return nil, status.Error(codes.Unimplemented, "method PeerReportEndpointStatus not implemented") +} +func (UnimplementedOpenShellServer) PeerGetSandboxProviderStatus(context.Context, *GetSandboxProviderStatusRequest) (*GetSandboxProviderStatusResponse, error) { + return nil, status.Error(codes.Unimplemented, "method PeerGetSandboxProviderStatus not implemented") +} func (UnimplementedOpenShellServer) WatchSandbox(*WatchSandboxRequest, grpc.ServerStreamingServer[SandboxStreamEvent]) error { return status.Error(codes.Unimplemented, "method WatchSandbox not implemented") } @@ -2663,6 +2713,60 @@ func _OpenShell_PeerRelay_Handler(srv interface{}, stream grpc.ServerStream) err // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type OpenShell_PeerRelayServer = grpc.BidiStreamingServer[PeerRelayFrame, PeerRelayFrame] +func _OpenShell_PeerReportProviderReadiness_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReportProviderReadinessRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(OpenShellServer).PeerReportProviderReadiness(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: OpenShell_PeerReportProviderReadiness_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(OpenShellServer).PeerReportProviderReadiness(ctx, req.(*ReportProviderReadinessRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _OpenShell_PeerReportEndpointStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReportEndpointStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(OpenShellServer).PeerReportEndpointStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: OpenShell_PeerReportEndpointStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(OpenShellServer).PeerReportEndpointStatus(ctx, req.(*ReportEndpointStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _OpenShell_PeerGetSandboxProviderStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetSandboxProviderStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(OpenShellServer).PeerGetSandboxProviderStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: OpenShell_PeerGetSandboxProviderStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(OpenShellServer).PeerGetSandboxProviderStatus(ctx, req.(*GetSandboxProviderStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _OpenShell_WatchSandbox_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(WatchSandboxRequest) if err := stream.RecvMsg(m); err != nil { @@ -3217,6 +3321,18 @@ var OpenShell_ServiceDesc = grpc.ServiceDesc{ MethodName: "FinalizeMainProcessExit", Handler: _OpenShell_FinalizeMainProcessExit_Handler, }, + { + MethodName: "PeerReportProviderReadiness", + Handler: _OpenShell_PeerReportProviderReadiness_Handler, + }, + { + MethodName: "PeerReportEndpointStatus", + Handler: _OpenShell_PeerReportEndpointStatus_Handler, + }, + { + MethodName: "PeerGetSandboxProviderStatus", + Handler: _OpenShell_PeerGetSandboxProviderStatus_Handler, + }, { MethodName: "SubmitPolicyAnalysis", Handler: _OpenShell_SubmitPolicyAnalysis_Handler,