Skip to content

Commit 424296b

Browse files
committed
feat(gateway): support runtime image env overrides
Closes #3502 Resolve trusted OCI runtime images with driver TOML, process environment, and compiled-default precedence across Docker, Podman, and Kubernetes. Signed-off-by: Evan Lezar <elezar@nvidia.com>
1 parent 36819f4 commit 424296b

14 files changed

Lines changed: 209 additions & 45 deletions

File tree

‎crates/openshell-core/src/config.rs‎

Lines changed: 48 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -89,24 +89,48 @@ pub const DEFAULT_SUPERVISOR_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/superv
8989
/// Default OCI repository for the sandbox runtime image (no tag).
9090
pub const DEFAULT_SANDBOX_RUNTIME_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/sandbox";
9191

92-
/// Return the default sandbox runtime image reference with a version-pinned tag.
93-
#[must_use]
94-
pub fn default_sandbox_runtime_image() -> String {
92+
/// Process-level default for the trusted sandbox runtime image.
93+
pub const SANDBOX_RUNTIME_IMAGE_ENV: &str = "OPENSHELL_SANDBOX_RUNTIME_IMAGE";
94+
95+
/// Process-level default for the trusted supervisor image.
96+
pub const SUPERVISOR_IMAGE_ENV: &str = "OPENSHELL_SUPERVISOR_IMAGE";
97+
98+
fn compiled_sandbox_runtime_image() -> String {
9599
format!(
96100
"{DEFAULT_SANDBOX_RUNTIME_IMAGE_REPO}:{}",
97101
default_supervisor_image_tag()
98102
)
99103
}
100104

101-
/// Return the default supervisor image reference with a version-pinned tag.
102-
#[must_use]
103-
pub fn default_supervisor_image() -> String {
105+
fn compiled_supervisor_image() -> String {
104106
format!(
105107
"{DEFAULT_SUPERVISOR_IMAGE_REPO}:{}",
106108
default_supervisor_image_tag()
107109
)
108110
}
109111

112+
fn runtime_image_default(environment_value: Option<String>, compiled_default: String) -> String {
113+
environment_value.unwrap_or(compiled_default)
114+
}
115+
116+
/// Return the process-configured sandbox runtime image, or the compiled default.
117+
#[must_use]
118+
pub fn default_sandbox_runtime_image() -> String {
119+
runtime_image_default(
120+
std::env::var(SANDBOX_RUNTIME_IMAGE_ENV).ok(),
121+
compiled_sandbox_runtime_image(),
122+
)
123+
}
124+
125+
/// Return the process-configured supervisor image, or the compiled default.
126+
#[must_use]
127+
pub fn default_supervisor_image() -> String {
128+
runtime_image_default(
129+
std::env::var(SUPERVISOR_IMAGE_ENV).ok(),
130+
compiled_supervisor_image(),
131+
)
132+
}
133+
110134
fn default_supervisor_image_tag() -> String {
111135
resolve_supervisor_image_tag(&[
112136
option_env!("OPENSHELL_IMAGE_TAG").unwrap_or(""),
@@ -1639,15 +1663,30 @@ mod tests {
16391663

16401664
#[test]
16411665
fn default_supervisor_image_is_version_pinned() {
1642-
use super::{default_sandbox_runtime_image, default_supervisor_image};
1643-
let image = default_supervisor_image();
1666+
use super::{compiled_sandbox_runtime_image, compiled_supervisor_image};
1667+
let image = compiled_supervisor_image();
16441668
assert!(image.starts_with("ghcr.io/nvidia/openshell/supervisor:"));
16451669
let tag = image.rsplit_once(':').unwrap().1;
16461670
assert!(!tag.is_empty());
16471671

1648-
let sandbox_image = default_sandbox_runtime_image();
1672+
let sandbox_image = compiled_sandbox_runtime_image();
16491673
assert!(sandbox_image.starts_with("ghcr.io/nvidia/openshell/sandbox:"));
16501674
let sandbox_tag = sandbox_image.rsplit_once(':').unwrap().1;
16511675
assert!(!sandbox_tag.is_empty());
16521676
}
1677+
1678+
#[test]
1679+
fn runtime_image_environment_value_replaces_compiled_default() {
1680+
use super::runtime_image_default;
1681+
1682+
let digest = format!("registry.example.com/sandbox@sha256:{}", "a".repeat(64));
1683+
assert_eq!(
1684+
runtime_image_default(Some(digest.clone()), "compiled:default".to_string()),
1685+
digest
1686+
);
1687+
assert_eq!(
1688+
runtime_image_default(None, "compiled:default".to_string()),
1689+
"compiled:default"
1690+
);
1691+
}
16531692
}

‎crates/openshell-driver-docker/src/tests.rs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2503,6 +2503,23 @@ fn validate_sandbox_rejects_unknown_driver_config_fields() {
25032503
assert!(err.message().contains("unknown field"));
25042504
}
25052505

2506+
#[test]
2507+
fn sandbox_driver_config_rejects_trusted_runtime_image_overrides() {
2508+
for field in ["sandbox_runtime_image", "supervisor_image"] {
2509+
let template = DriverSandboxTemplate {
2510+
driver_config: Some(json_struct(serde_json::json!({
2511+
(field): "registry.example.com/openshell/runtime:untrusted"
2512+
}))),
2513+
..Default::default()
2514+
};
2515+
2516+
let error = DockerSandboxDriverConfig::from_template(&template)
2517+
.expect_err("sandbox requests must not select trusted runtime images");
2518+
assert!(error.contains("unknown field"), "{error}");
2519+
assert!(error.contains(field), "{error}");
2520+
}
2521+
}
2522+
25062523
#[test]
25072524
fn validate_sandbox_accepts_gpu_count_request_shape() {
25082525
let mut config = runtime_config();

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8865,6 +8865,23 @@ mod tests {
88658865
assert!(err.contains("unknown field"));
88668866
}
88678867

8868+
#[test]
8869+
fn sandbox_driver_config_rejects_trusted_runtime_image_overrides() {
8870+
for field in ["sandbox_runtime_image", "supervisor_image"] {
8871+
let template = SandboxTemplate {
8872+
driver_config: Some(json_struct(serde_json::json!({
8873+
(field): "registry.example.com/openshell/runtime:untrusted"
8874+
}))),
8875+
..Default::default()
8876+
};
8877+
8878+
let error = KubernetesSandboxDriverConfig::from_template(&template)
8879+
.expect_err("sandbox requests must not select trusted runtime images");
8880+
assert!(error.contains("unknown field"), "{error}");
8881+
assert!(error.contains(field), "{error}");
8882+
}
8883+
}
8884+
88688885
#[test]
88698886
fn driver_config_for_spec_rejects_unknown_fields() {
88708887
let sandbox = Sandbox {

‎crates/openshell-driver-kubernetes/src/main.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,13 +116,13 @@ struct Args {
116116
#[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")]
117117
host_gateway_ip: Option<String>,
118118

119-
#[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE")]
119+
#[arg(long, env = openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV)]
120120
sandbox_runtime_image: Option<String>,
121121

122122
#[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE_PULL_POLICY")]
123123
sandbox_runtime_image_pull_policy: Option<KubernetesImagePullPolicy>,
124124

125-
#[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")]
125+
#[arg(long, env = openshell_core::config::SUPERVISOR_IMAGE_ENV)]
126126
supervisor_image: Option<String>,
127127

128128
#[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")]

‎crates/openshell-driver-podman/src/container.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2313,6 +2313,25 @@ mod tests {
23132313
assert!(err.to_string().contains("unknown field"));
23142314
}
23152315

2316+
#[test]
2317+
fn sandbox_driver_config_rejects_trusted_runtime_image_overrides() {
2318+
use openshell_core::proto::compute::v1::DriverSandboxTemplate;
2319+
2320+
for field in ["sandbox_runtime_image", "supervisor_image"] {
2321+
let template = DriverSandboxTemplate {
2322+
driver_config: Some(json_struct(serde_json::json!({
2323+
(field): "registry.example.com/openshell/runtime:untrusted"
2324+
}))),
2325+
..Default::default()
2326+
};
2327+
2328+
let error = PodmanSandboxDriverConfig::from_template(&template)
2329+
.expect_err("sandbox requests must not select trusted runtime images");
2330+
assert!(error.to_string().contains("unknown field"), "{error}");
2331+
assert!(error.to_string().contains(field), "{error}");
2332+
}
2333+
}
2334+
23162335
#[test]
23172336
fn container_spec_defaults_drop_capabilities_and_keep_runtime_seccomp() {
23182337
let sandbox = test_sandbox("test-id", "test-name");

‎crates/openshell-driver-podman/src/main.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,11 +110,11 @@ struct Args {
110110
health_check_interval_secs: Option<NonZeroU64>,
111111

112112
/// OCI image containing the `openshell-sandbox` runtime binary.
113-
#[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE")]
113+
#[arg(long, env = openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV)]
114114
sandbox_runtime_image: Option<String>,
115115

116116
/// OCI image containing the `openshell-supervisor` control binary.
117-
#[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")]
117+
#[arg(long, env = openshell_core::config::SUPERVISOR_IMAGE_ENV)]
118118
supervisor_image: Option<String>,
119119

120120
/// Host path to the CA certificate for supervisor-to-gateway TLS.

‎deploy/man/openshell-gateway.8.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,15 @@ need a custom bundle location.
171171
The gateway then starts from built-in defaults and reads
172172
*~/.config/openshell/gateway.toml* when that file exists.
173173

174+
The user service also reads *~/.config/openshell/gateway.env* for both config
175+
preflight and startup. **OPENSHELL_SANDBOX_RUNTIME_IMAGE** and
176+
**OPENSHELL_SUPERVISOR_IMAGE** accept complete tagged or digest-pinned OCI
177+
references for the built-in Docker, Podman, and Kubernetes drivers. Explicit
178+
image fields in the selected driver's TOML table take precedence over these
179+
variables; the variables take precedence over compiled release defaults.
180+
These are trusted gateway inputs and cannot be supplied by sandbox requests.
181+
Do not embed registry credentials in image references.
182+
174183
To persist the service across logouts:
175184

176185
sudo loginctl enable-linger $USER

‎deploy/rpm/CONFIGURATION.md‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,21 @@ editing the TOML file, add them to `~/.config/openshell/gateway.env`:
5151
OPENSHELL_BIND_ADDRESS=192.168.1.10
5252
```
5353

54+
To select exact trusted runtime artifacts across the built-in Docker, Podman,
55+
or Kubernetes driver, set complete tagged or digest-pinned references:
56+
57+
```shell
58+
OPENSHELL_SANDBOX_RUNTIME_IMAGE=registry.example.com/openshell/sandbox@sha256:<digest>
59+
OPENSHELL_SUPERVISOR_IMAGE=registry.example.com/openshell/supervisor@sha256:<digest>
60+
```
61+
62+
An explicit `sandbox_runtime_image` or `supervisor_image` in the selected
63+
driver's TOML table takes precedence over the environment; the environment
64+
takes precedence over the compiled release default. Preflight and startup use
65+
the same resolution. These values are trusted operator inputs, not sandbox
66+
request fields. Keep registry credentials in Podman's credential store rather
67+
than embedding them in image references.
68+
5469
To override the path to the TOML config file entirely:
5570

5671
```shell
@@ -216,8 +231,8 @@ overrides that persist across package upgrades.
216231
| `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. |
217232
| `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. |
218233
| `[openshell.drivers.podman].default_image` | `nvcr.io/nvidia/base/ubuntu:24.04` | Default sandbox image. |
219-
| `[openshell.drivers.podman].sandbox_runtime_image` | `ghcr.io/nvidia/openshell/sandbox:latest` | Static musl sandbox runtime image mounted into Podman workloads. |
220-
| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Dynamic glibc supervisor image used outside the workload. |
234+
| `[openshell.drivers.podman].sandbox_runtime_image` | `ghcr.io/nvidia/openshell/sandbox:latest` | Trusted sandbox runtime image. `OPENSHELL_SANDBOX_RUNTIME_IMAGE` supplies a process default when this field is omitted. |
235+
| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Trusted supervisor image. `OPENSHELL_SUPERVISOR_IMAGE` supplies a process default when this field is omitted. |
221236
| `[openshell.gateway].guest_tls_ca` | auto-generated path | Gateway CA injected into the selected local driver for supervisor-to-gateway TLS. Sandbox identity uses a bearer token. |
222237
| `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. |
223238
| `disable_tls` | unset | Set to `true` to disable TLS. |

‎docs/about/installation.mdx‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,21 @@ The script installs a Debian package on Debian and Ubuntu or an RPM package on F
9090

9191
The gateway runs as a systemd user service at `https://127.0.0.1:17670` and reads `~/.config/openshell/gateway.toml`.
9292

93+
Both Debian and RPM load `~/.config/openshell/gateway.env` for configuration
94+
preflight and gateway startup. Use it to select exact trusted runtime artifacts
95+
without creating driver-specific TOML:
96+
97+
```shell
98+
OPENSHELL_SANDBOX_RUNTIME_IMAGE=registry.example.com/openshell/sandbox@sha256:<digest>
99+
OPENSHELL_SUPERVISOR_IMAGE=registry.example.com/openshell/supervisor@sha256:<digest>
100+
```
101+
102+
Explicit image fields in the selected Docker, Podman, or Kubernetes TOML table
103+
take precedence. Restart the user service after editing the file. Treat these
104+
variables as trusted operator configuration; sandbox requests cannot set them,
105+
and registry credentials must remain in the container runtime's credential
106+
store or Kubernetes image-pull secrets.
107+
93108
```shell
94109
systemctl --user status openshell-gateway
95110
systemctl --user restart openshell-gateway

‎docs/how-it-works/gateways/configuration.mdx‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,38 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa
2020

2121
On-disk SQLite databases run in WAL mode with `synchronous=FULL`: the gateway keeps `openshell.db-wal` and `openshell.db-shm` next to the database file, the file must live on a local filesystem, and a backup must use `sqlite3 openshell.db ".backup <copy>"` rather than copying `openshell.db` on its own. Every acknowledged write, including SSH session revocations, survives a power loss. The one exception is SSH session issuance, which skips the per-commit sync; a power loss can drop a just-issued session token, and the client must request a new one.
2222

23+
### Trusted runtime image overrides
24+
25+
Docker, Podman, and Kubernetes consume two gateway-owned trusted runtime
26+
images. Set complete tagged or digest-pinned OCI references for the gateway
27+
process when you need to qualify exact artifacts or use a registry mirror:
28+
29+
```shell
30+
OPENSHELL_SANDBOX_RUNTIME_IMAGE=registry.example.com/openshell/sandbox@sha256:<digest>
31+
OPENSHELL_SUPERVISOR_IMAGE=registry.example.com/openshell/supervisor:1.2.3
32+
```
33+
34+
These two settings use driver-specific precedence because explicit operator
35+
configuration must remain authoritative:
36+
37+
```text
38+
[openshell.drivers.<selected>] TOML > process environment > compiled release default
39+
```
40+
41+
`openshell-gateway config preflight` and gateway startup use the same effective
42+
references. Environment values are handled exactly like the corresponding TOML
43+
fields and passed to the selected container runtime; this feature does not add
44+
separate OCI-reference validation. Configure registry credentials in Docker,
45+
Podman, or Kubernetes image-pull secrets instead of embedding them in a
46+
reference.
47+
48+
The variables affect only the built-in Docker, Podman, and Kubernetes drivers.
49+
They do not affect VM, MXC, or extension drivers. They are trusted gateway
50+
operator inputs and are never accepted from sandbox create requests. For
51+
Debian and RPM user services, place them in
52+
`~/.config/openshell/gateway.env`; the service supplies that file to both
53+
preflight and gateway startup.
54+
2355
`name` assigns an operator-facing identity to the gateway installation. Set it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. It defaults to `openshell`; the Helm chart defaults it to the chart fullname so all replicas in one installation share a name. Chart fullnames are only unique within their Kubernetes namespace, so set `server.name` explicitly when one collector receives telemetry from multiple namespaces or clusters. This identity is independent of client-side gateway aliases, TLS names, and `gateway_jwt.gateway_id`.
2456

2557
## Package-Managed Locations

0 commit comments

Comments
 (0)