Skip to content

Commit c32c785

Browse files
committed
fix(podman): trust an operator-supplied additional CA for direct egress
Since v0.1.2 the Podman supervisor runs as its own container, separate from the workload image, so it can no longer inherit a private CA baked into the workload's system trust store. Policy-allowed, inspected HTTPS requests to destinations signed by that CA fail upstream TLS establishment, and the only workaround is rebuilding the supervisor image with the CA baked in. Add a Podman-only additional_ca_bundle setting, independent of the existing proxy_ca_bundle (which requires https_proxy), that bind-mounts an operator PEM into the supervisor container and folds it into the upstream trust store alongside the system CA bundle. Validated on the gateway host at sandbox-create time, matching the existing corporate proxy CA bundle's validation, so a misconfigured path or certificate-free bundle fails closed immediately rather than producing a failed sandbox once the supervisor container starts. Scoped to Podman only, reusing the existing additive-bundle mechanism without new gateway.toml schema, proto fields, or cross-driver plumbing -- a narrower alternative to the in-flight cross-driver implementation in #3292, which remains the intended long-term direction. Fixes #3781. Signed-off-by: politerealism <burdcat17@gmail.com>
1 parent 07a486d commit c32c785

12 files changed

Lines changed: 413 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/openshell-core/src/driver_utils.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,18 @@ pub const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str = "/etc/openshell/auth/upstream-p
141141
/// mount is used rather than a driver secret.
142142
pub const PROXY_CA_MOUNT_PATH: &str = "/etc/openshell/tls/proxy/ca-bundle.pem";
143143

144+
/// Container-side mount path for an operator-supplied additional destination
145+
/// CA bundle.
146+
///
147+
/// Drivers with an `additional_ca_bundle` operator setting bind-mount the
148+
/// host PEM file here (read-only) and pass the path on the supervisor's argv
149+
/// via `--additional-ca-bundle`. The supervisor folds it into its upstream
150+
/// TLS trust store alongside the system CA bundle, for direct (non-proxied)
151+
/// inspected HTTPS egress to destinations signed by a private CA — unlike
152+
/// [`PROXY_CA_MOUNT_PATH`], this applies regardless of whether a corporate
153+
/// forward proxy is configured.
154+
pub const ADDITIONAL_CA_MOUNT_PATH: &str = "/etc/openshell/tls/additional/ca-bundle.pem";
155+
144156
/// A validated corporate upstream-proxy address.
145157
///
146158
/// Produced by [`parse_upstream_proxy_url`], which is the single source of

‎crates/openshell-driver-podman/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ url = { workspace = true }
5050
openshell-otel-test-support = { path = "../openshell-otel-test-support" }
5151
opentelemetry_sdk = { workspace = true, features = ["testing"] }
5252
prost-types = { workspace = true }
53+
rcgen = { workspace = true }
5354
temp-env = "0.3"
5455
tempfile = "3"
5556
tokio = { workspace = true, features = ["test-util"] }

‎crates/openshell-driver-podman/src/config.rs‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,23 @@ pub struct PodmanComputeConfig {
176176
/// and upstream verification. Only meaningful with `https_proxy` set; the
177177
/// bundle must exist and contain at least one certificate.
178178
pub proxy_ca_bundle: Option<String>,
179+
/// Path (on the gateway host) to a PEM CA bundle trusted for direct
180+
/// (non-proxied), policy-inspected HTTPS egress to destinations signed by
181+
/// a private CA.
182+
///
183+
/// Unlike [`proxy_ca_bundle`](Self::proxy_ca_bundle), this applies
184+
/// regardless of whether `https_proxy` is configured: it addresses
185+
/// supervisors that run as their own container, separate from the
186+
/// workload image, and therefore cannot inherit a private CA baked into
187+
/// the workload's system trust store. The gateway bind-mounts this file
188+
/// read-only into the supervisor container (at
189+
/// [`ADDITIONAL_CA_MOUNT_PATH`](openshell_core::driver_utils::ADDITIONAL_CA_MOUNT_PATH))
190+
/// and passes its path via `--additional-ca-bundle`. The supervisor folds
191+
/// it into its upstream TLS trust store alongside (not instead of) the
192+
/// system CA bundle. The bundle must exist and contain at least one
193+
/// certificate; this is validated on the gateway host at sandbox-create
194+
/// time, not deferred until the supervisor container starts.
195+
pub additional_ca_bundle: Option<String>,
179196
/// User namespace mode for sandbox containers (e.g. `auto`, `private`).
180197
/// When unset, containers use the default user namespace.
181198
pub userns: Option<String>,
@@ -340,6 +357,21 @@ impl PodmanComputeConfig {
340357
));
341358
}
342359
}
360+
// additional_ca_bundle is independent of the corporate proxy setting
361+
// above: it applies to direct, inspected egress and has no
362+
// https_proxy dependency. Read and validate the actual host file
363+
// here (this path is not yet mounted into any container), rather
364+
// than only checking it is non-empty: an operator with a mistyped
365+
// path or a certificate-free bundle should see a config-validation
366+
// error at sandbox-create time, not a failed/crash-looping sandbox
367+
// once the supervisor container starts and tries to read it.
368+
if let Some(path) = self.additional_ca_bundle.as_deref() {
369+
openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file(
370+
path,
371+
"additional_ca_bundle",
372+
)
373+
.map_err(crate::client::PodmanApiError::InvalidInput)?;
374+
}
343375
Ok(())
344376
}
345377

@@ -526,6 +558,7 @@ impl Default for PodmanComputeConfig {
526558
proxy_auth_allow_insecure: None,
527559
proxy_connect_by_hostname: None,
528560
proxy_ca_bundle: None,
561+
additional_ca_bundle: None,
529562
userns: None,
530563
uidmap: Vec::new(),
531564
gidmap: Vec::new(),
@@ -839,6 +872,65 @@ mod tests {
839872
assert!(err.to_string().contains("proxy_ca_bundle"), "{err}");
840873
}
841874

875+
#[test]
876+
fn validate_configuration_accepts_additional_ca_bundle_without_proxy() {
877+
// #3781: unlike proxy_ca_bundle, this must work with no corporate
878+
// proxy configured at all.
879+
let directory = tempfile::tempdir().unwrap();
880+
let ca_path = directory.path().join("additional-ca.pem");
881+
std::fs::write(&ca_path, generate_test_ca_pem()).unwrap();
882+
883+
let cfg = PodmanComputeConfig {
884+
additional_ca_bundle: Some(ca_path.to_str().unwrap().to_string()),
885+
..PodmanComputeConfig::default()
886+
};
887+
assert!(cfg.validate_proxy_config().is_ok());
888+
assert!(cfg.https_proxy.is_none());
889+
}
890+
891+
#[test]
892+
fn validate_configuration_rejects_empty_additional_ca_bundle() {
893+
let cfg = PodmanComputeConfig {
894+
additional_ca_bundle: Some(" ".to_string()),
895+
..PodmanComputeConfig::default()
896+
};
897+
let err = cfg.validate_proxy_config().unwrap_err();
898+
assert!(err.to_string().contains("additional_ca_bundle"), "{err}");
899+
}
900+
901+
#[test]
902+
fn validate_configuration_rejects_nonexistent_additional_ca_bundle() {
903+
// #3781 fix requirement: an operator with a mistyped path must see
904+
// this at config-validation time, not a failed sandbox later.
905+
let cfg = PodmanComputeConfig {
906+
additional_ca_bundle: Some("/nonexistent/additional-ca.pem".to_string()),
907+
..PodmanComputeConfig::default()
908+
};
909+
let err = cfg.validate_proxy_config().unwrap_err();
910+
assert!(err.to_string().contains("additional_ca_bundle"), "{err}");
911+
}
912+
913+
#[test]
914+
fn validate_configuration_rejects_certificate_free_additional_ca_bundle() {
915+
let directory = tempfile::tempdir().unwrap();
916+
let ca_path = directory.path().join("not-a-cert.pem");
917+
std::fs::write(&ca_path, "not a certificate\n").unwrap();
918+
919+
let cfg = PodmanComputeConfig {
920+
additional_ca_bundle: Some(ca_path.to_str().unwrap().to_string()),
921+
..PodmanComputeConfig::default()
922+
};
923+
let err = cfg.validate_proxy_config().unwrap_err();
924+
assert!(err.to_string().contains("additional_ca_bundle"), "{err}");
925+
}
926+
927+
/// Generate a real, valid self-signed CA certificate PEM for tests.
928+
fn generate_test_ca_pem() -> String {
929+
let key_pair = rcgen::KeyPair::generate().unwrap();
930+
let params = rcgen::CertificateParams::new(Vec::new()).unwrap();
931+
params.self_signed(&key_pair).unwrap().pem()
932+
}
933+
842934
#[test]
843935
fn validate_proxy_config_rejects_url_components() {
844936
for url in [

‎crates/openshell-driver-podman/src/container.rs‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str =
6767
openshell_core::driver_utils::UPSTREAM_PROXY_AUTH_MOUNT_PATH;
6868
const RESOLV_CONF_PATH: &str = "/etc/resolv.conf";
6969
const PROXY_CA_MOUNT_PATH: &str = openshell_core::driver_utils::PROXY_CA_MOUNT_PATH;
70+
const ADDITIONAL_CA_MOUNT_PATH: &str = openshell_core::driver_utils::ADDITIONAL_CA_MOUNT_PATH;
7071
const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str =
7172
openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR;
7273

@@ -515,6 +516,19 @@ fn upstream_proxy_cli_args(config: &PodmanComputeConfig) -> Vec<String> {
515516
args
516517
}
517518

519+
/// Build the additional-destination-CA command-line argument passed to the
520+
/// supervisor. Independent of `upstream_proxy_cli_args`: this applies to
521+
/// direct, inspected egress regardless of whether a corporate proxy is
522+
/// configured.
523+
fn additional_ca_cli_args(config: &PodmanComputeConfig) -> Vec<String> {
524+
let mut args = Vec::new();
525+
if config.additional_ca_bundle.is_some() {
526+
args.push("--additional-ca-bundle".to_string());
527+
args.push(ADDITIONAL_CA_MOUNT_PATH.to_string());
528+
}
529+
args
530+
}
531+
518532
fn build_env(
519533
sandbox: &DriverSandbox,
520534
config: &PodmanComputeConfig,
@@ -1178,6 +1192,7 @@ fn build_base_spec(
11781192
driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(),
11791193
];
11801194
command.extend(upstream_proxy_cli_args(config));
1195+
command.extend(additional_ca_cli_args(config));
11811196

11821197
let container_spec = ContainerSpec {
11831198
name,
@@ -1371,6 +1386,24 @@ fn build_base_spec(
13711386
options: ro,
13721387
});
13731388
}
1389+
// Bind-mount an operator-supplied additional destination CA
1390+
// bundle read-only when configured. Independent of the corporate
1391+
// proxy CA above: this applies to direct, inspected egress. The
1392+
// supervisor reads it via the --additional-ca-bundle argv path
1393+
// (see additional_ca_cli_args) and folds it into its upstream
1394+
// trust store alongside the system bundle.
1395+
if let Some(ca_bundle) = &config.additional_ca_bundle {
1396+
let mut ro = vec!["ro".into(), "rbind".into()];
1397+
if is_selinux_enabled() {
1398+
ro.push("z".into());
1399+
}
1400+
m.push(Mount {
1401+
kind: "bind".into(),
1402+
source: ca_bundle.clone(),
1403+
destination: ADDITIONAL_CA_MOUNT_PATH.into(),
1404+
options: ro,
1405+
});
1406+
}
13741407
if let Some(bin_path) = supervisor_bin_path {
13751408
let mut opts = vec!["ro".into(), "rbind".into()];
13761409
if is_selinux_enabled() {
@@ -2671,6 +2704,70 @@ mod tests {
26712704
);
26722705
}
26732706

2707+
#[test]
2708+
fn container_spec_binds_additional_ca_bundle_and_passes_argv_without_proxy() {
2709+
// #3781: the additional CA must work for direct, inspected egress
2710+
// with no corporate proxy configured at all — unlike proxy_ca_bundle,
2711+
// which requires https_proxy.
2712+
let sandbox = test_sandbox("additional-ca-id", "additional-ca-name");
2713+
let mut config = test_config();
2714+
assert!(
2715+
config.https_proxy.is_none(),
2716+
"test must exercise the no-proxy case"
2717+
);
2718+
config.additional_ca_bundle = Some("/host/additional-ca.pem".to_string());
2719+
2720+
let spec = build_container_spec(&sandbox, &config);
2721+
let command = spec_command(&spec);
2722+
2723+
let idx = command
2724+
.iter()
2725+
.position(|a| a == "--additional-ca-bundle")
2726+
.expect("additional CA bundle flag present");
2727+
assert_eq!(
2728+
command.get(idx + 1).map(String::as_str),
2729+
Some("/etc/openshell/tls/additional/ca-bundle.pem")
2730+
);
2731+
2732+
let mounts = spec["mounts"]
2733+
.as_array()
2734+
.expect("mounts should be an array");
2735+
let ca_mount = mounts
2736+
.iter()
2737+
.find(|m| {
2738+
m["type"].as_str() == Some("bind")
2739+
&& m["destination"].as_str()
2740+
== Some("/etc/openshell/tls/additional/ca-bundle.pem")
2741+
})
2742+
.expect("additional CA bundle bind mount present");
2743+
assert_eq!(ca_mount["source"].as_str(), Some("/host/additional-ca.pem"));
2744+
assert!(
2745+
ca_mount["options"]
2746+
.as_array()
2747+
.expect("options array")
2748+
.iter()
2749+
.any(|o| o.as_str() == Some("ro")),
2750+
"additional CA bundle mount must be read-only"
2751+
);
2752+
}
2753+
2754+
#[test]
2755+
fn container_spec_omits_additional_ca_bundle_when_unconfigured() {
2756+
let sandbox = test_sandbox("test-id", "test-name");
2757+
let spec = build_container_spec(&sandbox, &test_config());
2758+
let command = spec_command(&spec);
2759+
assert!(
2760+
!command.iter().any(|a| a == "--additional-ca-bundle"),
2761+
"no additional-ca-bundle flag without operator config"
2762+
);
2763+
let mounts = spec["mounts"].as_array().expect("mounts array");
2764+
assert!(
2765+
!mounts.iter().any(|m| m["destination"].as_str()
2766+
== Some("/etc/openshell/tls/additional/ca-bundle.pem")),
2767+
"no additional CA bundle mount without operator config"
2768+
);
2769+
}
2770+
26742771
#[test]
26752772
fn container_spec_sandbox_env_cannot_influence_proxy_argv() {
26762773
use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate};

‎crates/openshell-driver-podman/src/main.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,14 @@ struct Args {
174174
#[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CA_BUNDLE")]
175175
sandbox_proxy_ca_bundle: Option<String>,
176176

177+
/// Path (on the gateway host) to a PEM CA bundle trusted for direct
178+
/// (non-proxied), policy-inspected HTTPS egress to destinations signed by
179+
/// a private CA. Unlike `--sandbox-proxy-ca-bundle`, this applies
180+
/// regardless of whether a corporate proxy is configured. Bind-mounted
181+
/// read-only into the supervisor container.
182+
#[arg(long, env = "OPENSHELL_SANDBOX_ADDITIONAL_CA_BUNDLE")]
183+
sandbox_additional_ca_bundle: Option<String>,
184+
177185
/// User namespace mode for sandbox containers (e.g. `auto`).
178186
/// When unset, containers use the default user namespace.
179187
#[arg(long, env = "OPENSHELL_PODMAN_USERNS")]
@@ -240,6 +248,7 @@ async fn main() -> Result<()> {
240248
proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure,
241249
proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname,
242250
proxy_ca_bundle: args.sandbox_proxy_ca_bundle,
251+
additional_ca_bundle: args.sandbox_additional_ca_bundle,
243252
userns: args.userns,
244253
uidmap: args.uidmap,
245254
gidmap: args.gidmap,

‎crates/openshell-supervisor-network/src/l7/tls.rs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -585,11 +585,80 @@ mod tests {
585585
assert_eq!(config.alpn_protocols, vec![b"http/1.1".to_vec()]);
586586
}
587587

588+
/// Reproduces issue #3781 end to end with a real TLS handshake: an
589+
/// upstream client trying to reach a destination signed by an
590+
/// operator-owned private CA fails when that CA is unknown, and succeeds
591+
/// once it is appended to the bundle passed to
592+
/// `build_upstream_client_config` — the same mechanism the Podman driver
593+
/// fix (`additional_ca_bundle`) feeds into. No fakes: a real rustls
594+
/// `TlsAcceptor`/`TlsConnector` pair over a real loopback TCP connection.
595+
#[tokio::test]
596+
async fn additional_ca_enables_trust_of_privately_signed_upstream() {
597+
let hostname = "private.example";
598+
599+
// A CA the system bundle has never heard of, standing in for the
600+
// operator-owned private CA from #3781.
601+
let private_ca = SandboxCa::generate().unwrap();
602+
let private_ca_pem = private_ca.cert_pem().to_string();
603+
let cert_cache = CertCache::new(private_ca);
604+
let leaf = cert_cache.get_or_generate(hostname).unwrap();
605+
606+
let server_config = Arc::new(
607+
ServerConfig::builder()
608+
.with_no_client_auth()
609+
.with_single_cert(leaf.cert_chain.clone(), leaf.private_key.clone_key())
610+
.unwrap(),
611+
);
612+
let acceptor = TlsAcceptor::from(server_config);
613+
614+
// --- Without the private CA: must fail. This is the literal bug in
615+
// #3781 — "Upstream TLS establishment failed" against a
616+
// policy-allowed, private-CA-signed destination. ---
617+
let addr = serve_one(acceptor.clone()).await;
618+
let client_without = build_upstream_client_config("").unwrap();
619+
let tcp = tokio::net::TcpStream::connect(addr).await.unwrap();
620+
let result = tls_connect_upstream(tcp, hostname, &client_without).await;
621+
assert!(
622+
result.is_err(),
623+
"connection to a privately-signed destination must fail when the \
624+
private CA is not trusted, reproducing #3781"
625+
);
626+
627+
// --- With the private CA appended to the bundle: must succeed. This
628+
// is what additional_ca_bundle plumbs through in the Podman driver
629+
// fix. ---
630+
let addr = serve_one(acceptor).await;
631+
let client_with = build_upstream_client_config(&private_ca_pem).unwrap();
632+
let tcp = tokio::net::TcpStream::connect(addr).await.unwrap();
633+
let result = tls_connect_upstream(tcp, hostname, &client_with).await;
634+
assert!(
635+
result.is_ok(),
636+
"connection must succeed once the private CA is trusted: {:?}",
637+
result.err()
638+
);
639+
}
640+
588641
/// Helper: generate a self-signed CA and return its PEM string.
589642
fn generate_ca_pem() -> String {
590643
SandboxCa::generate().unwrap().ca_cert_pem
591644
}
592645

646+
/// Accept exactly one TLS connection on a fresh loopback listener and
647+
/// return its address. Used by `additional_ca_enables_trust_of_...` to
648+
/// give each connection attempt its own listener.
649+
async fn serve_one(acceptor: TlsAcceptor) -> std::net::SocketAddr {
650+
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
651+
let addr = listener.local_addr().unwrap();
652+
tokio::spawn(async move {
653+
let (stream, _) = listener.accept().await.unwrap();
654+
// Don't unwrap: when the client (correctly) rejects an untrusted
655+
// certificate, the server observes a failed handshake too. Only
656+
// the client's outcome matters here.
657+
let _ = acceptor.accept(stream).await;
658+
});
659+
addr
660+
}
661+
593662
#[test]
594663
fn load_pem_certs_single_ca() {
595664
let pem = generate_ca_pem();

0 commit comments

Comments
 (0)