fix(sandbox): reject local control peers when the kernel denies the ingress filter

Binding the boundary control listener attaches a classic BPF filter that drops
loopback-interface ingress. Some kernels gate SO_ATTACH_FILTER on CAP_NET_ADMIN,
and a sandbox pod holds no capabilities at all, so the attach returns EPERM and
every sandbox dies at startup with:

    bind boundary control listener: Operation not permitted (os error 1)

Granting the capability is not an option: the native Linux audit evidence
requires an empty capability set, so a sandbox that could attach the filter
would fail its own qualification instead.

Treat EPERM as "filter unavailable" and move the check to accept time. Any peer
the workload can reach the listener from arrives over `lo`, so its source
address is either loopback or the address the connection was accepted on; a
supervisor runs in a different pod and presents a different address. The serve
loop already drops PermissionDenied accepts, which is the same path the Unix
peer check uses, and the control channel still requires the per-sandbox mTLS
certificate.

Kernels that permit the filter keep the stronger standing guarantee.

Signed-off-by: divesh <dgude@nvidia.com>

Author: FrostGod

crates/openshell-isolation-interface/src/linux/socket_confinement.rs

@@ -57,6 +57,24 @@ pub fn reject_loopback_ingress(fd: impl AsFd) -> io::Result<()> {
     reject_ingress_interface(fd, index)
 }
 
+/// Install loopback-ingress rejection, reporting whether the kernel allowed it.
+///
+/// Some kernels gate `SO_ATTACH_FILTER` on `CAP_NET_ADMIN`, which a sandbox
+/// holding no capabilities cannot satisfy. `Ok(false)` means the filter is
+/// unavailable and the caller must reject loopback peers as it accepts them
+/// instead; every other error is still a hard failure.
+///
+/// # Errors
+///
+/// Returns the kernel error for any failure other than `EPERM`.
+pub fn try_reject_loopback_ingress(fd: impl AsFd) -> io::Result<bool> {
+    match reject_loopback_ingress(fd) {
+        Ok(()) => Ok(true),
+        Err(error) if error.raw_os_error() == Some(libc::EPERM) => Ok(false),
+        Err(error) => Err(error),
+    }
+}
+
 fn reject_ingress_interface(fd: impl AsFd, index: u32) -> io::Result<()> {
     // Ancillary loads use the documented negative offset encoding.
     let ifindex_offset = (libc::SKF_AD_OFF + libc::SKF_AD_IFINDEX).cast_unsigned();

crates/openshell-sandbox/src/boundary_server.rs

@@ -3277,6 +3277,9 @@ mod linux {
         Tcp {
             listener: std::net::TcpListener,
             server_config: Arc<rustls::ServerConfig>,
+            /// Set when the kernel refused the loopback-ingress filter, so
+            /// `accept` has to reject same-host peers itself.
+            reject_local_peers: bool,
         },
     }
 
@@ -3305,12 +3308,13 @@ mod linux {
                     })
                 }
                 BoundaryListenerConfig::TlsTcp { address, tls } => {
-                    let listener = Self::bind_tcp(*address)?;
+                    let (listener, reject_local_peers) = Self::bind_tcp(*address)?;
                     listener.set_nonblocking(true)?;
                     let server_config = Arc::new(load_tls_server_config(tls)?);
                     Ok(Self::Tcp {
                         listener,
                         server_config,
+                        reject_local_peers,
                     })
                 }
             }
@@ -3320,22 +3324,36 @@ mod linux {
         /// before it listens so workload sockets cannot reach it through
         /// loopback or the pod's own address. Configuration rejects loopback
         /// addresses; tests bind them without the filter.
-        fn bind_tcp(address: std::net::SocketAddr) -> io::Result<std::net::TcpListener> {
+        ///
+        /// Returns whether `accept` must reject same-host peers itself, which
+        /// is the case on kernels that gate `SO_ATTACH_FILTER` on
+        /// `CAP_NET_ADMIN`: a sandbox holds no capabilities, so the standing
+        /// filter is unavailable there and the check moves to accept time.
+        fn bind_tcp(address: std::net::SocketAddr) -> io::Result<(std::net::TcpListener, bool)> {
             let socket = socket2::Socket::new(
                 socket2::Domain::for_address(address),
                 socket2::Type::STREAM,
                 Some(socket2::Protocol::TCP),
             )?;
             socket.set_cloexec(true)?;
             socket.set_reuse_address(true)?;
+            let mut reject_local_peers = false;
             if !address.ip().is_loopback() {
-                openshell_isolation_interface::linux::socket_confinement::reject_loopback_ingress(
-                    &socket,
-                )?;
+                let filtered =
+                    openshell_isolation_interface::linux::socket_confinement::try_reject_loopback_ingress(
+                        &socket,
+                    )?;
+                reject_local_peers = !filtered;
+                if reject_local_peers {
+                    tracing::warn!(
+                        "Kernel refused the boundary loopback-ingress filter; rejecting \
+                         same-host control peers on accept instead"
+                    );
+                }
             }
             socket.bind(&address.into())?;
             socket.listen(128)?;
-            Ok(socket.into())
+            Ok((socket.into(), reject_local_peers))
         }
 
         fn bind_vsock(port: u32) -> io::Result<OwnedFd> {
@@ -3428,8 +3446,12 @@ mod linux {
                 Self::Tcp {
                     listener,
                     server_config,
+                    reject_local_peers,
                 } => {
                     let (stream, _) = listener.accept()?;
+                    if *reject_local_peers {
+                        reject_workload_tcp_peer(&stream)?;
+                    }
                     if let Err(error) = stream.set_nodelay(true) {
                         tracing::debug!(%error, "Failed to set boundary TCP_NODELAY");
                     }
@@ -3442,6 +3464,24 @@ mod linux {
         }
     }
 
+    /// Refuse a control connection that originated inside this sandbox.
+    ///
+    /// This is the accept-time stand-in for the loopback-ingress filter. Any
+    /// peer the workload can reach the listener from arrives over `lo`, so its
+    /// source address is either loopback or one of this network namespace's own
+    /// addresses, which is also the address the connection was accepted on. A
+    /// supervisor runs in a different pod and so presents a different address.
+    /// Rejection here is defence in depth; the control channel still requires
+    /// the per-sandbox mTLS certificate.
+    fn reject_workload_tcp_peer(stream: &std::net::TcpStream) -> io::Result<()> {
+        let peer = stream.peer_addr()?.ip().to_canonical();
+        let local = stream.local_addr()?.ip().to_canonical();
+        if peer.is_loopback() || peer == local {
+            return Err(io::Error::from_raw_os_error(libc::EACCES));
+        }
+        Ok(())
+    }
+
     fn reject_workload_unix_peer(stream: &std::os::unix::net::UnixStream) -> io::Result<()> {
         let mut credentials = libc::ucred {
             pid: 0,
@@ -5150,22 +5190,36 @@ mod linux {
                 .tcp_local_addr()
                 .expect("TLS listener address")
                 .port();
-            // Loopback and the host's own address both arrive on `lo`; the
-            // dropped SYN never completes a handshake.
-            let result = std::net::TcpStream::connect_timeout(
+            // Loopback and the host's own address both arrive on `lo`. With the
+            // standing filter the dropped SYN never completes a handshake; where
+            // the kernel gates `SO_ATTACH_FILTER` on `CAP_NET_ADMIN` the
+            // handshake completes and `accept` refuses the peer instead. Either
+            // way no loopback client reaches a control session.
+            let _client = std::net::TcpStream::connect_timeout(
                 &std::net::SocketAddr::from(([127, 0, 0, 1], port)),
                 Duration::from_millis(300),
             );
-            assert!(
-                result.is_err(),
-                "loopback client reached the control listener"
-            );
             assert!(matches!(
                 listener.accept().map(|_| ()),
-                Err(error) if error.kind() == io::ErrorKind::WouldBlock
+                Err(error) if matches!(
+                    error.kind(),
+                    io::ErrorKind::WouldBlock | io::ErrorKind::PermissionDenied
+                )
             ));
         }
 
+        #[test]
+        fn accept_time_fallback_refuses_a_same_host_control_peer() {
+            // Stands in for a kernel that gates `SO_ATTACH_FILTER` on
+            // `CAP_NET_ADMIN`, where the filter cannot be installed at all.
+            let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
+            let address = listener.local_addr().expect("listener address");
+            let _client = std::net::TcpStream::connect(address).expect("connect loopback client");
+            let (accepted, _) = listener.accept().expect("accept loopback client");
+            let error = reject_workload_tcp_peer(&accepted).expect_err("loopback peer refused");
+            assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
+        }
+
         #[test]
         fn tls_listener_preserves_session_when_control_switches_to_async_streaming() {
             let directory = tempfile::tempdir().expect("temporary directory");