diff --git a/crates/openshell-cli/src/edge_tunnel.rs b/crates/openshell-cli/src/edge_tunnel.rs index 814e245f3..72f54a285 100644 --- a/crates/openshell-cli/src/edge_tunnel.rs +++ b/crates/openshell-cli/src/edge_tunnel.rs @@ -101,6 +101,11 @@ async fn accept_loop(listener: TcpListener, config: Arc) { match listener.accept().await { Ok((stream, peer)) => { debug!(peer = %peer, "accepted local tunnel connection"); + // Small gRPC frames must flush immediately; Nagle's algorithm + // otherwise adds per-write latency on this loopback hop. + if let Err(e) = stream.set_nodelay(true) { + debug!(peer = %peer, error = %e, "failed to set TCP_NODELAY"); + } let config = Arc::clone(&config); tokio::spawn(async move { if let Err(e) = handle_connection(stream, &config).await { @@ -174,6 +179,21 @@ async fn open_ws(config: &TunnelConfig) -> Result Some(tcp), + MaybeTlsStream::Rustls(tls) => Some(tls.get_ref().0), + _ => None, + }; + if let Some(tcp) = tcp { + if let Err(e) = tcp.set_nodelay(true) { + debug!(error = %e, "failed to set TCP_NODELAY on WebSocket stream"); + } + } else { + debug!("unknown WebSocket stream variant; skipping TCP_NODELAY"); + } + debug!( status = %response.status(), "WebSocket connected to edge" diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 0182e1b66..452ff94d7 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -2891,6 +2891,11 @@ pub async fn service_forward_tcp( let (socket, peer) = accepted .into_diagnostic() .wrap_err("failed to accept local forward connection")?; + // Forwarded request/response bytes are relayed in small frames; + // Nagle's algorithm would add per-write latency. + if let Err(err) = socket.set_nodelay(true) { + tracing::debug!(peer = %peer, error = %err, "failed to set TCP_NODELAY"); + } let mut client = client.clone(); let sandbox_id = sandbox_id.clone(); let target_host = target_host.to_string(); diff --git a/crates/openshell-cli/src/tls.rs b/crates/openshell-cli/src/tls.rs index 10df401a5..28eee93c8 100644 --- a/crates/openshell-cli/src/tls.rs +++ b/crates/openshell-cli/src/tls.rs @@ -295,6 +295,11 @@ impl tower::Service for InsecureTlsConnector { let port = uri.port_u16().unwrap_or(443); let addr = format!("{host}:{port}"); let tcp = tokio::net::TcpStream::connect(addr).await?; + // set TCP_NODELAY so small gRPC frames are not delayed. Okay if it fails; + // things just go a bit slower in some cases, so we log and continue. + if let Err(e) = tcp.set_nodelay(true) { + debug!(error = %e, "failed to set TCP_NODELAY on TLS connection"); + } let server_name = ServerName::try_from(host)?; let tls_stream = tls_connector.connect(server_name, tcp).await?; Ok(hyper_util::rt::TokioIo::new(tls_stream)) diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 6462ccbbf..4cc39dcd8 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -549,6 +549,12 @@ async fn serve_gateway_listener( } }; + // Disable Nagle's algorithm: gateway responses and tunnel relay frames + // are small, latency-sensitive writes that must not wait for delayed ACKs. + if let Err(e) = stream.set_nodelay(true) { + debug!(error = %e, client = %addr, "Failed to set TCP_NODELAY on accepted connection"); + } + spawn_gateway_connection( stream, addr, diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 6d6053636..7f84dc04d 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -266,6 +266,11 @@ impl ProxyHandle { loop { match listener.accept().await { Ok((stream, _addr)) => { + // Proxied frames are relayed as they arrive; Nagle's + // algorithm would delay every small response. + if let Err(e) = stream.set_nodelay(true) { + tracing::debug!(error = %e, "failed to set TCP_NODELAY on accepted proxy connection"); + } let opa = opa_engine.clone(); let cache = identity_cache.clone(); let spid = entrypoint_pid.clone(); @@ -771,9 +776,7 @@ async fn handle_tcp_connection( // addresses (used by rootless Podman with pasta) are not hard-blocked. // Cloud metadata IPs and control-plane ports are still rejected. match resolve_and_check_trusted_gateway(&host, port, gw, sandbox_entrypoint_pid).await { - Ok(addrs) => TcpStream::connect(addrs.as_slice()) - .await - .into_diagnostic()?, + Ok(addrs) => connect_upstream(addrs.as_slice()).await.into_diagnostic()?, Err(reason) => { { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -827,9 +830,7 @@ async fn handle_tcp_connection( match resolve_and_check_allowed_ips(&host, port, &nets, sandbox_entrypoint_pid) .await { - Ok(addrs) => TcpStream::connect(addrs.as_slice()) - .await - .into_diagnostic()?, + Ok(addrs) => connect_upstream(addrs.as_slice()).await.into_diagnostic()?, Err(reason) => { { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -929,9 +930,7 @@ async fn handle_tcp_connection( // the resolved IP in allowed_ips. Always-blocked addresses and // control-plane ports remain denied. match resolve_and_check_declared_endpoint(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => TcpStream::connect(addrs.as_slice()) - .await - .into_diagnostic()?, + Ok(addrs) => connect_upstream(addrs.as_slice()).await.into_diagnostic()?, Err(reason) => { { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -981,9 +980,7 @@ async fn handle_tcp_connection( } else { // Default: reject all internal IPs (loopback, RFC 1918, link-local). match resolve_and_reject_internal(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => TcpStream::connect(addrs.as_slice()) - .await - .into_diagnostic()?, + Ok(addrs) => connect_upstream(addrs.as_slice()).await.into_diagnostic()?, Err(reason) => { { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -2097,6 +2094,20 @@ async fn write_all(writer: &mut (impl tokio::io::AsyncWrite + Unpin), data: &[u8 Ok(()) } +/// Connect to an upstream address with `TCP_NODELAY` set. +/// +/// Tunneled request/response bytes are relayed as they arrive; without +/// `TCP_NODELAY`, Nagle's algorithm holds back every sub-MSS write and adds +/// milliseconds of latency to each small request/response round trip. Setting +/// it is best-effort: a failure only costs latency, so we log and continue. +async fn connect_upstream(addrs: &[SocketAddr]) -> std::io::Result { + let stream = TcpStream::connect(addrs).await?; + if let Err(e) = stream.set_nodelay(true) { + tracing::debug!(error = %e, "failed to set TCP_NODELAY on upstream connection"); + } + Ok(stream) +} + #[derive(Debug, Clone)] struct L7ConfigSnapshot { config: crate::l7::L7EndpointConfig, @@ -4121,7 +4132,7 @@ async fn handle_forward_proxy( } // 6. Connect upstream - let mut upstream = match TcpStream::connect(addrs.as_slice()).await { + let mut upstream = match connect_upstream(addrs.as_slice()).await { Ok(s) => s, Err(e) => { let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -4379,6 +4390,18 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + /// Regression test: upstream connections must disable Nagle's algorithm, + /// otherwise every small tunneled request waits on delayed ACKs and adds + /// milliseconds of latency per round trip. + #[tokio::test] + async fn connect_upstream_sets_tcp_nodelay() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local addr"); + + let stream = connect_upstream(&[addr]).await.expect("connect"); + assert!(stream.nodelay().expect("query TCP_NODELAY")); + } + fn websocket_l7_config( protocol: crate::l7::L7Protocol, websocket_credential_rewrite: bool, diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 62d10f374..1d6ec1366 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -629,13 +629,17 @@ pub async fn connect_in_netns( .await .map_err(|_| std::io::Error::other("netns connect thread panicked"))??; std_stream.set_nonblocking(true)?; - return tokio::net::TcpStream::from_std(std_stream); + let stream = tokio::net::TcpStream::from_std(std_stream)?; + crate::supervisor_session::set_relay_nodelay(&stream); + return Ok(stream); } #[cfg(not(target_os = "linux"))] let _ = netns_fd; - tokio::net::TcpStream::connect(addr).await + let stream = tokio::net::TcpStream::connect(addr).await?; + crate::supervisor_session::set_relay_nodelay(&stream); + Ok(stream) } #[derive(Clone)] @@ -1275,6 +1279,22 @@ mod tests { use super::*; use std::process::Stdio; + /// Regression test: direct-tcpip forwards must disable Nagle's algorithm, + /// otherwise every small response from a sandbox service waits on delayed + /// ACKs and adds milliseconds of latency per round trip. + #[tokio::test] + async fn connect_in_netns_sets_tcp_nodelay() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind listener"); + let addr = listener.local_addr().expect("local addr"); + + let stream = connect_in_netns(&addr.to_string(), None) + .await + .expect("connect"); + assert!(stream.nodelay().expect("query TCP_NODELAY")); + } + /// Verify that dropping the input sender (the operation `channel_eof` /// performs) causes the stdin writer loop to exit and close the child's /// stdin pipe. Without this, commands like `cat | tar xf -` used by diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 7c524676c..e5b529a6a 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -620,10 +620,14 @@ async fn connect_tcp_target( .await .map_err(|_| "netns tcp connect thread panicked")??; stream.set_nonblocking(true)?; - return Ok(tokio::net::TcpStream::from_std(stream)?); + let stream = tokio::net::TcpStream::from_std(stream)?; + set_relay_nodelay(&stream); + return Ok(stream); } - Ok(tokio::net::TcpStream::connect((host.as_str(), port)).await?) + let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; + set_relay_nodelay(&stream); + Ok(stream) } #[cfg(not(target_os = "linux"))] @@ -632,7 +636,21 @@ async fn connect_tcp_target( port: u16, _netns_fd: Option, ) -> Result> { - Ok(tokio::net::TcpStream::connect((host.as_str(), port)).await?) + let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; + set_relay_nodelay(&stream); + Ok(stream) +} + +/// Disable Nagle's algorithm on a relayed TCP target, best-effort. +/// +/// Relayed request/response bytes are small and latency-sensitive; without +/// `TCP_NODELAY` every sub-MSS write waits on a delayed ACK, adding +/// milliseconds per round trip. A failure here only costs latency, so we log +/// and continue rather than fail the connection. +pub(crate) fn set_relay_nodelay(stream: &tokio::net::TcpStream) { + if let Err(e) = stream.set_nodelay(true) { + debug!(error = %e, "failed to set TCP_NODELAY on relay target"); + } } #[cfg(test)] @@ -674,6 +692,22 @@ mod target_tests { } } + /// Regression test: relayed TCP targets must disable Nagle's algorithm, + /// otherwise every small tunneled request waits on delayed ACKs and adds + /// milliseconds of latency per round trip. + #[tokio::test] + async fn connect_tcp_target_sets_tcp_nodelay() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind listener"); + let addr = listener.local_addr().expect("local addr"); + + let stream = connect_tcp_target(addr.ip().to_string(), addr.port(), None) + .await + .expect("connect"); + assert!(stream.nodelay().expect("query TCP_NODELAY")); + } + #[test] fn tcp_target_allows_loopback_hosts() { validate_tcp_target(&tcp("127.0.0.1", 8080)).expect("ipv4 loopback");