diff --git a/.gitignore b/.gitignore index 584b4c44..32610f71 100644 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,7 @@ python/*.data venv/ ENV/ env/ -.python-version +# .python-version is tracked — pins uv to match the sandbox base image Python # Installer logs pip-log.txt diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..44305e2f --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13.12 diff --git a/architecture/sandbox-custom-containers.md b/architecture/sandbox-custom-containers.md index 27ee2a91..b72cf047 100644 --- a/architecture/sandbox-custom-containers.md +++ b/architecture/sandbox-custom-containers.md @@ -98,7 +98,7 @@ The `openshell-sandbox` supervisor adapts to arbitrary environments: - **Log file fallback**: Attempts to open `/var/log/openshell.log` for append; silently falls back to stdout-only logging if the path is not writable. - **Command resolution**: Executes the command from CLI args, then the `OPENSHELL_SANDBOX_COMMAND` env var (set to `sleep infinity` by the server), then `/bin/bash` as a last resort. -- **Network namespace**: Requires successful namespace creation for proxy isolation; startup fails in proxy mode if required capabilities (`CAP_NET_ADMIN`, `CAP_SYS_ADMIN`) or `iproute2` are unavailable. +- **Network namespace**: Requires successful namespace creation for proxy isolation; startup fails in proxy mode if required capabilities (`CAP_NET_ADMIN`, `CAP_SYS_ADMIN`) or `iproute2` are unavailable. If the `iptables` package is present, the supervisor installs OUTPUT chain rules (LOG + REJECT) inside the namespace to provide fast-fail behavior (immediate `ECONNREFUSED` instead of a 30-second timeout) and diagnostic logging when processes attempt direct connections that bypass the HTTP CONNECT proxy. If `iptables` is absent, the supervisor logs a warning and continues — core network isolation still works via routing. ## Design Decisions @@ -114,6 +114,7 @@ The `openshell-sandbox` supervisor adapts to arbitrary environments: | Clear `run_as_user/group` for custom images | Prevents startup failure when the image lacks the default `sandbox` user | | Non-fatal log file init | `/var/log/openshell.log` may be unwritable in arbitrary images; falls back to stdout | | `docker save` / `ctr import` for push | Avoids requiring a registry for local dev; images land directly in the k3s containerd store | +| Optional `iptables` for bypass detection | Core network isolation works via routing alone (`iproute2`); `iptables` only adds fast-fail (`ECONNREFUSED`) and diagnostic LOG entries. Making it optional avoids hard failures in minimal images that lack `iptables` while giving better UX when it is available. | ## Limitations diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 62b2a554..a8e4d247 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -19,13 +19,14 @@ All paths are relative to `crates/openshell-sandbox/src/`. | `identity.rs` | `BinaryIdentityCache` -- SHA256 trust-on-first-use binary integrity | | `procfs.rs` | `/proc` filesystem reading for TCP peer identity resolution and ancestor chain walking | | `grpc_client.rs` | gRPC client for fetching policy, provider environment, inference route bundles, policy polling/status reporting, proposal submission, and log push (`CachedOpenShellClient`) | -| `denial_aggregator.rs` | `DenialAggregator` background task -- receives `DenialEvent`s from the proxy, deduplicates by `(host, port, binary)`, drains on flush interval | +| `denial_aggregator.rs` | `DenialAggregator` background task -- receives `DenialEvent`s from the proxy and bypass monitor, deduplicates by `(host, port, binary)`, drains on flush interval | | `mechanistic_mapper.rs` | Deterministic policy recommendation generator -- converts denial summaries to `PolicyChunk` proposals with confidence scores, rationale, and SSRF/private-IP detection | | `sandbox/mod.rs` | Platform abstraction -- dispatches to Linux or no-op | | `sandbox/linux/mod.rs` | Linux composition: Landlock then seccomp | | `sandbox/linux/landlock.rs` | Filesystem isolation via Landlock LSM (ABI V1) | | `sandbox/linux/seccomp.rs` | Syscall filtering via BPF on `SYS_socket` | -| `sandbox/linux/netns.rs` | Network namespace creation, veth pair setup, cleanup on drop | +| `bypass_monitor.rs` | Background `/dev/kmsg` reader for iptables bypass detection events | +| `sandbox/linux/netns.rs` | Network namespace creation, veth pair setup, bypass detection iptables rules, cleanup on drop | | `l7/mod.rs` | L7 types (`L7Protocol`, `TlsMode`, `EnforcementMode`, `L7EndpointConfig`), config parsing, validation, access preset expansion | | `l7/inference.rs` | Inference API pattern detection (`detect_inference_pattern()`), HTTP request/response parsing and formatting for intercepted inference connections | | `l7/tls.rs` | Ephemeral CA generation (`SandboxCa`), per-hostname leaf cert cache (`CertCache`), TLS termination/connection helpers | @@ -55,10 +56,12 @@ flowchart TD H --> I{Proxy mode?} I -- Yes --> J[Generate ephemeral CA + write TLS files] J --> K[Create network namespace] - K --> K2[Build InferenceContext] + K --> K1[Install bypass detection rules] + K1 --> K2[Build InferenceContext] K2 --> L[Start HTTP CONNECT proxy] I -- No --> M[Skip proxy setup] - L --> N{SSH enabled?} + L --> L2[Spawn bypass monitor] + L2 --> N{SSH enabled?} M --> N N -- Yes --> O[Spawn SSH server task] N -- No --> P[Spawn child process] @@ -96,7 +99,8 @@ flowchart TD 6. **Network namespace** (Linux, proxy mode only): - `NetworkNamespace::create()` builds the veth pair and namespace - Opens `/var/run/netns/sandbox-{uuid}` as an FD for later `setns()` - - On failure: return a fatal startup error (fail-closed) + - `install_bypass_rules(proxy_port)` installs iptables OUTPUT chain rules for bypass detection (fast-fail UX + diagnostic logging). See [Bypass detection](#bypass-detection). + - On failure: return a fatal startup error (fail-closed). Bypass rule failure is non-fatal (logged as warning). 7. **Proxy startup** (proxy mode only): - Validate that OPA engine and identity cache are present @@ -475,15 +479,123 @@ Each step has rollback on failure -- if any `ip` command fails, previously creat 2. Delete the host-side veth (`ip link delete veth-h-{id}`) -- this automatically removes the peer 3. Delete the namespace (`ip netns delete sandbox-{id}`) +#### Bypass detection + +**Files:** `crates/openshell-sandbox/src/sandbox/linux/netns.rs` (`install_bypass_rules()`), `crates/openshell-sandbox/src/bypass_monitor.rs` + +The network namespace routes all sandbox traffic through the veth pair, but a misconfigured process that ignores proxy environment variables can still attempt direct connections to the veth gateway IP or other addresses. Bypass detection catches these attempts, providing two benefits: immediate connection failure (fast-fail UX) instead of a 30-second TCP timeout, and structured diagnostic logging that identifies the offending process. + +##### iptables rules + +`install_bypass_rules()` installs OUTPUT chain rules inside the sandbox network namespace using `iptables` (IPv4) and `ip6tables` (IPv6, best-effort). Rules are installed via `ip netns exec {namespace} iptables ...`. The rules are evaluated in order: + +| # | Rule | Target | Purpose | +|---|------|--------|---------| +| 1 | `-d {host_ip}/32 -p tcp --dport {proxy_port}` | `ACCEPT` | Allow traffic to the proxy | +| 2 | `-o lo` | `ACCEPT` | Allow loopback traffic | +| 3 | `-m conntrack --ctstate ESTABLISHED,RELATED` | `ACCEPT` | Allow response packets for established connections | +| 4 | `-p tcp --syn -m limit --limit 5/sec --limit-burst 10 --log-prefix "openshell:bypass:{ns}:"` | `LOG` | Log TCP SYN bypass attempts (rate-limited) | +| 5 | `-p tcp` | `REJECT --reject-with icmp-port-unreachable` | Reject TCP bypass attempts (fast-fail) | +| 6 | `-p udp -m limit --limit 5/sec --limit-burst 10 --log-prefix "openshell:bypass:{ns}:"` | `LOG` | Log UDP bypass attempts, including DNS (rate-limited) | +| 7 | `-p udp` | `REJECT --reject-with icmp-port-unreachable` | Reject UDP bypass attempts (fast-fail) | + +The LOG rules use the `--log-uid` flag to include the UID of the process that initiated the connection. The log prefix `openshell:bypass:{namespace_name}:` enables the bypass monitor to filter `/dev/kmsg` for events belonging to a specific sandbox. + +The proxy port defaults to `3128` unless the policy specifies a different `http_addr`. IPv6 rules mirror the IPv4 rules via `ip6tables`; IPv6 rule installation failure is non-fatal (logged as warning) since IPv4 is the primary path. + +**Graceful degradation:** If iptables is not available (checked via `which iptables`), a warning is logged and rule installation is skipped entirely. The network namespace still provides isolation via routing — processes can only reach the proxy's IP, but without bypass rules they get a timeout rather than an immediate rejection. LOG rule failure is also non-fatal — if the `xt_LOG` kernel module is not loaded, the REJECT rules are still installed for fast-fail behavior. + +##### /dev/kmsg monitor + +`bypass_monitor::spawn()` starts a background tokio task (via `spawn_blocking`) that reads kernel log messages from `/dev/kmsg`. The monitor: + +1. Opens `/dev/kmsg` in read mode and seeks to end (skips historical messages) +2. Reads lines via `BufReader`, filtering for the namespace-specific prefix `openshell:bypass:{namespace_name}:` +3. Parses iptables LOG format via `parse_kmsg_line()`, extracting `DST`, `DPT`, `SPT`, `PROTO`, and `UID` fields +4. Resolves process identity for TCP events via `procfs::resolve_tcp_peer_identity()` (best-effort — requires a valid entrypoint PID and non-zero source port) +5. Emits a structured `tracing::warn!()` event with the tag `BYPASS_DETECT` +6. Sends a `DenialEvent` to the denial aggregator channel (if available) + +The `BypassEvent` struct holds the parsed fields: + +```rust +pub struct BypassEvent { + pub dst_addr: String, // Destination IP address + pub dst_port: u16, // Destination port + pub src_port: u16, // Source port (for process identity resolution) + pub proto: String, // "tcp" or "udp" + pub uid: Option, // UID from --log-uid (if present) +} +``` + +##### BYPASS_DETECT tracing event + +Each detected bypass attempt emits a `warn!()` log line with the following structured fields: + +| Field | Type | Description | +|-------|------|-------------| +| `dst_addr` | string | Destination IP address | +| `dst_port` | u16 | Destination port | +| `proto` | string | `"tcp"` or `"udp"` | +| `binary` | string | Binary path of the offending process (or `"-"` if unresolved) | +| `binary_pid` | string | PID of the offending process (or `"-"`) | +| `ancestors` | string | Ancestor chain (e.g., `"/usr/bin/bash -> /usr/bin/node"`) or `"-"` | +| `action` | string | Always `"reject"` | +| `reason` | string | `"direct connection bypassed HTTP CONNECT proxy"` | +| `hint` | string | Context-specific remediation hint (see below) | + +The `hint` field provides actionable guidance: + +| Condition | Hint | +|-----------|------| +| UDP + port 53 | `"DNS queries should route through the sandbox proxy; check resolver configuration"` | +| UDP (other) | `"UDP traffic must route through the sandbox proxy"` | +| TCP | `"ensure process honors HTTP_PROXY/HTTPS_PROXY; for Node.js set NODE_USE_ENV_PROXY=1"` | + +Process identity resolution is best-effort and TCP-only. For UDP events or when the entrypoint PID is not yet set (PID == 0), the binary, PID, and ancestors fields are reported as `"-"`. + +##### DenialEvent integration + +Each bypass event sends a `DenialEvent` to the denial aggregator with `denial_stage: "bypass"`. This integrates bypass detections into the same deduplication, aggregation, and policy proposal pipeline as proxy-level denials. The `DenialEvent` fields: + +| Field | Value | +|-------|-------| +| `host` | Destination IP address | +| `port` | Destination port | +| `binary` | Binary path (or `"-"`) | +| `ancestors` | Ancestor chain parsed from `" -> "` separator | +| `deny_reason` | `"direct connection bypassed HTTP CONNECT proxy"` | +| `denial_stage` | `"bypass"` | +| `l7_method` | `None` | +| `l7_path` | `None` | + +The denial aggregator deduplicates bypass events by the same `(host, port, binary)` key used for proxy denials, and flushes them to the gateway via `SubmitPolicyAnalysis` on the same interval. + +##### Lifecycle wiring + +The bypass detection subsystem is wired in `crates/openshell-sandbox/src/lib.rs`: + +1. After `NetworkNamespace::create()` succeeds, `install_bypass_rules(proxy_port)` is called. Failure is non-fatal (logged as warning). +2. The proxy's denial channel sender (`denial_tx`) is cloned as `bypass_denial_tx` before being passed to the proxy. +3. After proxy startup, `bypass_monitor::spawn()` is called with the namespace name, entrypoint PID, and `bypass_denial_tx`. Returns `Option` — `None` if `/dev/kmsg` is unavailable. + +The monitor runs for the lifetime of the sandbox. It exits when `/dev/kmsg` reaches EOF (process termination) or encounters an unrecoverable read error. + +**Graceful degradation:** If `/dev/kmsg` cannot be opened (e.g., restricted container environment without access to the kernel ring buffer), the monitor logs a one-time warning and returns `None`. The iptables REJECT rules still provide fast-fail UX — the monitor only adds diagnostic visibility. + +##### Dependencies + +Bypass detection requires the `iptables` package for rule installation (in addition to `iproute2` for namespace management). If iptables is not installed, bypass detection degrades to routing-only isolation. The `/dev/kmsg` device is required for the monitor but not for the REJECT rules. + #### Required capabilities | Capability | Purpose | |------------|---------| | `CAP_SYS_ADMIN` | Creating network namespaces, `setns()` | -| `CAP_NET_ADMIN` | Creating veth pairs, assigning IPs, configuring routes | +| `CAP_NET_ADMIN` | Creating veth pairs, assigning IPs, configuring routes, installing iptables bypass detection rules | | `CAP_SYS_PTRACE` | Proxy reading `/proc//fd/` and `/proc//exe` for processes running as a different user | -The `iproute2` package must be installed (provides the `ip` command). +The `iproute2` package must be installed (provides the `ip` command). The `iptables` package is required for bypass detection rules; if absent, the namespace still provides routing-based isolation but without fast-fail rejection or diagnostic logging for bypass attempts. If namespace creation fails (e.g., missing capabilities), startup fails in `Proxy` mode. This preserves fail-closed behavior: either network namespace isolation is active, or the sandbox does not run. @@ -1087,6 +1199,12 @@ The sandbox uses `miette` for error reporting and `thiserror` for typed errors. | Landlock failure + `HardRequirement` | Fatal | | Seccomp failure | Fatal | | Network namespace creation failure | Fatal in `Proxy` mode (sandbox startup aborts) | +| Bypass detection: iptables not available | Warn + skip rule installation (routing-only isolation) | +| Bypass detection: IPv4 rule installation failure | Warn + returned as error (non-fatal at call site) | +| Bypass detection: IPv6 rule installation failure | Warn + continue (IPv4 rules are the primary path) | +| Bypass detection: LOG rule installation failure | Warn + continue (REJECT rules still installed for fast-fail) | +| Bypass detection: `/dev/kmsg` not available | Warn + monitor not started (REJECT rules still provide fast-fail) | +| Bypass detection: `/dev/kmsg` read error (EPIPE/EIO) | Debug log + continue reading (kernel ring buffer overrun) | | Ephemeral CA generation failure | Warn + TLS termination disabled (L7 inspection on TLS endpoints will not work) | | CA file write failure | Warn + TLS termination disabled | | OPA engine Mutex lock poisoned | Error on the individual evaluation | @@ -1125,8 +1243,9 @@ Dual-output logging is configured in `main.rs`: Key structured log events: - `CONNECT`: One per proxy CONNECT request (for non-`inference.local` targets) with full identity context. Inference interception failures produce a separate `info!()` log with `action=deny` and the denial reason. +- `BYPASS_DETECT`: One per detected direct connection attempt that bypassed the HTTP CONNECT proxy. Includes destination, protocol, process identity (best-effort), and remediation hint. Emitted at `warn` level. - `L7_REQUEST`: One per L7-inspected request with method, path, and decision -- Sandbox lifecycle events: process start, exit, namespace creation/cleanup +- Sandbox lifecycle events: process start, exit, namespace creation/cleanup, bypass rule installation - Policy reload events: new version detected, reload success/failure, status report outcomes ## Log Streaming @@ -1359,6 +1478,7 @@ Platform-specific code is abstracted through `crates/openshell-sandbox/src/sandb | Landlock | Applied via `landlock` crate (ABI V1) | Warning + no-op | | Seccomp | Applied via `seccompiler` crate | No-op | | Network namespace | Full veth pair isolation | Not available | +| Bypass detection | iptables rules + `/dev/kmsg` monitor | Not available (no netns) | | `/proc` identity binding | Full support | `evaluate_opa_tcp()` always denies | | Proxy | Functional (binds to veth IP or loopback) | Functional (loopback only, no identity binding) | | SSH server | Full support (with netns for shell processes) | Functional (no netns isolation for shell processes) | diff --git a/crates/openshell-sandbox/src/bypass_monitor.rs b/crates/openshell-sandbox/src/bypass_monitor.rs new file mode 100644 index 00000000..f99d7493 --- /dev/null +++ b/crates/openshell-sandbox/src/bypass_monitor.rs @@ -0,0 +1,405 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bypass detection monitor — reads kernel log messages from `/dev/kmsg` to +//! detect and report direct connection attempts that bypass the HTTP CONNECT +//! proxy. +//! +//! When the sandbox network namespace has iptables LOG rules installed (see +//! `NetworkNamespace::install_bypass_rules`), the kernel writes a log line for +//! each dropped packet. This module reads those messages, parses the iptables +//! LOG format, and emits structured tracing events + denial aggregator entries. +//! +//! ## Graceful degradation +//! +//! If `/dev/kmsg` cannot be opened (e.g., restricted container environment), +//! the monitor logs a one-time warning and returns. The iptables REJECT rules +//! still provide fast-fail UX — the monitor only adds diagnostic visibility. + +use crate::denial_aggregator::DenialEvent; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; +use tokio::sync::mpsc; +use tracing::{debug, warn}; + +/// A parsed iptables LOG entry from `/dev/kmsg`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BypassEvent { + /// Destination IP address. + pub dst_addr: String, + /// Destination port. + pub dst_port: u16, + /// Source port (used for process identity resolution). + pub src_port: u16, + /// Protocol (TCP or UDP). + pub proto: String, + /// UID of the process that initiated the connection. + pub uid: Option, +} + +/// Parse an iptables LOG line from `/dev/kmsg`. +/// +/// Expected format (from the kernel LOG target): +/// ```text +/// ...,;openshell:bypass::IN= OUT=veth-s-... SRC=10.200.0.2 DST=93.184.216.34 +/// LEN=60 ... PROTO=TCP SPT=48012 DPT=443 ... UID=1000 +/// ``` +/// +/// Returns `None` if the line doesn't match the expected prefix or is malformed. +pub fn parse_kmsg_line(line: &str, namespace_prefix: &str) -> Option { + // Check that this line contains our namespace prefix. + let prefix_pos = line.find(namespace_prefix)?; + let relevant = &line[prefix_pos + namespace_prefix.len()..]; + + let dst_addr = extract_field(relevant, "DST=")?; + let dst_port = extract_field(relevant, "DPT=")?.parse::().ok()?; + let src_port = extract_field(relevant, "SPT=") + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + let proto = extract_field(relevant, "PROTO=") + .unwrap_or_else(|| "unknown".to_string()) + .to_lowercase(); + let uid = extract_field(relevant, "UID=").and_then(|s| s.parse::().ok()); + + Some(BypassEvent { + dst_addr, + dst_port, + src_port, + proto, + uid, + }) +} + +/// Extract a single space-delimited field value from an iptables LOG line. +/// +/// Given `"DST="` and a string like `"...DST=93.184.216.34 LEN=60..."`, +/// returns `Some("93.184.216.34")`. +fn extract_field(s: &str, key: &str) -> Option { + let start = s.find(key)? + key.len(); + let rest = &s[start..]; + let end = rest.find(' ').unwrap_or(rest.len()); + let value = &rest[..end]; + if value.is_empty() { + None + } else { + Some(value.to_string()) + } +} + +/// Generate a protocol-appropriate hint for the bypass event. +fn hint_for_event(event: &BypassEvent) -> &'static str { + if event.proto == "udp" && event.dst_port == 53 { + "DNS queries should route through the sandbox proxy; check resolver configuration" + } else if event.proto == "udp" { + "UDP traffic must route through the sandbox proxy" + } else { + "ensure process honors HTTP_PROXY/HTTPS_PROXY; for Node.js set NODE_USE_ENV_PROXY=1" + } +} + +/// Spawn the bypass monitor as a background tokio task. +/// +/// Uses `dmesg --follow` to tail the kernel ring buffer for iptables LOG +/// entries matching the given namespace. Falls back gracefully if `dmesg` +/// is not available. +/// +/// We use `dmesg` rather than reading `/dev/kmsg` directly because the +/// container runtime's device cgroup policy blocks direct `/dev/kmsg` access +/// even with `CAP_SYSLOG`. The `dmesg` command reads via the `syslog(2)` +/// syscall which is permitted with `CAP_SYSLOG`. +/// +/// Returns a `JoinHandle` if the monitor was started, or `None` if `dmesg` +/// is not available. +pub fn spawn( + namespace_name: String, + entrypoint_pid: Arc, + denial_tx: Option>, +) -> Option> { + use std::io::BufRead; + use std::process::{Command, Stdio}; + + // Verify dmesg is available before spawning the monitor. + let dmesg_check = Command::new("dmesg") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + + if !dmesg_check.is_ok_and(|s| s.success()) { + warn!( + "dmesg not available; bypass detection monitor will not run. \ + Bypass REJECT rules still provide fast-fail behavior." + ); + return None; + } + + let namespace_prefix = format!("openshell:bypass:{namespace_name}:"); + debug!( + namespace = %namespace_name, + "Starting bypass detection monitor via dmesg --follow" + ); + + let handle = tokio::task::spawn_blocking(move || { + // Start dmesg in follow mode to tail new kernel messages. + let mut child = match Command::new("dmesg") + .args(["--follow", "--notime"]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + { + Ok(c) => c, + Err(e) => { + warn!(error = %e, "Failed to start dmesg --follow; bypass monitor will not run"); + return; + } + }; + + let stdout = match child.stdout.take() { + Some(s) => s, + None => { + warn!("dmesg --follow produced no stdout; bypass monitor will not run"); + return; + } + }; + + let reader = std::io::BufReader::new(stdout); + for line in reader.lines() { + let line = match line { + Ok(l) => l, + Err(e) => { + debug!(error = %e, "Error reading dmesg line, continuing"); + continue; + } + }; + + let Some(event) = parse_kmsg_line(&line, &namespace_prefix) else { + continue; + }; + + // Attempt process identity resolution (best-effort, TCP only). + let pid = entrypoint_pid.load(Ordering::Acquire); + let (binary, binary_pid, ancestors) = + if event.proto == "tcp" && event.src_port > 0 && pid > 0 { + resolve_process_identity(pid, event.src_port) + } else { + ("-".to_string(), "-".to_string(), "-".to_string()) + }; + + let hint = hint_for_event(&event); + + warn!( + dst_addr = %event.dst_addr, + dst_port = event.dst_port, + proto = %event.proto, + binary = %binary, + binary_pid = %binary_pid, + ancestors = %ancestors, + action = "reject", + reason = "direct connection bypassed HTTP CONNECT proxy", + hint = hint, + "BYPASS_DETECT", + ); + + // Send to denial aggregator if available. + if let Some(ref tx) = denial_tx { + let ancestors_vec: Vec = if ancestors == "-" { + vec![] + } else { + ancestors.split(" -> ").map(String::from).collect() + }; + + let _ = tx.send(DenialEvent { + host: event.dst_addr.clone(), + port: event.dst_port, + binary: binary.clone(), + ancestors: ancestors_vec, + deny_reason: "direct connection bypassed HTTP CONNECT proxy".to_string(), + denial_stage: "bypass".to_string(), + l7_method: None, + l7_path: None, + }); + } + } + + // Clean up the dmesg child process. + let _ = child.kill(); + let _ = child.wait(); + debug!("Bypass monitor: dmesg reader exited"); + }); + + Some(handle) +} + +/// Resolve process identity from a TCP source port. +/// +/// Returns `(binary_path, pid, ancestors)` as display strings. +/// Falls back to `("-", "-", "-")` on any failure (race condition, etc.). +fn resolve_process_identity(entrypoint_pid: u32, src_port: u16) -> (String, String, String) { + #[cfg(target_os = "linux")] + { + use crate::procfs; + + match procfs::resolve_tcp_peer_identity(entrypoint_pid, src_port) { + Ok((binary_path, pid)) => { + let ancestors = procfs::collect_ancestor_binaries(pid, entrypoint_pid); + let ancestors_str = if ancestors.is_empty() { + "-".to_string() + } else { + ancestors + .iter() + .map(|p| p.display().to_string()) + .collect::>() + .join(" -> ") + }; + ( + binary_path.display().to_string(), + pid.to_string(), + ancestors_str, + ) + } + Err(_) => ("-".to_string(), "-".to_string(), "-".to_string()), + } + } + + #[cfg(not(target_os = "linux"))] + { + let _ = (entrypoint_pid, src_port); + ("-".to_string(), "-".to_string(), "-".to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_kmsg_line_tcp_bypass() { + let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ + SRC=10.200.0.2 DST=93.184.216.34 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=12345 \ + DF PROTO=TCP SPT=48012 DPT=443 WINDOW=65535 RES=0x00 SYN URGP=0 UID=1000"; + + let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); + assert_eq!(event.dst_addr, "93.184.216.34"); + assert_eq!(event.dst_port, 443); + assert_eq!(event.src_port, 48012); + assert_eq!(event.proto, "tcp"); + assert_eq!(event.uid, Some(1000)); + } + + #[test] + fn parse_kmsg_line_udp_dns_bypass() { + let line = "6,5678,9012,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ + SRC=10.200.0.2 DST=8.8.8.8 LEN=40 TOS=0x00 PREC=0x00 TTL=64 ID=0 \ + DF PROTO=UDP SPT=53421 DPT=53 LEN=32 UID=1000"; + + let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); + assert_eq!(event.dst_addr, "8.8.8.8"); + assert_eq!(event.dst_port, 53); + assert_eq!(event.src_port, 53421); + assert_eq!(event.proto, "udp"); + assert_eq!(event.uid, Some(1000)); + } + + #[test] + fn parse_kmsg_line_no_uid() { + let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ + SRC=10.200.0.2 DST=10.0.0.5 LEN=60 PROTO=TCP SPT=12345 DPT=6379"; + + let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); + assert_eq!(event.dst_addr, "10.0.0.5"); + assert_eq!(event.dst_port, 6379); + assert_eq!(event.proto, "tcp"); + assert_eq!(event.uid, None); + } + + #[test] + fn parse_kmsg_line_wrong_namespace_returns_none() { + let line = "6,1234,5678,-;openshell:bypass:sandbox-other:IN= OUT=veth \ + SRC=10.200.0.2 DST=1.2.3.4 PROTO=TCP SPT=1111 DPT=80"; + + let result = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:"); + assert!(result.is_none()); + } + + #[test] + fn parse_kmsg_line_unrelated_message_returns_none() { + let line = "6,1234,5678,-;audit: type=1400 audit(1234567890.123:1): something else"; + let result = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:"); + assert!(result.is_none()); + } + + #[test] + fn parse_kmsg_line_missing_dst_returns_none() { + let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth \ + SRC=10.200.0.2 PROTO=TCP SPT=1111 DPT=80"; + // Missing DST= field + let result = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:"); + assert!(result.is_none()); + } + + #[test] + fn parse_kmsg_line_ipv6_address() { + let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ + SRC=fd00::2 DST=2001:4860:4860::8888 LEN=60 PROTO=TCP SPT=55555 DPT=443 UID=1000"; + + let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); + assert_eq!(event.dst_addr, "2001:4860:4860::8888"); + assert_eq!(event.dst_port, 443); + assert_eq!(event.proto, "tcp"); + } + + #[test] + fn hint_for_tcp_event() { + let event = BypassEvent { + dst_addr: "1.2.3.4".to_string(), + dst_port: 443, + src_port: 12345, + proto: "tcp".to_string(), + uid: None, + }; + assert!(hint_for_event(&event).contains("HTTP_PROXY")); + } + + #[test] + fn hint_for_dns_bypass() { + let event = BypassEvent { + dst_addr: "8.8.8.8".to_string(), + dst_port: 53, + src_port: 12345, + proto: "udp".to_string(), + uid: None, + }; + assert!(hint_for_event(&event).contains("DNS")); + } + + #[test] + fn hint_for_non_dns_udp() { + let event = BypassEvent { + dst_addr: "1.2.3.4".to_string(), + dst_port: 5060, + src_port: 12345, + proto: "udp".to_string(), + uid: None, + }; + assert!(hint_for_event(&event).contains("UDP")); + } + + #[test] + fn extract_field_basic() { + let s = "DST=1.2.3.4 LEN=60"; + assert_eq!(extract_field(s, "DST="), Some("1.2.3.4".to_string())); + assert_eq!(extract_field(s, "LEN="), Some("60".to_string())); + } + + #[test] + fn extract_field_missing() { + let s = "DST=1.2.3.4 LEN=60"; + assert_eq!(extract_field(s, "PROTO="), None); + } + + #[test] + fn extract_field_at_end_of_string() { + let s = "DST=1.2.3.4"; + assert_eq!(extract_field(s, "DST="), Some("1.2.3.4".to_string())); + } +} diff --git a/crates/openshell-sandbox/src/denial_aggregator.rs b/crates/openshell-sandbox/src/denial_aggregator.rs index 6bd5a1da..175f31af 100644 --- a/crates/openshell-sandbox/src/denial_aggregator.rs +++ b/crates/openshell-sandbox/src/denial_aggregator.rs @@ -28,7 +28,7 @@ pub struct DenialEvent { pub ancestors: Vec, /// Reason for denial (e.g. "no matching policy", "internal address"). pub deny_reason: String, - /// Denial stage: "connect", "forward", "ssrf", "l7". + /// Denial stage: "connect", "forward", "ssrf", "l7", "bypass". pub denial_stage: String, /// L7 request details (method, path, decision) if this is an L7 denial. pub l7_method: Option, diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 8246555b..655acc86 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -5,6 +5,7 @@ //! //! This crate provides process sandboxing and monitoring capabilities. +pub mod bypass_monitor; mod child_env; pub mod denial_aggregator; mod grpc_client; @@ -257,7 +258,24 @@ pub async fn run_sandbox( #[cfg(target_os = "linux")] let netns = if matches!(policy.network.mode, NetworkMode::Proxy) { match NetworkNamespace::create() { - Ok(ns) => Some(ns), + Ok(ns) => { + // Install bypass detection rules (iptables LOG + REJECT). + // This provides fast-fail UX and diagnostic logging for direct + // connection attempts that bypass the HTTP CONNECT proxy. + let proxy_port = policy + .network + .proxy + .as_ref() + .and_then(|p| p.http_addr) + .map_or(3128, |addr| addr.port()); + if let Err(e) = ns.install_bypass_rules(proxy_port) { + warn!( + error = %e, + "Failed to install bypass detection rules (non-fatal)" + ); + } + Some(ns) + } Err(e) => { return Err(miette::miette!( "Network namespace creation failed and proxy mode requires isolation. \ @@ -279,7 +297,8 @@ pub async fn run_sandbox( // the entrypoint process's /proc/net/tcp for identity binding. let entrypoint_pid = Arc::new(AtomicU32::new(0)); - let (_proxy, denial_rx) = if matches!(policy.network.mode, NetworkMode::Proxy) { + let (_proxy, denial_rx, bypass_denial_tx) = if matches!(policy.network.mode, NetworkMode::Proxy) + { let proxy_policy = policy.network.proxy.as_ref().ok_or_else(|| { miette::miette!("Network mode is set to proxy but no proxy configuration was provided") })?; @@ -312,11 +331,13 @@ pub async fn run_sandbox( .await?; // Create denial aggregator channel if in gRPC mode (sandbox_id present). - let (denial_tx, denial_rx) = if sandbox_id.is_some() { + // Clone the sender for the bypass monitor before passing to the proxy. + let (denial_tx, denial_rx, bypass_denial_tx) = if sandbox_id.is_some() { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - (Some(tx), Some(rx)) + let bypass_tx = tx.clone(); + (Some(tx), Some(rx), Some(bypass_tx)) } else { - (None, None) + (None, None, None) }; let proxy_handle = ProxyHandle::start_with_bind_addr( @@ -331,11 +352,29 @@ pub async fn run_sandbox( denial_tx, ) .await?; - (Some(proxy_handle), denial_rx) + (Some(proxy_handle), denial_rx, bypass_denial_tx) } else { - (None, None) + (None, None, None) + }; + + // Spawn bypass detection monitor (Linux only, proxy mode only). + // Reads /dev/kmsg for iptables LOG entries and emits structured + // tracing events for direct connection attempts that bypass the proxy. + #[cfg(target_os = "linux")] + let _bypass_monitor = if netns.is_some() { + bypass_monitor::spawn( + netns.as_ref().expect("netns is Some").name().to_string(), + entrypoint_pid.clone(), + bypass_denial_tx, + ) + } else { + None }; + // On non-Linux, bypass_denial_tx is unused (no /dev/kmsg). + #[cfg(not(target_os = "linux"))] + drop(bypass_denial_tx); + // Compute the proxy URL and netns fd for SSH sessions. // SSH shell processes need both to enforce network policy: // - netns_fd: enter the network namespace via setns() so all traffic diff --git a/crates/openshell-sandbox/src/sandbox/linux/netns.rs b/crates/openshell-sandbox/src/sandbox/linux/netns.rs index 23a7d5d1..5e6907c5 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/netns.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/netns.rs @@ -223,6 +223,339 @@ impl NetworkNamespace { pub const fn ns_fd(&self) -> Option { self.ns_fd } + + /// Install iptables rules for bypass detection inside the namespace. + /// + /// Sets up OUTPUT chain rules that: + /// 1. ACCEPT traffic destined for the proxy (host_ip:proxy_port) + /// 2. ACCEPT loopback traffic + /// 3. ACCEPT established/related connections (response packets) + /// 4. LOG + REJECT all other TCP/UDP traffic (bypass attempts) + /// + /// This provides two benefits: + /// - **Fast-fail UX**: applications get immediate ECONNREFUSED instead of + /// a 30-second timeout when they bypass the proxy + /// - **Diagnostics**: iptables LOG entries are picked up by the bypass + /// monitor to emit structured tracing events + /// + /// Degrades gracefully if `iptables` is not available — the namespace + /// still provides isolation via routing, just without fast-fail and + /// diagnostic logging. + pub fn install_bypass_rules(&self, proxy_port: u16) -> Result<()> { + // Check if iptables is available before attempting to install rules. + let iptables_path = match find_iptables() { + Some(path) => path, + None => { + warn!( + namespace = %self.name, + search_paths = ?IPTABLES_SEARCH_PATHS, + "iptables not found; bypass detection rules will not be installed. \ + Install the iptables package for proxy bypass diagnostics." + ); + return Ok(()); + } + }; + + let host_ip_str = self.host_ip.to_string(); + let proxy_port_str = proxy_port.to_string(); + let log_prefix = format!("openshell:bypass:{}:", &self.name); + + info!( + namespace = %self.name, + iptables = iptables_path, + proxy_addr = %format!("{}:{}", host_ip_str, proxy_port), + "Installing bypass detection rules" + ); + + // Install IPv4 rules + if let Err(e) = + self.install_bypass_rules_for(iptables_path, &host_ip_str, &proxy_port_str, &log_prefix) + { + warn!( + namespace = %self.name, + error = %e, + "Failed to install IPv4 bypass detection rules" + ); + return Err(e); + } + + // Install IPv6 rules — best-effort. + // Skip the proxy ACCEPT rule for IPv6 since the proxy address is IPv4. + if let Some(ip6_path) = find_ip6tables(iptables_path) { + if let Err(e) = self.install_bypass_rules_for_v6(&ip6_path, &log_prefix) { + warn!( + namespace = %self.name, + error = %e, + "Failed to install IPv6 bypass detection rules (non-fatal)" + ); + } + } + + info!( + namespace = %self.name, + "Bypass detection rules installed" + ); + + Ok(()) + } + + /// Install bypass detection rules for a specific iptables variant (iptables or ip6tables). + fn install_bypass_rules_for( + &self, + iptables_cmd: &str, + host_ip: &str, + proxy_port: &str, + log_prefix: &str, + ) -> Result<()> { + // Rule 1: ACCEPT traffic to the proxy + run_iptables_netns( + &self.name, + iptables_cmd, + &[ + "-A", + "OUTPUT", + "-d", + &format!("{host_ip}/32"), + "-p", + "tcp", + "--dport", + proxy_port, + "-j", + "ACCEPT", + ], + )?; + + // Rule 2: ACCEPT loopback traffic + run_iptables_netns( + &self.name, + iptables_cmd, + &["-A", "OUTPUT", "-o", "lo", "-j", "ACCEPT"], + )?; + + // Rule 3: ACCEPT established/related connections (response packets) + run_iptables_netns( + &self.name, + iptables_cmd, + &[ + "-A", + "OUTPUT", + "-m", + "conntrack", + "--ctstate", + "ESTABLISHED,RELATED", + "-j", + "ACCEPT", + ], + )?; + + // Rule 4: LOG TCP SYN bypass attempts (rate-limited) + // LOG rule failure is non-fatal — the REJECT rule still provides fast-fail. + if let Err(e) = run_iptables_netns( + &self.name, + iptables_cmd, + &[ + "-A", + "OUTPUT", + "-p", + "tcp", + "--syn", + "-m", + "limit", + "--limit", + "5/sec", + "--limit-burst", + "10", + "-j", + "LOG", + "--log-prefix", + log_prefix, + "--log-uid", + ], + ) { + warn!( + error = %e, + "Failed to install LOG rule for TCP (xt_LOG module may not be loaded); \ + bypass REJECT rules will still be installed" + ); + } + + // Rule 5: REJECT TCP bypass attempts (fast-fail) + run_iptables_netns( + &self.name, + iptables_cmd, + &[ + "-A", + "OUTPUT", + "-p", + "tcp", + "-j", + "REJECT", + "--reject-with", + "icmp-port-unreachable", + ], + )?; + + // Rule 6: LOG UDP bypass attempts (rate-limited, covers DNS bypass) + if let Err(e) = run_iptables_netns( + &self.name, + iptables_cmd, + &[ + "-A", + "OUTPUT", + "-p", + "udp", + "-m", + "limit", + "--limit", + "5/sec", + "--limit-burst", + "10", + "-j", + "LOG", + "--log-prefix", + log_prefix, + "--log-uid", + ], + ) { + warn!( + error = %e, + "Failed to install LOG rule for UDP; bypass REJECT rules will still be installed" + ); + } + + // Rule 7: REJECT UDP bypass attempts (covers DNS bypass) + run_iptables_netns( + &self.name, + iptables_cmd, + &[ + "-A", + "OUTPUT", + "-p", + "udp", + "-j", + "REJECT", + "--reject-with", + "icmp-port-unreachable", + ], + )?; + + Ok(()) + } + + /// Install IPv6 bypass detection rules. + /// + /// Similar to `install_bypass_rules_for` but omits the proxy ACCEPT rule + /// (the proxy listens on an IPv4 address) and uses IPv6-appropriate + /// REJECT types. + fn install_bypass_rules_for_v6(&self, ip6tables_cmd: &str, log_prefix: &str) -> Result<()> { + // ACCEPT loopback traffic + run_iptables_netns( + &self.name, + ip6tables_cmd, + &["-A", "OUTPUT", "-o", "lo", "-j", "ACCEPT"], + )?; + + // ACCEPT established/related connections + run_iptables_netns( + &self.name, + ip6tables_cmd, + &[ + "-A", + "OUTPUT", + "-m", + "conntrack", + "--ctstate", + "ESTABLISHED,RELATED", + "-j", + "ACCEPT", + ], + )?; + + // LOG TCP SYN bypass attempts (rate-limited) + if let Err(e) = run_iptables_netns( + &self.name, + ip6tables_cmd, + &[ + "-A", + "OUTPUT", + "-p", + "tcp", + "--syn", + "-m", + "limit", + "--limit", + "5/sec", + "--limit-burst", + "10", + "-j", + "LOG", + "--log-prefix", + log_prefix, + "--log-uid", + ], + ) { + warn!(error = %e, "Failed to install IPv6 LOG rule for TCP"); + } + + // REJECT TCP bypass attempts + run_iptables_netns( + &self.name, + ip6tables_cmd, + &[ + "-A", + "OUTPUT", + "-p", + "tcp", + "-j", + "REJECT", + "--reject-with", + "icmp6-port-unreachable", + ], + )?; + + // LOG UDP bypass attempts (rate-limited) + if let Err(e) = run_iptables_netns( + &self.name, + ip6tables_cmd, + &[ + "-A", + "OUTPUT", + "-p", + "udp", + "-m", + "limit", + "--limit", + "5/sec", + "--limit-burst", + "10", + "-j", + "LOG", + "--log-prefix", + log_prefix, + "--log-uid", + ], + ) { + warn!(error = %e, "Failed to install IPv6 LOG rule for UDP"); + } + + // REJECT UDP bypass attempts + run_iptables_netns( + &self.name, + ip6tables_cmd, + &[ + "-A", + "OUTPUT", + "-p", + "udp", + "-j", + "REJECT", + "--reject-with", + "icmp6-port-unreachable", + ], + )?; + + Ok(()) + } } impl Drop for NetworkNamespace { @@ -256,7 +589,7 @@ impl Drop for NetworkNamespace { } } -/// Run an `ip` command. +/// Run an `ip` command on the host. fn run_ip(args: &[&str]) -> Result<()> { debug!(command = %format!("ip {}", args.join(" ")), "Running ip command"); @@ -299,6 +632,58 @@ fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { Ok(()) } +/// Run an iptables command inside a network namespace. +fn run_iptables_netns(netns: &str, iptables_cmd: &str, args: &[&str]) -> Result<()> { + let mut full_args = vec!["netns", "exec", netns, iptables_cmd]; + full_args.extend(args); + + debug!( + command = %format!("ip {}", full_args.join(" ")), + "Running iptables in namespace" + ); + + let output = Command::new("ip") + .args(&full_args) + .output() + .into_diagnostic()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(miette::miette!( + "ip netns exec {} {} failed: {}", + netns, + iptables_cmd, + stderr.trim() + )); + } + + Ok(()) +} + +/// Well-known paths where iptables may be installed. +/// The sandbox container PATH often excludes `/usr/sbin`, so we probe +/// explicit paths rather than relying on `which`. +const IPTABLES_SEARCH_PATHS: &[&str] = + &["/usr/sbin/iptables", "/sbin/iptables", "/usr/bin/iptables"]; + +/// Find the iptables binary path, checking well-known locations. +fn find_iptables() -> Option<&'static str> { + IPTABLES_SEARCH_PATHS + .iter() + .find(|path| std::path::Path::new(path).exists()) + .copied() +} + +/// Find the ip6tables binary path, deriving it from the iptables location. +fn find_ip6tables(iptables_path: &str) -> Option { + let ip6_path = iptables_path.replace("iptables", "ip6tables"); + if std::path::Path::new(&ip6_path).exists() { + Some(ip6_path) + } else { + None + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-server/src/sandbox/mod.rs b/crates/openshell-server/src/sandbox/mod.rs index 49051e93..7b1272a8 100644 --- a/crates/openshell-server/src/sandbox/mod.rs +++ b/crates/openshell-server/src/sandbox/mod.rs @@ -936,14 +936,15 @@ fn sandbox_template_to_k8s( // The sandbox process needs SYS_ADMIN (for seccomp filter installation and // network namespace creation), NET_ADMIN (for network namespace veth setup), - // and SYS_PTRACE (for the CONNECT proxy to read /proc//fd/ of sandbox-user - // processes to resolve binary identity for network policy enforcement). + // SYS_PTRACE (for the CONNECT proxy to read /proc//fd/ of sandbox-user + // processes to resolve binary identity for network policy enforcement), + // and SYSLOG (for reading /dev/kmsg to surface bypass detection diagnostics). // This mirrors the capabilities used by `mise run sandbox`. container.insert( "securityContext".to_string(), serde_json::json!({ "capabilities": { - "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE"] + "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] } }), ); @@ -1684,7 +1685,7 @@ mod tests { "image": "custom-image:latest", "securityContext": { "capabilities": { - "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE"] + "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] } } }] diff --git a/examples/bring-your-own-container/Dockerfile b/examples/bring-your-own-container/Dockerfile index 3b859545..296a22f2 100644 --- a/examples/bring-your-own-container/Dockerfile +++ b/examples/bring-your-own-container/Dockerfile @@ -8,8 +8,10 @@ FROM python:3.13-slim # System tools useful for sandbox networking and debugging. +# iproute2: required for network namespace management (ip netns, veth pairs) +# iptables: optional, enables bypass detection (LOG + REJECT for direct connections) RUN apt-get update && apt-get install -y --no-install-recommends \ - curl iproute2 \ + curl iproute2 iptables \ && rm -rf /var/lib/apt/lists/* # Create the sandbox user (uid/gid 1000) for non-root execution. diff --git a/tasks/scripts/cluster-deploy-fast.sh b/tasks/scripts/cluster-deploy-fast.sh index c9eba82d..213c2e25 100755 --- a/tasks/scripts/cluster-deploy-fast.sh +++ b/tasks/scripts/cluster-deploy-fast.sh @@ -405,6 +405,15 @@ if [[ "${needs_helm_upgrade}" == "1" ]]; then | cut -d'\"' -f4") || true SSH_HANDSHAKE_SECRET="${EXISTING_SECRET:-$(openssl rand -hex 32)}" + # Retrieve the host gateway IP from the entrypoint-rendered HelmChart CR so + # that hostAliases for host.openshell.internal are preserved across fast deploys. + HOST_GATEWAY_IP=$(cluster_exec "kubectl -n kube-system get helmchart openshell -o jsonpath='{.spec.valuesContent}' 2>/dev/null \ + | grep hostGatewayIP | awk '{print \$2}'" 2>/dev/null) || true + HOST_GATEWAY_ARGS="" + if [[ -n "${HOST_GATEWAY_IP}" ]]; then + HOST_GATEWAY_ARGS="--set server.hostGatewayIP=${HOST_GATEWAY_IP}" + fi + cluster_exec "helm upgrade openshell ${CONTAINER_CHART_DIR} \ --namespace openshell \ --set image.repository=${IMAGE_REPO_BASE}/gateway \ @@ -415,6 +424,7 @@ if [[ "${needs_helm_upgrade}" == "1" ]]; then --set server.tls.clientCaSecretName=openshell-server-client-ca \ --set server.tls.clientTlsSecretName=openshell-client-tls \ --set server.sshHandshakeSecret=${SSH_HANDSHAKE_SECRET} \ + ${HOST_GATEWAY_ARGS} \ ${helm_wait_args}" helm_end=$(date +%s) log_duration "Helm upgrade" "${helm_start}" "${helm_end}"