From ce45cbab1f20aa43a0b2d64c45b72fa3b2979183 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Tue, 17 Mar 2026 20:05:31 +0100 Subject: [PATCH 1/7] fix: docker access in updater error --- control-plane/csf-updater/src/verify.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/control-plane/csf-updater/src/verify.rs b/control-plane/csf-updater/src/verify.rs index 2edb833..c074b47 100644 --- a/control-plane/csf-updater/src/verify.rs +++ b/control-plane/csf-updater/src/verify.rs @@ -82,7 +82,7 @@ async fn remote_digest(client: &reqwest::Client, image: &str, tag: &str, ghcr_au fn local_digest(image: &str) -> Result { let output = std::process::Command::new("docker") - .args(["inspect", "--format", "{{index .RepoDigests 0}}", image]) + .args(["image", "inspect", "--format", "{{json .RepoDigests}}", image]) .output()?; if !output.status.success() { @@ -90,9 +90,11 @@ fn local_digest(image: &str) -> Result { } let raw = String::from_utf8(output.stdout)?; - raw.trim() - .split('@') - .nth(1) - .map(|s| s.to_string()) - .ok_or_else(|| anyhow::anyhow!("could not parse digest from docker inspect output for {}", image)) + let digests: Vec = serde_json::from_str(raw.trim()) + .map_err(|e| anyhow::anyhow!("failed to parse RepoDigests for {}: {}", image, e))?; + + digests + .into_iter() + .find_map(|d| d.split('@').nth(1).map(|s| s.to_string())) + .ok_or_else(|| anyhow::anyhow!("no repo digest found for {}", image)) } From fd01b361aa31869dd27daadb7694e54f6265c844 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Tue, 17 Mar 2026 20:09:12 +0100 Subject: [PATCH 2/7] feat: added update stop and resume --- .../api-gateway/src/routes/update.rs | 38 +++++++++++++++++++ control-plane/csf-updater/src/etcd.rs | 1 + control-plane/csf-updater/src/main.rs | 7 +++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/control-plane/api-gateway/src/routes/update.rs b/control-plane/api-gateway/src/routes/update.rs index 84deb37..58abb11 100644 --- a/control-plane/api-gateway/src/routes/update.rs +++ b/control-plane/api-gateway/src/routes/update.rs @@ -10,6 +10,7 @@ use crate::AppState; const ETCD_DESIRED_VERSION_KEY: &str = "/csf/config/desired_cp_version"; const ETCD_UPDATE_RESULT_KEY: &str = "/csf/config/last_update_result"; const ETCD_GHCR_TOKEN_KEY: &str = "/csf/config/ghcr_token"; +const ETCD_PAUSED_KEY: &str = "/csf/config/update_paused"; #[derive(Debug, Deserialize)] pub struct UpdateRequest { @@ -27,6 +28,7 @@ pub struct UpdateStatusResponse { pub current_version: String, pub desired_version: Option, pub last_result: Option, + pub paused: bool, } #[derive(Debug, Deserialize)] @@ -39,6 +41,8 @@ pub fn routes() -> Router { Router::new() .route("/system/update", post(trigger_update)) .route("/system/update/status", get(update_status)) + .route("/system/update/pause", post(pause_updates)) + .route("/system/update/resume", post(resume_updates)) .route("/system/ghcr-token", post(set_ghcr_token)) } @@ -98,11 +102,13 @@ async fn update_status( let desired = etcd_get(&mut client, ETCD_DESIRED_VERSION_KEY).await?; let last_result = etcd_get(&mut client, ETCD_UPDATE_RESULT_KEY).await?; + let paused = etcd_get(&mut client, ETCD_PAUSED_KEY).await?.as_deref() == Some("true"); Ok(Json(UpdateStatusResponse { current_version: env!("CARGO_PKG_VERSION").to_string(), desired_version: desired, last_result, + paused, })) } @@ -186,6 +192,38 @@ async fn write_result(result: &str) { } } +async fn pause_updates( + _auth: CanManageSystem, + State(_state): State, +) -> Result { + let mut client = etcd_client().await?; + client + .put(ETCD_PAUSED_KEY, b"true", None) + .await + .map_err(|e| { + tracing::error!(error = %e, "failed to set update_paused in etcd"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + tracing::info!("updates paused"); + Ok(StatusCode::NO_CONTENT) +} + +async fn resume_updates( + _auth: CanManageSystem, + State(_state): State, +) -> Result { + let mut client = etcd_client().await?; + client + .delete(ETCD_PAUSED_KEY, None) + .await + .map_err(|e| { + tracing::error!(error = %e, "failed to delete update_paused from etcd"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + tracing::info!("updates resumed"); + Ok(StatusCode::NO_CONTENT) +} + async fn set_ghcr_token( _auth: CanManageSystem, State(_state): State, diff --git a/control-plane/csf-updater/src/etcd.rs b/control-plane/csf-updater/src/etcd.rs index 274efc7..fff8468 100644 --- a/control-plane/csf-updater/src/etcd.rs +++ b/control-plane/csf-updater/src/etcd.rs @@ -5,6 +5,7 @@ use crate::config::Config; pub const DESIRED_VERSION_KEY: &str = "/csf/config/desired_cp_version"; pub const RESULT_KEY: &str = "/csf/config/last_update_result"; pub const GHCR_TOKEN_KEY: &str = "/csf/config/ghcr_token"; +pub const PAUSED_KEY: &str = "/csf/config/update_paused"; pub struct Client { inner: etcd_client::Client, diff --git a/control-plane/csf-updater/src/main.rs b/control-plane/csf-updater/src/main.rs index cd88180..d474148 100644 --- a/control-plane/csf-updater/src/main.rs +++ b/control-plane/csf-updater/src/main.rs @@ -39,6 +39,11 @@ async fn main() -> anyhow::Result<()> { async fn run_once(cfg: &config::Config, last_applied: &str) -> anyhow::Result> { let mut etcd = etcd::Client::connect(cfg).await?; + if etcd.get(etcd::PAUSED_KEY).await?.as_deref() == Some("true") { + tracing::info!("updates paused, skipping"); + return Ok(None); + } + let desired = match etcd.get(etcd::DESIRED_VERSION_KEY).await? { Some(v) => v, None => return Ok(None), @@ -66,7 +71,7 @@ async fn run_once(cfg: &config::Config, last_applied: &str) -> anyhow::Result { tracing::error!(error = %e, version = %desired, "update failed"); etcd.put(etcd::RESULT_KEY, "failed").await?; - Ok(None) + Ok(Some(desired)) } } } From 64e4326d174a440a911e8b46f024866dc607e2e8 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Tue, 17 Mar 2026 20:16:27 +0100 Subject: [PATCH 3/7] feat: use mutable binary paths for csf-agent and csf-updater to enable self-update --- nixos-node/modules/csf-daemon.nix | 8 ++++++- nixos-node/modules/server-configuration.nix | 24 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/nixos-node/modules/csf-daemon.nix b/nixos-node/modules/csf-daemon.nix index e81152e..6ce1c7d 100644 --- a/nixos-node/modules/csf-daemon.nix +++ b/nixos-node/modules/csf-daemon.nix @@ -13,6 +13,12 @@ in description = "The csf-agent package to use."; }; + binaryPath = lib.mkOption { + type = lib.types.str; + default = "/usr/local/bin/csf-agent"; + description = "Path to the csf-agent binary. Can be overwritten by the updater."; + }; + apiGateway = lib.mkOption { type = lib.types.str; example = "https://gateway.csf.example:8000"; @@ -69,7 +75,7 @@ in }; serviceConfig = { - ExecStart = "${cfg.package}/bin/csf-agent"; + ExecStart = cfg.binaryPath; Restart = "always"; RestartSec = "5s"; User = "csf-daemon"; diff --git a/nixos-node/modules/server-configuration.nix b/nixos-node/modules/server-configuration.nix index c2874e1..473b326 100644 --- a/nixos-node/modules/server-configuration.nix +++ b/nixos-node/modules/server-configuration.nix @@ -2,7 +2,9 @@ let composeDir = "/etc/csf-core"; + binDir = "/usr/local/bin"; csfUpdaterBin = csf.updaterPackage; + csfAgentBin = csf.agentPackage; in { system.stateVersion = "25.11"; @@ -63,6 +65,7 @@ in services.csf-daemon = { enable = true; package = csf.agentPackage; + binaryPath = "${binDir}/csf-agent"; apiGateway = "http://localhost:8000"; heartbeatInterval = 60; logLevel = "info"; @@ -101,13 +104,13 @@ in User = "csf-updater"; Group = "csf-updater"; EnvironmentFile = "/etc/csf-core/updater.env"; - ExecStart = "${csfUpdaterBin}/bin/csf-updater"; + ExecStart = "${binDir}/csf-updater"; Restart = "always"; RestartSec = "10"; NoNewPrivileges = true; ProtectSystem = "strict"; ProtectHome = true; - ReadWritePaths = [ composeDir "/tmp" ]; + ReadWritePaths = [ composeDir "/tmp" binDir ]; }; environment = { @@ -117,6 +120,8 @@ in GHCR_ORG = "csfx-cloud"; POLL_INTERVAL_SECS = "30"; RUST_LOG = "info"; + BINARY_DIR = binDir; + GITHUB_RELEASE_BASE_URL = "https://github.com/csfx-cloud/CSF-Core/releases/download"; PATH = lib.mkForce "/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin"; }; }; @@ -141,6 +146,21 @@ in }; }; + system.activationScripts.csf-binaries = { + text = '' + mkdir -p ${binDir} + if [ ! -f ${binDir}/csf-updater ]; then + cp ${csfUpdaterBin}/bin/csf-updater ${binDir}/csf-updater + chmod 755 ${binDir}/csf-updater + fi + if [ ! -f ${binDir}/csf-agent ]; then + cp ${csfAgentBin}/bin/csf-agent ${binDir}/csf-agent + chmod 755 ${binDir}/csf-agent + fi + ''; + deps = []; + }; + system.activationScripts.csf-core-setup = { text = '' mkdir -p ${composeDir} From 847dfe5b3b13a7764aae36715b751afb829e7142 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Tue, 17 Mar 2026 20:18:04 +0100 Subject: [PATCH 4/7] feat: add binary self-update for csf-agent and csf-updater via github releases --- control-plane/csf-updater/src/config.rs | 6 ++ control-plane/csf-updater/src/updater.rs | 63 ++++++++++++++++++++- nixos-node/modules/server-configuration.nix | 10 ++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/control-plane/csf-updater/src/config.rs b/control-plane/csf-updater/src/config.rs index b895bfe..2b18015 100644 --- a/control-plane/csf-updater/src/config.rs +++ b/control-plane/csf-updater/src/config.rs @@ -7,6 +7,8 @@ pub struct Config { pub compose_file: String, pub poll_interval_secs: u64, pub secret_encryption_key: String, + pub binary_dir: String, + pub github_release_base_url: String, } impl Config { @@ -26,6 +28,10 @@ impl Config { .unwrap_or(30), secret_encryption_key: env::var("SECRET_ENCRYPTION_KEY") .context("SECRET_ENCRYPTION_KEY must be set")?, + binary_dir: env::var("BINARY_DIR") + .unwrap_or_else(|_| "/usr/local/bin".to_string()), + github_release_base_url: env::var("GITHUB_RELEASE_BASE_URL") + .unwrap_or_else(|_| "https://github.com/csfx-cloud/CSF-Core/releases/download".to_string()), }) } } diff --git a/control-plane/csf-updater/src/updater.rs b/control-plane/csf-updater/src/updater.rs index a0046fe..82fb0ba 100644 --- a/control-plane/csf-updater/src/updater.rs +++ b/control-plane/csf-updater/src/updater.rs @@ -13,7 +13,9 @@ pub async fn run(cfg: &Config, version: &str, etcd: &mut etcd::Client) -> Result pull(cfg, version, docker_config_dir.as_deref()).await?; verify::verify_images(cfg, version, ghcr_auth.as_deref()).await?; up(cfg, version, docker_config_dir.as_deref()).await?; - health_check(cfg, version).await + health_check(cfg, version).await?; + update_agent_binary(cfg, version).await?; + update_self_binary(cfg, version).await } async fn setup_docker_auth(cfg: &Config, etcd: &mut etcd::Client) -> Result<(Option, Option)> { @@ -82,6 +84,65 @@ async fn health_check(cfg: &Config, version: &str) -> Result<()> { Ok(()) } +async fn update_agent_binary(cfg: &Config, version: &str) -> Result<()> { + info!(version = %version, "updating csf-agent binary"); + let arch = detect_arch(); + let url = format!( + "{}/v{}/csf-agent-{}", + cfg.github_release_base_url, version, arch + ); + let dest = format!("{}/csf-agent", cfg.binary_dir); + download_and_swap(&url, &dest).await?; + restart_unit("csf-daemon").await +} + +async fn update_self_binary(cfg: &Config, version: &str) -> Result<()> { + info!(version = %version, "updating csf-updater binary"); + let arch = detect_arch(); + let url = format!( + "{}/v{}/csf-updater-{}", + cfg.github_release_base_url, version, arch + ); + let dest = format!("{}/csf-updater", cfg.binary_dir); + download_and_swap(&url, &dest).await?; + restart_unit("csf-updater").await +} + +async fn download_and_swap(url: &str, dest: &str) -> Result<()> { + let tmp = format!("{}.new", dest); + + let resp = reqwest::get(url).await?; + if !resp.status().is_success() { + bail!("failed to download {}: {}", url, resp.status()); + } + + let bytes = resp.bytes().await?; + tokio::fs::write(&tmp, &bytes).await?; + + let mut perms = tokio::fs::metadata(&tmp).await?.permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755); + tokio::fs::set_permissions(&tmp, perms).await?; + + tokio::fs::rename(&tmp, dest).await?; + info!(dest = %dest, "binary swapped"); + Ok(()) +} + +async fn restart_unit(unit: &str) -> Result<()> { + let status = Command::new("sudo") + .args(["systemctl", "restart", unit]) + .status() + .await?; + if !status.success() { + bail!("systemctl restart {} failed: {}", unit, status); + } + Ok(()) +} + +fn detect_arch() -> &'static str { + if cfg!(target_arch = "aarch64") { "arm64" } else { "amd64" } +} + async fn compose(cfg: &Config, version: &str, docker_config_dir: Option<&str>, args: &[&str]) -> Result<()> { let mut cmd_args = vec!["compose", "-f", cfg.compose_file.as_str()]; cmd_args.extend_from_slice(args); diff --git a/nixos-node/modules/server-configuration.nix b/nixos-node/modules/server-configuration.nix index 473b326..5a69a40 100644 --- a/nixos-node/modules/server-configuration.nix +++ b/nixos-node/modules/server-configuration.nix @@ -57,6 +57,16 @@ in security.sudo.wheelNeedsPassword = false; + security.sudo.extraRules = [ + { + users = [ "csf-updater" ]; + commands = [ + { command = "/run/current-system/sw/bin/systemctl restart csf-daemon"; options = [ "NOPASSWD" ]; } + { command = "/run/current-system/sw/bin/systemctl restart csf-updater"; options = [ "NOPASSWD" ]; } + ]; + } + ]; + virtualisation.docker = { enable = true; enableOnBoot = true; From ddc22b83f374fae4c0e88c13b66158a6fd95f730 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Tue, 17 Mar 2026 20:18:56 +0100 Subject: [PATCH 5/7] fix: remove NoNewPrivileges to allow sudo systemctl for binary restart --- nixos-node/modules/server-configuration.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nixos-node/modules/server-configuration.nix b/nixos-node/modules/server-configuration.nix index 5a69a40..e67df3c 100644 --- a/nixos-node/modules/server-configuration.nix +++ b/nixos-node/modules/server-configuration.nix @@ -117,7 +117,6 @@ in ExecStart = "${binDir}/csf-updater"; Restart = "always"; RestartSec = "10"; - NoNewPrivileges = true; ProtectSystem = "strict"; ProtectHome = true; ReadWritePaths = [ composeDir "/tmp" binDir ]; From 08030a9bf54add66b3d8ad237a0b440cdd55ddc9 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Tue, 17 Mar 2026 20:20:16 +0100 Subject: [PATCH 6/7] feat: expose agent and updater binary versions in update-status --- control-plane/api-gateway/src/routes/update.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/control-plane/api-gateway/src/routes/update.rs b/control-plane/api-gateway/src/routes/update.rs index 58abb11..44db094 100644 --- a/control-plane/api-gateway/src/routes/update.rs +++ b/control-plane/api-gateway/src/routes/update.rs @@ -29,6 +29,8 @@ pub struct UpdateStatusResponse { pub desired_version: Option, pub last_result: Option, pub paused: bool, + pub agent_version: Option, + pub updater_version: Option, } #[derive(Debug, Deserialize)] @@ -104,14 +106,30 @@ async fn update_status( let last_result = etcd_get(&mut client, ETCD_UPDATE_RESULT_KEY).await?; let paused = etcd_get(&mut client, ETCD_PAUSED_KEY).await?.as_deref() == Some("true"); + let binary_dir = env::var("BINARY_DIR").unwrap_or_else(|_| "/usr/local/bin".to_string()); + let agent_version = binary_version(&format!("{}/csf-agent", binary_dir)).await; + let updater_version = binary_version(&format!("{}/csf-updater", binary_dir)).await; + Ok(Json(UpdateStatusResponse { current_version: env!("CARGO_PKG_VERSION").to_string(), desired_version: desired, last_result, paused, + agent_version, + updater_version, })) } +async fn binary_version(path: &str) -> Option { + let output = tokio::process::Command::new(path) + .arg("--version") + .output() + .await + .ok()?; + let raw = String::from_utf8(output.stdout).ok()?; + raw.split_whitespace().last().map(|s| s.trim().to_string()) +} + async fn etcd_get(client: &mut Client, key: &str) -> Result, StatusCode> { let resp = client.get(key, None).await.map_err(|e| { tracing::error!(error = %e, key = key, "failed to read from etcd"); From 2b3260fe91b3038cda9d7fb8edf9d8a498e6c643 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Tue, 17 Mar 2026 20:26:04 +0100 Subject: [PATCH 7/7] fix: restrict binary dir permissions and verify sha256 checksum on download --- control-plane/csf-updater/Cargo.toml | 3 ++ control-plane/csf-updater/src/updater.rs | 40 +++++++++++++++++---- nixos-node/modules/csf-daemon.nix | 1 + nixos-node/modules/server-configuration.nix | 10 ++++-- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/control-plane/csf-updater/Cargo.toml b/control-plane/csf-updater/Cargo.toml index e2aea74..2f63777 100644 --- a/control-plane/csf-updater/Cargo.toml +++ b/control-plane/csf-updater/Cargo.toml @@ -21,4 +21,7 @@ serde = { workspace = true } serde_json = { workspace = true } aes-gcm = { workspace = true } base64 = { workspace = true } +sha2 = { workspace = true } +hex = "0.4" +bytes = "1" tempfile = "3" diff --git a/control-plane/csf-updater/src/updater.rs b/control-plane/csf-updater/src/updater.rs index 82fb0ba..71e451a 100644 --- a/control-plane/csf-updater/src/updater.rs +++ b/control-plane/csf-updater/src/updater.rs @@ -1,4 +1,5 @@ use anyhow::{bail, Result}; +use sha2::{Digest, Sha256}; use std::process::Stdio; use tokio::process::Command; use tracing::info; @@ -111,16 +112,14 @@ async fn update_self_binary(cfg: &Config, version: &str) -> Result<()> { async fn download_and_swap(url: &str, dest: &str) -> Result<()> { let tmp = format!("{}.new", dest); - let resp = reqwest::get(url).await?; - if !resp.status().is_success() { - bail!("failed to download {}: {}", url, resp.status()); - } + let bytes = fetch(url).await?; + let expected = fetch_checksum(&format!("{}.sha256", url)).await?; + verify_checksum(&bytes, &expected)?; - let bytes = resp.bytes().await?; tokio::fs::write(&tmp, &bytes).await?; let mut perms = tokio::fs::metadata(&tmp).await?.permissions(); - std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755); + std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o750); tokio::fs::set_permissions(&tmp, perms).await?; tokio::fs::rename(&tmp, dest).await?; @@ -128,6 +127,35 @@ async fn download_and_swap(url: &str, dest: &str) -> Result<()> { Ok(()) } +async fn fetch(url: &str) -> Result { + let resp = reqwest::get(url).await?; + if !resp.status().is_success() { + bail!("failed to download {}: {}", url, resp.status()); + } + Ok(resp.bytes().await?) +} + +async fn fetch_checksum(url: &str) -> Result { + let resp = reqwest::get(url).await?; + if !resp.status().is_success() { + bail!("failed to download checksum {}: {}", url, resp.status()); + } + let text = resp.text().await?; + text.split_whitespace() + .next() + .map(|s| s.to_string()) + .ok_or_else(|| anyhow::anyhow!("empty checksum file at {}", url)) +} + +fn verify_checksum(data: &[u8], expected: &str) -> Result<()> { + let digest = hex::encode(Sha256::digest(data)); + if digest != expected { + bail!("checksum mismatch: expected={} got={}", expected, digest); + } + info!("checksum verified"); + Ok(()) +} + async fn restart_unit(unit: &str) -> Result<()> { let status = Command::new("sudo") .args(["systemctl", "restart", unit]) diff --git a/nixos-node/modules/csf-daemon.nix b/nixos-node/modules/csf-daemon.nix index 6ce1c7d..22f0626 100644 --- a/nixos-node/modules/csf-daemon.nix +++ b/nixos-node/modules/csf-daemon.nix @@ -48,6 +48,7 @@ in users.users.csf-daemon = { isSystemUser = true; group = "csf-daemon"; + extraGroups = [ "csf-updater" ]; home = "/var/lib/csf-daemon"; shell = pkgs.shadow; description = "CSF daemon service user"; diff --git a/nixos-node/modules/server-configuration.nix b/nixos-node/modules/server-configuration.nix index e67df3c..e50c760 100644 --- a/nixos-node/modules/server-configuration.nix +++ b/nixos-node/modules/server-configuration.nix @@ -2,7 +2,7 @@ let composeDir = "/etc/csf-core"; - binDir = "/usr/local/bin"; + binDir = "/var/lib/csf-updater/bin"; csfUpdaterBin = csf.updaterPackage; csfAgentBin = csf.agentPackage; in @@ -158,13 +158,17 @@ in system.activationScripts.csf-binaries = { text = '' mkdir -p ${binDir} + chown csf-updater:csf-updater ${binDir} + chmod 750 ${binDir} if [ ! -f ${binDir}/csf-updater ]; then cp ${csfUpdaterBin}/bin/csf-updater ${binDir}/csf-updater - chmod 755 ${binDir}/csf-updater + chown csf-updater:csf-updater ${binDir}/csf-updater + chmod 750 ${binDir}/csf-updater fi if [ ! -f ${binDir}/csf-agent ]; then cp ${csfAgentBin}/bin/csf-agent ${binDir}/csf-agent - chmod 755 ${binDir}/csf-agent + chown csf-updater:csf-updater ${binDir}/csf-agent + chmod 750 ${binDir}/csf-agent fi ''; deps = [];