diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ac6b6943..b93b2946d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + uses: Johan-Liebert1/bootc-actions/bootc-ubuntu-setup@main - name: Validate (default) run: just validate # Check for security vulnerabilities and license compliance @@ -106,7 +106,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + uses: Johan-Liebert1/bootc-actions/bootc-ubuntu-setup@main - name: Enable fsverity for / run: sudo tune2fs -O verity $(findmnt -vno SOURCE /) - name: Install utils @@ -171,7 +171,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + uses: Johan-Liebert1/bootc-actions/bootc-ubuntu-setup@main - name: Build mdbook run: just build-mdbook # Build packages for each test OS @@ -188,7 +188,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + uses: Johan-Liebert1/bootc-actions/bootc-ubuntu-setup@main - name: Setup env run: | @@ -252,7 +252,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + uses: Johan-Liebert1/bootc-actions/bootc-ubuntu-setup@main with: libvirt: true - name: Install tmt @@ -344,7 +344,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + uses: Johan-Liebert1/bootc-actions/bootc-ubuntu-setup@main with: libvirt: true - name: Install tmt @@ -399,7 +399,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + uses: Johan-Liebert1/bootc-actions/bootc-ubuntu-setup@main with: libvirt: true - name: Install tmt @@ -442,7 +442,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + uses: Johan-Liebert1/bootc-actions/bootc-ubuntu-setup@main with: libvirt: true diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index 0152285ef..488b6ec20 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -1,3 +1,7 @@ +use std::sync::mpsc; +use std::time::Duration; +use std::usize; + use anyhow::{Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; use fn_error_context::context; @@ -25,6 +29,10 @@ const FIELD_ADJUST: &str = "adjust"; const FIELD_FIXME_SKIP_IF_COMPOSEFS: &str = "fixme_skip_if_composefs"; const FIELD_FIXME_SKIP_IF_UKI: &str = "fixme_skip_if_uki"; +/// For tests that should only run for composefs systems +/// Ex. composefs-gc +const FIELD_SKIP_IF_OSTREE: &str = "skip_if_ostree"; + // bcvk options const BCVK_OPT_BIND_STORAGE_RO: &str = "--bind-storage-ro"; const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image"; @@ -32,6 +40,36 @@ const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image"; // Distro identifiers const DISTRO_CENTOS_9: &str = "centos-9"; +// Tests sorted by time taken (descending) +const TESTS_SORTED_BY_TIME: [&str; 23] = [ + // 10+ mins + "multi-device-esp", + "composefs-gc-uki", + "composefs-gc", + // 5+ mins + "loader-entries-source", + "download-only-upgrade", + "bib-build", + "rollback", + "logically-bound-switch", + "soft-reboot", + "switch-to-unified", + "image-pushpull-upgrade", + "install-no-boot-dir", + "upgrade-tag", + "custom-selinux-policy", + "factory-reset", + // 3+ mins + "upgrade-check-status", + "soft-reboot-selinux-policy", + "install-bootloader-none", + "install-outside-container", + "install-unified-flag", + "usroverlay", + "image-upgrade-reboot", + "install-karg-delete", +]; + // Import the argument types from xtask.rs use crate::bcvk::BcvkInstallOpts; use crate::{RunTmtArgs, SealState, TmtProvisionArgs}; @@ -207,10 +245,11 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result ) } -#[derive(Debug)] +#[derive(Debug, Default)] struct PlanMetadata { try_bind_storage: bool, skip_if_composefs: bool, + skip_if_ostree: bool, skip_if_uki: bool, } @@ -252,8 +291,7 @@ fn parse_plan_metadata( .and_modify(|m| m.try_bind_storage = b) .or_insert(PlanMetadata { try_bind_storage: b, - skip_if_uki: false, - skip_if_composefs: false, + ..Default::default() }); } } @@ -268,8 +306,7 @@ fn parse_plan_metadata( .and_modify(|m| m.skip_if_composefs = b) .or_insert(PlanMetadata { skip_if_composefs: b, - skip_if_uki: false, - try_bind_storage: false, + ..Default::default() }); } } @@ -284,8 +321,22 @@ fn parse_plan_metadata( .and_modify(|m| m.skip_if_uki = b) .or_insert(PlanMetadata { skip_if_uki: b, - skip_if_composefs: false, - try_bind_storage: false, + ..Default::default() + }); + } + } + + if let Some(skip_if_ostree) = plan_data.get(&serde_yaml::Value::String(format!( + "extra-{}", + FIELD_SKIP_IF_OSTREE + ))) { + if let Some(b) = skip_if_ostree.as_bool() { + plan_metadata + .entry(plan_name.to_string()) + .and_modify(|m| m.skip_if_ostree = b) + .or_insert(PlanMetadata { + skip_if_ostree: b, + ..Default::default() }); } } @@ -294,6 +345,214 @@ fn parse_plan_metadata( Ok(plan_metadata) } +struct RunPlanResult { + plan_name: String, + passed: bool, + time_taken: Option, + run_id: Option, +} + +impl RunPlanResult { + fn new( + plan_name: String, + passed: bool, + time_taken: Option, + run_id: Option, + ) -> Self { + Self { + plan_name, + passed, + time_taken, + run_id, + } + } +} + +fn run_plan( + plan: String, + vm_name: String, + image: String, + plan_bcvk_opts: Vec, + firmware_args: Vec, + context: Vec, + tmt_env_vars: Vec, + arg_env: Vec, + preserve_vm: bool, + vm_cpu: String, + vm_mem_mb: String, +) -> RunPlanResult { + let sh = match Shell::new() { + Ok(sh) => sh, + Err(err) => { + eprintln!("Failed to create new shell instance: {err:?}"); + return RunPlanResult::new(plan, false, None, None); + } + }; + + // Launch VM with bcvk + let firmware_args_slice = firmware_args.as_slice(); + let launch_result = cmd!( + sh, + "bcvk libvirt run --name {vm_name} --memory {vm_mem_mb} --cpus {vm_cpu} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {plan_bcvk_opts...} {image}" + ) + .run() + .context("Launching VM with bcvk"); + + if let Err(e) = launch_result { + eprintln!("Failed to launch VM for plan {}: {:#}", plan, e); + return RunPlanResult::new(plan, false, None, None); + } + + // Ensure VM cleanup happens even on error (unless --preserve-vm is set) + let cleanup_vm = || { + if preserve_vm { + return; + } + if let Err(e) = cmd!(sh, "bcvk libvirt rm --stop --force {vm_name}") + .ignore_stderr() + .ignore_status() + .run() + { + eprintln!("Warning: Failed to cleanup VM {}: {}", vm_name, e); + } + }; + + // Wait for VM to be ready and get SSH info + let vm_info = wait_for_vm_ready(&sh, &vm_name); + let (ssh_port, ssh_key) = match vm_info { + Ok((port, key)) => (port, key), + Err(e) => { + eprintln!("Failed to get VM info for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan, false, None, None); + } + }; + + println!("VM ready, SSH port: {}", ssh_port); + + // Save SSH private key to a temporary file + let key_file = tempfile::NamedTempFile::new().context("Creating temporary SSH key file"); + + let key_file = match key_file { + Ok(f) => f, + Err(e) => { + eprintln!("Failed to create SSH key file for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan, false, None, None); + } + }; + + let key_path = Utf8PathBuf::try_from(key_file.path().to_path_buf()) + .context("Converting key path to UTF-8"); + + let key_path = match key_path { + Ok(p) => p, + Err(e) => { + eprintln!("Failed to convert key path for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan, false, None, None); + } + }; + + if let Err(e) = std::fs::write(&key_path, ssh_key) { + eprintln!("Failed to write SSH key for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan, false, None, None); + } + + // Set proper permissions on the key file (SSH requires 0600) + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o600); + if let Err(e) = std::fs::set_permissions(&key_path, perms) { + eprintln!("Failed to set key permissions for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan, false, None, None); + } + } + + // Verify SSH connectivity + println!("Verifying SSH connectivity..."); + if let Err(e) = verify_ssh_connectivity(&sh, ssh_port, &key_path) { + eprintln!("SSH verification failed for plan {}: {:#}", plan, e); + cleanup_vm(); + return RunPlanResult::new(plan, false, None, None); + } + + println!("SSH connectivity verified"); + + let ssh_port_str = ssh_port.to_string(); + + let time_start = std::time::Instant::now(); + + // Run tmt for this specific plan using connect provisioner + println!("Running tmt tests for plan {}...", plan); + + // Generate a unique run ID for this test + // Use the VM name which already contains a random suffix for uniqueness + let run_id = vm_name.clone(); + + // Run tmt for this specific plan + // Note: provision must come before plan for connect to work properly + let how = ["--how=connect", "--guest=localhost", "--user=root"]; + let env = ["TMT_SCRIPTS_DIR=/var/lib/tmt/scripts", "BCVK_EXPORT=1"] + .into_iter() + .chain(arg_env.iter().map(|v| v.as_str())) + .chain(tmt_env_vars.iter().map(|v| v.as_str())) + .flat_map(|v| ["--environment", v]); + let test_result = cmd!( + sh, + "tmt {context...} run --id {run_id} --all {env...} provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}" + ) + .run(); + + let elapsed = time_start.elapsed(); + + // Log disk usage after each test run to help diagnose "no space left on device" failures + println!("Disk usage after plan {}:", plan); + let _ = cmd!(sh, "df -h").run(); + + // Clean up VM regardless of test result (unless --preserve-vm is set) + cleanup_vm(); + + let plan_result = match test_result { + Ok(_) => { + println!("Plan {} completed successfully", plan); + RunPlanResult::new(plan, true, Some(elapsed), Some(run_id)) + } + Err(e) => { + eprintln!("Plan {} failed: {:#}", plan, e); + RunPlanResult::new(plan, false, Some(elapsed), Some(run_id)) + } + }; + + // Print VM connection details if preserving + if preserve_vm { + // Copy SSH key to a persistent location + let persistent_key_path = Utf8Path::new("target").join(format!("{}.ssh-key", vm_name)); + if let Err(e) = std::fs::copy(&key_path, &persistent_key_path) { + eprintln!("Warning: Failed to save persistent SSH key: {}", e); + } else { + println!("\n========================================"); + println!("VM preserved for debugging:"); + println!("========================================"); + println!("VM name: {}", vm_name); + println!("SSH port: {}", ssh_port_str); + println!("SSH key: {}", persistent_key_path); + println!("\nTo connect via SSH:"); + println!( + " ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost", + persistent_key_path, ssh_port_str + ); + println!("\nTo cleanup:"); + println!(" bcvk libvirt rm --stop --force {}", vm_name); + println!("========================================\n"); + } + } + + plan_result +} + /// Run TMT tests using bcvk for VM management /// This spawns a separate VM per test plan to avoid state leakage between tests. #[context("Running TMT tests")] @@ -391,6 +650,14 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { .map(|(_, v)| v.skip_if_composefs) .unwrap_or(false) }); + } else { + plans.retain(|plan| { + !plan_metadata + .iter() + .find(|(key, _)| plan.ends_with(key.as_str())) + .map(|(_, v)| v.skip_if_ostree) + .unwrap_or(false) + }); } if matches!(args.boot_type, crate::BootType::Uki) { @@ -417,18 +684,77 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { return Ok(()); } + plans.sort_by_key(|full_plan_name| { + TESTS_SORTED_BY_TIME + .iter() + .position(|test_time| full_plan_name.contains(test_time)) + .unwrap_or(usize::MAX) + }); + println!("Found {} test plan(s): {:?}", plans.len(), plans); + let mut install_opts = Vec::new(); + + // Add --filesystem=xfs by default on fedora-coreos + if variant_id == "coreos" { + if distro.starts_with("fedora") { + install_opts.push("--filesystem=xfs".to_string()); + } + } + + if args.composefs_backend { + let filesystem = args.filesystem.as_deref().unwrap_or("ext4"); + install_opts.push(format!("--filesystem={}", filesystem)); + install_opts.push("--composefs-backend".into()); + + if let Some(b) = &args.bootloader { + install_opts.push(format!("--bootloader={b}")); + } + } + + for k in &args.karg { + install_opts.push(format!("--karg={k}")); + } + + let start = std::time::Instant::now(); + + println!("Creating base disk..."); + let opts = install_opts.clone(); + cmd!(sh, "bcvk libvirt to-base-disk {opts...} localhost/bootc").run()?; + + println!("Creating base disk took: {:#?}", start.elapsed()); + // Generate a random suffix for VM names let random_suffix = generate_random_suffix(); // Track overall success/failure let mut all_passed = true; - let mut test_results: Vec<(String, bool, Option)> = Vec::new(); + let mut test_results: Vec = Vec::new(); // Environment variables to pass to tmt (in addition to args.env) let mut tmt_env_vars = Vec::new(); + let mut active_threads = 0; + + let num_cpu = std::thread::available_parallelism() + .map(|c| c.get()) + .unwrap_or(1); + + println!("num_cpu: {num_cpu}"); + + let (vm_cpu, vm_mem) = (1, 1024); + + // Leave 1 cpu for the host + // If there's only 1 cpu (unlikely), then we only run 1 VM + let avail_cpu = (num_cpu - 1).max(1); + + // More than this and we bottleneck on IO + let parallel_vms = (avail_cpu / vm_cpu).min(6); + + println!("parallel_vms: {parallel_vms}"); + + let (tx, rx) = mpsc::channel::(); + // Run each plan in its own VM for plan in plans { let plan_name = sanitize_plan_name(plan); @@ -471,188 +797,74 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { distro ); } - // Add --filesystem=xfs by default on fedora-coreos - if variant_id == "coreos" { - if distro.starts_with("fedora") { - opts.push("--filesystem=xfs".to_string()); - } - } opts.extend(bcvk_opts.install_args()); opts }; - // Launch VM with bcvk - let firmware_args_slice = firmware_args.as_slice(); - let launch_result = cmd!( - sh, - "bcvk libvirt run --name {vm_name} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {plan_bcvk_opts...} {image}" - ) - .run() - .context("Launching VM with bcvk"); - - if let Err(e) = launch_result { - eprintln!("Failed to launch VM for plan {}: {:#}", plan, e); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } - - // Ensure VM cleanup happens even on error (unless --preserve-vm is set) - let cleanup_vm = || { - if preserve_vm { - return; - } - if let Err(e) = cmd!(sh, "bcvk libvirt rm --stop --force {vm_name}") - .ignore_stderr() - .ignore_status() - .run() - { - eprintln!("Warning: Failed to cleanup VM {}: {}", vm_name, e); - } - }; - - // Wait for VM to be ready and get SSH info - let vm_info = wait_for_vm_ready(sh, &vm_name); - let (ssh_port, ssh_key) = match vm_info { - Ok((port, key)) => (port, key), - Err(e) => { - eprintln!("Failed to get VM info for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } - }; - - println!("VM ready, SSH port: {}", ssh_port); - - // Save SSH private key to a temporary file - let key_file = tempfile::NamedTempFile::new().context("Creating temporary SSH key file"); - - let key_file = match key_file { - Ok(f) => f, - Err(e) => { - eprintln!("Failed to create SSH key file for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } - }; - - let key_path = Utf8PathBuf::try_from(key_file.path().to_path_buf()) - .context("Converting key path to UTF-8"); + let firmware_args = firmware_args.clone(); + let context = context.clone(); + let tmt_env_vars = tmt_env_vars.clone(); + let env = args.env.clone(); + let cloned_plan = plan.to_string(); + let cloned_vm_name = vm_name.to_string(); + let image = image.to_string(); + let vm_mem = vm_mem.to_string(); + let vm_cpu = vm_cpu.to_string(); + + let tx_clone = tx.clone(); + std::thread::spawn(move || { + let result = run_plan( + cloned_plan, + cloned_vm_name, + image, + plan_bcvk_opts, + firmware_args, + context, + tmt_env_vars, + env, + preserve_vm, + vm_cpu, + vm_mem, + ); - let key_path = match key_path { - Ok(p) => p, - Err(e) => { - eprintln!("Failed to convert key path for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; + if let Err(e) = tx_clone.send(result) { + eprintln!("Failed to send result through channel: {}", e); } - }; + }); - if let Err(e) = std::fs::write(&key_path, ssh_key) { - eprintln!("Failed to write SSH key for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } + active_threads += 1; - // Set proper permissions on the key file (SSH requires 0600) - { - use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(0o600); - if let Err(e) = std::fs::set_permissions(&key_path, perms) { - eprintln!("Failed to set key permissions for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; + // wait for a thread to complete if we've reached the parallel limit + if active_threads >= parallel_vms { + match rx.recv() { + Ok(plan_result) => { + test_results.push(plan_result); + active_threads -= 1; + } + Err(e) => { + eprintln!("Failed to receive result from channel: {}", e); + // still decrement to avoid infinite loop + // in theory this shouldn't happen as we loop over plans, but + // for sanity + active_threads -= 1; + } } } + } - // Verify SSH connectivity - println!("Verifying SSH connectivity..."); - if let Err(e) = verify_ssh_connectivity(sh, ssh_port, &key_path) { - eprintln!("SSH verification failed for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false, None)); - continue; - } - - println!("SSH connectivity verified"); - - let ssh_port_str = ssh_port.to_string(); - - // Run tmt for this specific plan using connect provisioner - println!("Running tmt tests for plan {}...", plan); - - // Generate a unique run ID for this test - // Use the VM name which already contains a random suffix for uniqueness - let run_id = vm_name.clone(); - - // Run tmt for this specific plan - // Note: provision must come before plan for connect to work properly - let context = context.clone(); - let how = ["--how=connect", "--guest=localhost", "--user=root"]; - let env = ["TMT_SCRIPTS_DIR=/var/lib/tmt/scripts", "BCVK_EXPORT=1"] - .into_iter() - .chain(args.env.iter().map(|v| v.as_str())) - .chain(tmt_env_vars.iter().map(|v| v.as_str())) - .flat_map(|v| ["--environment", v]); - let test_result = cmd!( - sh, - "tmt {context...} run --id {run_id} --all {env...} provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}" - ) - .run(); - - // Log disk usage after each test run to help diagnose "no space left on device" failures - println!("Disk usage after plan {}:", plan); - let _ = cmd!(sh, "df -h").run(); - - // Clean up VM regardless of test result (unless --preserve-vm is set) - cleanup_vm(); + // drop the sender to signal no more messages + drop(tx); - match test_result { - Ok(_) => { - println!("Plan {} completed successfully", plan); - test_results.push((plan.to_string(), true, Some(run_id))); + // remaining results from channel + for _ in 0..active_threads { + match rx.recv() { + Ok(plan_result) => { + test_results.push(plan_result); } Err(e) => { - eprintln!("Plan {} failed: {:#}", plan, e); - all_passed = false; - test_results.push((plan.to_string(), false, Some(run_id))); - } - } - - // Print VM connection details if preserving - if preserve_vm { - // Copy SSH key to a persistent location - let persistent_key_path = Utf8Path::new("target").join(format!("{}.ssh-key", vm_name)); - if let Err(e) = std::fs::copy(&key_path, &persistent_key_path) { - eprintln!("Warning: Failed to save persistent SSH key: {}", e); - } else { - println!("\n========================================"); - println!("VM preserved for debugging:"); - println!("========================================"); - println!("VM name: {}", vm_name); - println!("SSH port: {}", ssh_port_str); - println!("SSH key: {}", persistent_key_path); - println!("\nTo connect via SSH:"); - println!( - " ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost", - persistent_key_path, ssh_port_str - ); - println!("\nTo cleanup:"); - println!(" bcvk libvirt rm --stop --force {}", vm_name); - println!("========================================\n"); + eprintln!("Failed to receive remaining result from channel: {}", e); } } } @@ -661,16 +873,35 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { println!("\n========================================"); println!("Test Summary"); println!("========================================"); - for (plan, passed, _) in &test_results { - let status = if *passed { "PASSED" } else { "FAILED" }; - println!("{}: {}", plan, status); + + test_results.sort_by(|a, b| b.time_taken.cmp(&a.time_taken)); + + for RunPlanResult { + plan_name: plan, + passed, + time_taken, + .. + } in &test_results + { + let status = if *passed { + "PASSED" + } else { + all_passed = false; + "FAILED" + }; + println!( + "{}: {} ({:?})", + plan, + status, + time_taken.unwrap_or(Duration::from_secs(0)) + ); } println!("========================================\n"); // Print detailed error reports for failed tests let failed_tests: Vec<_> = test_results .iter() - .filter(|(_, passed, _)| !passed) + .filter(|plan_res| !plan_res.passed) .collect(); if !failed_tests.is_empty() { @@ -678,7 +909,12 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { println!("Detailed Error Reports"); println!("========================================\n"); - for (plan, _, run_id) in failed_tests { + for RunPlanResult { + plan_name: plan, + run_id, + .. + } in failed_tests + { println!("----------------------------------------"); println!("Plan: {}", plan); println!("----------------------------------------"); @@ -906,6 +1142,8 @@ struct TestDef { try_bind_storage: bool, /// Whether to skip this test for composefs backend skip_if_composefs: bool, + /// Whether to skip this test for ostree backend + skip_if_ostree: bool, /// Whether to skip this test for images with UKI skip_if_uki: bool, /// TMT fmf attributes to pass through (summary, duration, adjust, etc.) @@ -1077,6 +1315,13 @@ fn generate_integration() -> Result<(String, String)> { .and_then(|v| v.as_bool()) .unwrap_or(false); + let skip_if_ostree = metadata + .extra + .as_mapping() + .and_then(|m| m.get(&serde_yaml::Value::String(FIELD_SKIP_IF_OSTREE.to_string()))) + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let skip_if_uki = metadata .extra .as_mapping() @@ -1094,6 +1339,7 @@ fn generate_integration() -> Result<(String, String)> { test_command, try_bind_storage, skip_if_composefs, + skip_if_ostree, skip_if_uki, tmt: metadata.tmt, }); @@ -1217,6 +1463,13 @@ fn generate_integration() -> Result<(String, String)> { ); } + if test.skip_if_ostree { + plan_value.insert( + serde_yaml::Value::String(format!("extra-{}", FIELD_SKIP_IF_OSTREE)), + serde_yaml::Value::Bool(true), + ); + } + if test.skip_if_uki { plan_value.insert( serde_yaml::Value::String(format!("extra-{}", FIELD_FIXME_SKIP_IF_UKI)), diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 0997dde21..abae440ae 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -196,6 +196,8 @@ execute: how: fmf test: - /tmt/tests/tests/test-35-composefs-gc + extra-skip_if_ostree: true + extra-fixme_skip_if_uki: true /plan-35-upgrade-preflight-disk-check: summary: Verify pre-flight disk space check rejects images with inflated layer sizes @@ -254,6 +256,7 @@ execute: how: fmf test: - /tmt/tests/tests/test-41-composefs-gc-uki + extra-skip_if_ostree: true /plan-42-loader-entries-source: summary: Test bootc loader-entries set-options-for-source @@ -269,4 +272,5 @@ execute: how: fmf test: - /tmt/tests/tests/test-43-switch-same-digest + extra-skip_if_ostree: true # END GENERATED PLANS diff --git a/tmt/tests/booted/tap.nu b/tmt/tests/booted/tap.nu index b8b0b3f7e..63076d3c6 100644 --- a/tmt/tests/booted/tap.nu +++ b/tmt/tests/booted/tap.nu @@ -112,3 +112,15 @@ export def make_uki_containerfile [containerfile: string] { return $"($containerfile)\n($uki_stuff)" } + +# We share host container storage in the VM, so we usually already have localhost/bootc +# as an image in the VM. If we don't find it, only then do we copy from ostree/composefs +# storage +export def img_cp_to_store_smart [] { + let podman_digest = (podman image inspect localhost/bootc) | jq -r '.[0].Digest' + let bootc_digest = (bootc status --json | from json).status.booted.image.imageDigest + + if ($podman_digest != $bootc_digest) { + bootc image copy-to-storage + } +} diff --git a/tmt/tests/booted/test-bib-build.nu b/tmt/tests/booted/test-bib-build.nu index 765698204..ac26ce7eb 100644 --- a/tmt/tests/booted/test-bib-build.nu +++ b/tmt/tests/booted/test-bib-build.nu @@ -26,12 +26,14 @@ const BIB_IMAGE = "quay.io/centos-bootc/bootc-image-builder:latest" def main [] { tap begin "bootc-image-builder qcow2 build test" + print ">>>>>>>>>>>>>>>>>> the current working directory <<<<<<<<<<<<<<<<<" ($env.PWD) + let td = mktemp -d cd $td # Copy the currently booted image to podman storage print "=== Copying booted image to containers-storage ===" - bootc image copy-to-storage + tap img_cp_to_store_smart # Verify the image is in storage let images = podman images --format json | from json @@ -87,8 +89,10 @@ DISKEOF ' | save Dockerfile podman build -t localhost/bootc-bib-test . + let output_dir = "/var/output" + # Create output directory for bib - mkdir output + mkdir $output_dir # Run bootc-image-builder to create a qcow2 # We use --local to pull from local containers-storage @@ -97,11 +101,11 @@ DISKEOF let bib_image = $BIB_IMAGE # Note: we disable SELinux labeling since we're running in a test VM # and use unconfined_t to avoid permission issues - podman run --rm --privileged -v /var/lib/containers/storage:/var/lib/containers/storage --security-opt label=type:unconfined_t -v ./output:/output $bib_image --type qcow2 --rootfs xfs localhost/bootc-bib-test + podman run --rm --privileged -v /var/lib/containers/storage:/var/lib/containers/storage --security-opt label=type:unconfined_t -v $"($output_dir):/output" $bib_image --type qcow2 --rootfs xfs localhost/bootc-bib-test # Verify output was created print "=== Verifying output ===" - let disk_path = "output/qcow2/disk.qcow2" + let disk_path = $"($output_dir)/qcow2/disk.qcow2" assert ($disk_path | path exists) $"Expected disk image at ($disk_path)" # Check the disk has reasonable virtual size (at least 4GB as per disk.yaml) diff --git a/tmt/tests/booted/test-composefs-gc-uki.nu b/tmt/tests/booted/test-composefs-gc-uki.nu index b82c449bd..2a2244370 100644 --- a/tmt/tests/booted/test-composefs-gc-uki.nu +++ b/tmt/tests/booted/test-composefs-gc-uki.nu @@ -2,6 +2,8 @@ # tmt: # summary: Test composefs garbage collection for UKI # duration: 30m +# extra: +# skip_if_ostree: true use std assert use tap.nu @@ -24,7 +26,7 @@ if not $is_uki { # Create a large file in a new container image, then bootc switch to the image def first_boot [] { - bootc image copy-to-storage + tap img_cp_to_store_smart mut containerfile = $" FROM localhost/bootc as base diff --git a/tmt/tests/booted/test-composefs-gc.nu b/tmt/tests/booted/test-composefs-gc.nu index f1b5487b1..d030dcba5 100644 --- a/tmt/tests/booted/test-composefs-gc.nu +++ b/tmt/tests/booted/test-composefs-gc.nu @@ -2,6 +2,9 @@ # tmt: # summary: Test composefs garbage collection with same and different kernel+initrd # duration: 30m +# extra: +# skip_if_ostree: true +# fixme_skip_if_uki: true use std assert use tap.nu @@ -22,7 +25,7 @@ if ($st.status.booted.composefs.bootType | str downcase) == "uki" { # Create a large file in a new container image, then bootc switch to the image def first_boot [] { - bootc image copy-to-storage + tap img_cp_to_store_smart echo $" FROM localhost/bootc diff --git a/tmt/tests/booted/test-custom-selinux-policy.nu b/tmt/tests/booted/test-custom-selinux-policy.nu index 80f34be30..52edc4a32 100644 --- a/tmt/tests/booted/test-custom-selinux-policy.nu +++ b/tmt/tests/booted/test-custom-selinux-policy.nu @@ -24,7 +24,7 @@ def initial_build [] { let td = mktemp -d cd $td - bootc image copy-to-storage + tap img_cp_to_store_smart # A simple derived container that customizes selinux policy for random dir "FROM localhost/bootc diff --git a/tmt/tests/booted/test-download-only-upgrade.nu b/tmt/tests/booted/test-download-only-upgrade.nu index 59dcdc3f5..331027348 100644 --- a/tmt/tests/booted/test-download-only-upgrade.nu +++ b/tmt/tests/booted/test-download-only-upgrade.nu @@ -40,7 +40,7 @@ def initial_build [] { # This test only works in local mode assert ($imgsrc | str ends-with "-local") "This test requires local mode" - bootc image copy-to-storage + tap img_cp_to_store_smart # Create test file v1 on host "v1" | save testing-bootc-upgrade-apply diff --git a/tmt/tests/booted/test-image-pushpull-upgrade.nu b/tmt/tests/booted/test-image-pushpull-upgrade.nu index 708b868ec..d51260d67 100644 --- a/tmt/tests/booted/test-image-pushpull-upgrade.nu +++ b/tmt/tests/booted/test-image-pushpull-upgrade.nu @@ -39,7 +39,7 @@ def initial_build [] { let td = mktemp -d cd $td - bootc image copy-to-storage + tap img_cp_to_store_smart let img = podman image inspect localhost/bootc | from json mkdir usr/lib/bootc/kargs.d diff --git a/tmt/tests/booted/test-image-upgrade-reboot.nu b/tmt/tests/booted/test-image-upgrade-reboot.nu index f28cff1da..7fcab4283 100644 --- a/tmt/tests/booted/test-image-upgrade-reboot.nu +++ b/tmt/tests/booted/test-image-upgrade-reboot.nu @@ -46,7 +46,7 @@ def initial_build [] { let imgsrc = imgsrc # For the packit case, we build locally right now if ($imgsrc | str ends-with "-local") { - bootc image copy-to-storage + tap img_cp_to_store_smart # A simple derived container that adds a file ( diff --git a/tmt/tests/booted/test-install-bootloader-none.nu b/tmt/tests/booted/test-install-bootloader-none.nu index ef89deae7..2c0e52ed1 100644 --- a/tmt/tests/booted/test-install-bootloader-none.nu +++ b/tmt/tests/booted/test-install-bootloader-none.nu @@ -10,7 +10,7 @@ def main [] { tap begin "install with --bootloader=none" # Copy the booted image to container storage for use as install source - bootc image copy-to-storage + tap img_cp_to_store_smart let target_image = "containers-storage:localhost/bootc" truncate -s 10G disk.img diff --git a/tmt/tests/booted/test-install-no-boot-dir.nu b/tmt/tests/booted/test-install-no-boot-dir.nu index bc012fd66..a8859cd9b 100644 --- a/tmt/tests/booted/test-install-no-boot-dir.nu +++ b/tmt/tests/booted/test-install-no-boot-dir.nu @@ -10,7 +10,7 @@ def main [] { tap begin "install to-filesystem without /boot" # Copy the booted image to container storage for use as install source - bootc image copy-to-storage + tap img_cp_to_store_smart let target_image = "containers-storage:localhost/bootc" mkdir /var/mnt diff --git a/tmt/tests/booted/test-install-outside-container.nu b/tmt/tests/booted/test-install-outside-container.nu index 8c88cf17a..eccb2c1e3 100644 --- a/tmt/tests/booted/test-install-outside-container.nu +++ b/tmt/tests/booted/test-install-outside-container.nu @@ -10,7 +10,7 @@ use tap.nu # EFI update metadata. Export to OCI layout on a writable path since # containers-storage: transport can't work when the root fs is read-only # (composefs), and install-outside-container tests run directly on the host. -bootc image copy-to-storage +tap img_cp_to_store_smart skopeo copy containers-storage:localhost/bootc oci:/var/tmp/bootc-oci let target_image = "oci:/var/tmp/bootc-oci" diff --git a/tmt/tests/booted/test-loader-entries-source.nu b/tmt/tests/booted/test-loader-entries-source.nu index b52656798..164834b92 100644 --- a/tmt/tests/booted/test-loader-entries-source.nu +++ b/tmt/tests/booted/test-loader-entries-source.nu @@ -206,7 +206,7 @@ def fifth_boot [] { # Build a derived image and switch to it (this stages a deployment). # Then call set-options-for-source on top. The staged deployment should # be replaced with one that has the new image AND the source kargs. - bootc image copy-to-storage + tap img_cp_to_store_smart let td = mktemp -d $"FROM localhost/bootc diff --git a/tmt/tests/booted/test-logically-bound-switch.nu b/tmt/tests/booted/test-logically-bound-switch.nu index 298d7ff86..7da50dc2d 100644 --- a/tmt/tests/booted/test-logically-bound-switch.nu +++ b/tmt/tests/booted/test-logically-bound-switch.nu @@ -29,7 +29,7 @@ let booted = $st.status.booted.image echo '{}' | save -f /run/ostree/auth.json def initial_setup [] { - bootc image copy-to-storage + tap img_cp_to_store_smart podman images podman image inspect localhost/bootc | from json } diff --git a/tmt/tests/booted/test-multi-device-esp.nu b/tmt/tests/booted/test-multi-device-esp.nu index b3f69fcf3..d0d704311 100644 --- a/tmt/tests/booted/test-multi-device-esp.nu +++ b/tmt/tests/booted/test-multi-device-esp.nu @@ -168,7 +168,7 @@ def run_install [mountpoint: string] { def test_single_esp [] { tap begin "multi-device ESP detection tests" - bootc image copy-to-storage + tap img_cp_to_store_smart print "Starting single ESP test" diff --git a/tmt/tests/booted/test-rollback.nu b/tmt/tests/booted/test-rollback.nu index 0afe377ac..03dee51c4 100644 --- a/tmt/tests/booted/test-rollback.nu +++ b/tmt/tests/booted/test-rollback.nu @@ -39,7 +39,7 @@ def initial_switch [] { let imgsrc = imgsrc if ($imgsrc | str ends-with "-local") { - bootc image copy-to-storage + tap img_cp_to_store_smart print "Building derived container" let dockerfile = $"FROM localhost/bootc as base diff --git a/tmt/tests/booted/test-soft-reboot-selinux-policy.nu b/tmt/tests/booted/test-soft-reboot-selinux-policy.nu index 4e2706804..c3ca145f4 100644 --- a/tmt/tests/booted/test-soft-reboot-selinux-policy.nu +++ b/tmt/tests/booted/test-soft-reboot-selinux-policy.nu @@ -33,7 +33,7 @@ def initial_build [] { let td = mktemp -d cd $td - bootc image copy-to-storage + tap img_cp_to_store_smart # copy-to-storage does not copy repo file # but OSCI gating test needs repo to install package diff --git a/tmt/tests/booted/test-soft-reboot.nu b/tmt/tests/booted/test-soft-reboot.nu index 9be3fbcd5..3274161cc 100644 --- a/tmt/tests/booted/test-soft-reboot.nu +++ b/tmt/tests/booted/test-soft-reboot.nu @@ -24,7 +24,7 @@ def initial_build [] { let td = mktemp -d cd $td - bootc image copy-to-storage + tap img_cp_to_store_smart # A simple derived container that adds a file, but also injects some kargs let dockerfile = $"FROM localhost/bootc as base diff --git a/tmt/tests/booted/test-switch-same-digest.nu b/tmt/tests/booted/test-switch-same-digest.nu index eda56d467..de1dcd6d2 100644 --- a/tmt/tests/booted/test-switch-same-digest.nu +++ b/tmt/tests/booted/test-switch-same-digest.nu @@ -2,6 +2,8 @@ # tmt: # summary: Error on bootc switch to image with identical fs-verity digest # duration: 10m +# extra: +# skip_if_ostree: true # # Verify that `bootc switch` errors out when the target image produces the # same composefs fs-verity digest as an existing deployment. The simplest @@ -17,7 +19,7 @@ if not (tap is_composefs) { tap begin "bootc switch to same-digest image must error" # Copy the booted image into podman storage so we can retag it. -bootc image copy-to-storage +tap img_cp_to_store_smart # Tag the same image under a second name — identical bits, so the composefs # EROFS digest will be the same as the currently booted deployment. diff --git a/tmt/tests/booted/test-switch-to-unified.nu b/tmt/tests/booted/test-switch-to-unified.nu index 4e8b61442..9b7f9d497 100644 --- a/tmt/tests/booted/test-switch-to-unified.nu +++ b/tmt/tests/booted/test-switch-to-unified.nu @@ -29,7 +29,7 @@ def first_boot [] { tap begin "copy image to podman storage, switch, then onboard to unified storage" # Copy the currently booted image to podman storage - bootc image copy-to-storage + tap img_cp_to_store_smart # Switch to the base image using containers-storage transport bootc switch --transport containers-storage localhost/bootc diff --git a/tmt/tests/booted/test-upgrade-check-status.nu b/tmt/tests/booted/test-upgrade-check-status.nu index bde2813c2..731ffe149 100644 --- a/tmt/tests/booted/test-upgrade-check-status.nu +++ b/tmt/tests/booted/test-upgrade-check-status.nu @@ -30,7 +30,7 @@ def imgsrc [] { def initial_build [] { tap begin "upgrade --check cached update in status" - bootc image copy-to-storage + tap img_cp_to_store_smart # A simple derived container that adds a file with a version label "FROM localhost/bootc diff --git a/tmt/tests/booted/test-upgrade-tag.nu b/tmt/tests/booted/test-upgrade-tag.nu index 2125762c8..941170f59 100644 --- a/tmt/tests/booted/test-upgrade-tag.nu +++ b/tmt/tests/booted/test-upgrade-tag.nu @@ -23,7 +23,7 @@ def initial_build [] { cd $td # Copy bootc image to local storage - bootc image copy-to-storage + tap img_cp_to_store_smart # Build v1 image let dockerfile = $"FROM localhost/bootc as base