diff --git a/crates/integration-tests/src/main.rs b/crates/integration-tests/src/main.rs index fc7ba69..9623f41 100644 --- a/crates/integration-tests/src/main.rs +++ b/crates/integration-tests/src/main.rs @@ -13,6 +13,7 @@ pub(crate) use integration_tests::{ mod tests { pub mod libvirt_base_disks; + pub mod libvirt_ignition; pub mod libvirt_port_forward; pub mod libvirt_upload_disk; pub mod libvirt_verb; diff --git a/crates/integration-tests/src/tests/libvirt_ignition.rs b/crates/integration-tests/src/tests/libvirt_ignition.rs new file mode 100644 index 0000000..e13a238 --- /dev/null +++ b/crates/integration-tests/src/tests/libvirt_ignition.rs @@ -0,0 +1,172 @@ +//! Integration tests for Ignition config injection in libvirt VMs + +use integration_tests::integration_test; +use itest::TestResult; +use scopeguard::defer; +use tempfile::TempDir; +use xshell::cmd; + +use std::fs; + +use camino::Utf8Path; + +use crate::{get_bck_command, shell, LIBVIRT_INTEGRATION_TEST_LABEL}; + +/// Generate a random alphanumeric suffix for VM names to avoid collisions +fn random_suffix() -> String { + use rand::{distr::Alphanumeric, Rng}; + rand::rng() + .sample_iter(&Alphanumeric) + .take(8) + .map(char::from) + .collect() +} + +/// Fedora CoreOS image that supports Ignition +const FCOS_IMAGE: &str = "quay.io/fedora/fedora-coreos:stable"; + +/// Test that Ignition config injection mechanism works for libvirt +/// +/// This test verifies that the Ignition config injection mechanism is working +/// by checking that the VM can be created with --ignition flag and that the +/// config file is properly stored. +fn test_libvirt_ignition_works() -> TestResult { + let sh = shell()?; + let bck = get_bck_command()?; + let label = LIBVIRT_INTEGRATION_TEST_LABEL; + + // Pull FCOS image first + cmd!(sh, "podman pull -q {FCOS_IMAGE}").run()?; + + // Create a temporary Ignition config + let temp_dir = TempDir::new()?; + let config_path = Utf8Path::from_path(temp_dir.path()) + .expect("temp dir is not utf8") + .join("config.ign"); + + // Minimal valid Ignition config (v3.3.0 for FCOS) + let ignition_config = r#"{"ignition": {"version": "3.3.0"}}"#; + fs::write(&config_path, ignition_config)?; + + // Generate a unique VM name to avoid conflicts + let vm_name = format!("test-ignition-{}", random_suffix()); + + // Create VM with Ignition config + // We use --ssh-wait to wait for the VM to boot and verify SSH connectivity + // FCOS requires --filesystem to be specified + let output = cmd!( + sh, + "{bck} libvirt run --name {vm_name} --label {label} --ignition {config_path} --filesystem xfs --ssh-wait --memory 2G --cpus 2 {FCOS_IMAGE}" + ) + .ignore_status() + .output()?; + + // Cleanup: remove the VM + defer! { + let _ = cmd!(sh, "{bck} libvirt rm {vm_name} --force").run(); + } + + // Check that the command succeeded + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + panic!( + "Failed to create VM with Ignition config.\nStdout: {}\nStderr: {}", + stdout, stderr + ); + } + + // Verify the VM was created + let vm_list = cmd!(sh, "{bck} libvirt list").read()?; + assert!( + vm_list.contains(&vm_name), + "VM should be listed after creation" + ); + + println!("Ignition config injection test passed"); + Ok(()) +} +integration_test!(test_libvirt_ignition_works); + +/// Test that Ignition config validation rejects nonexistent files +fn test_libvirt_ignition_invalid_path() -> TestResult { + let sh = shell()?; + let bck = get_bck_command()?; + let label = LIBVIRT_INTEGRATION_TEST_LABEL; + + // Pull FCOS image first + cmd!(sh, "podman pull -q {FCOS_IMAGE}").run()?; + + let temp = TempDir::new()?; + let nonexistent_path = Utf8Path::from_path(temp.path()) + .expect("temp dir is not utf8") + .join("nonexistent-config.ign"); + + let vm_name = format!("test-ignition-invalid-{}", random_suffix()); + + let output = cmd!( + sh, + "{bck} libvirt run --name {vm_name} --label {label} --ignition {nonexistent_path} {FCOS_IMAGE}" + ) + .ignore_status() + .output()?; + + assert!( + !output.status.success(), + "Should fail with nonexistent Ignition config file" + ); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("not found"), + "Error should mention missing file: {}", + stderr + ); + + println!("Ignition invalid path test passed"); + Ok(()) +} +integration_test!(test_libvirt_ignition_invalid_path); + +/// Test that Ignition is rejected for images that don't support it +fn test_libvirt_ignition_unsupported_image() -> TestResult { + let sh = shell()?; + let bck = get_bck_command()?; + let label = LIBVIRT_INTEGRATION_TEST_LABEL; + + // Use standard bootc image that doesn't have Ignition support + let image = "quay.io/centos-bootc/centos-bootc:stream10"; + + let temp_dir = TempDir::new()?; + let config_path = Utf8Path::from_path(temp_dir.path()) + .expect("temp dir is not utf8") + .join("config.ign"); + + let ignition_config = r#"{"ignition": {"version": "3.3.0"}}"#; + fs::write(&config_path, ignition_config)?; + + let vm_name = format!("test-ignition-unsupported-{}", random_suffix()); + + let output = cmd!( + sh, + "{bck} libvirt run --name {vm_name} --label {label} --ignition {config_path} {image}" + ) + .ignore_status() + .output()?; + + assert!( + !output.status.success(), + "Should fail when using --ignition with non-Ignition image" + ); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("does not support Ignition"), + "Error should mention missing Ignition support: {}", + stderr + ); + + println!("Ignition unsupported image test passed"); + Ok(()) +} +integration_test!(test_libvirt_ignition_unsupported_image); diff --git a/crates/kit/src/libvirt/domain.rs b/crates/kit/src/libvirt/domain.rs index 399af75..1d6909b 100644 --- a/crates/kit/src/libvirt/domain.rs +++ b/crates/kit/src/libvirt/domain.rs @@ -55,6 +55,8 @@ pub struct DomainBuilder { nvram_template: Option, // Custom NVRAM template with enrolled keys nvram_format: Option, // Format of NVRAM template (raw, qcow2) firmware_log: Option, // OVMF debug log output via isa-debugcon + fw_cfg_entries: Vec<(String, String)>, // fw_cfg entries (name, file_path) + ignition_disk_path: Option, // Path to Ignition config for virtio-blk injection } impl Default for DomainBuilder { @@ -86,6 +88,8 @@ impl DomainBuilder { nvram_template: None, nvram_format: None, firmware_log: None, + fw_cfg_entries: Vec::new(), + ignition_disk_path: None, } } @@ -204,6 +208,21 @@ impl DomainBuilder { self } + /// Add a fw_cfg entry for passing config files to the guest + /// + /// This is used for Ignition config injection on x86_64/aarch64. + /// The entry will be converted to a QEMU commandline argument in the domain XML. + pub fn add_fw_cfg(mut self, name: String, file_path: String) -> Self { + self.fw_cfg_entries.push((name, file_path)); + self + } + + /// Set Ignition config disk path for virtio-blk injection (s390x/ppc64le) + pub fn with_ignition_disk(mut self, disk_path: String) -> Self { + self.ignition_disk_path = Some(disk_path); + self + } + /// Build the domain XML pub fn build_xml(self) -> Result { let name = self.name.ok_or_else(|| eyre!("Domain name is required"))?; @@ -221,7 +240,7 @@ impl DomainBuilder { let mut writer = XmlWriter::new(); // Root domain element - let domain_attrs = if self.qemu_args.is_empty() { + let domain_attrs = if self.qemu_args.is_empty() && self.fw_cfg_entries.is_empty() { vec![("type", "kvm")] } else { vec![ @@ -379,6 +398,17 @@ impl DomainBuilder { writer.end_element("disk")?; } + // Ignition config disk (virtio-blk with serial="ignition" for s390x/ppc64le) + if let Some(ref ignition_disk) = self.ignition_disk_path { + writer.start_element("disk", &[("type", "file"), ("device", "disk")])?; + writer.write_empty_element("driver", &[("name", "qemu"), ("type", "raw")])?; + writer.write_empty_element("source", &[("file", ignition_disk)])?; + writer.write_empty_element("target", &[("dev", "vdb"), ("bus", "virtio")])?; + writer.write_text_element("serial", "ignition")?; + writer.write_empty_element("readonly", &[])?; + writer.end_element("disk")?; + } + // Network let network_config = self.network.as_deref().unwrap_or("default"); match network_config { @@ -483,9 +513,22 @@ impl DomainBuilder { writer.end_element("devices")?; - // QEMU commandline section (if we have QEMU args) - if !self.qemu_args.is_empty() { + // QEMU commandline section (if we have QEMU args or fw_cfg entries) + if !self.qemu_args.is_empty() || !self.fw_cfg_entries.is_empty() { writer.start_element("qemu:commandline", &[])?; + + // Add fw_cfg entries first + // Format: -fw_cfg name=,file= + // Verified working: config accessible at /sys/firmware/qemu_fw_cfg/by_name//raw + for (name, file_path) in &self.fw_cfg_entries { + writer.write_empty_element("qemu:arg", &[("value", "-fw_cfg")])?; + writer.write_empty_element( + "qemu:arg", + &[("value", &format!("name={},file={}", name, file_path))], + )?; + } + + // Then add other QEMU args for arg in &self.qemu_args { writer.write_empty_element("qemu:arg", &[("value", arg)])?; } diff --git a/crates/kit/src/libvirt/rm.rs b/crates/kit/src/libvirt/rm.rs index cf742de..d6f5372 100644 --- a/crates/kit/src/libvirt/rm.rs +++ b/crates/kit/src/libvirt/rm.rs @@ -5,6 +5,7 @@ use clap::Parser; use color_eyre::Result; +use tracing::debug; /// Check if a domain is persistent (vs transient) /// @@ -108,6 +109,26 @@ fn remove_vm_impl( } } + // Remove Ignition config file if it exists (stored in metadata) + // Parse domain XML to get the ignition persistent path + if let Ok(xml_output) = global_opts + .virsh_command() + .args(&["dumpxml", vm_name]) + .output() + { + if let Ok(xml_str) = String::from_utf8(xml_output.stdout) { + if let Ok(dom) = crate::xml_utils::parse_xml_dom(&xml_str) { + if let Some(ignition_path_node) = dom.find("bootc:ignition-persistent-path") { + let ignition_path = ignition_path_node.text_content().trim(); + if !ignition_path.is_empty() && std::path::Path::new(ignition_path).exists() { + debug!("Removing Ignition config file: {}", ignition_path); + let _ = std::fs::remove_file(ignition_path); // Don't fail if this fails + } + } + } + } + } + // Remove libvirt domain with nvram and storage let output = global_opts .virsh_command() diff --git a/crates/kit/src/libvirt/run.rs b/crates/kit/src/libvirt/run.rs index a66b134..43e3b78 100644 --- a/crates/kit/src/libvirt/run.rs +++ b/crates/kit/src/libvirt/run.rs @@ -301,6 +301,10 @@ pub struct LibvirtRunOpts { #[clap(long)] pub transient: bool, + /// Path to Ignition config file (JSON format) for first-boot provisioning + #[clap(long = "ignition")] + pub ignition_config: Option, + /// Additional metadata key-value pairs (used internally, not exposed via CLI) #[clap(skip)] pub metadata: std::collections::HashMap, @@ -448,11 +452,36 @@ pub fn run(global_opts: &crate::libvirt::LibvirtOptions, mut opts: LibvirtRunOpt let image_digest = inspect.digest.to_string(); debug!("Image digest: {}", image_digest); + // Check Ignition support and validate config file path early + if let Some(ref ignition_path) = opts.ignition_config { + let has_ignition = check_ignition_support(&opts.image)?; + if !has_ignition { + return Err(eyre!( + "Image does not support Ignition. See man bcvk-libvirt-run for details." + )); + } + debug!("Image {} supports Ignition", opts.image); + + // Validate that the Ignition config file exists before proceeding + if !ignition_path.try_exists()? { + return Err(eyre!("Ignition config file not found: {}", ignition_path)); + } + } + if opts.update_from_host { opts.bind_storage_ro = true; opts.install.target_transport = Some(UPDATE_FROM_HOST_TRANSPORT.to_owned()); } + // Add Ignition kernel argument to install options if Ignition config is specified + // This ensures the kernel arg is baked into the installed system's GRUB configuration + if opts.ignition_config.is_some() { + opts.install + .karg + .push("ignition.platform.id=qemu".to_string()); + debug!("Added ignition.platform.id=qemu kernel argument to install options"); + } + // Phase 1: Find or create a base disk image let base_disk_path = crate::libvirt::base_disks::find_or_create_base_disk( &opts.image, @@ -1022,6 +1051,62 @@ mod tests { } } +/// Check if the container image has Ignition support +/// +/// Checks for labels indicating Ignition support: +/// - 'coreos.ignition' (future convention, not yet widely used) +/// - 'com.coreos.osname' (heuristic: CoreOS-based images likely have Ignition) +/// +/// Returns true if the image is likely to support Ignition. +fn check_ignition_support(image: &str) -> Result { + use std::collections::HashMap; + use std::process::Stdio; + + // Fetch all labels with a single podman inspect call + let output = std::process::Command::new("podman") + .args(["image", "inspect", "--format", "{{json .Labels}}", image]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .context("Failed to inspect image for labels")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(eyre!( + "Failed to inspect image {} for labels: {}", + image, + stderr.trim() + )); + } + + // Parse the JSON output + let labels: HashMap = + serde_json::from_slice(&output.stdout).context("Failed to parse image labels as JSON")?; + + // Check for coreos.ignition label (could contain version info or just "1") + if let Some(ignition_value) = labels.get("coreos.ignition") { + if !ignition_value.is_empty() { + debug!( + "Image {} has coreos.ignition={} label", + image, ignition_value + ); + return Ok(true); + } + } + + // Fallback: check for com.coreos.osname (CoreOS-based images) + if let Some(osname_value) = labels.get("com.coreos.osname").filter(|v| !v.is_empty()) { + debug!( + "Image {} has com.coreos.osname={}, assuming Ignition support", + image, osname_value + ); + return Ok(true); + } + + debug!("Image {} does not appear to support Ignition", image); + Ok(false) +} + /// Create a libvirt domain directly from a disk image file fn create_libvirt_domain_from_disk( domain_name: &str, @@ -1253,6 +1338,60 @@ fn create_libvirt_domain_from_disk( ); } + // Handle Ignition config injection if specified + if let Some(ref ignition_path) = opts.ignition_config { + debug!("Processing Ignition config from {}", ignition_path); + + // Copy Ignition config to libvirt pool for persistence + // (file existence already validated earlier in run()) + let pool_path = get_libvirt_storage_pool_path(global_opts.connect.as_deref()) + .context("Failed to get libvirt storage pool path for Ignition config")?; + let ignition_persistent_path = pool_path.join(format!("{}_ignition.json", domain_name)); + + std::fs::copy(ignition_path, &ignition_persistent_path).with_context(|| { + format!( + "Failed to copy Ignition config to {}", + ignition_persistent_path + ) + })?; + debug!("Copied Ignition config to {}", ignition_persistent_path); + + // Configure Ignition injection based on architecture + let arch = std::env::consts::ARCH; + match arch { + "x86_64" | "aarch64" => { + // Use fw_cfg for x86_64/aarch64 (standard QEMU platforms) + debug!("Adding Ignition config via fw_cfg for {}", arch); + const IGNITION_FW_CFG_NAME: &str = "opt/com.coreos/config"; + domain_builder = domain_builder.add_fw_cfg( + IGNITION_FW_CFG_NAME.to_string(), + ignition_persistent_path.to_string(), + ); + } + "s390x" | "powerpc64le" => { + // Use virtio-blk with serial="ignition" for s390x/ppc64le + debug!("Adding Ignition config via virtio-blk for {}", arch); + domain_builder = + domain_builder.with_ignition_disk(ignition_persistent_path.to_string()); + } + _ => { + return Err(eyre!( + "Ignition config injection not supported on architecture: {}\n\ + Supported architectures: x86_64, aarch64, s390x, powerpc64le", + arch + )); + } + } + + // Add metadata about Ignition + domain_builder = domain_builder + .with_metadata("bootc:ignition-config", ignition_path.as_str()) + .with_metadata( + "bootc:ignition-persistent-path", + ignition_persistent_path.as_str(), + ); + } + // Create a dropin for remote-fs.target that wants all virtiofs mount units. // We use remote-fs.target because virtiofs is conceptually similar to a remote // filesystem - it requires virtio transport infrastructure, like NFS needs network. diff --git a/crates/kit/src/run_ephemeral.rs b/crates/kit/src/run_ephemeral.rs index 140bda2..ea0c7a4 100644 --- a/crates/kit/src/run_ephemeral.rs +++ b/crates/kit/src/run_ephemeral.rs @@ -1397,7 +1397,7 @@ StandardOutput=file:/dev/virtio-ports/executestatus debug!("Adding Ignition config via fw_cfg: {}", ignition_path); qemu_config.add_fw_cfg(IGNITION_FW_CFG_NAME.to_string(), ignition_path.to_owned()); } - "s390x" | "powerpc64" => { + "s390x" | "powerpc64le" => { debug!("Adding Ignition config via virtio-blk: {}", ignition_path); qemu_config.add_virtio_blk_device_with_format_ro( ignition_path.to_string(), @@ -1409,7 +1409,7 @@ StandardOutput=file:/dev/virtio-ports/executestatus _ => { return Err(eyre!( "Ignition config injection not supported on architecture: {}\n\ - Supported architectures: x86_64, aarch64, s390x, powerpc64", + Supported architectures: x86_64, aarch64, s390x, powerpc64le", arch )); } diff --git a/docs/src/man/bcvk-libvirt-run.md b/docs/src/man/bcvk-libvirt-run.md index ca6096c..f62a3f9 100644 --- a/docs/src/man/bcvk-libvirt-run.md +++ b/docs/src/man/bcvk-libvirt-run.md @@ -189,6 +189,52 @@ Server management workflow: # Access for maintenance bcvk libvirt ssh production-server +## Ignition Configuration + +Inject [Ignition](https://coreos.github.io/ignition/) configuration files for first-boot provisioning on CoreOS-based images: + + # Create an Ignition config file (v3.3.0 format) + cat > config.ign <