MSR

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

Author: ludfjig

Cargo.lock

@@ -75,8 +75,8 @@ windows-version = "0.1"
 lazy_static = "1.4.0"
 
 [target.'cfg(unix)'.dependencies]
-kvm-bindings = { version = "0.14", features = ["fam-wrappers"], optional = true }
-kvm-ioctls = { version = "0.24", optional = true }
+kvm-bindings = { path = "../../../kvm/kvm-bindings", features = ["fam-wrappers"], optional = true }
+kvm-ioctls = { path = "../../../kvm/kvm-ioctls", optional = true }
 mshv-bindings = { version = "0.6", optional = true }
 mshv-ioctls = { version = "0.6", optional = true}
 

src/hyperlight_host/Cargo.toml

@@ -144,6 +144,14 @@ pub enum HyperlightError {
     #[error("Memory Allocation Failed with OS Error {0:?}.")]
     MemoryAllocationFailed(Option<i32>),
 
+    /// MSR Read Violation - Guest attempted to read from a Model-Specific Register
+    #[error("Guest attempted to read from MSR {0:#x}")]
+    MsrReadViolation(u32),
+
+    /// MSR Write Violation - Guest attempted to write to a Model-Specific Register
+    #[error("Guest attempted to write {1:#x} to MSR {0:#x}")]
+    MsrWriteViolation(u32, u64),
+
     /// Memory Protection Failed
     #[error("Memory Protection Failed with OS Error {0:?}.")]
     MemoryProtectionFailed(Option<i32>),
@@ -322,7 +330,9 @@ impl HyperlightError {
             | HyperlightError::PoisonedSandbox
             | HyperlightError::ExecutionAccessViolation(_)
             | HyperlightError::StackOverflow()
-            | HyperlightError::MemoryAccessViolation(_, _, _) => true,
+            | HyperlightError::MemoryAccessViolation(_, _, _)
+            | HyperlightError::MsrReadViolation(_)
+            | HyperlightError::MsrWriteViolation(_, _) => true,
 
             // All other errors do not poison the sandbox.
             HyperlightError::AnyhowError(_)

src/hyperlight_host/src/error.rs

@@ -46,7 +46,7 @@ use crate::hypervisor::hyperv_linux::MshvVm;
 use crate::hypervisor::hyperv_windows::WhpVm;
 #[cfg(kvm)]
 use crate::hypervisor::kvm::KvmVm;
-use crate::hypervisor::regs::CommonSpecialRegisters;
+use crate::hypervisor::regs::{CommonDebugRegs, CommonSpecialRegisters};
 #[cfg(target_os = "windows")]
 use crate::hypervisor::wrappers::HandleWrapper;
 use crate::hypervisor::{HyperlightExit, InterruptHandle, InterruptHandleImpl, get_max_log_level};
@@ -80,6 +80,9 @@ pub(crate) struct HyperlightVm {
     next_slot: u32,                     // Monotonically increasing slot number
     freed_slots: Vec<u32>,              // Reusable slots from unmapped regions
 
+    // pml4 saved to be able to restore it if needed
+    #[cfg(feature = "init-paging")]
+    pml4_addr: u64,
     #[cfg(gdb)]
     gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
     #[cfg(gdb)]
@@ -99,7 +102,7 @@ impl HyperlightVm {
         _pml4_addr: u64,
         entrypoint: u64,
         rsp: u64,
-        #[cfg_attr(not(any(kvm, mshv3)), allow(unused_variables))] config: &SandboxConfiguration,
+        config: &SandboxConfiguration,
         #[cfg(target_os = "windows")] handle: HandleWrapper,
         #[cfg(target_os = "windows")] raw_size: usize,
         #[cfg(gdb)] gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
@@ -122,6 +125,11 @@ impl HyperlightVm {
             None => return Err(NoHypervisorFound()),
         };
 
+        // MSR intercept can be unsafely disabled by the user in the config
+        if !config.get_allow_msr() {
+            vm.enable_msr_intercept()?;
+        }
+
         for (i, region) in mem_regions.iter().enumerate() {
             // Safety: slots are unique and region points to valid memory since we created the regions
             unsafe { vm.map_memory((i as u32, region))? };
@@ -191,6 +199,8 @@ impl HyperlightVm {
             mmap_regions: Vec::new(),
             freed_slots: Vec::new(),
 
+            #[cfg(feature = "init-paging")]
+            pml4_addr: _pml4_addr,
             #[cfg(gdb)]
             gdb_conn,
             #[cfg(gdb)]
@@ -273,9 +283,6 @@ impl HyperlightVm {
         };
         self.vm.set_regs(&regs)?;
 
-        // reset fpu
-        self.vm.set_fpu(&CommonFpu::default())?;
-
         self.run(
             mem_mgr,
             host_funcs,
@@ -497,6 +504,12 @@ impl HyperlightVm {
                         }
                     }
                 }
+                Ok(HyperlightExit::MsrRead(msr_index)) => {
+                    break Err(HyperlightError::MsrReadViolation(msr_index));
+                }
+                Ok(HyperlightExit::MsrWrite { msr_index, value }) => {
+                    break Err(HyperlightError::MsrWriteViolation(msr_index, value));
+                }
                 Ok(HyperlightExit::Cancelled()) => {
                     // If cancellation was not requested for this specific guest function call,
                     // the vcpu was interrupted by a stale cancellation. This can occur when:
@@ -565,6 +578,23 @@ impl HyperlightVm {
         self.interrupt_handle.clear_cancel();
     }
 
+    pub(crate) fn reset_vcpu(&self) -> Result<()> {
+        self.vm.set_regs(&CommonRegisters::default())?;
+        #[cfg(feature = "init-paging")]
+        self.vm
+            .set_sregs(&CommonSpecialRegisters::standard_64bit_defaults(
+                self.pml4_addr,
+            ))?;
+        #[cfg(not(feature = "init-paging"))]
+        self.vm
+            .set_sregs(&CommonSpecialRegisters::standard_real_mode_defaults())?;
+        self.vm.set_fpu(&CommonFpu::default())?;
+        self.vm.set_xsave(&[0; 1024])?;
+        self.vm.set_debug_regs(&CommonDebugRegs::default())?;
+        // MSRs don't need to be reset as they cannot be modified by guest (unless unsafely-allowed)
+        Ok(())
+    }
+
     #[cfg(gdb)]
     fn handle_debug(
         &mut self,

src/hyperlight_host/src/hypervisor/hyperlight_vm.rs

@@ -21,18 +21,23 @@ use std::sync::LazyLock;
 #[cfg(gdb)]
 use mshv_bindings::{DebugRegisters, hv_message_type_HVMSG_X64_EXCEPTION_INTERCEPT};
 use mshv_bindings::{
-    FloatingPointUnit, SpecialRegisters, StandardRegisters, hv_message_type,
+    FloatingPointUnit, HV_INTERCEPT_ACCESS_MASK_READ, HV_INTERCEPT_ACCESS_MASK_WRITE,
+    HV_INTERCEPT_ACCESS_READ, HV_INTERCEPT_ACCESS_WRITE, SpecialRegisters, StandardRegisters,
+    XSave, hv_intercept_type_HV_INTERCEPT_TYPE_X64_MSR, hv_message_type,
     hv_message_type_HVMSG_GPA_INTERCEPT, hv_message_type_HVMSG_UNMAPPED_GPA,
     hv_message_type_HVMSG_X64_HALT, hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT,
+    hv_message_type_HVMSG_X64_MSR_INTERCEPT,
     hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES,
     hv_partition_synthetic_processor_features, hv_register_assoc,
-    hv_register_name_HV_X64_REGISTER_RIP, hv_register_value, mshv_user_mem_region,
+    hv_register_name_HV_X64_REGISTER_RIP, hv_register_value, mshv_install_intercept,
+    mshv_user_mem_region,
 };
 use mshv_ioctls::{Mshv, VcpuFd, VmFd};
 use tracing::{Span, instrument};
 
 #[cfg(gdb)]
 use crate::hypervisor::gdb::DebuggableVm;
+use crate::hypervisor::regs::CommonDebugRegs;
 use crate::hypervisor::{HyperlightExit, Hypervisor};
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::{Result, new_error};
@@ -108,6 +113,7 @@ impl Hypervisor for MshvVm {
             hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT;
         const UNMAPPED_GPA_MESSAGE: hv_message_type = hv_message_type_HVMSG_UNMAPPED_GPA;
         const INVALID_GPA_ACCESS_MESSAGE: hv_message_type = hv_message_type_HVMSG_GPA_INTERCEPT;
+        const MSR_MESSAGE: hv_message_type = hv_message_type_HVMSG_X64_MSR_INTERCEPT;
         #[cfg(gdb)]
         const EXCEPTION_INTERCEPT: hv_message_type = hv_message_type_HVMSG_X64_EXCEPTION_INTERCEPT;
 
@@ -152,6 +158,21 @@ impl Hypervisor for MshvVm {
                         _ => HyperlightExit::Unknown("Unknown MMIO access".to_string()),
                     }
                 }
+                MSR_MESSAGE => {
+                    let msr_message = m.to_msr_info().map_err(mshv_ioctls::MshvError::from)?;
+                    let edx = msr_message.rdx;
+                    let eax = msr_message.rax;
+                    let written_value = (edx << 32) | eax;
+                    let access = msr_message.header.intercept_access_type as u32;
+                    match access {
+                        HV_INTERCEPT_ACCESS_READ => HyperlightExit::MsrRead(msr_message.msr_number),
+                        HV_INTERCEPT_ACCESS_WRITE => HyperlightExit::MsrWrite {
+                            msr_index: msr_message.msr_number,
+                            value: written_value,
+                        },
+                        _ => HyperlightExit::Unknown(format!("Unknown MSR access type={}", access)),
+                    }
+                }
                 #[cfg(gdb)]
                 EXCEPTION_INTERCEPT => {
                     let ex_info = m
@@ -213,6 +234,38 @@ impl Hypervisor for MshvVm {
         let xsave = self.vcpu_fd.get_xsave()?;
         Ok(xsave.buffer.to_vec())
     }
+
+    fn set_xsave(&self, xsave: &[u32; 1024]) -> Result<()> {
+        let mut buf = XSave::default();
+        let (prefix, bytes, suffix) = unsafe { xsave.align_to() };
+        assert!(prefix.is_empty());
+        assert!(suffix.is_empty());
+        buf.buffer.copy_from_slice(bytes);
+        self.vcpu_fd.set_xsave(&buf)?;
+        Ok(())
+    }
+
+    fn debug_regs(&self) -> Result<CommonDebugRegs> {
+        let debug_regs = self.vcpu_fd.get_debug_regs()?;
+        Ok(debug_regs.into())
+    }
+
+    fn set_debug_regs(&self, drs: &CommonDebugRegs) -> Result<()> {
+        let mshv_debug_regs = drs.into();
+        self.vcpu_fd.set_debug_regs(&mshv_debug_regs)?;
+        Ok(())
+    }
+
+    fn enable_msr_intercept(&mut self) -> Result<()> {
+        // Install MSR intercepts, will interrupt on all MSR accesses (READ & WRITE)
+        let intercept = mshv_install_intercept {
+            access_type_mask: HV_INTERCEPT_ACCESS_MASK_WRITE | HV_INTERCEPT_ACCESS_MASK_READ,
+            intercept_type: hv_intercept_type_HV_INTERCEPT_TYPE_X64_MSR,
+            intercept_parameter: Default::default(),
+        };
+        self.vm_fd.install_intercept(intercept)?;
+        Ok(())
+    }
 }
 
 #[cfg(gdb)]

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -33,7 +33,7 @@ use super::wrappers::HandleWrapper;
 #[cfg(gdb)]
 use crate::hypervisor::gdb::DebuggableVm;
 use crate::hypervisor::regs::{CommonFpu, CommonRegisters, CommonSpecialRegisters};
-use crate::hypervisor::{HyperlightExit, Hypervisor};
+use crate::hypervisor::{CommonDebugRegs, HyperlightExit, Hypervisor};
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::{Result, log_then_return, new_error};
 
@@ -266,6 +266,23 @@ impl Hypervisor for WhpVm {
             }
             // Execution was cancelled by the host.
             WHvRunVpExitReasonCanceled => HyperlightExit::Cancelled(),
+            // MSR access (read or write) - we configured all MSR writes to cause exits
+            WHvRunVpExitReasonX64MsrAccess => {
+                let msr_access = unsafe { exit_context.Anonymous.MsrAccess };
+                let eax = msr_access.Rax;
+                let edx = msr_access.Rdx;
+                let written_value = (edx << 32) | eax;
+                let access = unsafe { msr_access.AccessInfo.AsUINT32 };
+                // Missing from rust bindings for some reason, see https://github.com/MicrosoftDocs/Virtualization-Documentation/blob/265f685159dfd3b2fae8d1dcf1b7d206c31ee880/virtualization/api/hypervisor-platform/headers/WinHvPlatformDefs.h#L3020
+                match access {
+                    0 => HyperlightExit::MsrRead(msr_access.MsrNumber),
+                    1 => HyperlightExit::MsrWrite {
+                        msr_index: msr_access.MsrNumber,
+                        value: written_value,
+                    },
+                    _ => HyperlightExit::Unknown(format!("Unknown MSR access type={}", access)),
+                }
+            }
             #[cfg(gdb)]
             WHvRunVpExitReasonException => {
                 let exception = unsafe { exit_context.Anonymous.VpException };
@@ -452,6 +469,38 @@ impl Hypervisor for WhpVm {
         Ok(xsave_buffer)
     }
 
+    fn set_xsave(&self, _xsave: &[u32; 1024]) -> Result<()> {
+        todo!()
+    }
+
+    fn debug_regs(&self) -> Result<CommonDebugRegs> {
+        todo!()
+    }
+
+    fn set_debug_regs(&self, _drs: &CommonDebugRegs) -> Result<()> {
+        todo!()
+    }
+
+    fn enable_msr_intercept(&mut self) -> Result<()> {
+        // Enable MSR exits through Extended VM Exits
+        // Note: This must be set BEFORE WHvSetupPartition for WHP, so this implementation
+        // is a no-op since the setup is already done in the constructor.
+        // For WHP, MSR intercepts must be enabled during partition creation.
+        // X64MsrExit bit position, missing from rust bindings for some reason.
+        // See https://github.com/MicrosoftDocs/Virtualization-Documentation/blob/265f685159dfd3b2fae8d1dcf1b7d206c31ee880/virtualization/api/hypervisor-platform/headers/WinHvPlatformDefs.h#L1495
+        let mut extended_exits_property = WHV_PARTITION_PROPERTY::default();
+        extended_exits_property.ExtendedVmExits.AsUINT64 = 1 << 1;
+        unsafe {
+            WHvSetPartitionProperty(
+                self.partition,
+                WHvPartitionPropertyCodeExtendedVmExits,
+                &extended_exits_property as *const _ as *const _,
+                std::mem::size_of::<WHV_PARTITION_PROPERTY>() as _,
+            )?;
+        }
+        Ok(())
+    }
+
     /// Mark that initial memory setup is complete. After this, map_memory will fail.
     fn complete_initial_memory_setup(&mut self) {
         self.initial_memory_setup_done = true;