From 38b566ae926714acc5b8f3b295a6ac72f1d4f305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Mon, 28 Apr 2025 16:56:27 +0300 Subject: [PATCH 01/13] crashdump: create core dump file when a guest crashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the core dump file is an ELF file with special segments that describe the guest's memory when it crashed, the CPU register's values and other special notes that tell the debugger how to set up a debugging session starting from the core dump Signed-off-by: Doru Blânzeanu --- Cargo.lock | 25 ++ src/hyperlight_host/Cargo.toml | 1 + .../src/hypervisor/crashdump.rs | 265 ++++++++++++++++-- .../src/hypervisor/hyperv_linux.rs | 46 ++- .../src/hypervisor/hyperv_windows.rs | 46 ++- src/hyperlight_host/src/hypervisor/kvm.rs | 53 +++- src/hyperlight_host/src/hypervisor/mod.rs | 2 +- .../hypervisor/windows_hypervisor_platform.rs | 54 ++++ 8 files changed, 459 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eff2feaf8..700e97e47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,6 +663,19 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "elfcore" +version = "1.1.5" +source = "git+https://github.com/dblnz/elfcore.git?branch=split-linux-impl-from-elfcore#d3152cb3cbeee5f72b1b2ee8fd3b4c1ecf9ed5a7" +dependencies = [ + "libc", + "nix", + "smallvec", + "thiserror 1.0.69", + "tracing", + "zerocopy 0.7.35", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1222,6 +1235,7 @@ dependencies = [ "crossbeam", "crossbeam-channel", "crossbeam-queue", + "elfcore", "env_logger", "flatbuffers", "gdbstub", @@ -1897,6 +1911,17 @@ dependencies = [ "smallvec", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.3" diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 1352422d7..000e5aa3e 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -42,6 +42,7 @@ tempfile = { version = "3.20", optional = true } anyhow = "1.0" metrics = "0.24.2" serde_json = "1.0" +elfcore = { git = "https://github.com/dblnz/elfcore.git", branch = "split-linux-impl-from-elfcore" } [target.'cfg(windows)'.dependencies] windows = { version = "0.61", features = [ diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index 3467f5a81..b5acdbb06 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -14,47 +14,260 @@ See the License for the specific language governing permissions and limitations under the License. */ -use std::io::Write; +use std::cmp::min; +use elfcore::{ + ArchComponentState, ArchState, CoreDumpBuilder, CoreError, Elf64_Auxv, ProcessInfoSource, + ReadProcessMemory, ThreadView, VaProtection, VaRegion, +}; use tempfile::NamedTempFile; use super::Hypervisor; +use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::{Result, new_error}; -/// Dump registers + memory regions + raw memory to a tempfile -#[cfg(crashdump)] -pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { - let mut temp_file = NamedTempFile::with_prefix("mem")?; - let hv_details = format!("{:#x?}", hv); +const NT_X86_XSTATE: u32 = 0x202; +const AT_ENTRY: u64 = 9; +const AT_NULL: u64 = 0; + +/// Structure to hold the crash dump context +/// This structure contains the information needed to create a core dump +#[derive(Debug)] +pub(crate) struct CrashDumpContext<'a> { + regions: &'a [MemoryRegion], + regs: [u64; 27], + xsave: Vec, + entry: u64, +} + +impl<'a> CrashDumpContext<'a> { + pub(crate) fn new( + regions: &'a [MemoryRegion], + regs: [u64; 27], + xsave: Vec, + entry: u64, + ) -> Self { + Self { + regions, + regs, + xsave, + entry, + } + } +} + +/// Structure that contains the process information for the core dump +/// This serves as a source of information for `elfcore`'s [`CoreDumpBuilder`] +struct GuestView { + regions: Vec, + threads: Vec, + aux_vector: Vec, +} - // write hypervisor details such as registers, info about mapped memory regions, etc. - temp_file.write_all(hv_details.as_bytes())?; - temp_file.write_all(b"================ MEMORY DUMP =================\n")?; +impl GuestView { + fn new(ctx: &CrashDumpContext) -> Self { + // Map the regions to the format `CoreDumpBuilder` expects + let regions = ctx + .regions + .iter() + .filter(|r| !r.host_region.is_empty()) + .map(|r| VaRegion { + begin: r.guest_region.start as u64, + end: r.guest_region.end as u64, + offset: r.host_region.start as u64, + protection: VaProtection { + is_private: false, + read: r.flags.contains(MemoryRegionFlags::READ), + write: r.flags.contains(MemoryRegionFlags::WRITE), + execute: r.flags.contains(MemoryRegionFlags::EXECUTE), + }, + mapped_file_name: None, + }) + .collect(); - // write the raw memory dump for each memory region - for region in hv.get_memory_regions() { - if region.host_region.start == 0 || region.host_region.is_empty() { - continue; + // The xsave state is checked as it can be empty + let mut components = vec![]; + if !ctx.xsave.is_empty() { + components.push(ArchComponentState { + name: "XSAVE", + note_type: NT_X86_XSTATE, + note_name: b"LINUX", + data: ctx.xsave.clone(), + }); } - // SAFETY: we got this memory region from the hypervisor so should never be invalid - let region_slice = unsafe { - std::slice::from_raw_parts( - region.host_region.start as *const u8, - region.host_region.len(), - ) + + // Create the thread view + // The thread view contains the information about the thread + // NOTE: Some of these fields are not used in the current implementation + let thread = ThreadView { + flags: 0, // Kernel flags for the process + tid: 1, + uid: 0, // User ID + gid: 0, // Group ID + comm: "\0".to_string(), + ppid: 0, // Parent PID + pgrp: 0, // Process group ID + nice: 0, // Nice value + state: 0, // Process state + utime: 0, // User time + stime: 0, // System time + cutime: 0, // Children User time + cstime: 0, // Children User time + cursig: 0, // Current signal + session: 0, // Session ID of the process + sighold: 0, // Blocked signal + sigpend: 0, // Pending signal + cmd_line: "\0".to_string(), + + arch_state: Box::new(ArchState { + gpr_state: ctx.regs.to_vec(), + components, + }), }; - temp_file.write_all(region_slice)?; + + // Create the auxv vector + // The first entry is AT_ENTRY, which is the entry point of the program + // The entry point is the address where the program starts executing + // This helps the debugger to know that the entry is changed by an offset + // so the symbols can be loaded correctly. + // The second entry is AT_NULL, which marks the end of the vector + let auxv = vec![ + Elf64_Auxv { + a_type: AT_ENTRY, + a_val: ctx.entry, + }, + Elf64_Auxv { + a_type: AT_NULL, + a_val: 0, + }, + ]; + + Self { + regions, + threads: vec![thread], + aux_vector: auxv, + } + } +} + +impl ProcessInfoSource for GuestView { + fn pid(&self) -> i32 { + 1 + } + fn threads(&self) -> &[elfcore::ThreadView] { + &self.threads } - temp_file.flush()?; + fn page_size(&self) -> usize { + 0x1000 + } + fn aux_vector(&self) -> Option<&[elfcore::Elf64_Auxv]> { + Some(&self.aux_vector) + } + fn va_regions(&self) -> &[elfcore::VaRegion] { + &self.regions + } + fn mapped_files(&self) -> Option<&[elfcore::MappedFile]> { + None + } +} - // persist the tempfile to disk - let persist_path = temp_file.path().with_extension("dmp"); +/// Structure that reads the guest memory +/// This structure serves as a custom memory reader for `elfcore`'s +/// [`CoreDumpBuilder`] +struct GuestMemReader { + regions: Vec, +} + +impl GuestMemReader { + fn new(ctx: &CrashDumpContext) -> Self { + Self { + regions: ctx.regions.to_vec(), + } + } +} + +impl ReadProcessMemory for GuestMemReader { + fn read_process_memory( + &mut self, + base: usize, + buf: &mut [u8], + ) -> std::result::Result { + for r in self.regions.iter() { + // Check if the base address is within the guest region + if base >= r.guest_region.start && base < r.guest_region.end { + let offset = base - r.guest_region.start; + let region_slice = unsafe { + std::slice::from_raw_parts( + r.host_region.start as *const u8, + r.host_region.len(), + ) + }; + + // Calculate how much we can copy + let copy_size = min(buf.len(), region_slice.len() - offset); + if copy_size == 0 { + return std::result::Result::Ok(0); + } + + // Only copy the amount that fits in both buffers + buf[..copy_size].copy_from_slice(®ion_slice[offset..offset + copy_size]); + + // Return the number of bytes copied + return std::result::Result::Ok(copy_size); + } + } + + // If we reach here, we didn't find a matching region + std::result::Result::Ok(0) + } +} + +/// Create core dump file from the hypervisor information +/// +/// This function generates an ELF core dump file capturing the hypervisor's state, +/// which can be used for debugging when crashes occur. The file is created in the +/// system's temporary directory with extension '.elf' and the path is printed to stdout and logs. +/// +/// # Arguments +/// * `hv`: Reference to the hypervisor implementation +/// +/// # Returns +/// * `Result<()>`: Success or error +pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { + log::info!("Creating core dump file..."); + + // Create a temporary file with a recognizable prefix + let temp_file = NamedTempFile::with_prefix("hl_core_") + .map_err(|e| new_error!("Failed to create temporary file: {:?}", e))?; + + // Get crash context from hypervisor + let ctx = hv + .crashdump_context() + .map_err(|e| new_error!("Failed to get crashdump context: {:?}", e))?; + + // Set up data sources for the core dump + let guest_view = GuestView::new(&ctx); + let memory_reader = GuestMemReader::new(&ctx); + + // Create and write core dump + let core_builder = CoreDumpBuilder::from_source( + Box::new(guest_view) as Box, + Box::new(memory_reader) as Box, + ); + + core_builder + .write(&temp_file) + .map_err(|e| new_error!("Failed to write core dump: {:?}", e))?; + + let persist_path = temp_file.path().with_extension("elf"); temp_file .persist(&persist_path) - .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?; + .map_err(|e| new_error!("Failed to persist core dump file: {:?}", e))?; + + let path_string = persist_path.to_string_lossy().to_string(); - println!("Memory dumped to file: {:?}", persist_path); - log::error!("Memory dumped to file: {:?}", persist_path); + println!("Core dump created successfully: {}", path_string); + log::error!("Core dump file: {}", path_string); Ok(()) } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 21c1c9182..dd696435a 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -51,6 +51,8 @@ use mshv_bindings::{ use mshv_ioctls::{Mshv, MshvError, VcpuFd, VmFd}; use tracing::{Span, instrument}; +#[cfg(crashdump)] +use super::crashdump; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, MshvDebug}; @@ -750,8 +752,48 @@ impl Hypervisor for HypervLinuxDriver { } #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion] { - &self.mem_regions + fn crashdump_context(&self) -> Result { + let mut regs = [0; 27]; + + let vcpu_regs = self.vcpu_fd.get_regs()?; + let sregs = self.vcpu_fd.get_sregs()?; + let xsave = self.vcpu_fd.get_xsave()?; + + // Set up the registers for the crash dump + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = sregs.cs.selector as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = sregs.ss.selector as u64; // ss + regs[21] = sregs.fs.base; // fs_base + regs[22] = sregs.gs.base; // gs_base + regs[23] = sregs.ds.selector as u64; // ds + regs[24] = sregs.es.selector as u64; // es + regs[25] = sregs.fs.selector as u64; // fs + regs[26] = sregs.gs.selector as u64; // gs + + Ok(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave.buffer.to_vec(), + self.entrypoint, + )) } #[cfg(gdb)] diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index c732058d2..479438239 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -31,6 +31,8 @@ use windows::Win32::System::Hypervisor::{ WHvX64RegisterCs, WHvX64RegisterEfer, }; +#[cfg(crashdump)] +use super::crashdump; use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] use super::handlers::DbgMemAccessHandlerWrapper; @@ -514,8 +516,48 @@ impl Hypervisor for HypervWindowsDriver { } #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion] { - &self.mem_regions + fn crashdump_context(&self) -> Result { + let mut regs = [0; 27]; + + let vcpu_regs = self.processor.get_regs()?; + let sregs = self.processor.get_sregs()?; + let xsave = self.processor.get_xsave()?; + + // Set the registers in the order expected by the crashdump context + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = unsafe { sregs.cs.Segment.Selector } as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = unsafe { sregs.ss.Segment.Selector } as u64; // ss + regs[21] = unsafe { sregs.fs.Segment.Base }; // fs_base + regs[22] = unsafe { sregs.gs.Segment.Base }; // gs_base + regs[23] = unsafe { sregs.ds.Segment.Selector } as u64; // ds + regs[24] = unsafe { sregs.es.Segment.Selector } as u64; // es + regs[25] = unsafe { sregs.fs.Segment.Selector } as u64; // fs + regs[26] = unsafe { sregs.gs.Segment.Selector } as u64; // gs + + Ok(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave, + self.entrypoint, + )) } } diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index bf537fdb6..ed7e193c0 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -27,6 +27,8 @@ use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; use log::LevelFilter; use tracing::{Span, instrument}; +#[cfg(crashdump)] +use super::crashdump; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, KvmDebug, VcpuStopReason}; @@ -658,8 +660,55 @@ impl Hypervisor for KVMDriver { } #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion] { - &self.mem_regions + fn crashdump_context(&self) -> Result { + let mut regs = [0; 27]; + + let vcpu_regs = self.vcpu_fd.get_regs()?; + let sregs = self.vcpu_fd.get_sregs()?; + let xsave = self.vcpu_fd.get_xsave()?; + + // Set the registers in the order expected by the crashdump context + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = sregs.cs.selector as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = sregs.ss.selector as u64; // ss + regs[21] = sregs.fs.base; // fs_base + regs[22] = sregs.gs.base; // gs_base + regs[23] = sregs.ds.selector as u64; // ds + regs[24] = sregs.es.selector as u64; // es + regs[25] = sregs.fs.selector as u64; // fs + regs[26] = sregs.gs.selector as u64; // gs + + // The [`CrashDumpContext`] accepts xsave as a vector of u8, so we need to convert the + // xsave region to a vector of u8 + Ok(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave.region.into_iter().fold(vec![], |mut acc, item| { + let bytes = item.to_le_bytes(); + acc.append(&mut bytes.to_vec()); + + acc + }), + self.entrypoint, + )) } #[cfg(gdb)] diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index a7ca4761f..dbef4e32e 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -227,7 +227,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor; #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion]; + fn crashdump_context(&self) -> Result; #[cfg(gdb)] /// handles the cases when the vCPU stops due to a Debug event diff --git a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs index bb1877b4c..84e285f76 100644 --- a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs +++ b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs @@ -24,6 +24,7 @@ use windows::core::s; use windows_result::HRESULT; use super::wrappers::HandleWrapper; +use crate::HyperlightError; use crate::hypervisor::wrappers::{WHvFPURegisters, WHvGeneralRegisters, WHvSpecialRegisters}; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::{Result, new_error}; @@ -409,6 +410,59 @@ impl VMProcessor { } } + #[cfg(crashdump)] + #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] + pub(super) fn get_xsave(&self) -> Result> { + // Get the required buffer size by calling with NULL buffer + let mut buffer_size_needed: u32 = 0; + + unsafe { + // First call with NULL buffer to get required size + // If the buffer is not large enough, the return value is WHV_E_INSUFFICIENT_BUFFER. + // In this case, BytesWritten receives the required buffer size. + let result = WHvGetVirtualProcessorXsaveState( + self.get_partition_hdl(), + 0, + std::ptr::null_mut(), + 0, + &mut buffer_size_needed, + ); + + // If it failed for reasons other than insufficient buffer, return error + if let Err(e) = result { + if e.code() != windows::Win32::Foundation::WHV_E_INSUFFICIENT_BUFFER { + return Err(HyperlightError::WindowsAPIError(e)); + } + } + } + + // Create a buffer with the appropriate size + let mut xsave_buffer = vec![0; buffer_size_needed as usize]; + + // Get the Xsave state + let mut written_bytes = 0; + unsafe { + WHvGetVirtualProcessorXsaveState( + self.get_partition_hdl(), + 0, + xsave_buffer.as_mut_ptr() as *mut std::ffi::c_void, + buffer_size_needed, + &mut written_bytes, + ) + }?; + + // Check if the number of written bytes matches the expected size + if written_bytes != buffer_size_needed { + return Err(new_error!( + "Failed to get Xsave state: expected {} bytes, got {}", + buffer_size_needed, + written_bytes + )); + } + + Ok(xsave_buffer) + } + pub(super) fn set_fpu(&mut self, regs: &WHvFPURegisters) -> Result<()> { const LEN: usize = 26; From 54bb8b3f08ca5bcad14a94c3cdffd056afa46b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Fri, 2 May 2025 12:48:14 +0300 Subject: [PATCH 02/13] fixup: add constants instead of hardcoded values and add more comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- src/hyperlight_host/src/hypervisor/crashdump.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index b5acdbb06..cc629f419 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -26,9 +26,18 @@ use super::Hypervisor; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::{Result, new_error}; +/// This constant is used to identify the XSAVE state in the core dump const NT_X86_XSTATE: u32 = 0x202; +/// This constant identifies the entry point of the program in an Auxiliary Vector +/// note of ELF. This tells a debugger whether the entry point of the program changed +/// so it can load the symbols correctly. const AT_ENTRY: u64 = 9; +/// This constant is used to mark the end of the Auxiliary Vector note const AT_NULL: u64 = 0; +/// The PID of the core dump process - this is a placeholder value +const CORE_DUMP_PID: i32 = 1; +/// The page size of the core dump +const CORE_DUMP_PAGE_SIZE: usize = 0x1000; /// Structure to hold the crash dump context /// This structure contains the information needed to create a core dump @@ -152,13 +161,13 @@ impl GuestView { impl ProcessInfoSource for GuestView { fn pid(&self) -> i32 { - 1 + CORE_DUMP_PID } fn threads(&self) -> &[elfcore::ThreadView] { &self.threads } fn page_size(&self) -> usize { - 0x1000 + CORE_DUMP_PAGE_SIZE } fn aux_vector(&self) -> Option<&[elfcore::Elf64_Auxv]> { Some(&self.aux_vector) @@ -167,6 +176,7 @@ impl ProcessInfoSource for GuestView { &self.regions } fn mapped_files(&self) -> Option<&[elfcore::MappedFile]> { + // We don't have mapped files None } } From 7e3f90b552c8ff72f85d58017261e71acd9fa4f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Mon, 28 Apr 2025 16:50:19 +0300 Subject: [PATCH 03/13] crashdump: add SandboxMetadata field that store relevant data about the sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - only store the binary path for now Signed-off-by: Doru Blânzeanu --- .../src/hypervisor/crashdump.rs | 21 +++++++++++-- .../src/hypervisor/hyperv_linux.rs | 20 +++++++++++-- .../src/hypervisor/hyperv_windows.rs | 19 ++++++++++-- src/hyperlight_host/src/hypervisor/kvm.rs | 18 +++++++++-- src/hyperlight_host/src/hypervisor/mod.rs | 11 ++++++- .../src/sandbox/uninitialized.rs | 30 +++++++++++++++---- .../src/sandbox/uninitialized_evolve.rs | 16 +++++++++- 7 files changed, 120 insertions(+), 15 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index cc629f419..dd58af0a3 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -47,6 +47,8 @@ pub(crate) struct CrashDumpContext<'a> { regs: [u64; 27], xsave: Vec, entry: u64, + binary: Option, + filename: Option, } impl<'a> CrashDumpContext<'a> { @@ -55,12 +57,16 @@ impl<'a> CrashDumpContext<'a> { regs: [u64; 27], xsave: Vec, entry: u64, + binary: Option, + filename: Option, ) -> Self { Self { regions, regs, xsave, entry, + binary, + filename, } } } @@ -94,6 +100,17 @@ impl GuestView { }) .collect(); + // The filename and command line are set to null-terminated strings + let filename = ctx + .filename + .clone() + .map_or_else(|| "\0".to_string(), |s| format!("{}\0", s)); + + let cmd = ctx + .binary + .clone() + .map_or_else(|| "\0".to_string(), |s| format!("{}\0", s)); + // The xsave state is checked as it can be empty let mut components = vec![]; if !ctx.xsave.is_empty() { @@ -113,7 +130,7 @@ impl GuestView { tid: 1, uid: 0, // User ID gid: 0, // Group ID - comm: "\0".to_string(), + comm: filename, ppid: 0, // Parent PID pgrp: 0, // Process group ID nice: 0, // Nice value @@ -126,7 +143,7 @@ impl GuestView { session: 0, // Session ID of the process sighold: 0, // Blocked signal sigpend: 0, // Pending signal - cmd_line: "\0".to_string(), + cmd_line: cmd, arch_state: Box::new(ArchState { gpr_state: ctx.regs.to_vec(), diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index dd696435a..0f7f44e89 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -50,9 +50,9 @@ use mshv_bindings::{ }; use mshv_ioctls::{Mshv, MshvError, VcpuFd, VmFd}; use tracing::{Span, instrument}; - #[cfg(crashdump)] -use super::crashdump; +use {super::crashdump, crate::sandbox::uninitialized::SandboxMetadata, std::path::Path}; + use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, MshvDebug}; @@ -304,6 +304,8 @@ pub(crate) struct HypervLinuxDriver { debug: Option, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] + metadata: SandboxMetadata, } impl HypervLinuxDriver { @@ -323,6 +325,7 @@ impl HypervLinuxDriver { pml4_ptr: GuestPtr, config: &SandboxConfiguration, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] metadata: SandboxMetadata, ) -> Result { let mshv = Mshv::new()?; let pr = Default::default(); @@ -410,6 +413,8 @@ impl HypervLinuxDriver { debug, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + metadata, }) } @@ -788,11 +793,20 @@ impl Hypervisor for HypervLinuxDriver { regs[25] = sregs.fs.selector as u64; // fs regs[26] = sregs.gs.selector as u64; // gs + // Get the filename from the binary path + let filename = self.metadata.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + Ok(crashdump::CrashDumpContext::new( &self.mem_regions, regs, xsave.buffer.to_vec(), self.entrypoint, + self.metadata.binary_path.clone(), + filename, )) } @@ -917,6 +931,8 @@ mod tests { &config, #[cfg(gdb)] None, + #[cfg(crashdump)] + SandboxMetadata { binary_path: None }, ) .unwrap(); } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 479438239..978e2a1cd 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -30,9 +30,9 @@ use windows::Win32::System::Hypervisor::{ WHvCancelRunVirtualProcessor, WHvX64RegisterCr0, WHvX64RegisterCr3, WHvX64RegisterCr4, WHvX64RegisterCs, WHvX64RegisterEfer, }; - #[cfg(crashdump)] -use super::crashdump; +use {super::crashdump, crate::sandbox::uninitialized::SandboxMetadata, std::path::Path}; + use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] use super::handlers::DbgMemAccessHandlerWrapper; @@ -61,6 +61,8 @@ pub(crate) struct HypervWindowsDriver { orig_rsp: GuestPtr, mem_regions: Vec, interrupt_handle: Arc, + #[cfg(crashdump)] + metadata: SandboxMetadata, } /* This does not automatically impl Send/Sync because the host * address of the shared memory region is a raw pointer, which are @@ -71,6 +73,7 @@ unsafe impl Send for HypervWindowsDriver {} unsafe impl Sync for HypervWindowsDriver {} impl HypervWindowsDriver { + #[allow(clippy::too_many_arguments)] #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] pub(crate) fn new( mem_regions: Vec, @@ -80,6 +83,7 @@ impl HypervWindowsDriver { entrypoint: u64, rsp: u64, mmap_file_handle: HandleWrapper, + #[cfg(crashdump)] metadata: SandboxMetadata, ) -> Result { // create and setup hypervisor partition let mut partition = VMPartition::new(1)?; @@ -114,6 +118,8 @@ impl HypervWindowsDriver { partition_handle, dropped: AtomicBool::new(false), }), + #[cfg(crashdump)] + metadata, }) } @@ -552,11 +558,20 @@ impl Hypervisor for HypervWindowsDriver { regs[25] = unsafe { sregs.fs.Segment.Selector } as u64; // fs regs[26] = unsafe { sregs.gs.Segment.Selector } as u64; // gs + // Get the filename from the metadata + let filename = self.metadata.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + Ok(crashdump::CrashDumpContext::new( &self.mem_regions, regs, xsave, self.entrypoint, + self.metadata.binary_path.clone(), + filename, )) } } diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index ed7e193c0..784f013c0 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -26,9 +26,9 @@ use kvm_ioctls::Cap::UserMemory; use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; use log::LevelFilter; use tracing::{Span, instrument}; - #[cfg(crashdump)] -use super::crashdump; +use {super::crashdump, crate::sandbox::uninitialized::SandboxMetadata, std::path::Path}; + use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, KvmDebug, VcpuStopReason}; @@ -292,6 +292,8 @@ pub(crate) struct KVMDriver { debug: Option, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] + metadata: SandboxMetadata, } impl KVMDriver { @@ -306,6 +308,7 @@ impl KVMDriver { rsp: u64, config: &SandboxConfiguration, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] metadata: SandboxMetadata, ) -> Result { let kvm = Kvm::new()?; @@ -365,6 +368,8 @@ impl KVMDriver { debug, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + metadata, }; Ok(ret) } @@ -696,6 +701,13 @@ impl Hypervisor for KVMDriver { regs[25] = sregs.fs.selector as u64; // fs regs[26] = sregs.gs.selector as u64; // gs + // Get the filename from the metadata + let filename = self.metadata.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + // The [`CrashDumpContext`] accepts xsave as a vector of u8, so we need to convert the // xsave region to a vector of u8 Ok(crashdump::CrashDumpContext::new( @@ -708,6 +720,8 @@ impl Hypervisor for KVMDriver { acc }), self.entrypoint, + self.metadata.binary_path.clone(), + filename, )) } diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index dbef4e32e..e96ce9be3 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -455,6 +455,8 @@ pub(crate) mod tests { use crate::hypervisor::DbgMemAccessHandlerCaller; use crate::mem::ptr::RawPtr; use crate::sandbox::uninitialized::GuestBinary; + #[cfg(crashdump)] + use crate::sandbox::uninitialized::SandboxMetadata; use crate::sandbox::uninitialized_evolve::set_up_hypervisor_partition; use crate::sandbox::{SandboxConfiguration, UninitializedSandbox}; use crate::{Result, is_hypervisor_present, new_error}; @@ -501,7 +503,14 @@ pub(crate) mod tests { let sandbox = UninitializedSandbox::new(GuestBinary::FilePath(filename.clone()), Some(config))?; let (_hshm, mut gshm) = sandbox.mgr.build(); - let mut vm = set_up_hypervisor_partition(&mut gshm, &config)?; + #[cfg(crashdump)] + let metadata = SandboxMetadata { binary_path: None }; + let mut vm = set_up_hypervisor_partition( + &mut gshm, + &config, + #[cfg(crashdump)] + &metadata, + )?; vm.initialise( RawPtr::from(0x230000), 1234567890, diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs index 16df2bb05..de7658ab7 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized.rs @@ -52,6 +52,12 @@ const EXTRA_ALLOWED_SYSCALLS_FOR_WRITER_FUNC: &[super::ExtraAllowedSyscall] = &[ libc::SYS_close, ]; +#[cfg(crashdump)] +#[derive(Clone, Debug, Default)] +pub(crate) struct SandboxMetadata { + pub(crate) binary_path: Option, +} + /// A preliminary `Sandbox`, not yet ready to execute guest code. /// /// Prior to initializing a full-fledged `Sandbox`, you must create one of @@ -66,6 +72,8 @@ pub struct UninitializedSandbox { pub(crate) mgr: MemMgrWrapper, pub(crate) max_guest_log_level: Option, pub(crate) config: SandboxConfiguration, + #[cfg(crashdump)] + pub(crate) metadata: SandboxMetadata, } impl crate::sandbox_state::sandbox::UninitializedSandbox for UninitializedSandbox { @@ -146,15 +154,25 @@ impl UninitializedSandbox { let path = Path::new(&binary_path) .canonicalize() .map_err(|e| new_error!("GuestBinary not found: '{}': {}", binary_path, e))?; - GuestBinary::FilePath( - path.into_os_string() - .into_string() - .map_err(|e| new_error!("Error converting OsString to String: {:?}", e))?, - ) + let path = path + .into_os_string() + .into_string() + .map_err(|e| new_error!("Error converting OsString to String: {:?}", e))?; + + GuestBinary::FilePath(path) } buffer @ GuestBinary::Buffer(_) => buffer, }; + #[cfg(crashdump)] + let metadata = if let GuestBinary::FilePath(ref path) = guest_binary { + SandboxMetadata { + binary_path: Some(path.clone()), + } + } else { + SandboxMetadata::default() + }; + let sandbox_cfg = cfg.unwrap_or_default(); let mut mem_mgr_wrapper = { @@ -173,6 +191,8 @@ impl UninitializedSandbox { mgr: mem_mgr_wrapper, max_guest_log_level: None, config: sandbox_cfg, + #[cfg(crashdump)] + metadata, }; // If we were passed a writer for host print register it otherwise use the default. diff --git a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs index bdcf1a792..77adb4dae 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs @@ -36,6 +36,8 @@ use crate::sandbox::config::DebugInfo; use crate::sandbox::host_funcs::FunctionRegistry; use crate::sandbox::mem_access::mem_access_handler_wrapper; use crate::sandbox::outb::outb_handler_wrapper; +#[cfg(crashdump)] +use crate::sandbox::uninitialized::SandboxMetadata; use crate::sandbox::{HostSharedMemory, MemMgrWrapper}; use crate::sandbox_state::sandbox::Sandbox; #[cfg(target_os = "linux")] @@ -69,7 +71,12 @@ where ) -> Result, { let (hshm, mut gshm) = u_sbox.mgr.build(); - let mut vm = set_up_hypervisor_partition(&mut gshm, &u_sbox.config)?; + let mut vm = set_up_hypervisor_partition( + &mut gshm, + &u_sbox.config, + #[cfg(crashdump)] + &u_sbox.metadata, + )?; let outb_hdl = outb_handler_wrapper(hshm.clone(), u_sbox.host_funcs.clone()); let seed = { @@ -141,6 +148,7 @@ pub(super) fn evolve_impl_multi_use(u_sbox: UninitializedSandbox) -> Result, #[cfg_attr(target_os = "windows", allow(unused_variables))] config: &SandboxConfiguration, + #[cfg(crashdump)] metadata: &SandboxMetadata, ) -> Result> { let mem_size = u64::try_from(mgr.shared_mem.mem_size())?; let mut regions = mgr.layout.get_memory_regions(&mgr.shared_mem)?; @@ -206,6 +214,8 @@ pub(crate) fn set_up_hypervisor_partition( config, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + metadata.clone(), )?; Ok(Box::new(hv)) } @@ -220,6 +230,8 @@ pub(crate) fn set_up_hypervisor_partition( config, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + metadata.clone(), )?; Ok(Box::new(hv)) } @@ -241,6 +253,8 @@ pub(crate) fn set_up_hypervisor_partition( entrypoint_ptr.absolute()?, rsp_ptr.absolute()?, HandleWrapper::from(mmap_file_handle), + #[cfg(crashdump)] + metadata.clone(), )?; Ok(Box::new(hv)) } From 00cee41388e0907cca663cb704c38855ea952043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Fri, 2 May 2025 12:49:31 +0300 Subject: [PATCH 04/13] fixup: change default filename to "" and aother small updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- src/hyperlight_host/src/hypervisor/crashdump.rs | 9 ++++----- src/hyperlight_host/src/sandbox/uninitialized.rs | 3 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index dd58af0a3..d35167715 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -100,16 +100,15 @@ impl GuestView { }) .collect(); - // The filename and command line are set to null-terminated strings let filename = ctx .filename - .clone() - .map_or_else(|| "\0".to_string(), |s| format!("{}\0", s)); + .as_ref() + .map_or_else(|| "".to_string(), |s| s.to_string()); let cmd = ctx .binary - .clone() - .map_or_else(|| "\0".to_string(), |s| format!("{}\0", s)); + .as_ref() + .map_or_else(|| "".to_string(), |s| s.to_string()); // The xsave state is checked as it can be empty let mut components = vec![]; diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs index de7658ab7..25abac990 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized.rs @@ -153,8 +153,7 @@ impl UninitializedSandbox { GuestBinary::FilePath(binary_path) => { let path = Path::new(&binary_path) .canonicalize() - .map_err(|e| new_error!("GuestBinary not found: '{}': {}", binary_path, e))?; - let path = path + .map_err(|e| new_error!("GuestBinary not found: '{}': {}", binary_path, e))? .into_os_string() .into_string() .map_err(|e| new_error!("Error converting OsString to String: {:?}", e))?; From 8f60092df80938acc22c5ed2d959706975770002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Wed, 30 Apr 2025 17:18:45 +0300 Subject: [PATCH 05/13] crashdump: add documentation on how to load a guest core dump in a gdb and lldb using vscode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- docs/how-to-debug-a-hyperlight-guest.md | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index 8e5dd7d55..a58463712 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -201,3 +201,74 @@ involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or └─┘ │ | | | │ | └───────────────────────────────────────────────────────────────────────────────────────────────┘ ``` + +## Dumping the guest state to an ELF core dump when an unhandled crash occurs + +When a guest crashes, the vCPU state is dumped to an `ELF` core dump file. This can be used to inspect the state of the guest at the time of the crash. + +To dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. +The name and location of the dump file will be printed to the console and logged as an error message. + +### Inspecting the core dump + +After the core dump has been created, to inspect the state of the guest, load the core dump file using `gdb` or `lldb`. +A `gdb` version later than `15.0` and `lldb` version later than `17` have been used to test this feature. + +To do this in vscode, the following configuration can be used to add debug configurations: + +```vscode +{ + "version": "0.2.0", + "inputs": [ + { + "id": "core_dump", + "type": "promptString", + "description": "Path to the core dump file", + }, + { + "id": "program", + "type": "promptString", + "description": "Path to the program to debug", + } + ], + "configurations": [ + { + "name": "[GDB] Load core dump file", + "type": "cppdbg", + "request": "launch", + "program": "${input:program}", + "coreDumpPath": "${input:core_dump}", + "cwd": "${workspaceFolder}", + "MIMode": "gdb", + "externalConsole": false, + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ] + }, + { + "name": "[LLDB] Load core dump file", + "type": "lldb", + "request": "launch", + "stopOnEntry": true, + "processCreateCommands": [], + "targetCreateCommands": [ + "target create -c ${input:core_dump} ${input:program}", + ], + }, + ] +} +``` +NOTE: The `CodeLldb` debug session does not stop after launching. To see the code, stack frames and registers you need to +press the `pause` button. This is a known issue with the `CodeLldb` extension [#1245](https://github.com/vadimcn/codelldb/issues/1245). +The `cppdbg` extension works as expected and stops at the entry point of the program. + From e1a1c1a78a6661fc07eea49f6530389b920fb864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Wed, 7 May 2025 13:09:26 +0300 Subject: [PATCH 06/13] crashdump: update the use of CoreDumpBuilder to reflect elfcore change to generics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- Cargo.lock | 2 +- src/hyperlight_host/Cargo.toml | 2 +- src/hyperlight_host/src/hypervisor/crashdump.rs | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 700e97e47..6fdeb25d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -666,7 +666,7 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "elfcore" version = "1.1.5" -source = "git+https://github.com/dblnz/elfcore.git?branch=split-linux-impl-from-elfcore#d3152cb3cbeee5f72b1b2ee8fd3b4c1ecf9ed5a7" +source = "git+https://github.com/dblnz/elfcore.git?rev=1a57f04272dd54bc06df638f4027debe6d0f694f#1a57f04272dd54bc06df638f4027debe6d0f694f" dependencies = [ "libc", "nix", diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 000e5aa3e..8ae1b5521 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -42,7 +42,7 @@ tempfile = { version = "3.20", optional = true } anyhow = "1.0" metrics = "0.24.2" serde_json = "1.0" -elfcore = { git = "https://github.com/dblnz/elfcore.git", branch = "split-linux-impl-from-elfcore" } +elfcore = { git = "https://github.com/dblnz/elfcore.git", rev = "1a57f04272dd54bc06df638f4027debe6d0f694f" } [target.'cfg(windows)'.dependencies] windows = { version = "0.61", features = [ diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index d35167715..76c3b753e 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -276,10 +276,7 @@ pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { let memory_reader = GuestMemReader::new(&ctx); // Create and write core dump - let core_builder = CoreDumpBuilder::from_source( - Box::new(guest_view) as Box, - Box::new(memory_reader) as Box, - ); + let core_builder = CoreDumpBuilder::from_source(guest_view, memory_reader); core_builder .write(&temp_file) From 68d2606d0ef0831cc2cd3ba9e107d2abae3afa16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Mon, 19 May 2025 12:17:33 +0300 Subject: [PATCH 07/13] fixup: documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- docs/how-to-debug-a-hyperlight-guest.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index a58463712..f3611adb6 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -204,15 +204,17 @@ involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or ## Dumping the guest state to an ELF core dump when an unhandled crash occurs -When a guest crashes, the vCPU state is dumped to an `ELF` core dump file. This can be used to inspect the state of the guest at the time of the crash. +When a guest crashes because of an unknown VmExit or unhandled exception, the vCPU state is dumped to an `ELF` core dump file. +This can be used to inspect the state of the guest at the time of the crash. -To dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. +To make Hyperlight dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file, set the feature `crashdump` and run a debug build. +This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message. ### Inspecting the core dump After the core dump has been created, to inspect the state of the guest, load the core dump file using `gdb` or `lldb`. -A `gdb` version later than `15.0` and `lldb` version later than `17` have been used to test this feature. +**NOTE: This feature has been tested with version `15.0` of `gdb` and version `17` of `lldb`, earlier versions may not work, it is recommended to use these versions or later.** To do this in vscode, the following configuration can be used to add debug configurations: @@ -268,7 +270,7 @@ To do this in vscode, the following configuration can be used to add debug confi ] } ``` -NOTE: The `CodeLldb` debug session does not stop after launching. To see the code, stack frames and registers you need to +**NOTE: The `CodeLldb` debug session does not stop after launching. To see the code, stack frames and registers you need to press the `pause` button. This is a known issue with the `CodeLldb` extension [#1245](https://github.com/vadimcn/codelldb/issues/1245). -The `cppdbg` extension works as expected and stops at the entry point of the program. +The `cppdbg` extension works as expected and stops at the entry point of the program.** From cbabcf441d2256deec8d06c80c035ce39b7699f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Mon, 19 May 2025 12:20:52 +0300 Subject: [PATCH 08/13] fixup: map_or_else to map_or MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- src/hyperlight_host/src/hypervisor/crashdump.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index 76c3b753e..2a531d833 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -103,12 +103,12 @@ impl GuestView { let filename = ctx .filename .as_ref() - .map_or_else(|| "".to_string(), |s| s.to_string()); + .map_or(|| "".to_string(), |s| s.to_string()); let cmd = ctx .binary .as_ref() - .map_or_else(|| "".to_string(), |s| s.to_string()); + .map_or(|| "".to_string(), |s| s.to_string()); // The xsave state is checked as it can be empty let mut components = vec![]; From 1e364c69d874f7fada0eed8b43d3cf1a497e9f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Tue, 20 May 2025 17:50:53 +0300 Subject: [PATCH 09/13] crashdump: enable crashdump feature for release builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Improve documentation and describe how an user can create separate files for debug information - Change the output file directory to be configurable using HYPERLIGHT_CORE_DUMP_DIR environment variable - Change output file name to include a timestamp Signed-off-by: Doru Blânzeanu --- Cargo.lock | 3 +- docs/how-to-debug-a-hyperlight-guest.md | 146 ++++++++++++++++++ src/hyperlight_host/Cargo.toml | 7 +- src/hyperlight_host/build.rs | 3 +- .../src/hypervisor/crashdump.rs | 47 ++++-- 5 files changed, 186 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6fdeb25d6..068c24989 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -666,7 +666,7 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "elfcore" version = "1.1.5" -source = "git+https://github.com/dblnz/elfcore.git?rev=1a57f04272dd54bc06df638f4027debe6d0f694f#1a57f04272dd54bc06df638f4027debe6d0f694f" +source = "git+https://github.com/hyperlight-dev/elfcore.git?rev=cef4c80e26bf4b2a5599e50d2d1730965f942c13#cef4c80e26bf4b2a5599e50d2d1730965f942c13" dependencies = [ "libc", "nix", @@ -1231,6 +1231,7 @@ dependencies = [ "built", "cfg-if", "cfg_aliases", + "chrono", "criterion", "crossbeam", "crossbeam-channel", diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index f3611adb6..cb18d53d3 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -274,3 +274,149 @@ To do this in vscode, the following configuration can be used to add debug confi press the `pause` button. This is a known issue with the `CodeLldb` extension [#1245](https://github.com/vadimcn/codelldb/issues/1245). The `cppdbg` extension works as expected and stops at the entry point of the program.** +## Compiling guests with debug information for release builds + +This section explains how to compile a guest with debugging information but still have optimized code, and how to separate the debug information from the binary. + +### Creating a release build with debug information + +To create a release build with debug information, you can add a custom profile to your `Cargo.toml` file: + +```toml +[profile.release-with-debug] +inherits = "release" +debug = true +``` + +This creates a new profile called `release-with-debug` that inherits all settings from the release profile but adds debug information. + +### Splitting debug information from the binary + +To reduce the binary size while still having debug information available, you can split the debug information into a separate file. +This is useful for production environments where you want smaller binaries but still want to be able to debug crashes. + +Here's a step-by-step guide: + +1. Build your guest with the release-with-debug profile: + ```bash + cargo build --profile release-with-debug + ``` + +2. Locate your binary in the target directory: + ```bash + TARGET_DIR="target" + PROFILE="release-with-debug" + ARCH="x86_64-unknown-none" # Your target architecture + BUILD_DIR="${TARGET_DIR}/${ARCH}/${PROFILE}" + BINARY=$(find "${BUILD_DIR}" -type f -executable -name "guest-binary" | head -1) + ``` + +3. Extract debug information into a full debug file: + ```bash + DEBUG_FILE_FULL="${BINARY}.debug.full" + objcopy --only-keep-debug "${BINARY}" "${DEBUG_FILE_FULL}" + ``` + +4. Create a symbols-only debug file (smaller, but still useful for stack traces): + ```bash + DEBUG_FILE="${BINARY}.debug" + objcopy --keep-file-symbols "${DEBUG_FILE_FULL}" "${DEBUG_FILE}" + ``` + +5. Strip debug information from the original binary but keep function names: + ```bash + objcopy --strip-debug "${BINARY}" + ``` + +6. Add a debug link to the stripped binary: + ```bash + objcopy --add-gnu-debuglink="${DEBUG_FILE}" "${BINARY}" + ``` + +After these steps, you'll have: +- An optimized binary with function names for basic stack traces +- A symbols-only debug file for stack traces +- A full debug file for complete source-level debugging + +### Analyzing core dumps with the debug files + +When you have a core dump from a crashed guest, you can analyze it with different levels of detail using either GDB or LLDB. + +#### Using GDB + +1. For basic analysis with function names (stack traces): + ```bash + gdb ${BINARY} -c /path/to/core.dump + ``` + +2. For full source-level debugging: + ```bash + gdb -s ${DEBUG_FILE_FULL} ${BINARY} -c /path/to/core.dump + ``` + +#### Using LLDB + +LLDB provides similar capabilities with slightly different commands: + +1. For basic analysis with function names (stack traces): + ```bash + lldb ${BINARY} -c /path/to/core.dump + ``` + +2. For full source-level debugging: + ```bash + lldb -o "target create -c /path/to/core.dump ${BINARY}" -o "add-dsym ${DEBUG_FILE_FULL}" + ``` + +3. If your debug symbols are in a separate file: + ```bash + lldb ${BINARY} -c /path/to/core.dump + (lldb) add-dsym ${DEBUG_FILE_FULL} + ``` + +### VSCode Debug Configurations + +You can configure VSCode (in `.vscode/launch.json`) to use these files by modifying the debug configurations: + +#### For GDB + +```json +{ + "name": "[GDB] Load core dump with full debug symbols", + "type": "cppdbg", + "request": "launch", + "program": "${input:program}", + "coreDumpPath": "${input:core_dump}", + "cwd": "${workspaceFolder}", + "MIMode": "gdb", + "externalConsole": false, + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] +} +``` + +#### For LLDB + +```json +{ + "name": "[LLDB] Load core dump with full debug symbols", + "type": "lldb", + "request": "launch", + "program": "${input:program}", + "cwd": "${workspaceFolder}", + "processCreateCommands": [], + "targetCreateCommands": [ + "target create -c ${input:core_dump} ${input:program}" + ], + "postRunCommands": [ + // if debug symbols are in a different file + "add-dsym ${input:debug_file_path}" + ] +} +``` diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 8ae1b5521..d3d37d80a 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -38,11 +38,11 @@ vmm-sys-util = "0.14.0" crossbeam = "0.8.0" crossbeam-channel = "0.5.15" thiserror = "2.0.12" -tempfile = { version = "3.20", optional = true } +chrono = { version = "0.4", optional = true } anyhow = "1.0" metrics = "0.24.2" serde_json = "1.0" -elfcore = { git = "https://github.com/dblnz/elfcore.git", rev = "1a57f04272dd54bc06df638f4027debe6d0f694f" } +elfcore = { git = "https://github.com/hyperlight-dev/elfcore.git", rev = "cef4c80e26bf4b2a5599e50d2d1730965f942c13" } [target.'cfg(windows)'.dependencies] windows = { version = "0.61", features = [ @@ -125,7 +125,8 @@ function_call_metrics = [] executable_heap = [] # This feature enables printing of debug information to stdout in debug builds print_debug = [] -crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds. +# Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. +crashdump = ["dep:chrono"] kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"] mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"] mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"] diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 4484e27d5..75f9eba53 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -93,8 +93,7 @@ fn main() -> Result<()> { gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3"), target_os = "linux") }, kvm: { all(feature = "kvm", target_os = "linux") }, mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") }, - // crashdump feature is aliased with debug_assertions to make it only available in debug-builds. - crashdump: { all(feature = "crashdump", debug_assertions) }, + crashdump: { all(feature = "crashdump") }, // print_debug feature is aliased with debug_assertions to make it only available in debug-builds. print_debug: { all(feature = "print_debug", debug_assertions) }, // the following features are mutually exclusive but rather than enforcing that here we are enabling mshv3 to override mshv2 when both are enabled diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index 2a531d833..ee7c326a9 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -16,11 +16,11 @@ limitations under the License. use std::cmp::min; +use chrono; use elfcore::{ ArchComponentState, ArchState, CoreDumpBuilder, CoreError, Elf64_Auxv, ProcessInfoSource, ReadProcessMemory, ThreadView, VaProtection, VaRegion, }; -use tempfile::NamedTempFile; use super::Hypervisor; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; @@ -103,12 +103,12 @@ impl GuestView { let filename = ctx .filename .as_ref() - .map_or(|| "".to_string(), |s| s.to_string()); + .map_or("".to_string(), |s| s.to_string()); let cmd = ctx .binary .as_ref() - .map_or(|| "".to_string(), |s| s.to_string()); + .map_or("".to_string(), |s| s.to_string()); // The xsave state is checked as it can be empty let mut components = vec![]; @@ -262,10 +262,6 @@ impl ReadProcessMemory for GuestMemReader { pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { log::info!("Creating core dump file..."); - // Create a temporary file with a recognizable prefix - let temp_file = NamedTempFile::with_prefix("hl_core_") - .map_err(|e| new_error!("Failed to create temporary file: {:?}", e))?; - // Get crash context from hypervisor let ctx = hv .crashdump_context() @@ -278,16 +274,39 @@ pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { // Create and write core dump let core_builder = CoreDumpBuilder::from_source(guest_view, memory_reader); + // Generate timestamp string for the filename using chrono + let timestamp = chrono::Local::now() + .format("%Y%m%d_T%H%M%S%.3f") + .to_string(); + + // Determine the output directory based on environment variable + let output_dir = if let Ok(dump_dir) = std::env::var("HYPERLIGHT_CORE_DUMP_DIR") { + // Create the directory if it doesn't exist + let path = std::path::Path::new(&dump_dir); + if !path.exists() { + std::fs::create_dir_all(path) + .map_err(|e| new_error!("Failed to create core dump directory: {:?}", e))?; + } + std::path::PathBuf::from(dump_dir) + } else { + // Fall back to the system temp directory + std::env::temp_dir() + }; + + // Create the filename with timestamp + let filename = format!("hl_core_{}.elf", timestamp); + let file_path = output_dir.join(filename); + + // Create the file + let file = std::fs::File::create(&file_path) + .map_err(|e| new_error!("Failed to create core dump file: {:?}", e))?; + + // Write the core dump directly to the file core_builder - .write(&temp_file) + .write(&file) .map_err(|e| new_error!("Failed to write core dump: {:?}", e))?; - let persist_path = temp_file.path().with_extension("elf"); - temp_file - .persist(&persist_path) - .map_err(|e| new_error!("Failed to persist core dump file: {:?}", e))?; - - let path_string = persist_path.to_string_lossy().to_string(); + let path_string = file_path.to_string_lossy().to_string(); println!("Core dump created successfully: {}", path_string); log::error!("Core dump file: {}", path_string); From d4d4411a3bc36beea7518b17b359333196c30ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Wed, 21 May 2025 13:42:47 +0300 Subject: [PATCH 10/13] crashdump: allow crashdump toggle at sandbox level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - this allows a user to configure the crash dump feature at sandbox level - create a SandboxRuntimeConfig struct to contain all the configuration a sandbox would need at runtime to avoid passing the information as multiple functions arguments - add unit tests to verify crashdump behavior Signed-off-by: Doru Blânzeanu --- .github/workflows/dep_rust.yml | 6 + Justfile | 7 + .../src/hypervisor/crashdump.rs | 225 +++++++++++++++--- .../src/hypervisor/hyperv_linux.rs | 125 +++++----- .../src/hypervisor/hyperv_windows.rs | 116 ++++----- src/hyperlight_host/src/hypervisor/kvm.rs | 129 +++++----- src/hyperlight_host/src/hypervisor/mod.rs | 22 +- .../hypervisor/windows_hypervisor_platform.rs | 3 +- src/hyperlight_host/src/sandbox/config.rs | 32 +++ .../src/sandbox/uninitialized.rs | 49 ++-- .../src/sandbox/uninitialized_evolve.rs | 18 +- 11 files changed, 496 insertions(+), 236 deletions(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index e814d2771..7f00a777c 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -136,6 +136,12 @@ jobs: RUST_LOG: debug run: just test-rust-gdb-debugging ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + - name: Run Rust Crashdump tests + env: + CARGO_TERM_COLOR: always + RUST_LOG: debug + run: just test-rust-crashdump ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + ### Benchmarks ### - name: Install github-cli (Linux mariner) if: runner.os == 'Linux' && matrix.hypervisor == 'mshv' diff --git a/Justfile b/Justfile index b491d1db7..b14448166 100644 --- a/Justfile +++ b/Justfile @@ -81,6 +81,9 @@ test-like-ci config=default-target hypervisor="kvm": @# without any driver (should fail to compile) just test-compilation-fail {{config}} + @# test the crashdump feature + just test-rust-crashdump {{config}} + # runs all tests test target=default-target features="": (test-unit target features) (test-isolated target features) (test-integration "rust" target features) (test-integration "c" target features) (test-seccomp target features) @@ -123,6 +126,10 @@ test-rust-gdb-debugging target=default-target features="": cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --example guest-debugging {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} -- test_gdb +# rust test for crashdump +test-rust-crashdump target=default-target features="": + cargo test --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {'--features crashdump'} else { "--features crashdump," + features } }} -- test_crashdump + ################ ### LINTING #### diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index ee7c326a9..ac509cd90 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -15,6 +15,7 @@ limitations under the License. */ use std::cmp::min; +use std::io::Write; use chrono; use elfcore::{ @@ -248,18 +249,20 @@ impl ReadProcessMemory for GuestMemReader { } } -/// Create core dump file from the hypervisor information +/// Create core dump file from the hypervisor information if the sandbox is configured +/// to allow core dumps. /// /// This function generates an ELF core dump file capturing the hypervisor's state, -/// which can be used for debugging when crashes occur. The file is created in the -/// system's temporary directory with extension '.elf' and the path is printed to stdout and logs. +/// which can be used for debugging when crashes occur. +/// The location of the core dump file is determined by the `HYPERLIGHT_CORE_DUMP_DIR` +/// environment variable. If not set, it defaults to the system's temporary directory. /// /// # Arguments /// * `hv`: Reference to the hypervisor implementation /// /// # Returns /// * `Result<()>`: Success or error -pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { +pub(crate) fn generate_crashdump(hv: &dyn Hypervisor) -> Result<()> { log::info!("Creating core dump file..."); // Get crash context from hypervisor @@ -267,27 +270,59 @@ pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { .crashdump_context() .map_err(|e| new_error!("Failed to get crashdump context: {:?}", e))?; - // Set up data sources for the core dump - let guest_view = GuestView::new(&ctx); - let memory_reader = GuestMemReader::new(&ctx); + // Get env variable for core dump directory + let core_dump_dir = std::env::var("HYPERLIGHT_CORE_DUMP_DIR").ok(); - // Create and write core dump - let core_builder = CoreDumpBuilder::from_source(guest_view, memory_reader); + // Compute file path on the filesystem + let file_path = core_dump_file_path(core_dump_dir); + let create_dump_file = || { + // Create the file + Ok(Box::new( + std::fs::File::create(&file_path) + .map_err(|e| new_error!("Failed to create core dump file: {:?}", e))?, + ) as Box) + }; + + checked_core_dump(ctx, create_dump_file).map(|_| { + println!("Core dump created successfully: {}", file_path); + log::error!("Core dump file: {}", file_path); + }) +} + +/// Computes the file path for the core dump file. +/// +/// The file path is generated based on the current timestamp and an +/// output directory. +/// If the directory does not exist, it falls back to the system's temp directory. +/// If the variable is not set, it defaults to the system's temporary directory. +/// The filename is formatted as `hl_core_.elf`. +/// +/// Arguments: +/// * `dump_dir`: The environment variable value to check for the output directory. +/// +/// Returns: +/// * `String`: The file path for the core dump file. +fn core_dump_file_path(dump_dir: Option) -> String { // Generate timestamp string for the filename using chrono let timestamp = chrono::Local::now() .format("%Y%m%d_T%H%M%S%.3f") .to_string(); // Determine the output directory based on environment variable - let output_dir = if let Ok(dump_dir) = std::env::var("HYPERLIGHT_CORE_DUMP_DIR") { - // Create the directory if it doesn't exist - let path = std::path::Path::new(&dump_dir); - if !path.exists() { - std::fs::create_dir_all(path) - .map_err(|e| new_error!("Failed to create core dump directory: {:?}", e))?; + let output_dir = if let Some(dump_dir) = dump_dir { + // Check if the directory exists + // If it doesn't exist, fall back to the system temp directory + // This is to ensure that the core dump can be created even if the directory is not set + if std::path::Path::new(&dump_dir).exists() { + std::path::PathBuf::from(dump_dir) + } else { + log::warn!( + "Directory \"{}\" does not exist, falling back to temp directory", + dump_dir + ); + std::env::temp_dir() } - std::path::PathBuf::from(dump_dir) } else { // Fall back to the system temp directory std::env::temp_dir() @@ -297,19 +332,155 @@ pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { let filename = format!("hl_core_{}.elf", timestamp); let file_path = output_dir.join(filename); - // Create the file - let file = std::fs::File::create(&file_path) - .map_err(|e| new_error!("Failed to create core dump file: {:?}", e))?; + file_path.to_string_lossy().to_string() +} - // Write the core dump directly to the file - core_builder - .write(&file) - .map_err(|e| new_error!("Failed to write core dump: {:?}", e))?; +/// Create core dump from Hypervisor context if the sandbox is configured to allow core dumps. +/// +/// Arguments: +/// * `ctx`: Optional crash dump context from the hypervisor. This contains the information +/// needed to create the core dump. If `None`, no core dump will be created. +/// * `get_writer`: Closure that returns a writer to the output destination. +/// +/// Returns: +/// * `Result`: The number of bytes written to the core dump file. +fn checked_core_dump( + ctx: Option, + get_writer: impl FnOnce() -> Result>, +) -> Result { + let mut nbytes = 0; + // If the HV returned a context it means we can create a core dump + // This is the case when the sandbox has been configured at runtime to allow core dumps + if let Some(ctx) = ctx { + // Set up data sources for the core dump + let guest_view = GuestView::new(&ctx); + let memory_reader = GuestMemReader::new(&ctx); + + // Create and write core dump + let core_builder = CoreDumpBuilder::from_source(guest_view, memory_reader); + + let writer = get_writer()?; + // Write the core dump directly to the file + nbytes = core_builder + .write(writer) + .map_err(|e| new_error!("Failed to write core dump: {:?}", e))?; + } - let path_string = file_path.to_string_lossy().to_string(); + Ok(nbytes) +} + +/// Test module for the crash dump functionality +#[cfg(test)] +mod test { + use super::*; + + /// Test the core_dump_file_path function when the environment variable is set to an existing + /// directory + #[test] + fn test_crashdump_file_path_valid() { + // Get CWD + let valid_dir = std::env::current_dir() + .unwrap() + .to_string_lossy() + .to_string(); + + // Call the function + let path = core_dump_file_path(Some(valid_dir.clone())); + + // Check if the path is correct + assert!(path.contains(&valid_dir)); + } - println!("Core dump created successfully: {}", path_string); - log::error!("Core dump file: {}", path_string); + /// Test the core_dump_file_path function when the environment variable is set to an invalid + /// directory + #[test] + fn test_crashdump_file_path_invalid() { + // Call the function + let path = core_dump_file_path(Some("/tmp/not_existing_dir".to_string())); - Ok(()) + // Get the temp directory + let temp_dir = std::env::temp_dir().to_string_lossy().to_string(); + + // Check if the path is correct + assert!(path.contains(&temp_dir)); + } + + /// Test the core_dump_file_path function when the environment is not set + /// Check against the default temp directory by using the env::temp_dir() function + #[test] + fn test_crashdump_file_path_default() { + // Call the function + let path = core_dump_file_path(None); + + let temp_dir = std::env::temp_dir().to_string_lossy().to_string(); + + // Check if the path is correct + assert!(path.starts_with(&temp_dir)); + } + + /// Test core is not created when the context is None + #[test] + fn test_crashdump_not_created_when_context_is_none() { + // Call the function with None context + let result = checked_core_dump(None, || Ok(Box::new(std::io::empty()))); + + // Check if the result is ok and the number of bytes is 0 + assert!(result.is_ok()); + assert_eq!(result.unwrap(), 0); + } + + /// Test the core dump creation with no regions fails + #[test] + fn test_crashdump_write_fails_when_no_regions() { + // Create a dummy context + let ctx = CrashDumpContext::new( + &[], + [0; 27], + vec![], + 0, + Some("dummy_binary".to_string()), + Some("dummy_filename".to_string()), + ); + + let get_writer = || Ok(Box::new(std::io::empty()) as Box); + + // Call the function + let result = checked_core_dump(Some(ctx), get_writer); + + // Check if the result is an error + // This should fail because there are no regions + assert!(result.is_err()); + } + + /// Check core dump with a dummy region to local vec + /// This test checks if the core dump is created successfully + #[test] + fn test_crashdump_dummy_core_dump() { + let dummy_vec = vec![0; 0x1000]; + let regions = vec![MemoryRegion { + guest_region: 0x1000..0x2000, + host_region: dummy_vec.as_ptr() as usize..dummy_vec.as_ptr() as usize + dummy_vec.len(), + flags: MemoryRegionFlags::READ | MemoryRegionFlags::WRITE, + region_type: crate::mem::memory_region::MemoryRegionType::Code, + }]; + // Create a dummy context + let ctx = CrashDumpContext::new( + ®ions, + [0; 27], + vec![], + 0x1000, + Some("dummy_binary".to_string()), + Some("dummy_filename".to_string()), + ); + + let get_writer = || Ok(Box::new(std::io::empty()) as Box); + + // Call the function + let result = checked_core_dump(Some(ctx), get_writer); + + // Check if the result is ok and the number of bytes is 0 + assert!(result.is_ok()); + // Check the number of bytes written is more than 0x1000 (the size of the region) + assert_eq!(result.unwrap(), 0x2000); + } } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 0f7f44e89..a35ffe703 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -51,7 +51,7 @@ use mshv_bindings::{ use mshv_ioctls::{Mshv, MshvError, VcpuFd, VmFd}; use tracing::{Span, instrument}; #[cfg(crashdump)] -use {super::crashdump, crate::sandbox::uninitialized::SandboxMetadata, std::path::Path}; +use {super::crashdump, std::path::Path}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] @@ -70,6 +70,8 @@ use crate::hypervisor::HyperlightExit; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; use crate::sandbox::SandboxConfiguration; +#[cfg(crashdump)] +use crate::sandbox::uninitialized::SandboxRuntimeConfig; use crate::{Result, log_then_return, new_error}; #[cfg(gdb)] @@ -305,7 +307,7 @@ pub(crate) struct HypervLinuxDriver { #[cfg(gdb)] gdb_conn: Option>, #[cfg(crashdump)] - metadata: SandboxMetadata, + rt_cfg: SandboxRuntimeConfig, } impl HypervLinuxDriver { @@ -325,7 +327,7 @@ impl HypervLinuxDriver { pml4_ptr: GuestPtr, config: &SandboxConfiguration, #[cfg(gdb)] gdb_conn: Option>, - #[cfg(crashdump)] metadata: SandboxMetadata, + #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, ) -> Result { let mshv = Mshv::new()?; let pr = Default::default(); @@ -414,7 +416,7 @@ impl HypervLinuxDriver { #[cfg(gdb)] gdb_conn, #[cfg(crashdump)] - metadata, + rt_cfg, }) } @@ -757,57 +759,61 @@ impl Hypervisor for HypervLinuxDriver { } #[cfg(crashdump)] - fn crashdump_context(&self) -> Result { - let mut regs = [0; 27]; - - let vcpu_regs = self.vcpu_fd.get_regs()?; - let sregs = self.vcpu_fd.get_sregs()?; - let xsave = self.vcpu_fd.get_xsave()?; - - // Set up the registers for the crash dump - regs[0] = vcpu_regs.r15; // r15 - regs[1] = vcpu_regs.r14; // r14 - regs[2] = vcpu_regs.r13; // r13 - regs[3] = vcpu_regs.r12; // r12 - regs[4] = vcpu_regs.rbp; // rbp - regs[5] = vcpu_regs.rbx; // rbx - regs[6] = vcpu_regs.r11; // r11 - regs[7] = vcpu_regs.r10; // r10 - regs[8] = vcpu_regs.r9; // r9 - regs[9] = vcpu_regs.r8; // r8 - regs[10] = vcpu_regs.rax; // rax - regs[11] = vcpu_regs.rcx; // rcx - regs[12] = vcpu_regs.rdx; // rdx - regs[13] = vcpu_regs.rsi; // rsi - regs[14] = vcpu_regs.rdi; // rdi - regs[15] = 0; // orig rax - regs[16] = vcpu_regs.rip; // rip - regs[17] = sregs.cs.selector as u64; // cs - regs[18] = vcpu_regs.rflags; // eflags - regs[19] = vcpu_regs.rsp; // rsp - regs[20] = sregs.ss.selector as u64; // ss - regs[21] = sregs.fs.base; // fs_base - regs[22] = sregs.gs.base; // gs_base - regs[23] = sregs.ds.selector as u64; // ds - regs[24] = sregs.es.selector as u64; // es - regs[25] = sregs.fs.selector as u64; // fs - regs[26] = sregs.gs.selector as u64; // gs - - // Get the filename from the binary path - let filename = self.metadata.binary_path.clone().and_then(|path| { - Path::new(&path) - .file_name() - .and_then(|name| name.to_os_string().into_string().ok()) - }); - - Ok(crashdump::CrashDumpContext::new( - &self.mem_regions, - regs, - xsave.buffer.to_vec(), - self.entrypoint, - self.metadata.binary_path.clone(), - filename, - )) + fn crashdump_context(&self) -> Result> { + if self.rt_cfg.guest_core_dump { + let mut regs = [0; 27]; + + let vcpu_regs = self.vcpu_fd.get_regs()?; + let sregs = self.vcpu_fd.get_sregs()?; + let xsave = self.vcpu_fd.get_xsave()?; + + // Set up the registers for the crash dump + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = sregs.cs.selector as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = sregs.ss.selector as u64; // ss + regs[21] = sregs.fs.base; // fs_base + regs[22] = sregs.gs.base; // gs_base + regs[23] = sregs.ds.selector as u64; // ds + regs[24] = sregs.es.selector as u64; // es + regs[25] = sregs.fs.selector as u64; // fs + regs[26] = sregs.gs.selector as u64; // gs + + // Get the filename from the binary path + let filename = self.rt_cfg.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + + Ok(Some(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave.buffer.to_vec(), + self.entrypoint, + self.rt_cfg.binary_path.clone(), + filename, + ))) + } else { + Ok(None) + } } #[cfg(gdb)] @@ -932,7 +938,14 @@ mod tests { #[cfg(gdb)] None, #[cfg(crashdump)] - SandboxMetadata { binary_path: None }, + SandboxRuntimeConfig { + #[cfg(crashdump)] + binary_path: None, + #[cfg(gdb)] + debug_info: None, + #[cfg(crashdump)] + guest_core_dump: true, + }, ) .unwrap(); } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 978e2a1cd..0bc7886ad 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -31,7 +31,7 @@ use windows::Win32::System::Hypervisor::{ WHvX64RegisterCs, WHvX64RegisterEfer, }; #[cfg(crashdump)] -use {super::crashdump, crate::sandbox::uninitialized::SandboxMetadata, std::path::Path}; +use {super::crashdump, std::path::Path}; use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] @@ -49,6 +49,8 @@ use crate::hypervisor::fpu::FP_CONTROL_WORD_DEFAULT; use crate::hypervisor::wrappers::WHvGeneralRegisters; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; +#[cfg(crashdump)] +use crate::sandbox::uninitialized::SandboxRuntimeConfig; use crate::{Result, debug, new_error}; /// A Hypervisor driver for HyperV-on-Windows. @@ -62,7 +64,7 @@ pub(crate) struct HypervWindowsDriver { mem_regions: Vec, interrupt_handle: Arc, #[cfg(crashdump)] - metadata: SandboxMetadata, + rt_cfg: SandboxRuntimeConfig, } /* This does not automatically impl Send/Sync because the host * address of the shared memory region is a raw pointer, which are @@ -83,7 +85,7 @@ impl HypervWindowsDriver { entrypoint: u64, rsp: u64, mmap_file_handle: HandleWrapper, - #[cfg(crashdump)] metadata: SandboxMetadata, + #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, ) -> Result { // create and setup hypervisor partition let mut partition = VMPartition::new(1)?; @@ -119,7 +121,7 @@ impl HypervWindowsDriver { dropped: AtomicBool::new(false), }), #[cfg(crashdump)] - metadata, + rt_cfg, }) } @@ -522,57 +524,61 @@ impl Hypervisor for HypervWindowsDriver { } #[cfg(crashdump)] - fn crashdump_context(&self) -> Result { - let mut regs = [0; 27]; - - let vcpu_regs = self.processor.get_regs()?; - let sregs = self.processor.get_sregs()?; - let xsave = self.processor.get_xsave()?; - - // Set the registers in the order expected by the crashdump context - regs[0] = vcpu_regs.r15; // r15 - regs[1] = vcpu_regs.r14; // r14 - regs[2] = vcpu_regs.r13; // r13 - regs[3] = vcpu_regs.r12; // r12 - regs[4] = vcpu_regs.rbp; // rbp - regs[5] = vcpu_regs.rbx; // rbx - regs[6] = vcpu_regs.r11; // r11 - regs[7] = vcpu_regs.r10; // r10 - regs[8] = vcpu_regs.r9; // r9 - regs[9] = vcpu_regs.r8; // r8 - regs[10] = vcpu_regs.rax; // rax - regs[11] = vcpu_regs.rcx; // rcx - regs[12] = vcpu_regs.rdx; // rdx - regs[13] = vcpu_regs.rsi; // rsi - regs[14] = vcpu_regs.rdi; // rdi - regs[15] = 0; // orig rax - regs[16] = vcpu_regs.rip; // rip - regs[17] = unsafe { sregs.cs.Segment.Selector } as u64; // cs - regs[18] = vcpu_regs.rflags; // eflags - regs[19] = vcpu_regs.rsp; // rsp - regs[20] = unsafe { sregs.ss.Segment.Selector } as u64; // ss - regs[21] = unsafe { sregs.fs.Segment.Base }; // fs_base - regs[22] = unsafe { sregs.gs.Segment.Base }; // gs_base - regs[23] = unsafe { sregs.ds.Segment.Selector } as u64; // ds - regs[24] = unsafe { sregs.es.Segment.Selector } as u64; // es - regs[25] = unsafe { sregs.fs.Segment.Selector } as u64; // fs - regs[26] = unsafe { sregs.gs.Segment.Selector } as u64; // gs - - // Get the filename from the metadata - let filename = self.metadata.binary_path.clone().and_then(|path| { - Path::new(&path) - .file_name() - .and_then(|name| name.to_os_string().into_string().ok()) - }); - - Ok(crashdump::CrashDumpContext::new( - &self.mem_regions, - regs, - xsave, - self.entrypoint, - self.metadata.binary_path.clone(), - filename, - )) + fn crashdump_context(&self) -> Result> { + if self.rt_cfg.guest_core_dump { + let mut regs = [0; 27]; + + let vcpu_regs = self.processor.get_regs()?; + let sregs = self.processor.get_sregs()?; + let xsave = self.processor.get_xsave()?; + + // Set the registers in the order expected by the crashdump context + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = unsafe { sregs.cs.Segment.Selector } as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = unsafe { sregs.ss.Segment.Selector } as u64; // ss + regs[21] = unsafe { sregs.fs.Segment.Base }; // fs_base + regs[22] = unsafe { sregs.gs.Segment.Base }; // gs_base + regs[23] = unsafe { sregs.ds.Segment.Selector } as u64; // ds + regs[24] = unsafe { sregs.es.Segment.Selector } as u64; // es + regs[25] = unsafe { sregs.fs.Segment.Selector } as u64; // fs + regs[26] = unsafe { sregs.gs.Segment.Selector } as u64; // gs + + // Get the filename from the config + let filename = self.rt_cfg.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + + Ok(Some(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave, + self.entrypoint, + self.rt_cfg.binary_path.clone(), + filename, + ))) + } else { + Ok(None) + } } } diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index 784f013c0..03f03aeaa 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -27,7 +27,7 @@ use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; use log::LevelFilter; use tracing::{Span, instrument}; #[cfg(crashdump)] -use {super::crashdump, crate::sandbox::uninitialized::SandboxMetadata, std::path::Path}; +use {super::crashdump, std::path::Path}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] @@ -45,6 +45,8 @@ use crate::HyperlightError; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; use crate::sandbox::SandboxConfiguration; +#[cfg(crashdump)] +use crate::sandbox::uninitialized::SandboxRuntimeConfig; use crate::{Result, log_then_return, new_error}; /// Return `true` if the KVM API is available, version 12, and has UserMemory capability, or `false` otherwise @@ -293,7 +295,7 @@ pub(crate) struct KVMDriver { #[cfg(gdb)] gdb_conn: Option>, #[cfg(crashdump)] - metadata: SandboxMetadata, + rt_cfg: SandboxRuntimeConfig, } impl KVMDriver { @@ -308,7 +310,7 @@ impl KVMDriver { rsp: u64, config: &SandboxConfiguration, #[cfg(gdb)] gdb_conn: Option>, - #[cfg(crashdump)] metadata: SandboxMetadata, + #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, ) -> Result { let kvm = Kvm::new()?; @@ -369,7 +371,7 @@ impl KVMDriver { #[cfg(gdb)] gdb_conn, #[cfg(crashdump)] - metadata, + rt_cfg, }; Ok(ret) } @@ -665,64 +667,67 @@ impl Hypervisor for KVMDriver { } #[cfg(crashdump)] - fn crashdump_context(&self) -> Result { - let mut regs = [0; 27]; - - let vcpu_regs = self.vcpu_fd.get_regs()?; - let sregs = self.vcpu_fd.get_sregs()?; - let xsave = self.vcpu_fd.get_xsave()?; - - // Set the registers in the order expected by the crashdump context - regs[0] = vcpu_regs.r15; // r15 - regs[1] = vcpu_regs.r14; // r14 - regs[2] = vcpu_regs.r13; // r13 - regs[3] = vcpu_regs.r12; // r12 - regs[4] = vcpu_regs.rbp; // rbp - regs[5] = vcpu_regs.rbx; // rbx - regs[6] = vcpu_regs.r11; // r11 - regs[7] = vcpu_regs.r10; // r10 - regs[8] = vcpu_regs.r9; // r9 - regs[9] = vcpu_regs.r8; // r8 - regs[10] = vcpu_regs.rax; // rax - regs[11] = vcpu_regs.rcx; // rcx - regs[12] = vcpu_regs.rdx; // rdx - regs[13] = vcpu_regs.rsi; // rsi - regs[14] = vcpu_regs.rdi; // rdi - regs[15] = 0; // orig rax - regs[16] = vcpu_regs.rip; // rip - regs[17] = sregs.cs.selector as u64; // cs - regs[18] = vcpu_regs.rflags; // eflags - regs[19] = vcpu_regs.rsp; // rsp - regs[20] = sregs.ss.selector as u64; // ss - regs[21] = sregs.fs.base; // fs_base - regs[22] = sregs.gs.base; // gs_base - regs[23] = sregs.ds.selector as u64; // ds - regs[24] = sregs.es.selector as u64; // es - regs[25] = sregs.fs.selector as u64; // fs - regs[26] = sregs.gs.selector as u64; // gs - - // Get the filename from the metadata - let filename = self.metadata.binary_path.clone().and_then(|path| { - Path::new(&path) - .file_name() - .and_then(|name| name.to_os_string().into_string().ok()) - }); - - // The [`CrashDumpContext`] accepts xsave as a vector of u8, so we need to convert the - // xsave region to a vector of u8 - Ok(crashdump::CrashDumpContext::new( - &self.mem_regions, - regs, - xsave.region.into_iter().fold(vec![], |mut acc, item| { - let bytes = item.to_le_bytes(); - acc.append(&mut bytes.to_vec()); - - acc - }), - self.entrypoint, - self.metadata.binary_path.clone(), - filename, - )) + fn crashdump_context(&self) -> Result> { + if self.rt_cfg.guest_core_dump { + let mut regs = [0; 27]; + + let vcpu_regs = self.vcpu_fd.get_regs()?; + let sregs = self.vcpu_fd.get_sregs()?; + let xsave = self.vcpu_fd.get_xsave()?; + + // Set the registers in the order expected by the crashdump context + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = sregs.cs.selector as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = sregs.ss.selector as u64; // ss + regs[21] = sregs.fs.base; // fs_base + regs[22] = sregs.gs.base; // gs_base + regs[23] = sregs.ds.selector as u64; // ds + regs[24] = sregs.es.selector as u64; // es + regs[25] = sregs.fs.selector as u64; // fs + regs[26] = sregs.gs.selector as u64; // gs + + // Get the filename from the runtime config + let filename = self.rt_cfg.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + + // The [`CrashDumpContext`] accepts xsave as a vector of u8, so we need to convert the + // xsave region to a vector of u8 + Ok(Some(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave + .region + .iter() + .flat_map(|item| item.to_le_bytes()) + .collect::>(), + self.entrypoint, + self.rt_cfg.binary_path.clone(), + filename, + ))) + } else { + Ok(None) + } } #[cfg(gdb)] diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index e96ce9be3..bc6baa7cf 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -227,7 +227,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor; #[cfg(crashdump)] - fn crashdump_context(&self) -> Result; + fn crashdump_context(&self) -> Result>; #[cfg(gdb)] /// handles the cases when the vCPU stops due to a Debug event @@ -269,7 +269,7 @@ impl VirtualCPU { } Ok(HyperlightExit::Mmio(addr)) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; mem_access_fn .clone() @@ -281,7 +281,7 @@ impl VirtualCPU { } Ok(HyperlightExit::AccessViolation(addr, tried, region_permission)) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; if region_permission.intersects(MemoryRegionFlags::STACK_GUARD) { return Err(HyperlightError::StackOverflow()); @@ -300,14 +300,14 @@ impl VirtualCPU { } Ok(HyperlightExit::Unknown(reason)) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; log_then_return!("Unexpected VM Exit {:?}", reason); } Ok(HyperlightExit::Retry()) => continue, Err(e) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; return Err(e); } @@ -455,8 +455,8 @@ pub(crate) mod tests { use crate::hypervisor::DbgMemAccessHandlerCaller; use crate::mem::ptr::RawPtr; use crate::sandbox::uninitialized::GuestBinary; - #[cfg(crashdump)] - use crate::sandbox::uninitialized::SandboxMetadata; + #[cfg(any(crashdump, gdb))] + use crate::sandbox::uninitialized::SandboxRuntimeConfig; use crate::sandbox::uninitialized_evolve::set_up_hypervisor_partition; use crate::sandbox::{SandboxConfiguration, UninitializedSandbox}; use crate::{Result, is_hypervisor_present, new_error}; @@ -500,16 +500,16 @@ pub(crate) mod tests { let filename = dummy_guest_as_string().map_err(|e| new_error!("{}", e))?; let config: SandboxConfiguration = Default::default(); + #[cfg(any(crashdump, gdb))] + let rt_cfg: SandboxRuntimeConfig = Default::default(); let sandbox = UninitializedSandbox::new(GuestBinary::FilePath(filename.clone()), Some(config))?; let (_hshm, mut gshm) = sandbox.mgr.build(); - #[cfg(crashdump)] - let metadata = SandboxMetadata { binary_path: None }; let mut vm = set_up_hypervisor_partition( &mut gshm, &config, - #[cfg(crashdump)] - &metadata, + #[cfg(any(crashdump, gdb))] + &rt_cfg, )?; vm.initialise( RawPtr::from(0x230000), diff --git a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs index 84e285f76..3ea9d61d8 100644 --- a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs +++ b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs @@ -24,10 +24,9 @@ use windows::core::s; use windows_result::HRESULT; use super::wrappers::HandleWrapper; -use crate::HyperlightError; use crate::hypervisor::wrappers::{WHvFPURegisters, WHvGeneralRegisters, WHvSpecialRegisters}; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; -use crate::{Result, new_error}; +use crate::{HyperlightError, Result, new_error}; /// Interop calls for Windows Hypervisor Platform APIs /// diff --git a/src/hyperlight_host/src/sandbox/config.rs b/src/hyperlight_host/src/sandbox/config.rs index f0725be12..f535b385d 100644 --- a/src/hyperlight_host/src/sandbox/config.rs +++ b/src/hyperlight_host/src/sandbox/config.rs @@ -35,6 +35,14 @@ pub struct DebugInfo { #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(C)] pub struct SandboxConfiguration { + /// Guest core dump output directory + /// This field is by default set to true which means the value core dumps will be placed in: + /// - HYPERLIGHT_CORE_DUMP_DIR environment variable if it is set + /// - default value of the temporary directory + /// + /// The core dump files generation can be disabled by setting this field to false. + #[cfg(crashdump)] + guest_core_dump: bool, /// Guest gdb debug port #[cfg(gdb)] guest_debug_info: Option, @@ -100,6 +108,7 @@ impl SandboxConfiguration { interrupt_retry_delay: Duration, interrupt_vcpu_sigrtmin_offset: u8, #[cfg(gdb)] guest_debug_info: Option, + #[cfg(crashdump)] guest_core_dump: bool, ) -> Self { Self { input_data_size: max(input_data_size, Self::MIN_INPUT_SIZE), @@ -110,6 +119,8 @@ impl SandboxConfiguration { interrupt_vcpu_sigrtmin_offset, #[cfg(gdb)] guest_debug_info, + #[cfg(crashdump)] + guest_core_dump, } } @@ -174,6 +185,15 @@ impl SandboxConfiguration { Ok(()) } + /// Toggles the guest core dump generation for a sandbox + /// Setting this to false disables the core dump generation + /// This is only used when the `crashdump` feature is enabled + #[cfg(crashdump)] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub fn set_guest_core_dump(&mut self, enable: bool) { + self.guest_core_dump = enable; + } + /// Sets the configuration for the guest debug #[cfg(gdb)] #[instrument(skip_all, parent = Span::current(), level= "Trace")] @@ -191,6 +211,12 @@ impl SandboxConfiguration { self.output_data_size } + #[cfg(crashdump)] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub(crate) fn get_guest_core_dump(&self) -> bool { + self.guest_core_dump + } + #[cfg(gdb)] #[instrument(skip_all, parent = Span::current(), level= "Trace")] pub(crate) fn get_guest_debug_info(&self) -> Option { @@ -236,6 +262,8 @@ impl Default for SandboxConfiguration { Self::INTERRUPT_VCPU_SIGRTMIN_OFFSET, #[cfg(gdb)] None, + #[cfg(crashdump)] + true, ) } } @@ -260,6 +288,8 @@ mod tests { SandboxConfiguration::INTERRUPT_VCPU_SIGRTMIN_OFFSET, #[cfg(gdb)] None, + #[cfg(crashdump)] + true, ); let exe_info = simple_guest_exe_info().unwrap(); @@ -287,6 +317,8 @@ mod tests { SandboxConfiguration::INTERRUPT_VCPU_SIGRTMIN_OFFSET, #[cfg(gdb)] None, + #[cfg(crashdump)] + true, ); assert_eq!(SandboxConfiguration::MIN_INPUT_SIZE, cfg.input_data_size); assert_eq!(SandboxConfiguration::MIN_OUTPUT_SIZE, cfg.output_data_size); diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs index 25abac990..624d894d8 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized.rs @@ -52,10 +52,15 @@ const EXTRA_ALLOWED_SYSCALLS_FOR_WRITER_FUNC: &[super::ExtraAllowedSyscall] = &[ libc::SYS_close, ]; -#[cfg(crashdump)] +#[cfg(any(crashdump, gdb))] #[derive(Clone, Debug, Default)] -pub(crate) struct SandboxMetadata { +pub(crate) struct SandboxRuntimeConfig { + #[cfg(crashdump)] pub(crate) binary_path: Option, + #[cfg(gdb)] + pub(crate) debug_info: Option, + #[cfg(crashdump)] + pub(crate) guest_core_dump: bool, } /// A preliminary `Sandbox`, not yet ready to execute guest code. @@ -72,8 +77,8 @@ pub struct UninitializedSandbox { pub(crate) mgr: MemMgrWrapper, pub(crate) max_guest_log_level: Option, pub(crate) config: SandboxConfiguration, - #[cfg(crashdump)] - pub(crate) metadata: SandboxMetadata, + #[cfg(any(crashdump, gdb))] + pub(crate) rt_cfg: SandboxRuntimeConfig, } impl crate::sandbox_state::sandbox::UninitializedSandbox for UninitializedSandbox { @@ -163,17 +168,33 @@ impl UninitializedSandbox { buffer @ GuestBinary::Buffer(_) => buffer, }; - #[cfg(crashdump)] - let metadata = if let GuestBinary::FilePath(ref path) = guest_binary { - SandboxMetadata { - binary_path: Some(path.clone()), + let sandbox_cfg = cfg.unwrap_or_default(); + + #[cfg(any(crashdump, gdb))] + let rt_cfg = { + #[cfg(crashdump)] + let guest_core_dump = sandbox_cfg.get_guest_core_dump(); + + #[cfg(gdb)] + let debug_info = sandbox_cfg.get_guest_debug_info(); + + #[cfg(crashdump)] + let binary_path = if let GuestBinary::FilePath(ref path) = guest_binary { + Some(path.clone()) + } else { + None + }; + + SandboxRuntimeConfig { + #[cfg(crashdump)] + binary_path, + #[cfg(gdb)] + debug_info, + #[cfg(crashdump)] + guest_core_dump, } - } else { - SandboxMetadata::default() }; - let sandbox_cfg = cfg.unwrap_or_default(); - let mut mem_mgr_wrapper = { let mut mgr = UninitializedSandbox::load_guest_binary(sandbox_cfg, &guest_binary)?; let stack_guard = Self::create_stack_guard(); @@ -190,8 +211,8 @@ impl UninitializedSandbox { mgr: mem_mgr_wrapper, max_guest_log_level: None, config: sandbox_cfg, - #[cfg(crashdump)] - metadata, + #[cfg(any(crashdump, gdb))] + rt_cfg, }; // If we were passed a writer for host print register it otherwise use the default. diff --git a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs index 77adb4dae..9bda0cb73 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs @@ -23,6 +23,8 @@ use super::SandboxConfiguration; use super::hypervisor::{HypervisorType, get_available_hypervisor}; #[cfg(gdb)] use super::mem_access::dbg_mem_access_handler_wrapper; +#[cfg(any(crashdump, gdb))] +use super::uninitialized::SandboxRuntimeConfig; use crate::HyperlightError::NoHypervisorFound; use crate::hypervisor::Hypervisor; use crate::hypervisor::handlers::{MemAccessHandlerCaller, OutBHandlerCaller}; @@ -36,8 +38,6 @@ use crate::sandbox::config::DebugInfo; use crate::sandbox::host_funcs::FunctionRegistry; use crate::sandbox::mem_access::mem_access_handler_wrapper; use crate::sandbox::outb::outb_handler_wrapper; -#[cfg(crashdump)] -use crate::sandbox::uninitialized::SandboxMetadata; use crate::sandbox::{HostSharedMemory, MemMgrWrapper}; use crate::sandbox_state::sandbox::Sandbox; #[cfg(target_os = "linux")] @@ -74,8 +74,8 @@ where let mut vm = set_up_hypervisor_partition( &mut gshm, &u_sbox.config, - #[cfg(crashdump)] - &u_sbox.metadata, + #[cfg(any(crashdump, gdb))] + &u_sbox.rt_cfg, )?; let outb_hdl = outb_handler_wrapper(hshm.clone(), u_sbox.host_funcs.clone()); @@ -148,7 +148,7 @@ pub(super) fn evolve_impl_multi_use(u_sbox: UninitializedSandbox) -> Result, #[cfg_attr(target_os = "windows", allow(unused_variables))] config: &SandboxConfiguration, - #[cfg(crashdump)] metadata: &SandboxMetadata, + #[cfg(any(crashdump, gdb))] rt_cfg: &SandboxRuntimeConfig, ) -> Result> { let mem_size = u64::try_from(mgr.shared_mem.mem_size())?; let mut regions = mgr.layout.get_memory_regions(&mgr.shared_mem)?; @@ -184,7 +184,7 @@ pub(crate) fn set_up_hypervisor_partition( // Create gdb thread if gdb is enabled and the configuration is provided #[cfg(gdb)] - let gdb_conn = if let Some(DebugInfo { port }) = config.get_guest_debug_info() { + let gdb_conn = if let Some(DebugInfo { port }) = rt_cfg.debug_info { use crate::hypervisor::gdb::create_gdb_thread; let gdb_conn = create_gdb_thread(port, unsafe { libc::pthread_self() }); @@ -215,7 +215,7 @@ pub(crate) fn set_up_hypervisor_partition( #[cfg(gdb)] gdb_conn, #[cfg(crashdump)] - metadata.clone(), + rt_cfg.clone(), )?; Ok(Box::new(hv)) } @@ -231,7 +231,7 @@ pub(crate) fn set_up_hypervisor_partition( #[cfg(gdb)] gdb_conn, #[cfg(crashdump)] - metadata.clone(), + rt_cfg.clone(), )?; Ok(Box::new(hv)) } @@ -254,7 +254,7 @@ pub(crate) fn set_up_hypervisor_partition( rsp_ptr.absolute()?, HandleWrapper::from(mmap_file_handle), #[cfg(crashdump)] - metadata.clone(), + rt_cfg.clone(), )?; Ok(Box::new(hv)) } From 18c1804a7a625bcef20f6df55d794345ce5e5afa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Tue, 10 Jun 2025 00:38:54 +0300 Subject: [PATCH 11/13] cargo: move to official elfcore crates.io 2.0 version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- Cargo.lock | 5 +++-- src/hyperlight_host/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 068c24989..e95a6a8ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -665,8 +665,9 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "elfcore" -version = "1.1.5" -source = "git+https://github.com/hyperlight-dev/elfcore.git?rev=cef4c80e26bf4b2a5599e50d2d1730965f942c13#cef4c80e26bf4b2a5599e50d2d1730965f942c13" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824386967a6a98e7f99d5c15d40cd1534b0ebfc4193a7109689dcf5322e8d744" dependencies = [ "libc", "nix", diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index d3d37d80a..b94aebe0e 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -42,7 +42,7 @@ chrono = { version = "0.4", optional = true } anyhow = "1.0" metrics = "0.24.2" serde_json = "1.0" -elfcore = { git = "https://github.com/hyperlight-dev/elfcore.git", rev = "cef4c80e26bf4b2a5599e50d2d1730965f942c13" } +elfcore = "2.0" [target.'cfg(windows)'.dependencies] windows = { version = "0.61", features = [ From 55292df8bb7f3aca4e4693713cf05a7ee5b590e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Tue, 10 Jun 2025 12:11:02 +0300 Subject: [PATCH 12/13] Fix crashdump creation message print when the option is disabled for a sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update docs to add a note about how to disable crashdumps for specific sandboxes - Commit automatically update `Cargo.lock` for rust_guests Signed-off-by: Doru Blânzeanu --- docs/how-to-debug-a-hyperlight-guest.md | 7 +++++++ .../src/hypervisor/crashdump.rs | 18 ++++++++++++------ src/tests/rust_guests/witguest/Cargo.lock | 8 ++++---- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index cb18d53d3..36929e290 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -211,6 +211,13 @@ To make Hyperlight dump the state of the vCPU (general purpose registers, regist This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message. +**NOTE**: By enabling the `crashdump` feature, you instruct Hyperlight to create core dump files for all sandboxes when an unhandled crash occurs. +To selectively disable this feature for a specific sandbox, you can set the `guest_core_dump` field to `false` in the `SandboxConfiguration`. +```rust + let mut cfg = SandboxConfiguration::default(); + cfg.set_guest_core_dump(false); // Disable core dump for this sandbox +``` + ### Inspecting the core dump After the core dump has been created, to inspect the state of the guest, load the core dump file using `gdb` or `lldb`. diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index ac509cd90..e6e776c37 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -263,8 +263,6 @@ impl ReadProcessMemory for GuestMemReader { /// # Returns /// * `Result<()>`: Success or error pub(crate) fn generate_crashdump(hv: &dyn Hypervisor) -> Result<()> { - log::info!("Creating core dump file..."); - // Get crash context from hypervisor let ctx = hv .crashdump_context() @@ -284,10 +282,16 @@ pub(crate) fn generate_crashdump(hv: &dyn Hypervisor) -> Result<()> { ) as Box) }; - checked_core_dump(ctx, create_dump_file).map(|_| { - println!("Core dump created successfully: {}", file_path); - log::error!("Core dump file: {}", file_path); - }) + if let Ok(nbytes) = checked_core_dump(ctx, create_dump_file) { + if nbytes > 0 { + println!("Core dump created successfully: {}", file_path); + log::error!("Core dump file: {}", file_path); + } + } else { + log::error!("Failed to create core dump file"); + } + + Ok(()) } /// Computes the file path for the core dump file. @@ -352,6 +356,8 @@ fn checked_core_dump( // If the HV returned a context it means we can create a core dump // This is the case when the sandbox has been configured at runtime to allow core dumps if let Some(ctx) = ctx { + log::info!("Creating core dump file..."); + // Set up data sources for the core dump let guest_view = GuestView::new(&ctx); let memory_reader = GuestMemReader::new(&ctx); diff --git a/src/tests/rust_guests/witguest/Cargo.lock b/src/tests/rust_guests/witguest/Cargo.lock index 573a444ee..6a1aef84a 100644 --- a/src/tests/rust_guests/witguest/Cargo.lock +++ b/src/tests/rust_guests/witguest/Cargo.lock @@ -347,9 +347,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -477,9 +477,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.100" +version = "2.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "f6397daf94fa90f058bd0fd88429dd9e5738999cca8d701813c80723add80462" dependencies = [ "proc-macro2", "quote", From 97cd58afe97ffc11d4b870cc3ee71005d226f792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Wed, 11 Jun 2025 14:43:32 +0300 Subject: [PATCH 13/13] docs: update crashdump docs to signal support for both debug and release builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- docs/how-to-debug-a-hyperlight-guest.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index 36929e290..f3a565abc 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -207,10 +207,14 @@ involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or When a guest crashes because of an unknown VmExit or unhandled exception, the vCPU state is dumped to an `ELF` core dump file. This can be used to inspect the state of the guest at the time of the crash. -To make Hyperlight dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file, set the feature `crashdump` and run a debug build. -This will result in a dump file being created in the temporary directory. +To make Hyperlight dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file, enable the `crashdump` +feature and run. +The feature enables the creation of core dump files for both debug and release builds of Hyperlight hosts. +By default, Hyperlight places the core dumps in the temporary directory (platform specific). +To change this, use the `HYPERLIGHT_CORE_DUMP_DIR` environment variable to specify a directory. The name and location of the dump file will be printed to the console and logged as an error message. +**NOTE**: If the directory provided by `HYPERLIGHT_CORE_DUMP_DIR` does not exist, Hyperlight places the file in the temporary directory. **NOTE**: By enabling the `crashdump` feature, you instruct Hyperlight to create core dump files for all sandboxes when an unhandled crash occurs. To selectively disable this feature for a specific sandbox, you can set the `guest_core_dump` field to `false` in the `SandboxConfiguration`. ```rust