crashdump: create core dump file when a guest crashes

- the core dump file is an ELF file with special segments that describe
the guest's memory when it crashed, the CPU register's values and other
special notes that tell the debugger how to set up a debugging session
starting from the core dump

Signed-off-by: Doru Blânzeanu <[email protected]>

Author: dblnz

Cargo.lock

@@ -46,6 +46,7 @@ tempfile = { version = "3.19", optional = true }
 serde_yaml = "0.9"
 anyhow = "1.0"
 metrics = "0.24.2"
+elfcore = { git = "https://github.com/dblnz/elfcore.git", branch = "split-linux-impl-from-elfcore" }
 
 [target.'cfg(windows)'.dependencies]
 windows = { version = "0.61", features = [

src/hyperlight_host/Cargo.toml

@@ -1,44 +1,257 @@
-use std::io::Write;
+use std::cmp::min;
 
+use elfcore::{
+    ArchComponentState, ArchState, CoreDumpBuilder, CoreError, Elf64_Auxv, ProcessInfoSource,
+    ReadProcessMemory, ThreadView, VaProtection, VaRegion,
+};
 use tempfile::NamedTempFile;
 
 use super::Hypervisor;
+use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::{new_error, Result};
 
-/// Dump registers + memory regions + raw memory to a tempfile
-#[cfg(crashdump)]
-pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
-    let mut temp_file = NamedTempFile::with_prefix("mem")?;
-    let hv_details = format!("{:#x?}", hv);
+const NT_X86_XSTATE: u32 = 0x202;
+const AT_ENTRY: u64 = 9;
+const AT_NULL: u64 = 0;
+
+/// Structure to hold the crash dump context
+/// This structure contains the information needed to create a core dump
+#[derive(Debug)]
+pub(crate) struct CrashDumpContext<'a> {
+    regions: &'a [MemoryRegion],
+    regs: [u64; 27],
+    xsave: Vec<u8>,
+    entry: u64,
+}
+
+impl<'a> CrashDumpContext<'a> {
+    pub(crate) fn new(
+        regions: &'a [MemoryRegion],
+        regs: [u64; 27],
+        xsave: Vec<u8>,
+        entry: u64,
+    ) -> Self {
+        Self {
+            regions,
+            regs,
+            xsave,
+            entry,
+        }
+    }
+}
+
+/// Structure that contains the process information for the core dump
+/// This serves as a source of information for `elfcore`'s [`CoreDumpBuilder`]
+struct GuestView {
+    regions: Vec<VaRegion>,
+    threads: Vec<ThreadView>,
+    aux_vector: Vec<elfcore::Elf64_Auxv>,
+}
 
-    // write hypervisor details such as registers, info about mapped memory regions, etc.
-    temp_file.write_all(hv_details.as_bytes())?;
-    temp_file.write_all(b"================ MEMORY DUMP =================\n")?;
+impl GuestView {
+    fn new(ctx: &CrashDumpContext) -> Self {
+        // Map the regions to the format `CoreDumpBuilder` expects
+        let regions = ctx
+            .regions
+            .iter()
+            .filter(|r| !r.host_region.is_empty())
+            .map(|r| VaRegion {
+                begin: r.guest_region.start as u64,
+                end: r.guest_region.end as u64,
+                offset: r.host_region.start as u64,
+                protection: VaProtection {
+                    is_private: false,
+                    read: r.flags.contains(MemoryRegionFlags::READ),
+                    write: r.flags.contains(MemoryRegionFlags::WRITE),
+                    execute: r.flags.contains(MemoryRegionFlags::EXECUTE),
+                },
+                mapped_file_name: None,
+            })
+            .collect();
 
-    // write the raw memory dump for each memory region
-    for region in hv.get_memory_regions() {
-        if region.host_region.start == 0 || region.host_region.is_empty() {
-            continue;
+        // The xsave state is checked as it can be empty
+        let mut components = vec![];
+        if !ctx.xsave.is_empty() {
+            components.push(ArchComponentState {
+                name: "XSAVE",
+                note_type: NT_X86_XSTATE,
+                note_name: b"LINUX",
+                data: ctx.xsave.clone(),
+            });
         }
-        // SAFETY: we got this memory region from the hypervisor so should never be invalid
-        let region_slice = unsafe {
-            std::slice::from_raw_parts(
-                region.host_region.start as *const u8,
-                region.host_region.len(),
-            )
+
+        // Create the thread view
+        // The thread view contains the information about the thread
+        // NOTE: Some of these fields are not used in the current implementation
+        let thread = ThreadView {
+            flags: 0, // Kernel flags for the process
+            tid: 1,
+            uid: 0, // User ID
+            gid: 0, // Group ID
+            comm: "\0".to_string(),
+            ppid: 0,    // Parent PID
+            pgrp: 0,    // Process group ID
+            nice: 0,    // Nice value
+            state: 0,   // Process state
+            utime: 0,   // User time
+            stime: 0,   // System time
+            cutime: 0,  // Children User time
+            cstime: 0,  // Children User time
+            cursig: 0,  // Current signal
+            session: 0, // Session ID of the process
+            sighold: 0, // Blocked signal
+            sigpend: 0, // Pending signal
+            cmd_line: "\0".to_string(),
+
+            arch_state: Box::new(ArchState {
+                gpr_state: ctx.regs.to_vec(),
+                components,
+            }),
         };
-        temp_file.write_all(region_slice)?;
+
+        // Create the auxv vector
+        // The first entry is AT_ENTRY, which is the entry point of the program
+        // The entry point is the address where the program starts executing
+        // This helps the debugger to know that the entry is changed by an offset
+        // so the symbols can be loaded correctly.
+        // The second entry is AT_NULL, which marks the end of the vector
+        let auxv = vec![
+            Elf64_Auxv {
+                a_type: AT_ENTRY,
+                a_val: ctx.entry,
+            },
+            Elf64_Auxv {
+                a_type: AT_NULL,
+                a_val: 0,
+            },
+        ];
+
+        Self {
+            regions,
+            threads: vec![thread],
+            aux_vector: auxv,
+        }
+    }
+}
+
+impl ProcessInfoSource for GuestView {
+    fn pid(&self) -> i32 {
+        1
+    }
+    fn threads(&self) -> &[elfcore::ThreadView] {
+        &self.threads
     }
-    temp_file.flush()?;
+    fn page_size(&self) -> usize {
+        0x1000
+    }
+    fn aux_vector(&self) -> Option<&[elfcore::Elf64_Auxv]> {
+        Some(&self.aux_vector)
+    }
+    fn va_regions(&self) -> &[elfcore::VaRegion] {
+        &self.regions
+    }
+    fn mapped_files(&self) -> Option<&[elfcore::MappedFile]> {
+        None
+    }
+}
 
-    // persist the tempfile to disk
-    let persist_path = temp_file.path().with_extension("dmp");
+/// Structure that reads the guest memory
+/// This structure serves as a custom memory reader for `elfcore`'s
+/// [`CoreDumpBuilder`]
+struct GuestMemReader {
+    regions: Vec<MemoryRegion>,
+}
+
+impl GuestMemReader {
+    fn new(ctx: &CrashDumpContext) -> Self {
+        Self {
+            regions: ctx.regions.to_vec(),
+        }
+    }
+}
+
+impl ReadProcessMemory for GuestMemReader {
+    fn read_process_memory(
+        &mut self,
+        base: usize,
+        buf: &mut [u8],
+    ) -> std::result::Result<usize, CoreError> {
+        for r in self.regions.iter() {
+            // Check if the base address is within the guest region
+            if base >= r.guest_region.start && base < r.guest_region.end {
+                let offset = base - r.guest_region.start;
+                let region_slice = unsafe {
+                    std::slice::from_raw_parts(
+                        r.host_region.start as *const u8,
+                        r.host_region.len(),
+                    )
+                };
+
+                // Calculate how much we can copy
+                let copy_size = min(buf.len(), region_slice.len() - offset);
+                if copy_size == 0 {
+                    return std::result::Result::Ok(0);
+                }
+
+                // Only copy the amount that fits in both buffers
+                buf[..copy_size].copy_from_slice(&region_slice[offset..offset + copy_size]);
+
+                // Return the number of bytes copied
+                return std::result::Result::Ok(copy_size);
+            }
+        }
+
+        // If we reach here, we didn't find a matching region
+        std::result::Result::Ok(0)
+    }
+}
+
+/// Create core dump file from the hypervisor information
+///
+/// This function generates an ELF core dump file capturing the hypervisor's state,
+/// which can be used for debugging when crashes occur. The file is created in the
+/// system's temporary directory with extension '.elf' and the path is printed to stdout and logs.
+///
+/// # Arguments
+/// * `hv`: Reference to the hypervisor implementation
+///
+/// # Returns
+/// * `Result<()>`: Success or error
+pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
+    log::info!("Creating core dump file...");
+
+    // Create a temporary file with a recognizable prefix
+    let temp_file = NamedTempFile::with_prefix("hl_core_")
+        .map_err(|e| new_error!("Failed to create temporary file: {:?}", e))?;
+
+    // Get crash context from hypervisor
+    let ctx = hv
+        .crashdump_context()
+        .map_err(|e| new_error!("Failed to get crashdump context: {:?}", e))?;
+
+    // Set up data sources for the core dump
+    let guest_view = GuestView::new(&ctx);
+    let memory_reader = GuestMemReader::new(&ctx);
+
+    // Create and write core dump
+    let core_builder = CoreDumpBuilder::from_source(
+        Box::new(guest_view) as Box<dyn ProcessInfoSource>,
+        Box::new(memory_reader) as Box<dyn ReadProcessMemory>,
+    );
+
+    core_builder
+        .write(&temp_file)
+        .map_err(|e| new_error!("Failed to write core dump: {:?}", e))?;
+
+    let persist_path = temp_file.path().with_extension("elf");
     temp_file
         .persist(&persist_path)
-        .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?;
+        .map_err(|e| new_error!("Failed to persist core dump file: {:?}", e))?;
+
+    let path_string = persist_path.to_string_lossy().to_string();
 
-    println!("Memory dumped to file: {:?}", persist_path);
-    log::error!("Memory dumped to file: {:?}", persist_path);
+    println!("Core dump created successfully: {}", path_string);
+    log::error!("Core dump file: {}", path_string);
 
     Ok(())
 }

src/hyperlight_host/src/hypervisor/crashdump.rs

@@ -49,6 +49,8 @@ use mshv_bindings::{
 use mshv_ioctls::{Mshv, VcpuFd, VmFd};
 use tracing::{instrument, Span};
 
+#[cfg(crashdump)]
+use super::crashdump;
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
 #[cfg(gdb)]
 use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, MshvDebug};
@@ -658,8 +660,48 @@ impl Hypervisor for HypervLinuxDriver {
     }
 
     #[cfg(crashdump)]
-    fn get_memory_regions(&self) -> &[MemoryRegion] {
-        &self.mem_regions
+    fn crashdump_context(&self) -> Result<super::crashdump::CrashDumpContext> {
+        let mut regs = [0; 27];
+
+        let vcpu_regs = self.vcpu_fd.get_regs()?;
+        let sregs = self.vcpu_fd.get_sregs()?;
+        let xsave = self.vcpu_fd.get_xsave()?;
+
+        // Set up the registers for the crash dump
+        regs[0] = vcpu_regs.r15; // r15
+        regs[1] = vcpu_regs.r14; // r14
+        regs[2] = vcpu_regs.r13; // r13
+        regs[3] = vcpu_regs.r12; // r12
+        regs[4] = vcpu_regs.rbp; // rbp
+        regs[5] = vcpu_regs.rbx; // rbx
+        regs[6] = vcpu_regs.r11; // r11
+        regs[7] = vcpu_regs.r10; // r10
+        regs[8] = vcpu_regs.r9; // r9
+        regs[9] = vcpu_regs.r8; // r8
+        regs[10] = vcpu_regs.rax; // rax
+        regs[11] = vcpu_regs.rcx; // rcx
+        regs[12] = vcpu_regs.rdx; // rdx
+        regs[13] = vcpu_regs.rsi; // rsi
+        regs[14] = vcpu_regs.rdi; // rdi
+        regs[15] = 0; // orig rax
+        regs[16] = vcpu_regs.rip; // rip
+        regs[17] = sregs.cs.selector as u64; // cs
+        regs[18] = vcpu_regs.rflags; // eflags
+        regs[19] = vcpu_regs.rsp; // rsp
+        regs[20] = sregs.ss.selector as u64; // ss
+        regs[21] = sregs.fs.base; // fs_base
+        regs[22] = sregs.gs.base; // gs_base
+        regs[23] = sregs.ds.selector as u64; // ds
+        regs[24] = sregs.es.selector as u64; // es
+        regs[25] = sregs.fs.selector as u64; // fs
+        regs[26] = sregs.gs.selector as u64; // gs
+
+        Ok(crashdump::CrashDumpContext::new(
+            &self.mem_regions,
+            regs,
+            xsave.buffer.to_vec(),
+            self.entrypoint,
+        ))
     }
 
     #[cfg(gdb)]