Skip to content

Commit 1c4354a

Browse files
committed
[performance] Move set_up_hypervisor_partition to inside the hv handler thread
As per KVM docs, vCPU ioctls should be issued from the same thread that was used to create the vCPU. In accordance to that, this PR moves set_up_hypervisor_partition to inside the hv handler thread, which manages vCPU interactions like init and dispatching guest funciton calls.
1 parent 20d8f35 commit 1c4354a

File tree

3 files changed

+54
-14
lines changed

3 files changed

+54
-14
lines changed

src/hyperlight_host/src/hypervisor/hypervisor_handler.rs

Lines changed: 45 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ limitations under the License.
1717
#[cfg(target_os = "windows")]
1818
use core::ffi::c_void;
1919
use std::sync::atomic::{AtomicBool, Ordering};
20-
use std::sync::{Arc, Mutex};
20+
use std::sync::{Arc, Mutex, MutexGuard};
2121
use std::thread;
2222
use std::thread::{sleep, JoinHandle};
2323
use std::time::Duration;
@@ -64,6 +64,7 @@ pub(crate) struct HypervisorHandler {
6464
communication_channels: HvHandlerCommChannels,
6565
configuration: HvHandlerConfig,
6666
execution_variables: HvHandlerExecVars,
67+
hypervisor: Option<Arc<Mutex<Box<dyn Hypervisor>>>>,
6768
}
6869

6970
impl HypervisorHandler {
@@ -215,9 +216,23 @@ impl HypervisorHandler {
215216
communication_channels,
216217
configuration,
217218
execution_variables,
219+
hypervisor: None,
218220
}
219221
}
220222

223+
/// Sets the hypervisor for the Hypervisor Handler.
224+
pub(crate) fn set_hypervisor(&mut self, hv: Box<dyn Hypervisor>) {
225+
self.hypervisor = Some(Arc::new(Mutex::new(hv)));
226+
}
227+
228+
/// Gets the hypervisor for the Hypervisor Handler.
229+
pub(crate) fn get_hypervisor(&self) -> Result<MutexGuard<Box<dyn Hypervisor>>> {
230+
self.hypervisor
231+
.as_ref()
232+
.ok_or_else(|| new_error!("Failed to get hypervisor: {}:{}:", file!(), line!()))
233+
.map(|hv| hv.lock().unwrap())
234+
}
235+
221236
/// Sets up a Hypervisor 'handler', designed to listen to messages to execute a specific action,
222237
/// such as:
223238
/// - `initialise` resources,
@@ -228,13 +243,10 @@ impl HypervisorHandler {
228243
#[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
229244
pub(crate) fn start_hypervisor_handler(
230245
&mut self,
231-
mut sandbox_memory_manager: SandboxMemoryManager<GuestSharedMemory>,
246+
sandbox_memory_manager: SandboxMemoryManager<GuestSharedMemory>,
232247
) -> Result<()> {
248+
let mut sandbox_memory_manager_clone = sandbox_memory_manager.clone();
233249
let configuration = self.configuration.clone();
234-
let mut hv = set_up_hypervisor_partition(
235-
&mut sandbox_memory_manager,
236-
configuration.outb_handler.clone(),
237-
)?;
238250
#[cfg(target_os = "windows")]
239251
let in_process = sandbox_memory_manager.is_in_process();
240252

@@ -267,20 +279,14 @@ impl HypervisorHandler {
267279
#[cfg(target_os = "linux")]
268280
self.execution_variables.run_cancelled.store(false);
269281

270-
#[cfg(target_os = "windows")]
271-
if !in_process {
272-
self.execution_variables
273-
.set_partition_handle(hv.get_partition_handle())?;
274-
}
275-
276282
let to_handler_rx = self.communication_channels.to_handler_rx.clone();
277283
#[cfg(target_os = "windows")]
278284
let execution_variables = self.execution_variables.clone();
279285
#[cfg(target_os = "linux")]
280286
let mut execution_variables = self.execution_variables.clone();
281287
// ^^^ this needs to be mut on linux to set_thread_id
282288
let from_handler_tx = self.communication_channels.from_handler_tx.clone();
283-
let hv_handler_clone = self.clone();
289+
let mut hv_handler_clone = self.clone();
284290

285291
// Hyperlight has two signal handlers:
286292
// (1) for timeouts, and
@@ -298,6 +304,23 @@ impl HypervisorHandler {
298304
for action in to_handler_rx {
299305
match action {
300306
HypervisorHandlerAction::Initialise => {
307+
{
308+
let hv = set_up_hypervisor_partition(
309+
&mut sandbox_memory_manager_clone,
310+
configuration.outb_handler.clone(),
311+
)?;
312+
313+
hv_handler_clone.set_hypervisor(hv);
314+
}
315+
316+
let mut hv = hv_handler_clone.get_hypervisor()?;
317+
318+
#[cfg(target_os = "windows")]
319+
if !in_process {
320+
execution_variables
321+
.set_partition_handle(hv.get_partition_handle())?;
322+
}
323+
301324
#[cfg(target_os = "linux")]
302325
{
303326
// We cannot use the Killable trait, so we get the `pthread_t` via a libc
@@ -362,6 +385,8 @@ impl HypervisorHandler {
362385
}
363386
}
364387
HypervisorHandlerAction::DispatchCallFromHost(function_name) => {
388+
let mut hv = hv_handler_clone.get_hypervisor()?;
389+
365390
// Lock to indicate an action is being performed in the hypervisor
366391
execution_variables.running.store(true, Ordering::SeqCst);
367392

@@ -944,6 +969,13 @@ mod tests {
944969
}
945970
}
946971

972+
#[test]
973+
fn create_10_sandboxes() {
974+
for _ in 0..10 {
975+
create_multi_use_sandbox();
976+
}
977+
}
978+
947979
#[test]
948980
fn hello_world() -> Result<()> {
949981
let mut sandbox = create_multi_use_sandbox();

src/hyperlight_host/src/hypervisor/kvm.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,15 @@ impl KVMDriver {
8282
};
8383
let kvm = Kvm::new()?;
8484

85+
#[cfg(debug_assertions)]
86+
let _now = std::time::Instant::now();
8587
let vm_fd = kvm.create_vm_with_type(0)?;
88+
#[cfg(debug_assertions)]
89+
println!(
90+
"Time to create_vm_with_type: {:?} from thread {:?}",
91+
_now.elapsed(),
92+
std::thread::current().name()
93+
);
8694

8795
let perm_flags =
8896
MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE;

src/hyperlight_host/src/mem/shared_mem.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ unsafe impl Send for ExclusiveSharedMemory {}
122122
/// unit that likely can't be discovered by the compiler) that _rust_
123123
/// users do not perform racy accesses to the guest communication
124124
/// buffers that are also accessed by HostSharedMemory.
125-
#[derive(Debug)]
125+
#[derive(Debug, Clone)]
126126
pub struct GuestSharedMemory {
127127
region: Arc<HostMapping>,
128128
/// The lock that indicates this shared memory is being used by non-Rust code

0 commit comments

Comments
 (0)