[vm/ffi] FFI callbacks on ARM64.

For design context and motivation, see go/dart-ffi-callbacks

Change-Id: Ie463a462c8676c4a1973f377acee067253aca9f0
Cq-Include-Trybots: luci.dart.try:vm-kernel-linux-debug-simdbc64-try, vm-kernel-linux-release-simdbc64-try, vm-kernel-mac-debug-simdbc64-try, vm-kernel-mac-release-simdbc64-try, vm-kernel-reload-mac-debug-simdbc64-try, vm-kernel-reload-mac-release-simdbc64-try, vm-kernel-linux-debug-ia32-try, vm-dartkb-linux-debug-simarm64-try, vm-kernel-win-debug-ia32-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/101825
Commit-Queue: Samir Jindel <[email protected]>
Reviewed-by: Aart Bik <[email protected]>
Reviewed-by: Daco Harkes <[email protected]>

Author: sjindel-google

runtime/lib/ffi.cc

@@ -552,11 +552,12 @@ static uword CompileNativeCallback(const Function& c_signature,
                                    const Function& dart_target) {
 #if defined(DART_PRECOMPILED_RUNTIME) || defined(DART_PRECOMPILER)
   UNREACHABLE();
-#elif !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32)
+#elif !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32) &&               \
+    !defined(TARGET_ARCH_ARM64)
   // https://github.com/dart-lang/sdk/issues/35774
   // FFI is supported, but callbacks are not.
   Exceptions::ThrowUnsupportedError(
-      "FFI callbacks are currently supported on Intel only.");
+      "FFI callbacks are currently supported on Intel and 64-bit ARM only.");
 #else
   Thread* const thread = Thread::Current();
   const int32_t callback_id = thread->AllocateFfiCallbackId();

runtime/vm/compiler/assembler/assembler_arm64.cc

@@ -1164,6 +1164,23 @@ void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) {
   }
 }
 
+void Assembler::EmitEntryFrameVerification() {
+#if defined(DEBUG)
+  Label done;
+  ASSERT(!constant_pool_allowed());
+  LoadImmediate(TMP,
+                compiler::target::frame_layout.exit_link_slot_from_entry_fp *
+                    compiler::target::kWordSize);
+  add(TMP, TMP, Operand(FPREG));
+  cmp(TMP, Operand(SPREG));
+  b(&done, EQ);
+
+  Breakpoint();
+
+  Bind(&done);
+#endif
+}
+
 void Assembler::RestoreCodePointer() {
   ldr(CODE_REG, Address(FP, compiler::target::frame_layout.code_from_fp *
                                 target::kWordSize));
@@ -1295,11 +1312,12 @@ void Assembler::LeaveDartFrame(RestorePP restore_pp) {
 }
 
 void Assembler::TransitionGeneratedToNative(Register destination,
+                                            Register new_exit_rame,
                                             Register state) {
   Register addr = TMP2;
 
   // Save exit frame information to enable stack walking.
-  StoreToOffset(FPREG, THR,
+  StoreToOffset(new_exit_rame, THR,
                 compiler::target::Thread::top_exit_frame_info_offset());
 
   // Mark that the thread is executing native code.
@@ -1826,6 +1844,41 @@ void Assembler::PopRegisters(const RegisterSet& regs) {
   }
 }
 
+void Assembler::PushNativeCalleeSavedRegisters() {
+  // Save the callee-saved registers.
+  for (int i = kAbiFirstPreservedCpuReg; i <= kAbiLastPreservedCpuReg; i++) {
+    const Register r = static_cast<Register>(i);
+    // We use str instead of the Push macro because we will be pushing the PP
+    // register when it is not holding a pool-pointer since we are coming from
+    // C++ code.
+    str(r, Address(SP, -1 * target::kWordSize, Address::PreIndex));
+  }
+
+  // Save the bottom 64-bits of callee-saved V registers.
+  for (int i = kAbiFirstPreservedFpuReg; i <= kAbiLastPreservedFpuReg; i++) {
+    const VRegister r = static_cast<VRegister>(i);
+    PushDouble(r);
+  }
+}
+
+void Assembler::PopNativeCalleeSavedRegisters() {
+  // Restore the bottom 64-bits of callee-saved V registers.
+  for (int i = kAbiLastPreservedFpuReg; i >= kAbiFirstPreservedFpuReg; i--) {
+    const VRegister r = static_cast<VRegister>(i);
+    PopDouble(r);
+  }
+
+  // Restore C++ ABI callee-saved registers.
+  for (int i = kAbiLastPreservedCpuReg; i >= kAbiFirstPreservedCpuReg; i--) {
+    Register r = static_cast<Register>(i);
+    // We use ldr instead of the Pop macro because we will be popping the PP
+    // register when it is not holding a pool-pointer since we are returning to
+    // C++ code. We also skip the dart stack pointer SP, since we are still
+    // using it as the stack pointer.
+    ldr(r, Address(SP, 1 * target::kWordSize, Address::PostIndex));
+  }
+}
+
 }  // namespace compiler
 
 }  // namespace dart

runtime/vm/compiler/assembler/assembler_arm64.h

@@ -438,6 +438,12 @@ class Assembler : public AssemblerBase {
   void PushRegisters(const RegisterSet& registers);
   void PopRegisters(const RegisterSet& registers);
 
+  // Push all registers which are callee-saved according to the ARM64 ABI.
+  void PushNativeCalleeSavedRegisters();
+
+  // Pop all registers which are callee-saved according to the ARM64 ABI.
+  void PopNativeCalleeSavedRegisters();
+
   void MoveRegister(Register rd, Register rn) {
     if (rd != rn) {
       mov(rd, rn);
@@ -481,6 +487,11 @@ class Assembler : public AssemblerBase {
 
   void ReserveAlignedFrameSpace(intptr_t frame_space);
 
+  // In debug mode, this generates code to check that:
+  //   FP + kExitLinkSlotFromEntryFp == SP
+  // or triggers breakpoint otherwise.
+  void EmitEntryFrameVerification();
+
   // Instruction pattern from entrypoint is used in Dart frame prologs
   // to set up the frame and save a PC which can be used to figure out the
   // RawInstruction object corresponding to the code running in the frame.
@@ -1524,9 +1535,13 @@ class Assembler : public AssemblerBase {
   void LeaveFrame();
   void Ret() { ret(LR); }
 
-  // These require that CSP and SP are equal and aligned.
-  // These require a scratch register (in addition to TMP/TMP2).
+  // Emit code to transition between generated mode and native mode.
+  //
+  // These require that CSP and SP are equal and aligned and require a scratch
+  // register (in addition to TMP/TMP2).
+
   void TransitionGeneratedToNative(Register destination_address,
+                                   Register new_exit_frame,
                                    Register scratch);
   void TransitionNativeToGenerated(Register scratch);
 

runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc

@@ -1160,8 +1160,8 @@ void FlowGraphCompiler::EmitOptimizedStaticCall(
   }
   // Do not use the code from the function, but let the code be patched so that
   // we can record the outgoing edges to other code.
-  GenerateStaticDartCall(deopt_id, token_pos,
-                         RawPcDescriptors::kOther, locs, function);
+  GenerateStaticDartCall(deopt_id, token_pos, RawPcDescriptors::kOther, locs,
+                         function);
   __ Drop(count_with_type_args);
 }
 
@@ -1311,6 +1311,10 @@ void FlowGraphCompiler::EmitMove(Location destination,
     if (destination.IsRegister()) {
       const intptr_t source_offset = source.ToStackSlotOffset();
       __ LoadFromOffset(destination.reg(), source.base_reg(), source_offset);
+    } else if (destination.IsFpuRegister()) {
+      const intptr_t src_offset = source.ToStackSlotOffset();
+      VRegister dst = destination.fpu_reg();
+      __ LoadDFromOffset(dst, source.base_reg(), src_offset);
     } else {
       ASSERT(destination.IsStackSlot());
       const intptr_t source_offset = source.ToStackSlotOffset();

runtime/vm/compiler/backend/il.cc

@@ -3960,7 +3960,8 @@ LocationSummary* NativeEntryInstr::MakeLocationSummary(Zone* zone,
   UNREACHABLE();
 }
 
-#if !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32)
+#if !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32) &&                 \
+    !defined(TARGET_ARCH_ARM64)
 void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
   UNREACHABLE();
 }
@@ -4079,8 +4080,8 @@ void NativeParameterInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
 #if !defined(TARGET_ARCH_DBC)
   // The native entry frame has size -kExitLinkSlotFromFp. In order to access
   // the top of stack from above the entry frame, we add a constant to account
-  // for the the two frame pointers and return address of the entry frame.
-  constexpr intptr_t kEntryFramePadding = 3;
+  // for the the two frame pointers and two return addresses of the entry frame.
+  constexpr intptr_t kEntryFramePadding = 4;
   FrameRebase rebase(/*old_base=*/SPREG, /*new_base=*/FPREG,
                      -kExitLinkSlotFromEntryFp + kEntryFramePadding);
   const Location dst = locs()->out(0);

runtime/vm/compiler/backend/il_arm64.cc

@@ -916,7 +916,7 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
   __ mov(CSP, SP);
 
   // Update information in the thread object and enter a safepoint.
-  __ TransitionGeneratedToNative(branch, temp);
+  __ TransitionGeneratedToNative(branch, FPREG, temp);
 
   __ blr(branch);
 
@@ -944,6 +944,177 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
   __ set_constant_pool_allowed(true);
 }
 
+void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
+  __ LeaveDartFrame();
+
+  // The dummy return address is in LR, no need to pop it as on Intel.
+
+  // These can be anything besides the return register (R0).
+  const Register vm_tag_reg = R1, old_exit_frame_reg = R2, tmp = R3;
+
+  __ Pop(old_exit_frame_reg);
+
+  // Restore top_resource.
+  __ Pop(tmp);
+  __ StoreToOffset(tmp, THR, compiler::target::Thread::top_resource_offset());
+
+  __ Pop(vm_tag_reg);
+
+  // Reset the exit frame info to
+  // old_exit_frame_reg *before* entering the safepoint.
+  __ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg, tmp);
+
+  __ PopNativeCalleeSavedRegisters();
+
+  // Leave the entry frame.
+  __ LeaveFrame();
+
+  // Leave the dummy frame holding the pushed arguments.
+  __ LeaveFrame();
+
+  // Restore the actual stack pointer from SPREG.
+  __ RestoreCSP();
+
+  __ Ret();
+
+  // For following blocks.
+  __ set_constant_pool_allowed(true);
+}
+
+void NativeEntryInstr::SaveArgument(FlowGraphCompiler* compiler,
+                                    Location loc) const {
+  ASSERT(!loc.IsPairLocation());
+
+  if (loc.HasStackIndex()) return;
+
+  if (loc.IsRegister()) {
+    __ Push(loc.reg());
+  } else if (loc.IsFpuRegister()) {
+    __ PushDouble(loc.fpu_reg());
+  } else {
+    UNREACHABLE();
+  }
+}
+
+void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
+  if (FLAG_precompiled_mode) {
+    UNREACHABLE();
+  }
+
+  // Constant pool cannot be used until we enter the actual Dart frame.
+  __ set_constant_pool_allowed(false);
+
+  __ Bind(compiler->GetJumpLabel(this));
+
+  // We don't use the regular stack pointer in ARM64, so we have to copy the
+  // native stack pointer into the Dart stack pointer.
+  __ SetupDartSP();
+
+  // Create a dummy frame holding the pushed arguments. This simplifies
+  // NativeReturnInstr::EmitNativeCode.
+  __ EnterFrame(0);
+
+  // Save the argument registers, in reverse order.
+  for (intptr_t i = argument_locations_->length(); i-- > 0;) {
+    SaveArgument(compiler, argument_locations_->At(i));
+  }
+
+  // Enter the entry frame.
+  __ EnterFrame(0);
+
+  // Save a space for the code object.
+  __ PushImmediate(0);
+
+  __ PushNativeCalleeSavedRegisters();
+
+  // Load the thread object.
+  // TODO(35765): Fix linking issue on AOT.
+  // TOOD(35934): Exclude native callbacks from snapshots.
+  //
+  // Create another frame to align the frame before continuing in "native" code.
+  {
+    __ EnterFrame(0);
+    __ ReserveAlignedFrameSpace(0);
+
+    __ LoadImmediate(
+        R0, reinterpret_cast<int64_t>(DLRT_GetThreadForNativeCallback));
+    __ blr(R0);
+    __ mov(THR, R0);
+
+    __ LeaveFrame();
+  }
+
+  // Refresh write barrier mask.
+  __ ldr(BARRIER_MASK,
+         Address(THR, compiler::target::Thread::write_barrier_mask_offset()));
+
+  // Save the current VMTag on the stack.
+  __ LoadFromOffset(R0, THR, compiler::target::Thread::vm_tag_offset());
+  __ Push(R0);
+
+  // Save the top resource.
+  __ LoadFromOffset(R0, THR, compiler::target::Thread::top_resource_offset());
+  __ Push(R0);
+  __ StoreToOffset(ZR, THR, compiler::target::Thread::top_resource_offset());
+
+  // Save the top exit frame info. We don't set it to 0 yet in Thread because we
+  // need to leave the safepoint first.
+  __ LoadFromOffset(R0, THR,
+                    compiler::target::Thread::top_exit_frame_info_offset());
+  __ Push(R0);
+
+  // In debug mode, verify that we've pushed the top exit frame info at the
+  // correct offset from FP.
+  __ EmitEntryFrameVerification();
+
+  // TransitionNativeToGenerated will reset top exit frame info to 0 *after*
+  // leaving the safepoint.
+  __ TransitionNativeToGenerated(R0);
+
+  // Now that the safepoint has ended, we can touch Dart objects without
+  // handles.
+
+  // Otherwise we'll clobber the argument sent from the caller.
+  ASSERT(CallingConventions::ArgumentRegisters[0] != TMP &&
+         CallingConventions::ArgumentRegisters[0] != TMP2 &&
+         CallingConventions::ArgumentRegisters[0] != R1);
+  __ LoadImmediate(CallingConventions::ArgumentRegisters[0], callback_id_);
+  __ LoadFromOffset(
+      R1, THR,
+      compiler::target::Thread::verify_callback_isolate_entry_point_offset());
+  __ blr(R1);
+
+  // Load the code object.
+  __ LoadFromOffset(R0, THR, compiler::target::Thread::callback_code_offset());
+  __ LoadFieldFromOffset(R0, R0,
+                         compiler::target::GrowableObjectArray::data_offset());
+  __ LoadFieldFromOffset(CODE_REG, R0,
+                         compiler::target::Array::data_offset() +
+                             callback_id_ * compiler::target::kWordSize);
+
+  // Put the code object in the reserved slot.
+  __ StoreToOffset(CODE_REG, FPREG,
+                   kPcMarkerSlotFromFp * compiler::target::kWordSize);
+  if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
+    __ ldr(PP,
+           Address(THR, compiler::target::Thread::global_object_pool_offset()));
+    __ sub(PP, PP, Operand(kHeapObjectTag));  // Pool in PP is untagged!
+  } else {
+    // We now load the pool pointer (PP) with a GC safe value as we are about to
+    // invoke dart code. We don't need a real object pool here.
+    // Smi zero does not work because ARM64 assumes PP to be untagged.
+    __ LoadObject(PP, compiler::NullObject());
+  }
+
+  // Load a dummy return address which suggests that we are inside of
+  // InvokeDartCodeStub. This is how the stack walker detects an entry frame.
+  __ LoadFromOffset(LR, THR,
+                    compiler::target::Thread::invoke_dart_code_stub_offset());
+  __ LoadFieldFromOffset(LR, LR, compiler::target::Code::entry_point_offset());
+
+  FunctionEntryInstr::EmitNativeCode(compiler);
+}
+
 LocationSummary* OneByteStringFromCharCodeInstr::MakeLocationSummary(
     Zone* zone,
     bool opt) const {

runtime/vm/compiler/backend/il_x64.cc

@@ -1012,7 +1012,9 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
     SaveArgument(compiler, argument_locations_->At(i));
   }
 
-  // Enter the entry frame.
+  // Enter the entry frame. Push a dummy return address for consistency with
+  // EnterFrame on ARM(64).
+  __ PushImmediate(Immediate(0));
   __ EnterFrame(0);
 
   // Save a space for the code object.

runtime/vm/compiler/frontend/scope_builder.cc

@@ -392,6 +392,7 @@ ScopeBuildingResult* ScopeBuilder::BuildScopes() {
       // Callbacks need try/catch variables.
       if (function.IsFfiTrampoline() &&
           function.FfiCallbackTarget() != Function::null()) {
+        current_function_async_marker_ = FunctionNodeHelper::kSync;
         ++depth_.try_;
         AddTryVariables();
         --depth_.try_;