8193260: AArch64: JVMCI: Implement trampoline calls

Reviewed-by: adinn

src/hotspot/cpu/aarch64/assembler_aarch64.hpp

@@ -848,7 +848,7 @@ public:
   // architecture.  In debug mode we shrink it in order to test
   // trampolines, but not so small that branches in the interpreter
   // are out of range.
-  static const unsigned long branch_range = INCLUDE_JVMCI ? 128 * M : NOT_DEBUG(128 * M) DEBUG_ONLY(2 * M);
+  static const unsigned long branch_range = NOT_DEBUG(128 * M) DEBUG_ONLY(2 * M);
 
   static bool reachable_from_branch_at(address branch, address target) {
     return uabs(target - branch) < branch_range;

src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp

@@ -71,6 +71,13 @@ int CompiledStaticCall::to_interp_stub_size() {
   return 7 * NativeInstruction::instruction_size;
 }
 
+int CompiledStaticCall::to_trampoline_stub_size() {
+  // Somewhat pessimistically, we count 3 instructions here (although
+  // there are only two) because we sometimes emit an alignment nop.
+  // Trampoline stubs are always word aligned.
+  return 3 * NativeInstruction::instruction_size + wordSize;
+}
+
 // Relocation entries for call stub, compiled java to interpreter.
 int CompiledStaticCall::reloc_to_interp_stub() {
   return 4; // 3 in emit_to_interp_stub + 1 in emit_call

src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp

@@ -109,7 +109,7 @@ void CodeInstaller::pd_relocate_ForeignCall(NativeInstruction* inst, jlong forei
   TRACE_jvmci_3("relocating (foreign call) at " PTR_FORMAT, p2i(inst));
 }
 
-void CodeInstaller::pd_relocate_JavaMethod(Handle hotspot_method, jint pc_offset, TRAPS) {
+void CodeInstaller::pd_relocate_JavaMethod(CodeBuffer &cbuf, Handle hotspot_method, jint pc_offset, TRAPS) {
 #ifdef ASSERT
   Method* method = NULL;
   // we need to check, this might also be an unresolved method
@@ -124,22 +124,22 @@ void CodeInstaller::pd_relocate_JavaMethod(Handle hotspot_method, jint pc_offset
     case INVOKEINTERFACE: {
       assert(method == NULL || !method->is_static(), "cannot call static method with invokeinterface");
       NativeCall* call = nativeCall_at(_instructions->start() + pc_offset);
-      call->set_destination(SharedRuntime::get_resolve_virtual_call_stub());
       _instructions->relocate(call->instruction_address(), virtual_call_Relocation::spec(_invoke_mark_pc));
+      call->trampoline_jump(cbuf, SharedRuntime::get_resolve_virtual_call_stub());
       break;
     }
     case INVOKESTATIC: {
       assert(method == NULL || method->is_static(), "cannot call non-static method with invokestatic");
       NativeCall* call = nativeCall_at(_instructions->start() + pc_offset);
-      call->set_destination(SharedRuntime::get_resolve_static_call_stub());
       _instructions->relocate(call->instruction_address(), relocInfo::static_call_type);
+      call->trampoline_jump(cbuf, SharedRuntime::get_resolve_static_call_stub());
       break;
     }
     case INVOKESPECIAL: {
       assert(method == NULL || !method->is_static(), "cannot call static method with invokespecial");
       NativeCall* call = nativeCall_at(_instructions->start() + pc_offset);
-      call->set_destination(SharedRuntime::get_resolve_opt_virtual_call_stub());
       _instructions->relocate(call->instruction_address(), relocInfo::opt_virtual_call_type);
+      call->trampoline_jump(cbuf, SharedRuntime::get_resolve_opt_virtual_call_stub());
       break;
     }
     default:

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -801,7 +801,7 @@ address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset,
   assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
 
   end_a_stub();
-  return stub;
+  return stub_start_addr;
 }
 
 address MacroAssembler::ic_call(address entry, jint method_index) {

src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp

@@ -367,3 +367,24 @@ void NativeCallTrampolineStub::set_destination(address new_destination) {
   set_ptr_at(data_offset, new_destination);
   OrderAccess::release();
 }
+
+// Generate a trampoline for a branch to dest.  If there's no need for a
+// trampoline, simply patch the call directly to dest.
+address NativeCall::trampoline_jump(CodeBuffer &cbuf, address dest) {
+  MacroAssembler a(&cbuf);
+  address stub = NULL;
+
+  if (a.far_branches()
+      && ! is_NativeCallTrampolineStub_at(instruction_address() + displacement())) {
+    stub = a.emit_trampoline_stub(instruction_address() - cbuf.insts()->start(), dest);
+  }
+
+  if (stub == NULL) {
+    // If we generated no stub, patch this call directly to dest.
+    // This will happen if we don't need far branches or if there
+    // already was a trampoline.
+    set_destination(dest);
+  }
+
+  return stub;
+}

src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp

@@ -61,7 +61,7 @@ class NativeInstruction VALUE_OBJ_CLASS_SPEC {
     return uint_at(0);
   }
 
-  bool is_blr()                      const { return (encoding() & 0xfffffc1f) == 0xd63f0000; }
+  bool is_blr()                      const { return (encoding() & 0xff9ffc1f) == 0xd61f0000; } // blr(register) or br(register)
   bool is_adr_aligned()              const { return (encoding() & 0xff000000) == 0x10000000; } // adr Xn, <label>, where label is aligned to 4 bytes (address of instruction).
 
   inline bool is_nop();
@@ -143,8 +143,9 @@ inline NativeInstruction* nativeInstruction_at(uint32_t *address) {
 }
 
 inline NativeCall* nativeCall_at(address address);
-// The NativeCall is an abstraction for accessing/manipulating native call imm32/rel32off
-// instructions (used to manipulate inline caches, primitive & dll calls, etc.).
+// The NativeCall is an abstraction for accessing/manipulating native
+// call instructions (used to manipulate inline caches, primitive &
+// DSO calls, etc.).
 
 class NativeCall: public NativeInstruction {
  public:
@@ -155,7 +156,6 @@ class NativeCall: public NativeInstruction {
     return_address_offset       =    4
   };
 
-  enum { cache_line_size = BytesPerWord };  // conservative estimate!
   address instruction_address() const       { return addr_at(instruction_offset); }
   address next_instruction_address() const  { return addr_at(return_address_offset); }
   int   displacement() const                { return (int_at(displacement_offset) << 6) >> 4; }
@@ -206,6 +206,7 @@ class NativeCall: public NativeInstruction {
   void set_destination_mt_safe(address dest, bool assert_lock = true);
 
   address get_trampoline();
+  address trampoline_jump(CodeBuffer &cbuf, address dest);
 };
 
 inline NativeCall* nativeCall_at(address address) {

src/hotspot/cpu/sparc/compiledIC_sparc.cpp

@@ -73,6 +73,11 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
 }
 #undef __
 
+int CompiledStaticCall::to_trampoline_stub_size() {
+  // SPARC doesn't use trampolines.
+  return 0;
+}
+
 int CompiledStaticCall::to_interp_stub_size() {
   // This doesn't need to be accurate but it must be larger or equal to
   // the real size of the stub.

src/hotspot/cpu/sparc/jvmciCodeInstaller_sparc.cpp

@@ -115,7 +115,7 @@ void CodeInstaller::pd_relocate_ForeignCall(NativeInstruction* inst, jlong forei
   TRACE_jvmci_3("relocating (foreign call) at " PTR_FORMAT, p2i(inst));
 }
 
-void CodeInstaller::pd_relocate_JavaMethod(Handle hotspot_method, jint pc_offset, TRAPS) {
+void CodeInstaller::pd_relocate_JavaMethod(CodeBuffer &, Handle hotspot_method, jint pc_offset, TRAPS) {
 #ifdef ASSERT
   Method* method = NULL;
   // we need to check, this might also be an unresolved method

src/hotspot/cpu/x86/compiledIC_x86.cpp

@@ -73,6 +73,11 @@ int CompiledStaticCall::to_interp_stub_size() {
          LP64_ONLY(15);  // movq (1+1+8); jmp (1+4)
 }
 
+int CompiledStaticCall::to_trampoline_stub_size() {
+  // x86 doesn't use trampolines.
+  return 0;
+}
+
 // Relocation entries for call stub, compiled java to interpreter.
 int CompiledStaticCall::reloc_to_interp_stub() {
   return 4; // 3 in emit_to_interp_stub + 1 in emit_call

src/hotspot/cpu/x86/jvmciCodeInstaller_x86.cpp

@@ -144,7 +144,7 @@ void CodeInstaller::pd_relocate_ForeignCall(NativeInstruction* inst, jlong forei
   TRACE_jvmci_3("relocating (foreign call)  at " PTR_FORMAT, p2i(inst));
 }
 
-void CodeInstaller::pd_relocate_JavaMethod(Handle hotspot_method, jint pc_offset, TRAPS) {
+void CodeInstaller::pd_relocate_JavaMethod(CodeBuffer &, Handle hotspot_method, jint pc_offset, TRAPS) {
 #ifdef ASSERT
   Method* method = NULL;
   // we need to check, this might also be an unresolved method

src/hotspot/share/code/compiledIC.hpp

@@ -344,6 +344,7 @@ class CompiledStaticCall : public ResourceObj {
   // Code
   static address emit_to_interp_stub(CodeBuffer &cbuf, address mark = NULL);
   static int to_interp_stub_size();
+  static int to_trampoline_stub_size();
   static int reloc_to_interp_stub();
   static void emit_to_aot_stub(CodeBuffer &cbuf, address mark = NULL);
   static int to_aot_stub_size();

src/hotspot/share/jvmci/jvmciCodeInstaller.cpp

@@ -696,6 +696,7 @@ int CodeInstaller::estimate_stubs_size(TRAPS) {
   // Estimate the number of static and aot call stubs that might be emitted.
   int static_call_stubs = 0;
   int aot_call_stubs = 0;
+  int trampoline_stubs = 0;
   objArrayOop sites = this->sites();
   for (int i = 0; i < sites->length(); i++) {
     oop site = sites->obj_at(i);
@@ -707,8 +708,18 @@ int CodeInstaller::estimate_stubs_size(TRAPS) {
             JVMCI_ERROR_0("expected Integer id, got %s", id_obj->klass()->signature_name());
           }
           jint id = id_obj->int_field(java_lang_boxing_object::value_offset_in_bytes(T_INT));
-          if (id == INVOKESTATIC || id == INVOKESPECIAL) {
+          switch (id) {
+          case INVOKEINTERFACE:
+          case INVOKEVIRTUAL:
+            trampoline_stubs++;
+            break;
+          case INVOKESTATIC:
+          case INVOKESPECIAL:
             static_call_stubs++;
+            trampoline_stubs++;
+            break;
+          default:
+            break;
           }
         }
       }
@@ -723,6 +734,7 @@ int CodeInstaller::estimate_stubs_size(TRAPS) {
     }
   }
   int size = static_call_stubs * CompiledStaticCall::to_interp_stub_size();
+  size += trampoline_stubs * CompiledStaticCall::to_trampoline_stub_size();
 #if INCLUDE_AOT
   size += aot_call_stubs * CompiledStaticCall::to_aot_stub_size();
 #endif
@@ -1168,7 +1180,7 @@ void CodeInstaller::site_Call(CodeBuffer& buffer, jint pc_offset, Handle site, T
     }
 
     TRACE_jvmci_3("method call");
-    CodeInstaller::pd_relocate_JavaMethod(hotspot_method, pc_offset, CHECK);
+    CodeInstaller::pd_relocate_JavaMethod(buffer, hotspot_method, pc_offset, CHECK);
     if (_next_call_type == INVOKESTATIC || _next_call_type == INVOKESPECIAL) {
       // Need a static call stub for transitions from compiled to interpreted.
       CompiledStaticCall::emit_to_interp_stub(buffer, _instructions->start() + pc_offset);
@@ -1279,4 +1291,3 @@ void CodeInstaller::site_Mark(CodeBuffer& buffer, jint pc_offset, Handle site, T
     }
   }
 }
-

src/hotspot/share/jvmci/jvmciCodeInstaller.hpp

@@ -185,7 +185,7 @@ private:
   void pd_patch_MetaspaceConstant(int pc_offset, Handle constant, TRAPS);
   void pd_patch_DataSectionReference(int pc_offset, int data_offset, TRAPS);
   void pd_relocate_ForeignCall(NativeInstruction* inst, jlong foreign_call_destination, TRAPS);
-  void pd_relocate_JavaMethod(Handle method, jint pc_offset, TRAPS);
+  void pd_relocate_JavaMethod(CodeBuffer &cbuf, Handle method, jint pc_offset, TRAPS);
   void pd_relocate_poll(address pc, jint mark, TRAPS);
 
   objArrayOop sites() { return (objArrayOop) JNIHandles::resolve(_sites_handle); }