8229971: Arm32: implementation for Thread-local handshakes

Reviewed-by: rehn, dsamersoff

src/hotspot/cpu/arm/arm.ad

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -351,10 +351,7 @@ void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 
   // If this does safepoint polling, then do it here
   if (do_polling() && ra_->C->is_method_compilation()) {
-    // mov_slow here is usually one or two instruction
-    __ mov_address(Rtemp, (address)os::get_polling_page());
-    __ relocate(relocInfo::poll_return_type);
-    __ ldr(Rtemp, Address(Rtemp));
+    __ read_polling_page(Rtemp, relocInfo::poll_return_type);
   }
 }
 

src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -285,23 +285,16 @@ int LIR_Assembler::emit_deopt_handler() {
 void LIR_Assembler::return_op(LIR_Opr result) {
   // Pop the frame before safepoint polling
   __ remove_frame(initial_frame_size_in_bytes());
-
-  // mov_slow here is usually one or two instruction
-  __ mov_address(Rtemp, os::get_polling_page());
-  __ relocate(relocInfo::poll_return_type);
-  __ ldr(Rtemp, Address(Rtemp));
+  __ read_polling_page(Rtemp, relocInfo::poll_return_type);
   __ ret();
 }
 
-
 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
-  __ mov_address(Rtemp, os::get_polling_page());
   if (info != NULL) {
     add_debug_info_for_branch(info);
   }
   int offset = __ offset();
-  __ relocate(relocInfo::poll_type);
-  __ ldr(Rtemp, Address(Rtemp));
+  __ read_polling_page(Rtemp, relocInfo::poll_type);
   return offset;
 }
 

src/hotspot/cpu/arm/globalDefinitions_arm.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -62,4 +62,6 @@ const bool HaveVFP = true;
 #endif
 #endif
 
+#define THREAD_LOCAL_POLL
+
 #endif // CPU_ARM_GLOBALDEFINITIONS_ARM_HPP

src/hotspot/cpu/arm/interp_masm_arm.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -42,6 +42,7 @@
 #include "runtime/basicLock.hpp"
 #include "runtime/biasedLocking.hpp"
 #include "runtime/frame.inline.hpp"
+#include "runtime/safepointMechanism.hpp"
 #include "runtime/sharedRuntime.hpp"
 
 //--------------------------------------------------------------------
@@ -556,7 +557,7 @@ void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
 
 void InterpreterMacroAssembler::dispatch_base(TosState state,
                                               DispatchTableMode table_mode,
-                                              bool verifyoop) {
+                                              bool verifyoop, bool generate_poll) {
   if (VerifyActivationFrameSize) {
     Label L;
     sub(Rtemp, FP, SP);
@@ -571,6 +572,18 @@ void InterpreterMacroAssembler::dispatch_base(TosState state,
     interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
   }
 
+  Label safepoint;
+  address* const safepoint_table = Interpreter::safept_table(state);
+  address* const table           = Interpreter::dispatch_table(state);
+  bool needs_thread_local_poll = generate_poll &&
+    SafepointMechanism::uses_thread_local_poll() && table != safepoint_table;
+
+  if (needs_thread_local_poll) {
+    NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
+    ldr(Rtemp, Address(Rthread, Thread::polling_page_offset()));
+    tbnz(Rtemp, exact_log2(SafepointMechanism::poll_bit()), safepoint);
+  }
+
   if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) {
     zap_high_non_significant_bits(R0_tos);
   }
@@ -600,12 +613,18 @@ void InterpreterMacroAssembler::dispatch_base(TosState state,
     indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
   }
 
+  if (needs_thread_local_poll) {
+    bind(safepoint);
+    lea(Rtemp, ExternalAddress((address)safepoint_table));
+    indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
+  }
+
   nop(); // to avoid filling CPU pipeline with invalid instructions
   nop();
 }
 
-void InterpreterMacroAssembler::dispatch_only(TosState state) {
-  dispatch_base(state, DispatchDefault);
+void InterpreterMacroAssembler::dispatch_only(TosState state, bool generate_poll) {
+  dispatch_base(state, DispatchDefault, true, generate_poll);
 }
 
 
@@ -617,10 +636,10 @@ void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
   dispatch_base(state, DispatchNormal, false);
 }
 
-void InterpreterMacroAssembler::dispatch_next(TosState state, int step) {
+void InterpreterMacroAssembler::dispatch_next(TosState state, int step, bool generate_poll) {
   // load next bytecode and advance Rbcp
   ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed));
-  dispatch_base(state, DispatchDefault);
+  dispatch_base(state, DispatchDefault, true, generate_poll);
 }
 
 void InterpreterMacroAssembler::narrow(Register result) {

src/hotspot/cpu/arm/interp_masm_arm.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -54,7 +54,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
 
   // base routine for all dispatches
   typedef enum { DispatchDefault, DispatchNormal } DispatchTableMode;
-  void dispatch_base(TosState state, DispatchTableMode table_mode, bool verifyoop = true);
+  void dispatch_base(TosState state, DispatchTableMode table_mode, bool verifyoop = true, bool generate_poll = false);
 
  public:
   InterpreterMacroAssembler(CodeBuffer* code);
@@ -160,10 +160,10 @@ class InterpreterMacroAssembler: public MacroAssembler {
   // Dispatching
   void dispatch_prolog(TosState state, int step = 0);
   void dispatch_epilog(TosState state, int step = 0);
-  void dispatch_only(TosState state);                      // dispatch by R3_bytecode
-  void dispatch_only_normal(TosState state);               // dispatch normal table by R3_bytecode
+  void dispatch_only(TosState state, bool generate_poll = false);  // dispatch by R3_bytecode
+  void dispatch_only_normal(TosState state);                       // dispatch normal table by R3_bytecode
   void dispatch_only_noverify(TosState state);
-  void dispatch_next(TosState state, int step = 0);        // load R3_bytecode from [Rbcp + step] and dispatch by R3_bytecode
+  void dispatch_next(TosState state, int step = 0, bool generate_poll = false); // load R3_bytecode from [Rbcp + step] and dispatch by R3_bytecode
 
   // jump to an invoked target
   void prepare_to_jump_from_interpreted();

src/hotspot/cpu/arm/macroAssembler_arm.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2053,4 +2053,32 @@ void MacroAssembler::fast_unlock(Register Roop, Register Rbox, Register Rscratch
   bind(done);
 
 }
+
+void MacroAssembler::safepoint_poll(Register tmp1, Label& slow_path) {
+  if (SafepointMechanism::uses_thread_local_poll()) {
+    ldr_u32(tmp1, Address(Rthread, Thread::polling_page_offset()));
+    tst(tmp1, exact_log2(SafepointMechanism::poll_bit()));
+    b(slow_path, eq);
+  } else {
+    ldr_global_s32(tmp1, SafepointSynchronize::address_of_state());
+    cmp(tmp1, SafepointSynchronize::_not_synchronized);
+    b(slow_path, ne);
+  }
+}
+
+void MacroAssembler::get_polling_page(Register dest) {
+  if (SafepointMechanism::uses_thread_local_poll()) {
+    ldr(dest, Address(Rthread, Thread::polling_page_offset()));
+  } else {
+    mov_address(dest, os::get_polling_page());
+  }
+}
+
+void MacroAssembler::read_polling_page(Register dest, relocInfo::relocType rtype) {
+  get_polling_page(dest);
+  relocate(rtype);
+  ldr(dest, Address(dest));
+}
+
+
 #endif // COMPILER2

src/hotspot/cpu/arm/macroAssembler_arm.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1074,7 +1074,9 @@ public:
   void fast_unlock(Register obj, Register box, Register scratch, Register scratch2);
 #endif
 
-
+  void safepoint_poll(Register tmp1, Label& slow_path);
+  void get_polling_page(Register dest);
+  void read_polling_page(Register dest, relocInfo::relocType rtype);
 };
 
 

src/hotspot/cpu/arm/sharedRuntime_arm.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,6 +34,7 @@
 #include "oops/compiledICHolder.hpp"
 #include "oops/klass.inline.hpp"
 #include "runtime/sharedRuntime.hpp"
+#include "runtime/safepointMechanism.hpp"
 #include "runtime/vframeArray.hpp"
 #include "utilities/align.hpp"
 #include "vmreg_arm.inline.hpp"
@@ -1219,20 +1220,18 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   }
 
   // Do a safepoint check while thread is in transition state
-  InlinedAddress safepoint_state(SafepointSynchronize::address_of_state());
   Label call_safepoint_runtime, return_to_java;
   __ mov(Rtemp, _thread_in_native_trans);
-  __ ldr_literal(R2, safepoint_state);
   __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
 
   // make sure the store is observed before reading the SafepointSynchronize state and further mem refs
   __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
 
-  __ ldr_s32(R2, Address(R2));
+  __ safepoint_poll(R2, call_safepoint_runtime);
   __ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
-  __ cmp(R2, SafepointSynchronize::_not_synchronized);
-  __ cond_cmp(R3, 0, eq);
+  __ cmp(R3, 0);
   __ b(call_safepoint_runtime, ne);
+
   __ bind(return_to_java);
 
   // Perform thread state transition and reguard stack yellow pages if needed
@@ -1303,8 +1302,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   pop_result_registers(masm, ret_type);
   __ b(return_to_java);
 
-  __ bind_literal(safepoint_state);
-
   // Reguard stack pages. Save native results around a call to C runtime.
   __ bind(reguard);
   push_result_registers(masm, ret_type);
@@ -1806,15 +1803,29 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t
   oop_maps->add_gc_map(pc_offset, map);
   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
 
-  // Check for pending exception
-  __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
-  __ cmp(Rtemp, 0);
-
   if (!cause_return) {
+    if (SafepointMechanism::uses_thread_local_poll()) {
+      // If our stashed return pc was modified by the runtime we avoid touching it
+      __ ldr(R3_tmp, Address(Rthread, JavaThread::saved_exception_pc_offset()));
+      __ ldr(R2_tmp, Address(SP, RegisterSaver::LR_offset * wordSize));
+      __ cmp(R2_tmp, R3_tmp);
+      // Adjust return pc forward to step over the safepoint poll instruction
+      __ add(R2_tmp, R2_tmp, 4, eq);
+      __ str(R2_tmp, Address(SP, RegisterSaver::LR_offset * wordSize), eq);
+    }
+
+    // Check for pending exception
+    __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
+    __ cmp(Rtemp, 0);
+
     RegisterSaver::restore_live_registers(masm, false);
     __ pop(PC, eq);
     __ pop(Rexception_pc);
   } else {
+    // Check for pending exception
+    __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
+    __ cmp(Rtemp, 0);
+
     RegisterSaver::restore_live_registers(masm);
     __ bx(LR, eq);
     __ mov(Rexception_pc, LR);

src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -959,8 +959,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
     // Force this write out before the read below
   __ membar(MacroAssembler::StoreLoad, Rtemp);
 
-  __ ldr_global_s32(Rtemp, SafepointSynchronize::address_of_state());
-
   // Protect the return value in the interleaved code: save it to callee-save registers.
   __ mov(Rsaved_result_lo, R0);
   __ mov(Rsaved_result_hi, R1);
@@ -973,12 +971,16 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
 #endif // __ABI_HARD__
 
   {
-    __ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
-    __ cmp(Rtemp, SafepointSynchronize::_not_synchronized);
-    __ cond_cmp(R3, 0, eq);
+  Label call, skip_call;
+  __ safepoint_poll(Rtemp, call);
+  __ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
+  __ cmp(R3, 0);
+  __ b(skip_call, eq);
+  __ bind(call);
+  __ mov(R0, Rthread);
+  __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none);
+  __ bind(skip_call);
 
-  __ mov(R0, Rthread, ne);
-  __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none, ne);
 #if R9_IS_SCRATCHED
   __ restore_method();
 #endif

src/hotspot/cpu/arm/templateTable_arm.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2168,7 +2168,7 @@ void TemplateTable::branch(bool is_jsr, bool is_wide) {
   }
 
   // continue with the bytecode @ target
-  __ dispatch_only(vtos);
+  __ dispatch_only(vtos, true);
 
   if (UseLoopCounter) {
     if (ProfileInterpreter) {
@@ -2362,7 +2362,7 @@ void TemplateTable::tableswitch() {
 
   // load the next bytecode to R3_bytecode and advance Rbcp
   __ ldrb(R3_bytecode, Address(Rbcp, Roffset, lsl, 0, pre_indexed));
-  __ dispatch_only(vtos);
+  __ dispatch_only(vtos, true);
 
 }
 
@@ -2439,7 +2439,7 @@ void TemplateTable::fast_linearswitch() {
 
   // load the next bytecode to R3_bytecode and advance Rbcp
   __ ldrb(R3_bytecode, Address(Rbcp, Roffset, lsl, 0, pre_indexed));
-  __ dispatch_only(vtos);
+  __ dispatch_only(vtos, true);
 }
 
 
@@ -2533,7 +2533,7 @@ void TemplateTable::fast_binaryswitch() {
   __ profile_switch_case(R0, i, R1, i);
   __ byteswap_u32(offset, temp1, temp2);
   __ ldrb(R3_bytecode, Address(Rbcp, offset, lsl, 0, pre_indexed));
-  __ dispatch_only(vtos);
+  __ dispatch_only(vtos, true);
 
   // default case
   __ bind(default_case);
@@ -2541,7 +2541,7 @@ void TemplateTable::fast_binaryswitch() {
   __ ldr_s32(offset, Address(array, -2*BytesPerInt));
   __ byteswap_u32(offset, temp1, temp2);
   __ ldrb(R3_bytecode, Address(Rbcp, offset, lsl, 0, pre_indexed));
-  __ dispatch_only(vtos);
+  __ dispatch_only(vtos, true);
 }