8221554: aarch64 cross-modifying code

Reviewed-by: rehn, aph

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -2922,7 +2922,6 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* arg
   if (info != NULL) {
     add_call_info_here(info);
   }
-  __ maybe_isb();
 }
 
 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {

src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp

@@ -80,7 +80,6 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, addre
   pop(r0, sp);
 #endif
   reset_last_Java_frame(true);
-  maybe_isb();
 
   // check for pending exceptions
   { Label L;

src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp

@@ -1617,7 +1617,7 @@ void InterpreterMacroAssembler::call_VM_base(Register oop_result,
     Label L;
     ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
     cbz(rscratch1, L);
-    stop("InterpreterMacroAssembler::call_VM_leaf_base:"
+    stop("InterpreterMacroAssembler::call_VM_base:"
          " last_sp != NULL");
     bind(L);
   }

src/hotspot/cpu/aarch64/jniFastGetField_aarch64.cpp

@@ -159,7 +159,6 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
     __ enter();
     __ lea(rscratch1, ExternalAddress(slow_case_addr));
     __ blr(rscratch1);
-    __ maybe_isb();
     __ leave();
     __ ret(lr);
   }

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -1381,7 +1381,6 @@ void MacroAssembler::call_VM_leaf_base(address entry_point,
     bind(*retaddr);
 
   ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize)));
-  maybe_isb();
 }
 
 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
@@ -4387,10 +4386,15 @@ void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype)
 // Read the polling page.  The address of the polling page must
 // already be in r.
 address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) {
+  address mark;
+  {
     InstructionMark im(this);
     code_section()->relocate(inst_mark(), rtype);
     ldrw(zr, Address(r, 0));
-  return inst_mark();
+    mark = inst_mark();
+  }
+  verify_cross_modify_fence_not_required();
+  return mark;
 }
 
 void MacroAssembler::adrp(Register reg1, const Address &dest, uint64_t &byte_offset) {
@@ -4455,6 +4459,7 @@ void MacroAssembler::build_frame(int framesize) {
       sub(sp, sp, rscratch1);
     }
   }
+  verify_cross_modify_fence_not_required();
 }
 
 void MacroAssembler::remove_frame(int framesize) {
@@ -5315,3 +5320,29 @@ void MacroAssembler::verify_ptrue() {
   stop("Error: the preserved predicate register (p7) elements are not all true");
   bind(verify_ok);
 }
+
+void MacroAssembler::safepoint_isb() {
+  isb();
+#ifndef PRODUCT
+  if (VerifyCrossModifyFence) {
+    // Clear the thread state.
+    strb(zr, Address(rthread, in_bytes(JavaThread::requires_cross_modify_fence_offset())));
+  }
+#endif
+}
+
+#ifndef PRODUCT
+void MacroAssembler::verify_cross_modify_fence_not_required() {
+  if (VerifyCrossModifyFence) {
+    // Check if thread needs a cross modify fence.
+    ldrb(rscratch1, Address(rthread, in_bytes(JavaThread::requires_cross_modify_fence_offset())));
+    Label fence_not_required;
+    cbz(rscratch1, fence_not_required);
+    // If it does then fail.
+    lea(rscratch1, CAST_FROM_FN_PTR(address, JavaThread::verify_cross_modify_fence_failure));
+    mov(c_rarg0, rthread);
+    blr(rscratch1);
+    bind(fence_not_required);
+  }
+}
+#endif

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -1315,8 +1315,9 @@ public:
                        Register zlen, Register tmp1, Register tmp2, Register tmp3,
                        Register tmp4, Register tmp5, Register tmp6, Register tmp7);
   void mul_add(Register out, Register in, Register offs, Register len, Register k);
-  // ISB may be needed because of a safepoint
+
-  void maybe_isb() { isb(); }
+  // Place an ISB after code may have been modified due to a safepoint.
+  void safepoint_isb();
 
 private:
   // Return the effective address r + (r1 << ext) + offset.
@@ -1392,6 +1393,11 @@ public:
   }
   void cache_wb(Address line);
   void cache_wbsync(bool is_pre);
+
+private:
+  // Check the current thread doesn't need a cross modify fence.
+  void verify_cross_modify_fence_not_required() PRODUCT_RETURN;
+
 };
 
 #ifdef ASSERT

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -373,7 +373,10 @@ static void patch_callers_callsite(MacroAssembler *masm) {
   __ mov(c_rarg1, lr);
   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
   __ blr(rscratch1);
-  __ maybe_isb();
+
+  // Explicit isb required because fixup_callers_callsite may change the code
+  // stream.
+  __ safepoint_isb();
 
   __ pop_CPU_state();
   // restore sp
@@ -1150,7 +1153,6 @@ static void rt_call(MacroAssembler* masm, address dest) {
   } else {
     __ lea(rscratch1, RuntimeAddress(dest));
     __ blr(rscratch1);
-    __ maybe_isb();
   }
 }
 
@@ -1857,7 +1859,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     __ verify_sve_vector_length();
   }
 
-  // check for safepoint operation in progress and/or pending suspend requests
+  // Check for safepoint operation in progress and/or pending suspend requests.
   {
     // We need an acquire here to ensure that any subsequent load of the
     // global SafepointSynchronize::_state flag is ordered after this load
@@ -2081,7 +2083,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 #endif
     __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
     __ blr(rscratch1);
-    __ maybe_isb();
+
     // Restore any method result value
     restore_native_result(masm, ret_type, stack_slots);
 
@@ -2787,7 +2789,6 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t
 
   __ reset_last_Java_frame(false);
 
-  __ maybe_isb();
   __ membar(Assembler::LoadLoad | Assembler::LoadStore);
 
   if (UseSVE > 0 && save_vectors) {
@@ -2894,8 +2895,6 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const cha
 
   oop_maps->add_gc_map( __ offset() - start, map);
 
-  __ maybe_isb();
-
   // r0 contains the address we are going to jump to assuming no exception got installed
 
   // clear last_Java_sp
@@ -3017,7 +3016,8 @@ void OptoRuntime::generate_exception_blob() {
   __ mov(c_rarg0, rthread);
   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));
   __ blr(rscratch1);
-  __ maybe_isb();
+  // handle_exception_C is a special VM call which does not require an explicit
+  // instruction sync afterwards.
 
   // Set an oopmap for the call site.  This oopmap will only be used if we
   // are unwinding the stack.  Hence, all locations will be dead.

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -5629,7 +5629,6 @@ class StubGenerator: public StubCodeGenerator {
     oop_maps->add_gc_map(the_pc - start, map);
 
     __ reset_last_Java_frame(true);
-    __ maybe_isb();
 
     if (UseSVE > 0) {
       // Reinitialize the ptrue predicate register, in case the external runtime

src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp

@@ -1357,7 +1357,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   // Call the native method.
   __ blr(r10);
   __ bind(native_return);
-  __ maybe_isb();
   __ get_method(rmethod);
   // result potentially in r0 or v0
 
@@ -1410,7 +1409,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
     __ mov(c_rarg0, rthread);
     __ mov(rscratch2, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
     __ blr(rscratch2);
-    __ maybe_isb();
     __ get_method(rmethod);
     __ reinit_heapbase();
     __ bind(Continue);

src/hotspot/os_cpu/aix_ppc/orderAccess_aix_ppc.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2019 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -73,7 +73,7 @@ inline void OrderAccess::storeload()  { inlasm_sync();   }
 inline void OrderAccess::acquire()    { inlasm_lwsync(); }
 inline void OrderAccess::release()    { inlasm_lwsync(); }
 inline void OrderAccess::fence()      { inlasm_sync();   }
-inline void OrderAccess::cross_modify_fence()
+inline void OrderAccess::cross_modify_fence_impl()
                                       { inlasm_isync();  }
 
 #undef inlasm_sync

src/hotspot/os_cpu/bsd_x86/orderAccess_bsd_x86.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -59,7 +59,7 @@ inline void OrderAccess::fence() {
   compiler_barrier();
 }
 
-inline void OrderAccess::cross_modify_fence() {
+inline void OrderAccess::cross_modify_fence_impl() {
   int idx = 0;
   __asm__ volatile ("cpuid " : "+a" (idx) : : "ebx", "ecx", "edx", "memory");
 }

src/hotspot/os_cpu/bsd_zero/orderAccess_bsd_zero.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2007, 2008, 2009 Red Hat, Inc.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -77,6 +77,6 @@ inline void OrderAccess::storeload()  { FULL_MEM_BARRIER;  }
 inline void OrderAccess::acquire()    { LIGHT_MEM_BARRIER; }
 inline void OrderAccess::release()    { LIGHT_MEM_BARRIER; }
 inline void OrderAccess::fence()      { FULL_MEM_BARRIER;  }
-inline void OrderAccess::cross_modify_fence()            { }
+inline void OrderAccess::cross_modify_fence_impl()             { }
 
 #endif // OS_CPU_BSD_ZERO_ORDERACCESS_BSD_ZERO_HPP

src/hotspot/os_cpu/linux_aarch64/orderAccess_linux_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -53,6 +53,8 @@ inline void OrderAccess::fence() {
   FULL_MEM_BARRIER;
 }
 
-inline void OrderAccess::cross_modify_fence() { }
+inline void OrderAccess::cross_modify_fence_impl() {
+  asm volatile("isb" : : : "memory");
+}
 
 #endif // OS_CPU_LINUX_AARCH64_ORDERACCESS_LINUX_AARCH64_HPP

src/hotspot/os_cpu/linux_arm/orderAccess_linux_arm.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -101,6 +101,6 @@ inline void OrderAccess::storestore() { dmb_st(); }
 inline void OrderAccess::storeload()  { dmb_sy(); }
 inline void OrderAccess::release()    { dmb_sy(); }
 inline void OrderAccess::fence()      { dmb_sy(); }
-inline void OrderAccess::cross_modify_fence()   { }
+inline void OrderAccess::cross_modify_fence_impl()   { }
 
 #endif // OS_CPU_LINUX_ARM_ORDERACCESS_LINUX_ARM_HPP

src/hotspot/os_cpu/linux_ppc/orderAccess_linux_ppc.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -77,7 +77,7 @@ inline void   OrderAccess::storeload()  { inlasm_sync();   }
 inline void   OrderAccess::acquire()    { inlasm_lwsync(); }
 inline void   OrderAccess::release()    { inlasm_lwsync(); }
 inline void   OrderAccess::fence()      { inlasm_sync();   }
-inline void   OrderAccess::cross_modify_fence()
+inline void   OrderAccess::cross_modify_fence_impl()
                                         { inlasm_isync();  }
 
 #undef inlasm_sync

src/hotspot/os_cpu/linux_s390/orderAccess_linux_s390.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2016, 2019 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -74,7 +74,7 @@ inline void OrderAccess::storeload()  { inlasm_zarch_sync(); }
 inline void OrderAccess::acquire()    { inlasm_zarch_acquire(); }
 inline void OrderAccess::release()    { inlasm_zarch_release(); }
 inline void OrderAccess::fence()      { inlasm_zarch_sync(); }
-inline void OrderAccess::cross_modify_fence() { inlasm_zarch_sync(); }
+inline void OrderAccess::cross_modify_fence_impl() { inlasm_zarch_sync(); }
 
 #undef inlasm_compiler_barrier
 #undef inlasm_zarch_sync

src/hotspot/os_cpu/linux_x86/orderAccess_linux_x86.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -55,7 +55,7 @@ inline void OrderAccess::fence() {
   compiler_barrier();
 }
 
-inline void OrderAccess::cross_modify_fence() {
+inline void OrderAccess::cross_modify_fence_impl() {
   int idx = 0;
 #ifdef AMD64
   __asm__ volatile ("cpuid " : "+a" (idx) : : "ebx", "ecx", "edx", "memory");

src/hotspot/os_cpu/linux_zero/orderAccess_linux_zero.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2007, 2008, 2009 Red Hat, Inc.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -78,6 +78,6 @@ inline void OrderAccess::acquire()    { LIGHT_MEM_BARRIER; }
 inline void OrderAccess::release()    { LIGHT_MEM_BARRIER; }
 
 inline void OrderAccess::fence()      { FULL_MEM_BARRIER;  }
-inline void OrderAccess::cross_modify_fence()            { }
+inline void OrderAccess::cross_modify_fence_impl()            { }
 
 #endif // OS_CPU_LINUX_ZERO_ORDERACCESS_LINUX_ZERO_HPP

src/hotspot/os_cpu/windows_aarch64/orderAccess_windows_aarch64.hpp

@@ -28,7 +28,7 @@
 // Included in orderAccess.hpp header file.
 #include <atomic>
 using std::atomic_thread_fence;
-#include <intrin.h>
+#include <arm64intr.h>
 #include "vm_version_aarch64.hpp"
 #include "runtime/vm_version.hpp"
 
@@ -55,6 +55,8 @@ inline void OrderAccess::fence() {
   FULL_MEM_BARRIER;
 }
 
-inline void OrderAccess::cross_modify_fence() { }
+inline void OrderAccess::cross_modify_fence_impl() {
+  __isb(_ARM64_BARRIER_SY);
+}
 
 #endif // OS_CPU_WINDOWS_AARCH64_ORDERACCESS_WINDOWS_AARCH64_HPP

src/hotspot/os_cpu/windows_x86/orderAccess_windows_x86.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -58,7 +58,7 @@ inline void OrderAccess::fence() {
   compiler_barrier();
 }
 
-inline void OrderAccess::cross_modify_fence() {
+inline void OrderAccess::cross_modify_fence_impl() {
   int regs[4];
   __cpuid(regs, 0);
 }

src/hotspot/share/runtime/globals.hpp

@@ -2490,7 +2490,12 @@ const intx ObjectAlignmentInBytes = 8;
                 "Allow allocating fields in empty slots of super-classes")  \
                                                                             \
   product(bool, DeoptimizeNMethodBarriersALot, false, DIAGNOSTIC,           \
-                "Make nmethod barriers deoptimise a lot.")
+                "Make nmethod barriers deoptimise a lot.")                  \
+                                                                            \
+  develop(bool, VerifyCrossModifyFence,                                     \
+          false AARCH64_ONLY(DEBUG_ONLY(||true)),                           \
+             "Mark all threads after a safepoint, and clear on a modify "   \
+             "fence. Add cleanliness checks.")                              \
 
 // end of RUNTIME_FLAGS
 

src/hotspot/share/runtime/orderAccess.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,10 @@
 #include "precompiled.hpp"
 #include "runtime/orderAccess.hpp"
 #include "runtime/stubRoutines.hpp"
+
+#ifndef PRODUCT
 #include "runtime/thread.hpp"
+#endif
 
 void OrderAccess::StubRoutines_fence() {
   // Use a stub if it exists.  It may not exist during bootstrap so do
@@ -38,3 +41,11 @@ void OrderAccess::StubRoutines_fence() {
   }
   assert(Threads::number_of_threads() == 0, "for bootstrap only");
 }
+
+#ifndef PRODUCT
+void OrderAccess::cross_modify_fence_verify() {
+    if (VerifyCrossModifyFence) {
+      JavaThread::current()->set_requires_cross_modify_fence(false);
+    }
+}
+#endif

src/hotspot/share/runtime/orderAccess.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -112,7 +112,7 @@
 // may be more conservative in implementations. We advise using the bound
 // variants whenever possible.
 //
-// Finally, we define a "fence" operation, as a bidirectional barrier.
+// We define a "fence" operation, as a bidirectional barrier.
 // It guarantees that any memory access preceding the fence is not
 // reordered w.r.t. any memory accesses subsequent to the fence in program
 // order. This may be used to prevent sequences of loads from floating up
@@ -229,6 +229,10 @@
 // order*.  And that their destructors do a release and unlock, in *that*
 // order.  If their implementations change such that these assumptions
 // are violated, a whole lot of code will break.
+//
+// Finally, we define an "instruction_fence" operation, which ensures that all
+// instructions that come after the fence in program order are fetched
+// from the cache or memory after the fence has completed.
 
 class OrderAccess : public AllStatic {
  public:
@@ -242,7 +246,10 @@ class OrderAccess : public AllStatic {
   static void     release();
   static void     fence();
 
-  static void     cross_modify_fence();
+  static void     cross_modify_fence() {
+    cross_modify_fence_impl();
+    cross_modify_fence_verify();
+  }
 
   // Processors which are not multi-copy-atomic require a full fence
   // to enforce a globally consistent order of Independent Reads of
@@ -259,6 +266,10 @@ private:
   // routine if it exists, It should only be used by platforms that
   // don't have another way to do the inline assembly.
   static void StubRoutines_fence();
+
+  static void cross_modify_fence_impl();
+
+  static void cross_modify_fence_verify() PRODUCT_RETURN;
 };
 
 #include OS_CPU_HEADER(orderAccess)

src/hotspot/share/runtime/safepoint.cpp

@@ -381,6 +381,14 @@ void SafepointSynchronize::begin() {
   assert(_waiting_to_block == 0, "No thread should be running");
 
 #ifndef PRODUCT
+  // Mark all threads
+  if (VerifyCrossModifyFence) {
+    JavaThreadIteratorWithHandle jtiwh;
+    for (; JavaThread *cur = jtiwh.next(); ) {
+      cur->set_requires_cross_modify_fence(true);
+    }
+  }
+
   if (safepoint_limit_time != 0) {
     jlong current_time = os::javaTimeNanos();
     if (safepoint_limit_time < current_time) {

src/hotspot/share/runtime/thread.cpp

@@ -1580,6 +1580,9 @@ JavaThread::JavaThread() :
   ThreadSafepointState::create(this);
 
   SafepointMechanism::initialize_header(this);
+
+  set_requires_cross_modify_fence(false);
+
   pd_initialize();
   assert(deferred_card_mark().is_empty(), "Default MemRegion ctor");
 }
@@ -4646,3 +4649,9 @@ void Threads::verify() {
   VMThread* thread = VMThread::vm_thread();
   if (thread != NULL) thread->verify();
 }
+
+#ifndef PRODUCT
+void JavaThread::verify_cross_modify_fence_failure(JavaThread *thread) {
+   report_vm_error(__FILE__, __LINE__, "Cross modify fence failure", "%p", thread);
+}
+#endif

src/hotspot/share/runtime/thread.hpp

@@ -1093,6 +1093,7 @@ class JavaThread: public Thread {
  private:
   ThreadSafepointState* _safepoint_state;        // Holds information about a thread during a safepoint
   address               _saved_exception_pc;     // Saved pc of instruction where last implicit exception happened
+  NOT_PRODUCT(bool      _requires_cross_modify_fence;) // State used by VerifyCrossModifyFence
 
   // JavaThread termination support
   enum TerminatedTypes {
@@ -1324,6 +1325,8 @@ class JavaThread: public Thread {
 
   SafepointMechanism::ThreadData* poll_data() { return &_poll_data; }
 
+  void set_requires_cross_modify_fence(bool val) PRODUCT_RETURN NOT_PRODUCT({ _requires_cross_modify_fence = val; })
+
  private:
   // Support for thread handshake operations
   HandshakeState _handshake;
@@ -1599,6 +1602,7 @@ class JavaThread: public Thread {
     return byte_offset_of(JavaThread, _should_post_on_exceptions_flag);
   }
   static ByteSize doing_unsafe_access_offset() { return byte_offset_of(JavaThread, _doing_unsafe_access); }
+  NOT_PRODUCT(static ByteSize requires_cross_modify_fence_offset()  { return byte_offset_of(JavaThread, _requires_cross_modify_fence); })
 
   // Returns the jni environment for this thread
   JNIEnv* jni_environment()                      { return &_jni_environment; }
@@ -1888,6 +1892,8 @@ public:
   bool is_interrupted(bool clear_interrupted);
 
   static OopStorage* thread_oop_storage();
+
+  static void verify_cross_modify_fence_failure(JavaThread *thread) PRODUCT_RETURN;
 };
 
 // Inline implementation of JavaThread::current