8154736: enhancement of cmpxchg and copy_to_survivor for ppc64

Reviewed-by: eosterlund, mdoerr, kbarrett

src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

@@ -213,7 +213,8 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
     Copy::aligned_disjoint_words((HeapWord*)o, (HeapWord*)new_obj, new_obj_size);
 
     // Now we have to CAS in the header.
-    if (o->cas_forward_to(new_obj, test_mark)) {
+    // Make copy visible to threads reading the forwardee.
+    if (o->cas_forward_to(new_obj, test_mark, memory_order_release)) {
       // We won any races, we "own" this object.
       assert(new_obj == o->forwardee(), "Sanity");
 
@@ -256,11 +257,12 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
       }
 
       // don't update this before the unallocation!
-      new_obj = o->forwardee();
+      // Using acquire though consume would be accurate for accessing new_obj.
+      new_obj = o->forwardee_acquire();
     }
   } else {
     assert(o->is_forwarded(), "Sanity");
-    new_obj = o->forwardee();
+    new_obj = o->forwardee_acquire();
   }
 
   // This code must come after the CAS test, or it will print incorrect

src/hotspot/share/oops/oop.hpp

@@ -30,6 +30,7 @@
 #include "memory/memRegion.hpp"
 #include "oops/access.hpp"
 #include "oops/metadata.hpp"
+#include "runtime/atomic.hpp"
 #include "utilities/macros.hpp"
 
 // oopDesc is the top baseclass for objects classes. The {name}Desc classes describe
@@ -72,7 +73,7 @@ class oopDesc {
 
   inline void release_set_mark(markOop m);
   inline markOop cas_set_mark(markOop new_mark, markOop old_mark);
-  inline markOop cas_set_mark_raw(markOop new_mark, markOop old_mark);
+  inline markOop cas_set_mark_raw(markOop new_mark, markOop old_mark, atomic_memory_order order = memory_order_conservative);
 
   // Used only to re-initialize the mark word (e.g., of promoted
   // objects during a GC) -- requires a valid klass pointer
@@ -259,7 +260,7 @@ class oopDesc {
   inline bool is_forwarded() const;
 
   inline void forward_to(oop p);
-  inline bool cas_forward_to(oop p, markOop compare);
+  inline bool cas_forward_to(oop p, markOop compare, atomic_memory_order order = memory_order_conservative);
 
   // Like "forward_to", but inserts the forwarding pointer atomically.
   // Exactly one thread succeeds in inserting the forwarding pointer, and
@@ -268,6 +269,7 @@ class oopDesc {
   inline oop forward_to_atomic(oop p);
 
   inline oop forwardee() const;
+  inline oop forwardee_acquire() const;
 
   // Age of object during scavenge
   inline uint age() const;

src/hotspot/share/oops/oop.inline.hpp

@@ -71,8 +71,8 @@ markOop oopDesc::cas_set_mark(markOop new_mark, markOop old_mark) {
   return HeapAccess<>::atomic_cmpxchg_at(new_mark, as_oop(), mark_offset_in_bytes(), old_mark);
 }
 
-markOop oopDesc::cas_set_mark_raw(markOop new_mark, markOop old_mark) {
+markOop oopDesc::cas_set_mark_raw(markOop new_mark, markOop old_mark, atomic_memory_order order) {
-  return Atomic::cmpxchg(new_mark, &_mark, old_mark);
+  return Atomic::cmpxchg(new_mark, &_mark, old_mark, order);
 }
 
 void oopDesc::init_mark() {
@@ -342,14 +342,14 @@ void oopDesc::forward_to(oop p) {
 }
 
 // Used by parallel scavengers
-bool oopDesc::cas_forward_to(oop p, markOop compare) {
+bool oopDesc::cas_forward_to(oop p, markOop compare, atomic_memory_order order) {
   assert(check_obj_alignment(p),
          "forwarding to something not aligned");
   assert(Universe::heap()->is_in_reserved(p),
          "forwarding to something not in heap");
   markOop m = markOopDesc::encode_pointer_as_mark(p);
   assert(m->decode_pointer() == p, "encoding must be reversable");
-  return cas_set_mark_raw(m, compare) == compare;
+  return cas_set_mark_raw(m, compare, order) == compare;
 }
 
 oop oopDesc::forward_to_atomic(oop p) {
@@ -381,6 +381,14 @@ oop oopDesc::forwardee() const {
   return (oop) mark_raw()->decode_pointer();
 }
 
+// Note that the forwardee is not the same thing as the displaced_mark.
+// The forwardee is used when copying during scavenge and mark-sweep.
+// It does need to clear the low two locking- and GC-related bits.
+oop oopDesc::forwardee_acquire() const {
+  markOop m = OrderAccess::load_acquire(&_mark);
+  return (oop) m->decode_pointer();
+}
+
 // The following method needs to be MT safe.
 uint oopDesc::age() const {
   assert(!is_forwarded(), "Attempt to read age from forwarded mark");