8212753: Improve oopDesc::forward_to_atomic

Avoid multiple unnecessary reloads of the mark oop in oopDesc::forward_to_atomic

Reviewed-by: kbarrett, mdoerr

src/hotspot/share/gc/cms/parNewGeneration.cpp

@@ -1118,7 +1118,7 @@ oop ParNewGeneration::copy_to_survivor_space(ParScanThreadState* par_scan_state,
 
     // Attempt to install a null forwarding pointer (atomically),
     // to claim the right to install the real forwarding pointer.
-    forward_ptr = old->forward_to_atomic(ClaimedForwardPtr);
+    forward_ptr = old->forward_to_atomic(ClaimedForwardPtr, m);
     if (forward_ptr != NULL) {
       // someone else beat us to it.
         return real_forwardee(old);
@@ -1144,7 +1144,7 @@ oop ParNewGeneration::copy_to_survivor_space(ParScanThreadState* par_scan_state,
     // Is in to-space; do copying ourselves.
     Copy::aligned_disjoint_words((HeapWord*)old, (HeapWord*)new_obj, sz);
     assert(CMSHeap::heap()->is_in_reserved(new_obj), "illegal forwarding pointer value.");
-    forward_ptr = old->forward_to_atomic(new_obj);
+    forward_ptr = old->forward_to_atomic(new_obj, m);
     // Restore the mark word copied above.
     new_obj->set_mark_raw(m);
     // Increment age if obj still in new generation

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -265,7 +265,7 @@ oop G1ParScanThreadState::copy_to_survivor_space(InCSetState const state,
   Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes);
 
   const oop obj = oop(obj_ptr);
-  const oop forward_ptr = old->forward_to_atomic(obj, memory_order_relaxed);
+  const oop forward_ptr = old->forward_to_atomic(obj, old_mark, memory_order_relaxed);
   if (forward_ptr == NULL) {
     Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz);
 
@@ -355,7 +355,7 @@ void G1ParScanThreadStateSet::flush() {
 oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markOop m) {
   assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
 
-  oop forward_ptr = old->forward_to_atomic(old, memory_order_relaxed);
+  oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed);
   if (forward_ptr == NULL) {
     // Forward-to-self succeeded. We are the "owner" of the object.
     HeapRegion* r = _g1h->heap_region_containing(old);

src/hotspot/share/oops/oop.hpp

@@ -273,7 +273,7 @@ class oopDesc {
   // Exactly one thread succeeds in inserting the forwarding pointer, and
   // this call returns "NULL" for that thread; any other thread has the
   // value of the forwarding pointer returned and does not modify "this".
-  inline oop forward_to_atomic(oop p, atomic_memory_order order = memory_order_conservative);
+  inline oop forward_to_atomic(oop p, markOop compare, atomic_memory_order order = memory_order_conservative);
 
   inline oop forwardee() const;
   inline oop forwardee_acquire() const;

src/hotspot/share/oops/oop.inline.hpp

@@ -370,26 +370,21 @@ bool oopDesc::cas_forward_to(oop p, markOop compare, atomic_memory_order order)
   return cas_set_mark_raw(m, compare, order) == compare;
 }
 
-oop oopDesc::forward_to_atomic(oop p, atomic_memory_order order) {
+oop oopDesc::forward_to_atomic(oop p, markOop compare, atomic_memory_order order) {
-  markOop oldMark = mark_raw();
+  // CMS forwards some non-heap value into the mark oop to reserve oops during
-  markOop forwardPtrMark = markOopDesc::encode_pointer_as_mark(p);
+  // promotion, so the next two asserts do not hold.
-  markOop curMark;
+  assert(UseConcMarkSweepGC || check_obj_alignment(p),
-
+         "forwarding to something not aligned");
-  assert(forwardPtrMark->decode_pointer() == p, "encoding must be reversable");
+  assert(UseConcMarkSweepGC || Universe::heap()->is_in_reserved(p),
-  assert(sizeof(markOop) == sizeof(intptr_t), "CAS below requires this.");
+         "forwarding to something not in heap");
-
+  markOop m = markOopDesc::encode_pointer_as_mark(p);
-  while (!oldMark->is_marked()) {
+  assert(m->decode_pointer() == p, "encoding must be reversable");
-    curMark = cas_set_mark_raw(forwardPtrMark, oldMark, order);
+  markOop old_mark = cas_set_mark_raw(m, compare, order);
-    assert(is_forwarded(), "object should have been forwarded");
+  if (old_mark == compare) {
-    if (curMark == oldMark) {
     return NULL;
+  } else {
+    return (oop)old_mark->decode_pointer();
   }
-    // If the CAS was unsuccessful then curMark->is_marked()
-    // should return true as another thread has CAS'd in another
-    // forwarding pointer.
-    oldMark = curMark;
-  }
-  return forwardee();
 }
 
 // Note that the forwardee is not the same thing as the displaced_mark.