8146991: Introduce per-worker preserved mark stacks in ParallelGC

Reviewed-by: tschatzl, ysr

hotspot/src/share/vm/gc/parallel/psPromotionManager.cpp

@@ -29,6 +29,7 @@
 #include "gc/parallel/psPromotionManager.inline.hpp"
 #include "gc/parallel/psScavenge.inline.hpp"
 #include "gc/shared/gcTrace.hpp"
+#include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 #include "logging/log.hpp"
 #include "memory/allocation.inline.hpp"
@@ -41,6 +42,7 @@
 
 PaddedEnd<PSPromotionManager>* PSPromotionManager::_manager_array = NULL;
 OopStarTaskQueueSet*           PSPromotionManager::_stack_array_depth = NULL;
+PreservedMarksSet*             PSPromotionManager::_preserved_marks_set = NULL;
 PSOldGen*                      PSPromotionManager::_old_gen = NULL;
 MutableSpace*                  PSPromotionManager::_young_space = NULL;
 
@@ -50,10 +52,12 @@ void PSPromotionManager::initialize() {
   _old_gen = heap->old_gen();
   _young_space = heap->young_gen()->to_space();
 
+  const uint promotion_manager_num = ParallelGCThreads + 1;
+
   // To prevent false sharing, we pad the PSPromotionManagers
   // and make sure that the first instance starts at a cache line.
   assert(_manager_array == NULL, "Attempt to initialize twice");
-  _manager_array = PaddedArray<PSPromotionManager, mtGC>::create_unfreeable(ParallelGCThreads + 1);
+  _manager_array = PaddedArray<PSPromotionManager, mtGC>::create_unfreeable(promotion_manager_num);
   guarantee(_manager_array != NULL, "Could not initialize promotion manager");
 
   _stack_array_depth = new OopStarTaskQueueSet(ParallelGCThreads);
@@ -65,6 +69,14 @@ void PSPromotionManager::initialize() {
   }
   // The VMThread gets its own PSPromotionManager, which is not available
   // for work stealing.
+
+  assert(_preserved_marks_set == NULL, "Attempt to initialize twice");
+  _preserved_marks_set = new PreservedMarksSet(true /* in_c_heap */);
+  guarantee(_preserved_marks_set != NULL, "Could not initialize preserved marks set");
+  _preserved_marks_set->init(promotion_manager_num);
+  for (uint i = 0; i < promotion_manager_num; i += 1) {
+    _manager_array[i].register_preserved_marks(_preserved_marks_set->get(i));
+  }
 }
 
 // Helper functions to get around the circular dependency between
@@ -90,6 +102,7 @@ PSPromotionManager* PSPromotionManager::vm_thread_promotion_manager() {
 void PSPromotionManager::pre_scavenge() {
   ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
 
+  _preserved_marks_set->assert_empty();
   _young_space = heap->young_gen()->to_space();
 
   for(uint i=0; i<ParallelGCThreads+1; i++) {
@@ -110,6 +123,11 @@ bool PSPromotionManager::post_scavenge(YoungGCTracer& gc_tracer) {
     }
     manager->flush_labs();
   }
+  if (!promotion_failure_occurred) {
+    // If there was no promotion failure, the preserved mark stacks
+    // should be empty.
+    _preserved_marks_set->assert_empty();
+  }
   return promotion_failure_occurred;
 }
 
@@ -187,6 +205,8 @@ PSPromotionManager::PSPromotionManager() {
   // let's choose 1.5x the chunk size
   _min_array_size_for_chunking = 3 * _array_chunk_size / 2;
 
+  _preserved_marks = NULL;
+
   reset();
 }
 
@@ -211,6 +231,10 @@ void PSPromotionManager::reset() {
   TASKQUEUE_STATS_ONLY(reset_stats());
 }
 
+void PSPromotionManager::register_preserved_marks(PreservedMarks* preserved_marks) {
+  assert(_preserved_marks == NULL, "do not set it twice");
+  _preserved_marks = preserved_marks;
+}
 
 void PSPromotionManager::drain_stacks_depth(bool totally_drain) {
   totally_drain = totally_drain || _totally_drain;
@@ -422,8 +446,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markOop obj_mark) {
 
     push_contents(obj);
 
-    // Save the mark if needed
+    _preserved_marks->push_if_necessary(obj, obj_mark);
-    PSScavenge::oop_promotion_failed(obj, obj_mark);
   }  else {
     // We lost, someone else "owns" this object
     guarantee(obj->is_forwarded(), "Object must be forwarded if the cas failed.");

hotspot/src/share/vm/gc/parallel/psPromotionManager.hpp

@@ -28,6 +28,7 @@
 #include "gc/parallel/psPromotionLAB.hpp"
 #include "gc/shared/copyFailedInfo.hpp"
 #include "gc/shared/gcTrace.hpp"
+#include "gc/shared/preservedMarks.hpp"
 #include "gc/shared/taskqueue.hpp"
 #include "memory/allocation.hpp"
 #include "memory/padded.hpp"
@@ -55,6 +56,7 @@ class PSPromotionManager VALUE_OBJ_CLASS_SPEC {
  private:
   static PaddedEnd<PSPromotionManager>* _manager_array;
   static OopStarTaskQueueSet*           _stack_array_depth;
+  static PreservedMarksSet*             _preserved_marks_set;
   static PSOldGen*                      _old_gen;
   static MutableSpace*                  _young_space;
 
@@ -84,6 +86,7 @@ class PSPromotionManager VALUE_OBJ_CLASS_SPEC {
   uint                                _array_chunk_size;
   uint                                _min_array_size_for_chunking;
 
+  PreservedMarks*                     _preserved_marks;
   PromotionFailedInfo                 _promotion_failed_info;
 
   // Accessors
@@ -176,6 +179,8 @@ class PSPromotionManager VALUE_OBJ_CLASS_SPEC {
   oop oop_promotion_failed(oop obj, markOop obj_mark);
 
   void reset();
+  void register_preserved_marks(PreservedMarks* preserved_marks);
+  static void restore_preserved_marks() { _preserved_marks_set->restore(); }
 
   void flush_labs();
   void drain_stacks(bool totally_drain) {

hotspot/src/share/vm/gc/parallel/psScavenge.cpp

@@ -68,8 +68,6 @@ uintptr_t                  PSScavenge::_young_generation_boundary_compressed = 0
 elapsedTimer               PSScavenge::_accumulated_time;
 STWGCTimer                 PSScavenge::_gc_timer;
 ParallelScavengeTracer     PSScavenge::_gc_tracer;
-Stack<markOop, mtGC>       PSScavenge::_preserved_mark_stack;
-Stack<oop, mtGC>           PSScavenge::_preserved_oop_stack;
 CollectorCounters*         PSScavenge::_counters = NULL;
 
 // Define before use
@@ -123,14 +121,6 @@ class PSEvacuateFollowersClosure: public VoidClosure {
   }
 };
 
-class PSPromotionFailedClosure : public ObjectClosure {
-  virtual void do_object(oop obj) {
-    if (obj->is_forwarded()) {
-      obj->init_mark();
-    }
-  }
-};
-
 class PSRefProcTaskProxy: public GCTask {
   typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
   ProcessTask & _rp_task;
@@ -257,9 +247,6 @@ bool PSScavenge::invoke_no_policy() {
   assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
   assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread");
 
-  assert(_preserved_mark_stack.is_empty(), "should be empty");
-  assert(_preserved_oop_stack.is_empty(), "should be empty");
-
   _gc_timer.register_gc_start();
 
   TimeStamp scavenge_entry;
@@ -656,52 +643,20 @@ bool PSScavenge::invoke_no_policy() {
 }
 
 // This method iterates over all objects in the young generation,
-// unforwarding markOops. It then restores any preserved mark oops,
+// removing all forwarding references. It then restores any preserved marks.
-// and clears the _preserved_mark_stack.
 void PSScavenge::clean_up_failed_promotion() {
   ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
   PSYoungGen* young_gen = heap->young_gen();
 
-  {
+  RemoveForwardedPointerClosure remove_fwd_ptr_closure;
-    ResourceMark rm;
+  young_gen->object_iterate(&remove_fwd_ptr_closure);
 
-    // Unforward all pointers in the young gen.
+  PSPromotionManager::restore_preserved_marks();
-    PSPromotionFailedClosure unforward_closure;
-    young_gen->object_iterate(&unforward_closure);
-
-    log_trace(gc, ergo)("Restoring " SIZE_FORMAT " marks", _preserved_oop_stack.size());
-
-    // Restore any saved marks.
-    while (!_preserved_oop_stack.is_empty()) {
-      oop obj      = _preserved_oop_stack.pop();
-      markOop mark = _preserved_mark_stack.pop();
-      obj->set_mark(mark);
-    }
-
-    // Clear the preserved mark and oop stack caches.
-    _preserved_mark_stack.clear(true);
-    _preserved_oop_stack.clear(true);
-  }
 
   // Reset the PromotionFailureALot counters.
   NOT_PRODUCT(heap->reset_promotion_should_fail();)
 }
 
-// This method is called whenever an attempt to promote an object
-// fails. Some markOops will need preservation, some will not. Note
-// that the entire eden is traversed after a failed promotion, with
-// all forwarded headers replaced by the default markOop. This means
-// it is not necessary to preserve most markOops.
-void PSScavenge::oop_promotion_failed(oop obj, markOop obj_mark) {
-  if (obj_mark->must_be_preserved_for_promotion_failure(obj)) {
-    // Should use per-worker private stacks here rather than
-    // locking a common pair of stacks.
-    ThreadCritical tc;
-    _preserved_oop_stack.push(obj);
-    _preserved_mark_stack.push(obj_mark);
-  }
-}
-
 bool PSScavenge::should_attempt_scavenge() {
   ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
   PSGCAdaptivePolicyCounters* counters = heap->gc_policy_counters();

hotspot/src/share/vm/gc/parallel/psScavenge.hpp

@@ -79,8 +79,6 @@ class PSScavenge: AllStatic {
   static HeapWord*            _young_generation_boundary;
   // Used to optimize compressed oops young gen boundary checking.
   static uintptr_t            _young_generation_boundary_compressed;
-  static Stack<markOop, mtGC> _preserved_mark_stack; // List of marks to be restored after failed promotion
-  static Stack<oop, mtGC>     _preserved_oop_stack;  // List of oops that need their mark restored.
   static CollectorCounters*   _counters;             // collector performance counters
 
   static void clean_up_failed_promotion();
@@ -127,9 +125,6 @@ class PSScavenge: AllStatic {
   // Return true if a collection was done; false otherwise.
   static bool invoke_no_policy();
 
-  // If an attempt to promote fails, this method is invoked
-  static void oop_promotion_failed(oop obj, markOop obj_mark);
-
   template <class T> static inline bool should_scavenge(T* p);
 
   // These call should_scavenge() above and, if it returns true, also check that

hotspot/src/share/vm/gc/shared/preservedMarks.cpp

@@ -62,9 +62,14 @@ void PreservedMarksSet::init(uint num) {
 }
 
 void PreservedMarksSet::restore() {
+  size_t total_size = 0;
   for (uint i = 0; i < _num; i += 1) {
+    total_size += get(i)->size();
     get(i)->restore();
   }
+  assert_empty();
+
+  log_trace(gc)("Restored " SIZE_FORMAT " marks", total_size);
 }
 
 void PreservedMarksSet::reclaim() {

hotspot/src/share/vm/gc/shared/preservedMarks.hpp

@@ -53,6 +53,7 @@ private:
 
 public:
   bool is_empty() const { return _stack.is_empty(); }
+  size_t size() const { return _stack.size(); }
   inline void push_if_necessary(oop obj, markOop m);
   // Iterate over the stack, restore the preserved marks, then reclaim
   // the memory taken up by stack chunks.
@@ -65,7 +66,7 @@ public:
   virtual void do_object(oop obj);
 };
 
-class PreservedMarksSet VALUE_OBJ_CLASS_SPEC {
+class PreservedMarksSet : public CHeapObj<mtGC> {
 private:
   // true -> _stacks will be allocated in the C heap
   // false -> _stacks will be allocated in the resource arena