8226311: Shenandoah: Concurrent evacuation of OopStorage backed weak roots

Reviewed-by: rkennke

src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp

@@ -31,6 +31,7 @@
 #include "gc/shared/gcTimer.hpp"
 #include "gc/shared/referenceProcessor.hpp"
 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
+#include "gc/shared/strongRootsScope.hpp"
 
 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
 #include "gc/shenandoah/shenandoahClosures.inline.hpp"

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1523,7 +1523,8 @@ void ShenandoahHeap::op_final_mark() {
 
       if (ShenandoahVerify) {
         if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
-          verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::JNIHandleRoots);
+          ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
+          verifier()->verify_roots_no_forwarded_except(types);
         } else {
           verifier()->verify_roots_no_forwarded();
         }
@@ -1590,6 +1591,7 @@ void ShenandoahHeap::op_cleanup() {
 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
 private:
   ShenandoahJNIHandleRoots<true /*concurrent*/> _jni_roots;
+  ShenandoahWeakRoots<true /*concurrent*/>      _weak_roots;
 
 public:
   ShenandoahConcurrentRootsEvacUpdateTask() :
@@ -1599,7 +1601,9 @@ public:
   void work(uint worker_id) {
     ShenandoahEvacOOMScope oom;
     ShenandoahEvacuateUpdateRootsClosure cl;
+
     _jni_roots.oops_do<ShenandoahEvacuateUpdateRootsClosure>(&cl);
+    _weak_roots.oops_do<ShenandoahEvacuateUpdateRootsClosure>(&cl);
   }
 };
 

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp

@@ -27,16 +27,13 @@
 #include "classfile/stringTable.hpp"
 #include "classfile/systemDictionary.hpp"
 #include "code/codeCache.hpp"
-#include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
-#include "gc/shenandoah/shenandoahHeuristics.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 #include "gc/shenandoah/shenandoahStringDedup.hpp"
 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
 #include "gc/shenandoah/shenandoahVMOperations.hpp"
-#include "gc/shared/weakProcessor.inline.hpp"
-#include "memory/allocation.inline.hpp"
+#include "jfr/jfr.hpp"
 #include "memory/iterator.hpp"
 #include "memory/resourceArea.hpp"
 #include "memory/universe.hpp"
@@ -71,6 +68,40 @@ void ShenandoahSerialRoots::oops_do(OopClosure* cl, uint worker_id) {
   _jvmti_root.oops_do(cl, worker_id);
 }
 
+ShenandoahWeakSerialRoot::ShenandoahWeakSerialRoot(ShenandoahWeakSerialRoot::WeakOopsDo weak_oops_do, ShenandoahPhaseTimings::GCParPhases phase) :
+  _weak_oops_do(weak_oops_do), _phase(phase) {
+}
+
+void ShenandoahWeakSerialRoot::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id) {
+  if (_claimed.try_set()) {
+    ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
+    ShenandoahWorkerTimingsTracker timer(worker_times, _phase, worker_id);
+    _weak_oops_do(is_alive, keep_alive);
+  }
+}
+
+#if INCLUDE_JVMTI
+ShenandoahJVMTIWeakRoot::ShenandoahJVMTIWeakRoot() :
+  ShenandoahWeakSerialRoot(&JvmtiExport::weak_oops_do, ShenandoahPhaseTimings::JVMTIWeakRoots) {
+}
+#endif // INCLUDE_JVMTI
+
+#if INCLUDE_JFR
+ShenandoahJFRWeakRoot::ShenandoahJFRWeakRoot() :
+  ShenandoahWeakSerialRoot(&Jfr::weak_oops_do, ShenandoahPhaseTimings::JFRWeakRoots) {
+}
+#endif // INCLUDE_JFR
+
+void ShenandoahSerialWeakRoots::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id) {
+  JVMTI_ONLY(_jvmti_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);)
+  JFR_ONLY(_jfr_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);)
+}
+
+void ShenandoahSerialWeakRoots::weak_oops_do(OopClosure* cl, uint worker_id) {
+  AlwaysTrueClosure always_true;
+  weak_oops_do(&always_true, cl, worker_id);
+}
+
 ShenandoahThreadRoots::ShenandoahThreadRoots(bool is_par) : _is_par(is_par) {
   Threads::change_thread_claim_token();
 }
@@ -93,17 +124,6 @@ ShenandoahThreadRoots::~ShenandoahThreadRoots() {
   Threads::assert_all_threads_claimed();
 }
 
-ShenandoahWeakRoots::ShenandoahWeakRoots(uint n_workers) :
-  _process_timings(n_workers),
-  _task(&_process_timings, n_workers) {
-}
-
-ShenandoahWeakRoots::~ShenandoahWeakRoots() {
-  ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
-  ShenandoahTimingConverter::weak_processing_timing_to_shenandoah_timing(&_process_timings,
-                                                                         worker_times);
-}
-
 ShenandoahStringDedupRoots::ShenandoahStringDedupRoots() {
   if (ShenandoahStringDedup::is_enabled()) {
     StringDedup::gc_prologue(false);
@@ -137,7 +157,6 @@ ShenandoahRootProcessor::~ShenandoahRootProcessor() {
 ShenandoahRootEvacuator::ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool include_concurrent_roots) :
   ShenandoahRootProcessor(phase),
   _thread_roots(n_workers > 1),
-  _weak_roots(n_workers),
   _include_concurrent_roots(include_concurrent_roots) {
 }
 
@@ -147,29 +166,28 @@ void ShenandoahRootEvacuator::roots_do(uint worker_id, OopClosure* oops) {
   AlwaysTrueClosure always_true;
 
   _serial_roots.oops_do(oops, worker_id);
+  _serial_weak_roots.weak_oops_do(oops, worker_id);
   if (_include_concurrent_roots) {
     _jni_roots.oops_do<OopClosure>(oops, worker_id);
+    _weak_roots.oops_do<OopClosure>(oops, worker_id);
   }
 
   _thread_roots.oops_do(oops, NULL, worker_id);
   _cld_roots.cld_do(&clds, worker_id);
   _code_roots.code_blobs_do(&blobsCl, worker_id);
 
-  _weak_roots.oops_do<AlwaysTrueClosure, OopClosure>(&always_true, oops, worker_id);
   _dedup_roots.oops_do(&always_true, oops, worker_id);
 }
 
 ShenandoahRootUpdater::ShenandoahRootUpdater(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool update_code_cache) :
   ShenandoahRootProcessor(phase),
   _thread_roots(n_workers > 1),
-  _weak_roots(n_workers),
   _update_code_cache(update_code_cache) {
 }
 
 ShenandoahRootAdjuster::ShenandoahRootAdjuster(uint n_workers, ShenandoahPhaseTimings::Phase phase) :
   ShenandoahRootProcessor(phase),
-  _thread_roots(n_workers > 1),
-  _weak_roots(n_workers) {
+  _thread_roots(n_workers > 1) {
   assert(ShenandoahHeap::heap()->is_full_gc_in_progress(), "Full GC only");
 }
 
@@ -185,7 +203,8 @@ void ShenandoahRootAdjuster::roots_do(uint worker_id, OopClosure* oops) {
   _cld_roots.cld_do(&adjust_cld_closure, worker_id);
   _code_roots.code_blobs_do(&adjust_code_closure, worker_id);
 
-  _weak_roots.oops_do<AlwaysTrueClosure, OopClosure>(&always_true, oops, worker_id);
+  _serial_weak_roots.weak_oops_do(oops, worker_id);
+  _weak_roots.oops_do<OopClosure>(oops, worker_id);
   _dedup_roots.oops_do(&always_true, oops, worker_id);
 }
 

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp

@@ -29,11 +29,7 @@
 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
-#include "gc/shared/strongRootsScope.hpp"
-#include "gc/shared/weakProcessor.hpp"
-#include "gc/shared/weakProcessorPhaseTimes.hpp"
-#include "gc/shared/workgroup.hpp"
-#include "memory/allocation.hpp"
+#include "gc/shenandoah/shenandoahSharedVariables.hpp"
 #include "memory/iterator.hpp"
 
 class ShenandoahSerialRoot {
@@ -61,10 +57,102 @@ public:
   void oops_do(OopClosure* cl, uint worker_id);
 };
 
+class ShenandoahWeakSerialRoot {
+  typedef void (*WeakOopsDo)(BoolObjectClosure*, OopClosure*);
+private:
+  ShenandoahSharedFlag                      _claimed;
+  const WeakOopsDo                          _weak_oops_do;
+  const ShenandoahPhaseTimings::GCParPhases _phase;
+
+public:
+  ShenandoahWeakSerialRoot(WeakOopsDo oops_do, ShenandoahPhaseTimings::GCParPhases);
+  void weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id);
+};
+
+#if INCLUDE_JVMTI
+class ShenandoahJVMTIWeakRoot : public ShenandoahWeakSerialRoot {
+public:
+  ShenandoahJVMTIWeakRoot();
+};
+#endif // INCLUDE_JVMTI
+
+#if INCLUDE_JFR
+class ShenandoahJFRWeakRoot : public ShenandoahWeakSerialRoot {
+public:
+  ShenandoahJFRWeakRoot();
+};
+#endif // INCLUDE_JFR
+
+class ShenandoahSerialWeakRoots {
+private:
+  JVMTI_ONLY(ShenandoahJVMTIWeakRoot _jvmti_weak_roots;)
+  JFR_ONLY(ShenandoahJFRWeakRoot     _jfr_weak_roots;)
+public:
+  void weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id);
+  void weak_oops_do(OopClosure* cl, uint worker_id);
+};
+
+template <bool CONCURRENT>
+class ShenandoahWeakRoot {
+private:
+  OopStorage::ParState<CONCURRENT, false /* is_const */> _itr;
+  const ShenandoahPhaseTimings::GCParPhases _phase;
+public:
+  ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase);
+
+  template <typename Closure>
+  void oops_do(Closure* cl, uint worker_id);
+};
+
+template <>
+class ShenandoahWeakRoot<false /*concurrent*/> {
+private:
+  OopStorage::ParState<false /*concurrent*/, false /*is_const*/> _itr;
+  const ShenandoahPhaseTimings::GCParPhases _phase;
+
+public:
+  ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase);
+
+  template <typename IsAliveClosure, typename KeepAliveClosure>
+  void weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id);
+};
+
+template <bool CONCURRENT>
+class ShenandoahWeakRoots {
+private:
+  ShenandoahWeakRoot<CONCURRENT>  _jni_roots;
+  ShenandoahWeakRoot<CONCURRENT>  _string_table_roots;
+  ShenandoahWeakRoot<CONCURRENT>  _resolved_method_table_roots;
+  ShenandoahWeakRoot<CONCURRENT>  _vm_roots;
+
+public:
+  ShenandoahWeakRoots();
+
+  template <typename Closure>
+  void oops_do(Closure* cl, uint worker_id = 0);
+};
+
+template <>
+class ShenandoahWeakRoots<false /*concurrent */> {
+private:
+  ShenandoahWeakRoot<false /*concurrent*/>  _jni_roots;
+  ShenandoahWeakRoot<false /*concurrent*/>  _string_table_roots;
+  ShenandoahWeakRoot<false /*concurrent*/>  _resolved_method_table_roots;
+  ShenandoahWeakRoot<false /*concurrent*/>  _vm_roots;
+public:
+  ShenandoahWeakRoots();
+
+  template <typename Closure>
+  void oops_do(Closure* cl, uint worker_id = 0);
+
+  template <typename IsAliveClosure, typename KeepAliveClosure>
+  void weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id);
+};
+
 template <bool CONCURRENT>
 class ShenandoahJNIHandleRoots {
 private:
-  OopStorage::ParState<CONCURRENT, false /* is_const */> _itr;
+  OopStorage::ParState<CONCURRENT, false /*is_const*/> _itr;
 public:
   ShenandoahJNIHandleRoots();
 
@@ -83,18 +171,6 @@ public:
   void threads_do(ThreadClosure* tc, uint worker_id);
 };
 
-class ShenandoahWeakRoots {
-private:
-  WeakProcessorPhaseTimes _process_timings;
-  WeakProcessor::Task     _task;
-public:
-  ShenandoahWeakRoots(uint n_workers);
-  ~ShenandoahWeakRoots();
-
-  template <typename IsAlive, typename KeepAlive>
-  void oops_do(IsAlive* is_alive, KeepAlive* keep_alive, uint worker_id);
-};
-
 class ShenandoahStringDedupRoots {
 public:
   ShenandoahStringDedupRoots();
@@ -183,7 +259,8 @@ private:
   ShenandoahJNIHandleRoots<false /*concurrent*/>            _jni_roots;
   ShenandoahClassLoaderDataRoots<false /*single threaded*/> _cld_roots;
   ShenandoahThreadRoots                                     _thread_roots;
-  ShenandoahWeakRoots                                       _weak_roots;
+  ShenandoahSerialWeakRoots                                 _serial_weak_roots;
+  ShenandoahWeakRoots<false /*concurrent*/>                 _weak_roots;
   ShenandoahStringDedupRoots                                _dedup_roots;
   ShenandoahCodeCacheRoots<ShenandoahCsetCodeRootsIterator> _code_roots;
   bool                                                      _include_concurrent_roots;
@@ -201,7 +278,8 @@ private:
   ShenandoahJNIHandleRoots<false /*concurrent*/>            _jni_roots;
   ShenandoahClassLoaderDataRoots<false /*single threaded*/> _cld_roots;
   ShenandoahThreadRoots                                     _thread_roots;
-  ShenandoahWeakRoots                                       _weak_roots;
+  ShenandoahSerialWeakRoots                                 _serial_weak_roots;
+  ShenandoahWeakRoots<false /*concurrent*/>                 _weak_roots;
   ShenandoahStringDedupRoots                                _dedup_roots;
   ShenandoahCodeCacheRoots<ShenandoahCsetCodeRootsIterator> _code_roots;
   const bool                                                _update_code_cache;
@@ -220,7 +298,8 @@ private:
   ShenandoahJNIHandleRoots<false /*concurrent*/>            _jni_roots;
   ShenandoahClassLoaderDataRoots<false /*single threaded*/> _cld_roots;
   ShenandoahThreadRoots                                     _thread_roots;
-  ShenandoahWeakRoots                                       _weak_roots;
+  ShenandoahSerialWeakRoots                                 _serial_weak_roots;
+  ShenandoahWeakRoots<false /*concurrent*/>                 _weak_roots;
   ShenandoahStringDedupRoots                                _dedup_roots;
   ShenandoahCodeCacheRoots<ShenandoahAllCodeRootsIterator>  _code_roots;
 

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp

@@ -25,14 +25,83 @@
 #define SHARE_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_INLINE_HPP
 
 #include "classfile/classLoaderDataGraph.hpp"
+#include "classfile/stringTable.hpp"
+#include "classfile/systemDictionary.hpp"
 #include "gc/shared/oopStorageParState.inline.hpp"
 #include "gc/shenandoah/shenandoahHeuristics.hpp"
 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
 #include "memory/resourceArea.hpp"
+#include "prims/resolvedMethodTable.hpp"
 #include "runtime/safepoint.hpp"
 
+template <bool CONCURRENT>
+inline ShenandoahWeakRoot<CONCURRENT>::ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase) :
+  _itr(storage), _phase(phase) {
+}
+
+template <bool CONCURRENT>
+template <typename Closure>
+inline void ShenandoahWeakRoot<CONCURRENT>::oops_do(Closure* cl, uint worker_id) {
+  if (CONCURRENT) {
+    _itr.oops_do(cl);
+  } else {
+    ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
+    ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ThreadRoots, worker_id);
+    _itr.oops_do(cl);
+  }
+}
+
+inline ShenandoahWeakRoot<false>::ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase) :
+  _itr(storage), _phase(phase) {
+}
+
+template <typename IsAliveClosure, typename KeepAliveClosure>
+void ShenandoahWeakRoot<false /* concurrent */>::weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id) {
+  ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
+  ShenandoahWorkerTimingsTracker timer(worker_times, _phase, worker_id);
+  _itr.weak_oops_do(is_alive, keep_alive);
+}
+
+template <bool CONCURRENT>
+ShenandoahWeakRoots<CONCURRENT>::ShenandoahWeakRoots() :
+  _jni_roots(JNIHandles::weak_global_handles(), ShenandoahPhaseTimings::JNIWeakRoots),
+  _string_table_roots(StringTable::weak_storage(), ShenandoahPhaseTimings::StringTableRoots),
+  _resolved_method_table_roots(ResolvedMethodTable::weak_storage(), ShenandoahPhaseTimings::ResolvedMethodTableRoots),
+  _vm_roots(SystemDictionary::vm_weak_oop_storage(), ShenandoahPhaseTimings::VMWeakRoots) {
+}
+
+template <bool CONCURRENT>
+template <typename Closure>
+void ShenandoahWeakRoots<CONCURRENT>::oops_do(Closure* cl, uint worker_id) {
+  _jni_roots.oops_do(cl, worker_id);
+  _string_table_roots.oops_do(cl, worker_id);
+  _resolved_method_table_roots.oops_do(cl, worker_id);
+  _vm_roots.oops_do(cl, worker_id);
+}
+
+inline ShenandoahWeakRoots<false /* concurrent */>::ShenandoahWeakRoots() :
+  _jni_roots(JNIHandles::weak_global_handles(), ShenandoahPhaseTimings::JNIWeakRoots),
+  _string_table_roots(StringTable::weak_storage(), ShenandoahPhaseTimings::StringTableRoots),
+  _resolved_method_table_roots(ResolvedMethodTable::weak_storage(), ShenandoahPhaseTimings::ResolvedMethodTableRoots),
+  _vm_roots(SystemDictionary::vm_weak_oop_storage(), ShenandoahPhaseTimings::VMWeakRoots) {
+}
+
+template <typename IsAliveClosure, typename KeepAliveClosure>
+void ShenandoahWeakRoots<false /* concurrent*/>::weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id) {
+  _jni_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+  _string_table_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+  _resolved_method_table_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+  _vm_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+}
+
+template <typename Closure>
+void ShenandoahWeakRoots<false /* concurrent */>::oops_do(Closure* cl, uint worker_id) {
+  AlwaysTrueClosure always_true;
+  weak_oops_do<AlwaysTrueClosure, Closure>(&always_true, cl, worker_id);
+}
+
 template <bool CONCURRENT>
 ShenandoahJNIHandleRoots<CONCURRENT>::ShenandoahJNIHandleRoots() :
   _itr(JNIHandles::global_handles()) {
@@ -50,11 +119,6 @@ void ShenandoahJNIHandleRoots<CONCURRENT>::oops_do(T* cl, uint worker_id) {
   }
 }
 
-template <typename IsAlive, typename KeepAlive>
-void ShenandoahWeakRoots::oops_do(IsAlive* is_alive, KeepAlive* keep_alive, uint worker_id) {
-  _task.work<IsAlive, KeepAlive>(worker_id, is_alive, keep_alive);
-}
-
 template <bool SINGLE_THREADED>
 ShenandoahClassLoaderDataRoots<SINGLE_THREADED>::ShenandoahClassLoaderDataRoots() {
   if (!SINGLE_THREADED) {
@@ -197,7 +261,8 @@ void ShenandoahRootUpdater::roots_do(uint worker_id, IsAlive* is_alive, KeepAliv
     _code_roots.code_blobs_do(&update_blobs, worker_id);
   }
 
-  _weak_roots.oops_do<IsAlive, KeepAlive>(is_alive, keep_alive, worker_id);
+  _serial_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);
+  _weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);
   _dedup_roots.oops_do(is_alive, keep_alive, worker_id);
 }
 

src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp

@@ -54,6 +54,10 @@ bool ShenandoahRootVerifier::verify(RootTypes type) const {
   return (_types & type) != 0;
 }
 
+ShenandoahRootVerifier::RootTypes ShenandoahRootVerifier::combine(RootTypes t1, RootTypes t2) {
+  return static_cast<ShenandoahRootVerifier::RootTypes>(static_cast<uint>(t1) | static_cast<uint>(t2));
+}
+
 void ShenandoahRootVerifier::oops_do(OopClosure* oops) {
   CodeBlobToOopClosure blobs(oops, !CodeBlobToOopClosure::FixRelocations);
   if (verify(CodeRoots)) {

src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp

@@ -52,6 +52,8 @@ public:
   // Used to seed ShenandoahVerifier, do not honor root type filter
   void roots_do(OopClosure* cl);
   void strong_roots_do(OopClosure* cl);
+
+  static RootTypes combine(RootTypes t1, RootTypes t2);
 private:
   bool verify(RootTypes type) const;
 };