8144714: Add extension point to G1 evacuation closures

Reviewed-by: ehelin, jmasa

hotspot/src/share/vm/gc/g1/g1CollectedHeap.hpp

@@ -573,6 +573,9 @@ public:
   void register_old_region_with_cset(HeapRegion* r) {
     _in_cset_fast_test.set_in_old(r->hrm_index());
   }
+  inline void register_ext_region_with_cset(HeapRegion* r) {
+    _in_cset_fast_test.set_ext(r->hrm_index());
+  }
   void clear_in_cset(const HeapRegion* hr) {
     _in_cset_fast_test.clear(hr);
   }

hotspot/src/share/vm/gc/g1/g1InCSetState.hpp

@@ -53,8 +53,12 @@ struct InCSetState {
     // frequency of the checks.
     // The most common check is whether the region is in the collection set or not,
     // this encoding allows us to use an > 0 check.
-    // The other values are simply encoded in increasing generation order, which
+    // The positive values are encoded in increasing generation order, which
-    // makes getting the next generation fast by a simple increment.
+    // makes getting the next generation fast by a simple increment. They are also
+    // used to index into arrays.
+    // The negative values are used for objects requiring various special cases,
+    // for example eager reclamation of humongous objects.
+    Ext          = -2,    // Extension point
     Humongous    = -1,    // The region is humongous
     NotInCSet    =  0,    // The region is not in the collection set.
     Young        =  1,    // The region is in the collection set and a young region.
@@ -76,10 +80,11 @@ struct InCSetState {
   bool is_humongous() const            { return _value == Humongous; }
   bool is_young() const                { return _value == Young; }
   bool is_old() const                  { return _value == Old; }
+  bool is_ext() const                  { return _value == Ext; }
 
 #ifdef ASSERT
-  bool is_default() const              { return !is_in_cset_or_humongous(); }
+  bool is_default() const              { return _value == NotInCSet; }
-  bool is_valid() const                { return (_value >= Humongous) && (_value < Num); }
+  bool is_valid() const                { return (_value >= Ext) && (_value < Num); }
   bool is_valid_gen() const            { return (_value >= Young && _value <= Old); }
 #endif
 };
@@ -105,6 +110,12 @@ class G1InCSetStateFastTestBiasedMappedArray : public G1BiasedMappedArray<InCSet
     set_by_index(index, InCSetState::Humongous);
   }
 
+  void set_ext(uintptr_t index) {
+    assert(get_by_index(index).is_default(),
+           "State at index " INTPTR_FORMAT " should be default but is " CSETSTATE_FORMAT, index, get_by_index(index).value());
+    set_by_index(index, InCSetState::Ext);
+  }
+
   void clear_humongous(uintptr_t index) {
     set_by_index(index, InCSetState::NotInCSet);
   }

hotspot/src/share/vm/gc/g1/g1OopClosures.hpp

@@ -121,7 +121,7 @@ enum G1Mark {
   G1MarkPromotedFromRoot
 };
 
-template <G1Barrier barrier, G1Mark do_mark_object>
+template <G1Barrier barrier, G1Mark do_mark_object, bool use_ext>
 class G1ParCopyClosure : public G1ParCopyHelper {
 public:
   G1ParCopyClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :

hotspot/src/share/vm/gc/g1/g1OopClosures.inline.hpp

@@ -90,6 +90,8 @@ inline void G1ParScanClosure::do_oop_nv(T* p) {
     } else {
       if (state.is_humongous()) {
         _g1->set_humongous_is_live(obj);
+      } else if (state.is_ext()) {
+        _par_scan_state->do_oop_ext(p);
       }
       _par_scan_state->update_rs(_from, p, obj);
     }
@@ -102,12 +104,15 @@ inline void G1ParPushHeapRSClosure::do_oop_nv(T* p) {
 
   if (!oopDesc::is_null(heap_oop)) {
     oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
-    if (_g1->is_in_cset_or_humongous(obj)) {
+    const InCSetState state = _g1->in_cset_state(obj);
+    if (state.is_in_cset_or_humongous()) {
       Prefetch::write(obj->mark_addr(), 0);
       Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
 
       // Place on the references queue
       _par_scan_state->push_on_queue(p);
+    } else if (state.is_ext()) {
+      _par_scan_state->do_oop_ext(p);
     } else {
       assert(!_g1->obj_in_cs(obj), "checking");
     }
@@ -249,9 +254,9 @@ void G1ParCopyHelper::mark_forwarded_object(oop from_obj, oop to_obj) {
   _cm->grayRoot(to_obj, (size_t) from_obj->size(), _worker_id);
 }
 
-template <G1Barrier barrier, G1Mark do_mark_object>
+template <G1Barrier barrier, G1Mark do_mark_object, bool use_ext>
 template <class T>
-void G1ParCopyClosure<barrier, do_mark_object>::do_oop_nv(T* p) {
+void G1ParCopyClosure<barrier, do_mark_object, use_ext>::do_oop_nv(T* p) {
   T heap_oop = oopDesc::load_heap_oop(p);
 
   if (oopDesc::is_null(heap_oop)) {
@@ -286,6 +291,10 @@ void G1ParCopyClosure<barrier, do_mark_object>::do_oop_nv(T* p) {
     if (state.is_humongous()) {
       _g1->set_humongous_is_live(obj);
     }
+
+    if (use_ext && state.is_ext()) {
+      _par_scan_state->do_oop_ext(p);
+    }
     // The object is not in collection set. If we're a root scanning
     // closure during an initial mark pause then attempt to mark the object.
     if (do_mark_object == G1MarkFromRoot) {

hotspot/src/share/vm/gc/g1/g1ParScanThreadState.hpp

@@ -96,6 +96,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   bool verify_task(StarTask ref) const;
 #endif // ASSERT
 
+  template <class T> void do_oop_ext(T* ref);
   template <class T> void push_on_queue(T* ref);
 
   template <class T> void update_rs(HeapRegion* from, T* p, oop o) {

hotspot/src/share/vm/gc/g1/g1ParScanThreadState.inline.hpp

@@ -50,8 +50,8 @@ template <class T> void G1ParScanThreadState::do_oop_evac(T* p, HeapRegion* from
   } else if (in_cset_state.is_humongous()) {
     _g1h->set_humongous_is_live(obj);
   } else {
-    assert(!in_cset_state.is_in_cset_or_humongous(),
+    assert(in_cset_state.is_default() || in_cset_state.is_ext(),
-           "In_cset_state must be NotInCSet here, but is " CSETSTATE_FORMAT, in_cset_state.value());
+           "In_cset_state must be NotInCSet or Ext here, but is " CSETSTATE_FORMAT, in_cset_state.value());
   }
 
   assert(obj != NULL, "Must be");

hotspot/src/share/vm/gc/g1/g1ParScanThreadState_ext.cpp

@@ -29,3 +29,10 @@
 G1ParScanThreadState* G1ParScanThreadStateSet::new_par_scan_state(uint worker_id, size_t young_cset_length) {
   return new G1ParScanThreadState(_g1h, worker_id, young_cset_length);
 }
+
+template <typename T>
+void G1ParScanThreadState::do_oop_ext(T* ref) {
+}
+
+template void G1ParScanThreadState::do_oop_ext<oop>(oop* ref);
+template void G1ParScanThreadState::do_oop_ext<narrowOop>(narrowOop* ref);

hotspot/src/share/vm/gc/g1/g1SharedClosures.hpp

@@ -31,11 +31,11 @@ class G1CollectedHeap;
 class G1ParScanThreadState;
 
 // Simple holder object for a complete set of closures used by the G1 evacuation code.
-template <G1Mark Mark>
+template <G1Mark Mark, bool use_ext = false>
 class G1SharedClosures VALUE_OBJ_CLASS_SPEC {
 public:
-  G1ParCopyClosure<G1BarrierNone,  Mark> _oops;
+  G1ParCopyClosure<G1BarrierNone,  Mark, use_ext> _oops;
-  G1ParCopyClosure<G1BarrierKlass, Mark> _oop_in_klass;
+  G1ParCopyClosure<G1BarrierKlass, Mark, use_ext> _oop_in_klass;
   G1KlassScanClosure                              _klass_in_cld_closure;
   CLDToKlassAndOopClosure                         _clds;
   G1CodeBlobClosure                               _codeblobs;