8145442: Add the facility to verify remembered sets for G1

Implement remembered sets verification for G1 with option VerifyRememberedSets

Reviewed-by: jmasa, mgerdin

hotspot/src/share/vm/gc/g1/g1CollectedHeap.cpp

@@ -3535,6 +3535,16 @@ void G1CollectedHeap::register_humongous_regions_with_cset() {
   cl.flush_rem_set_entries();
 }
 
+class VerifyRegionRemSetClosure : public HeapRegionClosure {
+  public:
+    bool doHeapRegion(HeapRegion* hr) {
+      if (!hr->is_archive() && !hr->is_continues_humongous()) {
+        hr->verify_rem_set();
+      }
+      return false;
+    }
+};
+
 #ifdef ASSERT
 class VerifyCSetClosure: public HeapRegionClosure {
 public:
@@ -3724,6 +3734,12 @@ G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) {
       increment_total_collections(false /* full gc */);
       increment_gc_time_stamp();
 
+      if (VerifyRememberedSets) {
+        log_info(gc, verify)("[Verifying RemSets before GC]");
+        VerifyRegionRemSetClosure v_cl;
+        heap_region_iterate(&v_cl);
+      }
+
       verify_before_gc();
 
       check_bitmaps("GC Start");
@@ -3928,6 +3944,12 @@ G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) {
         // scanning cards (see CR 7039627).
         increment_gc_time_stamp();
 
+        if (VerifyRememberedSets) {
+          log_info(gc, verify)("[Verifying RemSets after GC]");
+          VerifyRegionRemSetClosure v_cl;
+          heap_region_iterate(&v_cl);
+        }
+
         verify_after_gc();
         check_bitmaps("GC End");
 

hotspot/src/share/vm/gc/g1/heapRegion.cpp

@@ -599,8 +599,8 @@ void HeapRegion::print_on(outputStream* st) const {
                p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start()));
 }
 
-class VerifyLiveClosure: public OopClosure {
+class G1VerificationClosure : public OopClosure {
-private:
+protected:
   G1CollectedHeap* _g1h;
   CardTableModRefBS* _bs;
   oop _containing_obj;
@@ -611,10 +611,10 @@ public:
   // _vo == UsePrevMarking -> use "prev" marking information,
   // _vo == UseNextMarking -> use "next" marking information,
   // _vo == UseMarkWord    -> use mark word from object header.
-  VerifyLiveClosure(G1CollectedHeap* g1h, VerifyOption vo) :
+  G1VerificationClosure(G1CollectedHeap* g1h, VerifyOption vo) :
     _g1h(g1h), _bs(barrier_set_cast<CardTableModRefBS>(g1h->barrier_set())),
-    _containing_obj(NULL), _failures(false), _n_failures(0), _vo(vo)
+    _containing_obj(NULL), _failures(false), _n_failures(0), _vo(vo) {
-  { }
+  }
 
   void set_containing_obj(oop obj) {
     _containing_obj = obj;
@@ -623,9 +623,6 @@ public:
   bool failures() { return _failures; }
   int n_failures() { return _n_failures; }
 
-  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
-  virtual void do_oop(      oop* p) { do_oop_work(p); }
-
   void print_object(outputStream* out, oop obj) {
 #ifdef PRODUCT
     Klass* k = obj->klass();
@@ -635,12 +632,24 @@ public:
     obj->print_on(out);
 #endif // PRODUCT
   }
+};
+
+class VerifyLiveClosure : public G1VerificationClosure {
+public:
+  VerifyLiveClosure(G1CollectedHeap* g1h, VerifyOption vo) : G1VerificationClosure(g1h, vo) {}
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+  virtual void do_oop(oop* p) { do_oop_work(p); }
 
   template <class T>
   void do_oop_work(T* p) {
     assert(_containing_obj != NULL, "Precondition");
     assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo),
-           "Precondition");
+      "Precondition");
+    verify_liveness(p);
+  }
+
+  template <class T>
+  void verify_liveness(T* p) {
     T heap_oop = oopDesc::load_heap_oop(p);
     LogHandle(gc, verify) log;
     if (!oopDesc::is_null(heap_oop)) {
@@ -648,7 +657,7 @@ public:
       bool failed = false;
       if (!_g1h->is_in_closed_subset(obj) || _g1h->is_obj_dead_cond(obj, _vo)) {
         MutexLockerEx x(ParGCRareEvent_lock,
-                        Mutex::_no_safepoint_check_flag);
+          Mutex::_no_safepoint_check_flag);
 
         if (!_failures) {
           log.info("----------");
@@ -657,17 +666,17 @@ public:
         if (!_g1h->is_in_closed_subset(obj)) {
           HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
           log.info("Field " PTR_FORMAT " of live obj " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ")",
-                   p2i(p), p2i(_containing_obj), p2i(from->bottom()), p2i(from->end()));
+            p2i(p), p2i(_containing_obj), p2i(from->bottom()), p2i(from->end()));
           print_object(log.info_stream(), _containing_obj);
           log.info("points to obj " PTR_FORMAT " not in the heap", p2i(obj));
         } else {
           HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
-          HeapRegion* to   = _g1h->heap_region_containing((HeapWord*)obj);
+          HeapRegion* to = _g1h->heap_region_containing((HeapWord*)obj);
           log.info("Field " PTR_FORMAT " of live obj " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ")",
-                   p2i(p), p2i(_containing_obj), p2i(from->bottom()), p2i(from->end()));
+            p2i(p), p2i(_containing_obj), p2i(from->bottom()), p2i(from->end()));
           print_object(log.info_stream(), _containing_obj);
           log.info("points to dead obj " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ")",
-                   p2i(obj), p2i(to->bottom()), p2i(to->end()));
+            p2i(obj), p2i(to->bottom()), p2i(to->end()));
           print_object(log.info_stream(), obj);
         }
         log.info("----------");
@@ -675,42 +684,64 @@ public:
         failed = true;
         _n_failures++;
       }
+    }
+  }
+};
 
-      if (!_g1h->collector_state()->full_collection() || G1VerifyRSetsDuringFullGC) {
+class VerifyRemSetClosure : public G1VerificationClosure {
-        HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
+public:
-        HeapRegion* to   = _g1h->heap_region_containing(obj);
+  VerifyRemSetClosure(G1CollectedHeap* g1h, VerifyOption vo) : G1VerificationClosure(g1h, vo) {}
-        if (from != NULL && to != NULL &&
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
-            from != to &&
+  virtual void do_oop(oop* p) { do_oop_work(p); }
-            !to->is_pinned()) {
-          jbyte cv_obj = *_bs->byte_for_const(_containing_obj);
-          jbyte cv_field = *_bs->byte_for_const(p);
-          const jbyte dirty = CardTableModRefBS::dirty_card_val();
 
-          bool is_bad = !(from->is_young()
+  template <class T>
-                          || to->rem_set()->contains_reference(p)
+  void do_oop_work(T* p) {
-                          || !G1HRRSFlushLogBuffersOnVerify && // buffers were not flushed
+    assert(_containing_obj != NULL, "Precondition");
-                              (_containing_obj->is_objArray() ?
+    assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo),
-                                  cv_field == dirty
+      "Precondition");
-                               : cv_obj == dirty || cv_field == dirty));
+    verify_remembered_set(p);
-          if (is_bad) {
+  }
-            MutexLockerEx x(ParGCRareEvent_lock,
-                            Mutex::_no_safepoint_check_flag);
 
-            if (!_failures) {
+  template <class T>
-              log.info("----------");
+  void verify_remembered_set(T* p) {
-            }
+    T heap_oop = oopDesc::load_heap_oop(p);
-            log.info("Missing rem set entry:");
+    LogHandle(gc, verify) log;
-            log.info("Field " PTR_FORMAT " of obj " PTR_FORMAT ", in region " HR_FORMAT,
+    if (!oopDesc::is_null(heap_oop)) {
-                     p2i(p), p2i(_containing_obj), HR_FORMAT_PARAMS(from));
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
-            ResourceMark rm;
+      bool failed = false;
-            _containing_obj->print_on(log.info_stream());
+      HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
-            log.info("points to obj " PTR_FORMAT " in region " HR_FORMAT, p2i(obj), HR_FORMAT_PARAMS(to));
+      HeapRegion* to = _g1h->heap_region_containing(obj);
-            obj->print_on(log.info_stream());
+      if (from != NULL && to != NULL &&
-            log.info("Obj head CTE = %d, field CTE = %d.", cv_obj, cv_field);
+        from != to &&
+        !to->is_pinned()) {
+        jbyte cv_obj = *_bs->byte_for_const(_containing_obj);
+        jbyte cv_field = *_bs->byte_for_const(p);
+        const jbyte dirty = CardTableModRefBS::dirty_card_val();
+
+        bool is_bad = !(from->is_young()
+          || to->rem_set()->contains_reference(p)
+          || !G1HRRSFlushLogBuffersOnVerify && // buffers were not flushed
+          (_containing_obj->is_objArray() ?
+          cv_field == dirty
+          : cv_obj == dirty || cv_field == dirty));
+        if (is_bad) {
+          MutexLockerEx x(ParGCRareEvent_lock,
+            Mutex::_no_safepoint_check_flag);
+
+          if (!_failures) {
             log.info("----------");
-            _failures = true;
-            if (!failed) _n_failures++;
           }
+          log.info("Missing rem set entry:");
+          log.info("Field " PTR_FORMAT " of obj " PTR_FORMAT ", in region " HR_FORMAT,
+            p2i(p), p2i(_containing_obj), HR_FORMAT_PARAMS(from));
+          ResourceMark rm;
+          _containing_obj->print_on(log.info_stream());
+          log.info("points to obj " PTR_FORMAT " in region " HR_FORMAT, p2i(obj), HR_FORMAT_PARAMS(to));
+          obj->print_on(log.info_stream());
+          log.info("Obj head CTE = %d, field CTE = %d.", cv_obj, cv_field);
+          log.info("----------");
+          _failures = true;
+          if (!failed) _n_failures++;
         }
       }
     }
@@ -727,6 +758,7 @@ void HeapRegion::verify(VerifyOption vo,
   HeapWord* p = bottom();
   HeapWord* prev_p = NULL;
   VerifyLiveClosure vl_cl(g1, vo);
+  VerifyRemSetClosure vr_cl(g1, vo);
   bool is_region_humongous = is_humongous();
   size_t object_num = 0;
   while (p < top()) {
@@ -752,7 +784,23 @@ void HeapRegion::verify(VerifyOption vo,
           return;
         } else {
           vl_cl.set_containing_obj(obj);
-          obj->oop_iterate_no_header(&vl_cl);
+          if (!g1->collector_state()->full_collection() || G1VerifyRSetsDuringFullGC) {
+            // verify liveness and rem_set
+            vr_cl.set_containing_obj(obj);
+            G1Mux2Closure mux(&vl_cl, &vr_cl);
+            obj->oop_iterate_no_header(&mux);
+
+            if (vr_cl.failures()) {
+              *failures = true;
+            }
+            if (G1MaxVerifyFailures >= 0 &&
+              vr_cl.n_failures() >= G1MaxVerifyFailures) {
+              return;
+            }
+          } else {
+            // verify only liveness
+            obj->oop_iterate_no_header(&vl_cl);
+          }
           if (vl_cl.failures()) {
             *failures = true;
           }
@@ -762,7 +810,7 @@ void HeapRegion::verify(VerifyOption vo,
           }
         }
       } else {
-        log_info(gc, verify)(PTR_FORMAT " no an oop", p2i(obj));
+        log_info(gc, verify)(PTR_FORMAT " not an oop", p2i(obj));
         *failures = true;
         return;
       }
@@ -852,6 +900,46 @@ void HeapRegion::verify() const {
   verify(VerifyOption_G1UsePrevMarking, /* failures */ &dummy);
 }
 
+void HeapRegion::verify_rem_set(VerifyOption vo, bool* failures) const {
+  G1CollectedHeap* g1 = G1CollectedHeap::heap();
+  *failures = false;
+  HeapWord* p = bottom();
+  HeapWord* prev_p = NULL;
+  VerifyRemSetClosure vr_cl(g1, vo);
+  while (p < top()) {
+    oop obj = oop(p);
+    size_t obj_size = block_size(p);
+
+    if (!g1->is_obj_dead_cond(obj, this, vo)) {
+      if (obj->is_oop()) {
+        vr_cl.set_containing_obj(obj);
+        obj->oop_iterate_no_header(&vr_cl);
+
+        if (vr_cl.failures()) {
+          *failures = true;
+        }
+        if (G1MaxVerifyFailures >= 0 &&
+          vr_cl.n_failures() >= G1MaxVerifyFailures) {
+          return;
+        }
+      } else {
+        log_info(gc, verify)(PTR_FORMAT " not an oop", p2i(obj));
+        *failures = true;
+        return;
+      }
+    }
+
+    prev_p = p;
+    p += obj_size;
+  }
+}
+
+void HeapRegion::verify_rem_set() const {
+  bool failures = false;
+  verify_rem_set(VerifyOption_G1UsePrevMarking, &failures);
+  guarantee(!failures, "HeapRegion RemSet verification failed");
+}
+
 void HeapRegion::prepare_for_compaction(CompactPoint* cp) {
   scan_and_forward(this, cp);
 }

hotspot/src/share/vm/gc/g1/heapRegion.hpp

@@ -745,6 +745,9 @@ class HeapRegion: public G1ContiguousSpace {
 
   // Override; it uses the "prev" marking information
   virtual void verify() const;
+
+  void verify_rem_set(VerifyOption vo, bool *failures) const;
+  void verify_rem_set() const;
 };
 
 // HeapRegionClosure is used for iterating over regions.