8227175: ZGC: ZHeapIterator visits potentially dead objects

Reviewed-by: zgu, eosterlund

src/hotspot/share/classfile/classLoaderData.cpp

@@ -266,6 +266,19 @@ bool ClassLoaderData::ChunkedHandleList::owner_of(oop* oop_handle) {
 }
 #endif // PRODUCT
 
+void ClassLoaderData::clear_claim(int claim) {
+  for (;;) {
+    int old_claim = Atomic::load(&_claim);
+    if ((old_claim & claim) == 0) {
+      return;
+    }
+    int new_claim = old_claim & ~claim;
+    if (Atomic::cmpxchg(new_claim, &_claim, old_claim) == old_claim) {
+      return;
+    }
+  }
+}
+
 bool ClassLoaderData::try_claim(int claim) {
   for (;;) {
     int old_claim = Atomic::load(&_claim);

src/hotspot/share/classfile/classLoaderData.hpp

@@ -206,16 +206,17 @@ class ClassLoaderData : public CHeapObj<mtClass> {
 
   // The "claim" is typically used to check if oops_do needs to be applied on
   // the CLD or not. Most GCs only perform strong marking during the marking phase.
-  enum {
+  enum Claim {
-    _claim_none        = 0,
+    _claim_none         = 0,
-    _claim_finalizable = 2,
+    _claim_finalizable  = 2,
-    _claim_strong      = 3
+    _claim_strong       = 3,
+    _claim_other        = 4
   };
   void clear_claim() { _claim = 0; }
+  void clear_claim(int claim);
   bool claimed() const { return _claim != 0; }
+  bool claimed(int claim) const { return (_claim & claim) == claim; }
   bool try_claim(int claim);
-  int get_claim() const { return _claim; }
-  void set_claim(int claim) { _claim = claim; }
 
   // Computes if the CLD is alive or not. This is safe to call in concurrent
   // contexts.

src/hotspot/share/classfile/classLoaderDataGraph.cpp

@@ -64,6 +64,11 @@ void ClassLoaderDataGraph::clear_claimed_marks() {
   }
 }
 
+void ClassLoaderDataGraph::clear_claimed_marks(int claim) {
+ for (ClassLoaderData* cld = OrderAccess::load_acquire(&_head); cld != NULL; cld = cld->next()) {
+    cld->clear_claim(claim);
+  }
+}
 // Class iterator used by the compiler.  It gets some number of classes at
 // a safepoint to decay invocation counters on the methods.
 class ClassLoaderDataGraphKlassIteratorStatic {
@@ -471,7 +476,7 @@ GrowableArray<ClassLoaderData*>* ClassLoaderDataGraph::new_clds() {
   // The CLDs in [_head, _saved_head] were all added during last call to remember_new_clds(true);
   ClassLoaderData* curr = _head;
   while (curr != _saved_head) {
-    if (!curr->claimed()) {
+    if (!curr->claimed(ClassLoaderData::_claim_strong)) {
       array->push(curr);
       LogTarget(Debug, class, loader, data) lt;
       if (lt.is_enabled()) {

src/hotspot/share/classfile/classLoaderDataGraph.hpp

@@ -68,6 +68,7 @@ class ClassLoaderDataGraph : public AllStatic {
   static void clean_module_and_package_info();
   static void purge();
   static void clear_claimed_marks();
+  static void clear_claimed_marks(int claim);
   // Iteration through CLDG inside a safepoint; GC support
   static void cld_do(CLDClosure* cl);
   static void cld_unloading_do(CLDClosure* cl);

src/hotspot/share/gc/z/zArguments.cpp

@@ -92,6 +92,11 @@ void ZArguments::initialize() {
   // same reason we need fixup_partial_loads
   FLAG_SET_DEFAULT(VerifyBeforeIteration, false);
 
+  if (VerifyBeforeGC || VerifyDuringGC || VerifyAfterGC) {
+    FLAG_SET_DEFAULT(ZVerifyRoots, true);
+    FLAG_SET_DEFAULT(ZVerifyObjects, true);
+  }
+
   // Verification of stacks not (yet) supported, for the same reason
   // we need fixup_partial_loads
   DEBUG_ONLY(FLAG_SET_DEFAULT(VerifyStack, false));

src/hotspot/share/gc/z/zCollectedHeap.cpp

@@ -232,11 +232,11 @@ GrowableArray<MemoryPool*> ZCollectedHeap::memory_pools() {
 }
 
 void ZCollectedHeap::object_iterate(ObjectClosure* cl) {
-  _heap.object_iterate(cl, true /* visit_referents */);
+  _heap.object_iterate(cl, true /* visit_weaks */);
 }
 
 void ZCollectedHeap::safe_object_iterate(ObjectClosure* cl) {
-  _heap.object_iterate(cl, true /* visit_referents */);
+  _heap.object_iterate(cl, true /* visit_weaks */);
 }
 
 HeapWord* ZCollectedHeap::block_start(const void* addr) const {

src/hotspot/share/gc/z/zDriver.cpp

@@ -31,6 +31,7 @@
 #include "gc/z/zMessagePort.inline.hpp"
 #include "gc/z/zServiceability.hpp"
 #include "gc/z/zStat.hpp"
+#include "gc/z/zVerify.hpp"
 #include "logging/log.hpp"
 #include "memory/universe.hpp"
 #include "runtime/vmOperations.hpp"
@@ -86,6 +87,9 @@ public:
     GCIdMark gc_id_mark(_gc_id);
     IsGCActiveMark gc_active_mark;
 
+    // Verify roots
+    ZVerify::roots_strong();
+
     // Execute operation
     _success = do_operation();
 
@@ -301,8 +305,14 @@ void ZDriver::concurrent_reset_relocation_set() {
 
 void ZDriver::pause_verify() {
   if (VerifyBeforeGC || VerifyDuringGC || VerifyAfterGC) {
+    // Full verification
     VM_Verify op;
     VMThread::execute(&op);
+
+  } else if (ZVerifyRoots || ZVerifyObjects) {
+    // Limited verification
+    VM_ZVerifyOperation op;
+    VMThread::execute(&op);
   }
 }
 

src/hotspot/share/gc/z/zHeap.cpp

@@ -41,6 +41,7 @@
 #include "gc/z/zTask.hpp"
 #include "gc/z/zThread.hpp"
 #include "gc/z/zTracer.inline.hpp"
+#include "gc/z/zVerify.hpp"
 #include "gc/z/zVirtualMemory.inline.hpp"
 #include "gc/z/zWorkers.inline.hpp"
 #include "logging/log.hpp"
@@ -340,6 +341,9 @@ bool ZHeap::mark_end() {
   // Enter mark completed phase
   ZGlobalPhase = ZPhaseMarkCompleted;
 
+  // Verify after mark
+  ZVerify::after_mark();
+
   // Update statistics
   ZStatSample(ZSamplerHeapUsedAfterMark, used());
   ZStatHeap::set_at_mark_end(capacity(), allocated(), used());
@@ -468,11 +472,11 @@ void ZHeap::relocate() {
                                  used(), used_high(), used_low());
 }
 
-void ZHeap::object_iterate(ObjectClosure* cl, bool visit_referents) {
+void ZHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) {
   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 
   ZHeapIterator iter;
-  iter.objects_do(cl, visit_referents);
+  iter.objects_do(cl, visit_weaks);
 }
 
 void ZHeap::serviceability_initialize() {
@@ -518,40 +522,11 @@ void ZHeap::print_extended_on(outputStream* st) const {
   st->cr();
 }
 
-class ZVerifyRootsTask : public ZTask {
-private:
-  ZStatTimerDisable  _disable;
-  ZRootsIterator     _strong_roots;
-  ZWeakRootsIterator _weak_roots;
-
-public:
-  ZVerifyRootsTask() :
-      ZTask("ZVerifyRootsTask"),
-      _disable(),
-      _strong_roots(),
-      _weak_roots() {}
-
-  virtual void work() {
-    ZStatTimerDisable disable;
-    ZVerifyOopClosure cl;
-    _strong_roots.oops_do(&cl);
-    _weak_roots.oops_do(&cl);
-  }
-};
-
 void ZHeap::verify() {
   // Heap verification can only be done between mark end and
   // relocate start. This is the only window where all oop are
   // good and the whole heap is in a consistent state.
   guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
 
-  {
+  ZVerify::after_weak_processing();
-    ZVerifyRootsTask task;
-    _workers.run_parallel(&task);
-  }
-
-  {
-    ZVerifyObjectClosure cl;
-    object_iterate(&cl, false /* visit_referents */);
-  }
 }

src/hotspot/share/gc/z/zHeap.hpp

@@ -161,7 +161,7 @@ public:
   void relocate();
 
   // Iteration
-  void object_iterate(ObjectClosure* cl, bool visit_referents);
+  void object_iterate(ObjectClosure* cl, bool visit_weaks);
 
   // Serviceability
   void serviceability_initialize();

src/hotspot/share/gc/z/zHeapIterator.cpp

@@ -22,6 +22,8 @@
  */
 
 #include "precompiled.hpp"
+#include "classfile/classLoaderData.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
 #include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zGranuleMap.inline.hpp"
@@ -83,7 +85,7 @@ public:
 };
 
 template <bool VisitReferents>
-class ZHeapIteratorOopClosure : public BasicOopIterateClosure {
+class ZHeapIteratorOopClosure : public ClaimMetadataVisitingOopIterateClosure {
 private:
   ZHeapIterator* const _iter;
   const oop            _base;
@@ -98,6 +100,7 @@ private:
 
 public:
   ZHeapIteratorOopClosure(ZHeapIterator* iter, oop base) :
+      ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_other),
       _iter(iter),
       _base(base) {}
 
@@ -130,6 +133,7 @@ ZHeapIterator::~ZHeapIterator() {
   for (ZHeapIteratorBitMap* map; iter.next(&map);) {
     delete map;
   }
+  ClassLoaderDataGraph::clear_claimed_marks(ClassLoaderData::_claim_other);
 }
 
 static size_t object_index_max() {
@@ -184,15 +188,23 @@ void ZHeapIterator::push_fields(oop obj) {
   obj->oop_iterate(&cl);
 }
 
-template <bool VisitReferents>
+class ZHeapIterateConcurrentRootsIterator : public ZConcurrentRootsIterator {
+public:
+  ZHeapIterateConcurrentRootsIterator() :
+      ZConcurrentRootsIterator(ClassLoaderData::_claim_other) {}
+};
+
+template <bool VisitWeaks>
 void ZHeapIterator::objects_do(ObjectClosure* cl) {
   ZStatTimerDisable disable;
 
   // Push roots to visit
-  push_roots<ZRootsIterator,               false /* Concurrent */, false /* Weak */>();
+  push_roots<ZRootsIterator,                      false /* Concurrent */, false /* Weak */>();
-  push_roots<ZConcurrentRootsIterator,     true  /* Concurrent */, false /* Weak */>();
+  push_roots<ZHeapIterateConcurrentRootsIterator, true  /* Concurrent */, false /* Weak */>();
-  push_roots<ZWeakRootsIterator,           false /* Concurrent */, true  /* Weak */>();
+  if (VisitWeaks) {
-  push_roots<ZConcurrentWeakRootsIterator, true  /* Concurrent */, true  /* Weak */>();
+    push_roots<ZWeakRootsIterator,           false /* Concurrent */, true  /* Weak */>();
+    push_roots<ZConcurrentWeakRootsIterator, true  /* Concurrent */, true  /* Weak */>();
+  }
 
   // Drain stack
   while (!_visit_stack.is_empty()) {
@@ -202,14 +214,14 @@ void ZHeapIterator::objects_do(ObjectClosure* cl) {
     cl->do_object(obj);
 
     // Push fields to visit
-    push_fields<VisitReferents>(obj);
+    push_fields<VisitWeaks>(obj);
   }
 }
 
-void ZHeapIterator::objects_do(ObjectClosure* cl, bool visit_referents) {
+void ZHeapIterator::objects_do(ObjectClosure* cl, bool visit_weaks) {
-  if (visit_referents) {
+  if (visit_weaks) {
-    objects_do<true /* VisitReferents */>(cl);
+    objects_do<true /* VisitWeaks */>(cl);
   } else {
-    objects_do<false /* VisitReferents */>(cl);
+    objects_do<false /* VisitWeaks */>(cl);
   }
 }

src/hotspot/share/gc/z/zHeapIterator.hpp

@@ -54,7 +54,7 @@ public:
   ZHeapIterator();
   ~ZHeapIterator();
 
-  void objects_do(ObjectClosure* cl, bool visit_referents);
+  void objects_do(ObjectClosure* cl, bool visit_weaks);
 };
 
 #endif // SHARE_GC_Z_ZHEAPITERATOR_HPP

src/hotspot/share/gc/z/zMark.cpp

@@ -22,6 +22,7 @@
  */
 
 #include "precompiled.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
 #include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zMark.inline.hpp"
 #include "gc/z/zMarkCache.inline.hpp"
@@ -632,14 +633,23 @@ public:
 
 class ZMarkConcurrentRootsTask : public ZTask {
 private:
+  SuspendibleThreadSetJoiner          _sts_joiner;
   ZConcurrentRootsIterator            _roots;
   ZMarkConcurrentRootsIteratorClosure _cl;
 
 public:
   ZMarkConcurrentRootsTask(ZMark* mark) :
       ZTask("ZMarkConcurrentRootsTask"),
-      _roots(true /* marking */),
+      _sts_joiner(true /* active */),
-      _cl() {}
+      _roots(ClassLoaderData::_claim_strong),
+      _cl() {
+    ClassLoaderDataGraph_lock->lock();
+    ClassLoaderDataGraph::clear_claimed_marks();
+  }
+
+  ~ZMarkConcurrentRootsTask() {
+    ClassLoaderDataGraph_lock->unlock();
+  }
 
   virtual void work() {
     _roots.oops_do(&_cl);

src/hotspot/share/gc/z/zOopClosures.cpp

@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/z/zHeap.hpp"
-#include "gc/z/zOopClosures.inline.hpp"
-#include "gc/z/zOop.inline.hpp"
-#include "memory/iterator.inline.hpp"
-#include "oops/access.inline.hpp"
-#include "oops/oop.inline.hpp"
-#include "runtime/safepoint.hpp"
-#include "utilities/debug.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-void ZVerifyOopClosure::do_oop(oop* p) {
-  guarantee(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
-  guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
-  guarantee(!ZResurrection::is_blocked(), "Invalid phase");
-
-  const oop o = RawAccess<>::oop_load(p);
-  if (o != NULL) {
-    const uintptr_t addr = ZOop::to_address(o);
-    const uintptr_t good_addr = ZAddress::good(addr);
-    guarantee(ZAddress::is_good(addr) || ZAddress::is_finalizable_good(addr),
-              "Bad oop " PTR_FORMAT " found at " PTR_FORMAT ", expected " PTR_FORMAT,
-              addr, p2i(p), good_addr);
-    guarantee(oopDesc::is_oop(ZOop::from_address(good_addr)),
-              "Bad object " PTR_FORMAT " found at " PTR_FORMAT,
-              addr, p2i(p));
-  }
-}
-
-void ZVerifyOopClosure::do_oop(narrowOop* p) {
-  ShouldNotReachHere();
-}
-
-void ZVerifyObjectClosure::do_object(oop o) {
-  ZVerifyOopClosure cl;
-  o->oop_iterate(&cl);
-}

src/hotspot/share/gc/z/zOopClosures.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -46,16 +46,13 @@ public:
 };
 
 template <bool finalizable>
-class ZMarkBarrierOopClosure : public MetadataVisitingOopIterateClosure {
+class ZMarkBarrierOopClosure : public ClaimMetadataVisitingOopIterateClosure {
 public:
   ZMarkBarrierOopClosure();
 
   virtual void do_oop(oop* p);
   virtual void do_oop(narrowOop* p);
 
-  virtual void do_klass(Klass* k);
-  virtual void do_cld(ClassLoaderData* cld);
-
 #ifdef ASSERT
   virtual bool should_verify_oops() {
     return false;
@@ -80,26 +77,4 @@ public:
   virtual void do_oop(narrowOop* p);
 };
 
-class ZVerifyOopClosure : public ZRootsIteratorClosure, public BasicOopIterateClosure {
-public:
-  virtual void do_oop(oop* p);
-  virtual void do_oop(narrowOop* p);
-
-  virtual ReferenceIterationMode reference_iteration_mode() {
-    return DO_FIELDS;
-  }
-
-#ifdef ASSERT
-  // Verification handled by the closure itself
-  virtual bool should_verify_oops() {
-    return false;
-  }
-#endif
-};
-
-class ZVerifyObjectClosure : public ObjectClosure {
-public:
-  virtual void do_object(oop o);
-};
-
 #endif // SHARE_GC_Z_ZOOPCLOSURES_HPP

src/hotspot/share/gc/z/zOopClosures.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -55,7 +55,12 @@ inline void ZNMethodOopClosure::do_oop(narrowOop* p) {
 
 template <bool finalizable>
 inline ZMarkBarrierOopClosure<finalizable>::ZMarkBarrierOopClosure() :
-    MetadataVisitingOopIterateClosure(finalizable ? NULL : ZHeap::heap()->reference_discoverer()) {}
+    ClaimMetadataVisitingOopIterateClosure(finalizable
+                                               ? ClassLoaderData::_claim_finalizable
+                                               : ClassLoaderData::_claim_strong,
+                                           finalizable
+                                               ? NULL
+                                               : ZHeap::heap()->reference_discoverer()) {}
 
 template <bool finalizable>
 inline void ZMarkBarrierOopClosure<finalizable>::do_oop(oop* p) {
@@ -67,18 +72,6 @@ inline void ZMarkBarrierOopClosure<finalizable>::do_oop(narrowOop* p) {
   ShouldNotReachHere();
 }
 
-template <bool finalizable>
-inline void ZMarkBarrierOopClosure<finalizable>::do_klass(Klass* k) {
-  ClassLoaderData* const cld = k->class_loader_data();
-  ZMarkBarrierOopClosure<finalizable>::do_cld(cld);
-}
-
-template <bool finalizable>
-inline void ZMarkBarrierOopClosure<finalizable>::do_cld(ClassLoaderData* cld) {
-  const int claim = finalizable ? ClassLoaderData::_claim_finalizable : ClassLoaderData::_claim_strong;
-  cld->oops_do(this, claim);
-}
-
 inline bool ZPhantomIsAliveObjectClosure::do_object_b(oop o) {
   return ZBarrier::is_alive_barrier_on_phantom_oop(o);
 }

src/hotspot/share/gc/z/zRootsIterator.cpp

@@ -263,24 +263,16 @@ void ZRootsIterator::oops_do(ZRootsIteratorClosure* cl, bool visit_jvmti_weak_ex
   }
 }
 
-ZConcurrentRootsIterator::ZConcurrentRootsIterator(bool marking) :
+ZConcurrentRootsIterator::ZConcurrentRootsIterator(int cld_claim) :
-    _marking(marking),
-    _sts_joiner(marking /* active */),
     _jni_handles_iter(JNIHandles::global_handles()),
+    _cld_claim(cld_claim),
     _jni_handles(this),
     _class_loader_data_graph(this) {
   ZStatTimer timer(ZSubPhaseConcurrentRootsSetup);
-  if (_marking) {
-    ClassLoaderDataGraph_lock->lock();
-    ClassLoaderDataGraph::clear_claimed_marks();
-  }
 }
 
 ZConcurrentRootsIterator::~ZConcurrentRootsIterator() {
   ZStatTimer timer(ZSubPhaseConcurrentRootsTeardown);
-  if (_marking) {
-    ClassLoaderDataGraph_lock->unlock();
-  }
 }
 
 void ZConcurrentRootsIterator::do_jni_handles(ZRootsIteratorClosure* cl) {
@@ -290,13 +282,8 @@ void ZConcurrentRootsIterator::do_jni_handles(ZRootsIteratorClosure* cl) {
 
 void ZConcurrentRootsIterator::do_class_loader_data_graph(ZRootsIteratorClosure* cl) {
   ZStatTimer timer(ZSubPhaseConcurrentRootsClassLoaderDataGraph);
-  if (_marking) {
+  CLDToOopClosure cld_cl(cl, _cld_claim);
-    CLDToOopClosure cld_cl(cl, ClassLoaderData::_claim_strong);
+  ClassLoaderDataGraph::always_strong_cld_do(&cld_cl);
-    ClassLoaderDataGraph::always_strong_cld_do(&cld_cl);
-  } else {
-    CLDToOopClosure cld_cl(cl, ClassLoaderData::_claim_none);
-    ClassLoaderDataGraph::cld_do(&cld_cl);
-  }
 }
 
 void ZConcurrentRootsIterator::oops_do(ZRootsIteratorClosure* cl) {

src/hotspot/share/gc/z/zRootsIterator.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -111,9 +111,8 @@ public:
 
 class ZConcurrentRootsIterator {
 private:
-  const bool                 _marking;
-  SuspendibleThreadSetJoiner _sts_joiner;
   ZOopStorageIterator        _jni_handles_iter;
+  int                        _cld_claim;
 
   void do_jni_handles(ZRootsIteratorClosure* cl);
   void do_class_loader_data_graph(ZRootsIteratorClosure* cl);
@@ -122,7 +121,7 @@ private:
   ZParallelOopsDo<ZConcurrentRootsIterator, &ZConcurrentRootsIterator::do_class_loader_data_graph> _class_loader_data_graph;
 
 public:
-  ZConcurrentRootsIterator(bool marking = false);
+  ZConcurrentRootsIterator(int cld_claim);
   ~ZConcurrentRootsIterator();
 
   void oops_do(ZRootsIteratorClosure* cl);

src/hotspot/share/gc/z/zVerify.cpp

@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/classLoaderData.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zOop.hpp"
+#include "gc/z/zResurrection.hpp"
+#include "gc/z/zRootsIterator.hpp"
+#include "gc/z/zStat.hpp"
+#include "gc/z/zVerify.hpp"
+#include "memory/allocation.hpp"
+#include "memory/iterator.inline.hpp"
+#include "oops/oop.inline.hpp"
+
+#define BAD_OOP_REPORT(addr)                                                \
+    "Bad oop " PTR_FORMAT " found at " PTR_FORMAT ", expected " PTR_FORMAT, \
+    addr, p2i(p), ZAddress::good(addr)
+
+class ZVerifyRootsClosure : public ZRootsIteratorClosure {
+public:
+  virtual void do_oop(oop* p) {
+    uintptr_t value = ZOop::to_address(*p);
+
+    if (value == 0) {
+      return;
+    }
+
+    guarantee(!ZAddress::is_finalizable(value), BAD_OOP_REPORT(value));
+    guarantee(ZAddress::is_good(value), BAD_OOP_REPORT(value));
+    guarantee(oopDesc::is_oop(ZOop::from_address(value)), BAD_OOP_REPORT(value));
+  }
+  virtual void do_oop(narrowOop*) { ShouldNotReachHere(); }
+};
+
+template <bool VisitReferents>
+class ZVerifyOopClosure : public ClaimMetadataVisitingOopIterateClosure, public ZRootsIteratorClosure  {
+public:
+  ZVerifyOopClosure() :
+      ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_other) {}
+
+  virtual void do_oop(oop* p);
+  virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); }
+
+  virtual ReferenceIterationMode reference_iteration_mode() {
+    return VisitReferents ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT;
+  }
+
+#ifdef ASSERT
+  // Verification handled by the closure itself
+  virtual bool should_verify_oops() {
+    return false;
+  }
+#endif
+};
+
+class ZVerifyObjectClosure : public ObjectClosure {
+private:
+  bool _visit_referents;
+
+public:
+  ZVerifyObjectClosure(bool visit_referents) : _visit_referents(visit_referents) {}
+  virtual void do_object(oop o);
+};
+
+template <typename RootsIterator>
+void ZVerify::roots_impl() {
+  if (ZVerifyRoots) {
+    ZVerifyRootsClosure cl;
+    RootsIterator iter;
+    iter.oops_do(&cl);
+  }
+}
+
+void ZVerify::roots_strong() {
+  roots_impl<ZRootsIterator>();
+}
+
+class ZVerifyConcurrentRootsIterator : public ZConcurrentRootsIterator {
+public:
+  ZVerifyConcurrentRootsIterator()
+      : ZConcurrentRootsIterator(ClassLoaderData::_claim_none) {}
+};
+
+void ZVerify::roots_concurrent() {
+  roots_impl<ZVerifyConcurrentRootsIterator>();
+}
+
+void ZVerify::roots_weak() {
+  assert(!ZResurrection::is_blocked(), "Invalid phase");
+
+  roots_impl<ZWeakRootsIterator>();
+}
+
+void ZVerify::roots(bool verify_weaks) {
+  roots_strong();
+  roots_concurrent();
+  if (verify_weaks) {
+    roots_weak();
+    roots_concurrent_weak();
+  }
+}
+
+void ZVerify::objects(bool verify_weaks) {
+  if (ZVerifyObjects) {
+    ZVerifyObjectClosure cl(verify_weaks);
+    ZHeap::heap()->object_iterate(&cl, verify_weaks);
+  }
+}
+
+void ZVerify::roots_concurrent_weak() {
+  assert(!ZResurrection::is_blocked(), "Invalid phase");
+
+  roots_impl<ZConcurrentWeakRootsIterator>();
+}
+
+void ZVerify::roots_and_objects(bool verify_weaks) {
+  ZStatTimerDisable  _disable;
+
+  roots(verify_weaks);
+  objects(verify_weaks);
+}
+
+void ZVerify::after_mark() {
+  // Only verify strong roots and references.
+  roots_and_objects(false /* verify_weaks */);
+}
+
+void ZVerify::after_weak_processing() {
+  // Also verify weaks - all should have been processed at this point.
+  roots_and_objects(true /* verify_weaks */);
+}
+
+template <bool VisitReferents>
+void ZVerifyOopClosure<VisitReferents>::do_oop(oop* p) {
+  guarantee(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
+  guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
+  guarantee(!ZResurrection::is_blocked(), "Invalid phase");
+
+  const oop o = RawAccess<>::oop_load(p);
+  if (o == NULL) {
+    return;
+  }
+
+  const uintptr_t addr = ZOop::to_address(o);
+  if (VisitReferents) {
+    guarantee(ZAddress::is_good(addr) || ZAddress::is_finalizable_good(addr), BAD_OOP_REPORT(addr));
+  } else {
+    // Should not encounter finalizable oops through strong-only paths. Assumes only strong roots are visited.
+    guarantee(ZAddress::is_good(addr), BAD_OOP_REPORT(addr));
+  }
+
+  const uintptr_t good_addr = ZAddress::good(addr);
+  guarantee(oopDesc::is_oop(ZOop::from_address(good_addr)), BAD_OOP_REPORT(addr));
+}
+
+void ZVerifyObjectClosure::do_object(oop o) {
+  if (_visit_referents) {
+    ZVerifyOopClosure<true /* VisitReferents */> cl;
+    o->oop_iterate((OopIterateClosure*)&cl);
+  } else {
+    ZVerifyOopClosure<false /* VisitReferents */> cl;
+    o->oop_iterate(&cl);
+  }
+}

src/hotspot/share/gc/z/zVerify.hpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZVERIFY_HPP
+#define SHARE_GC_Z_ZVERIFY_HPP
+
+#include "memory/allocation.hpp"
+
+class ZVerify : public AllStatic {
+private:
+  template <typename RootsIterator>
+  static void roots_impl();
+  static void roots(bool verify_weaks);
+
+  static void roots_weak();
+  static void roots_concurrent();
+  static void roots_concurrent_weak();
+
+  static void objects(bool verify_weaks);
+
+  static void roots_and_objects(bool visit_weaks);
+
+public:
+  // Verify strong (non-concurrent) roots. Should always be good.
+  static void roots_strong();
+
+  // Verify all strong roots and references after marking.
+  static void after_mark();
+
+  // Verify strong and weak roots and references.
+  static void after_weak_processing();
+};
+
+class VM_ZVerifyOperation : public VM_Operation {
+public:
+  virtual bool needs_inactive_gc_locker() const {
+    // An inactive GC locker is needed in operations where we change the bad
+    // mask or move objects. Changing the bad mask will invalidate all oops,
+    // which makes it conceptually the same thing as moving all objects.
+    return false;
+  }
+
+  virtual void doit() {
+    ZVerify::after_weak_processing();
+  }
+
+  bool success() const {
+    return true;
+  }
+
+  virtual VMOp_Type type() const { return VMOp_ZVerify; }
+};
+
+#endif // SHARE_GC_Z_ZVERIFY_HPP

src/hotspot/share/gc/z/z_globals.hpp

@@ -76,6 +76,12 @@
   diagnostic(bool, ZVerifyViews, false,                                     \
           "Verify heap view accesses")                                      \
                                                                             \
+  diagnostic(bool, ZVerifyRoots, trueInDebug,                               \
+          "Verify roots")                                                   \
+                                                                            \
+  diagnostic(bool, ZVerifyObjects, false,                                   \
+          "Verify objects")                                                 \
+                                                                            \
   diagnostic(bool, ZVerifyMarking, false,                                   \
           "Verify marking stacks")                                          \
                                                                             \

src/hotspot/share/memory/iterator.hpp

@@ -144,18 +144,28 @@ class CLDToOopClosure : public CLDClosure {
   void do_cld(ClassLoaderData* cld);
 };
 
-// The base class for all concurrent marking closures,
+class ClaimMetadataVisitingOopIterateClosure : public OopIterateClosure {
-// that participates in class unloading.
+ protected:
-// It's used to proxy through the metadata to the oops defined in them.
+  const int _claim;
-class MetadataVisitingOopIterateClosure: public OopIterateClosure {
+
  public:
-  MetadataVisitingOopIterateClosure(ReferenceDiscoverer* rd = NULL) : OopIterateClosure(rd) { }
+  ClaimMetadataVisitingOopIterateClosure(int claim, ReferenceDiscoverer* rd = NULL) :
+      OopIterateClosure(rd),
+      _claim(claim) { }
 
   virtual bool do_metadata() { return true; }
   virtual void do_klass(Klass* k);
   virtual void do_cld(ClassLoaderData* cld);
 };
 
+// The base class for all concurrent marking closures,
+// that participates in class unloading.
+// It's used to proxy through the metadata to the oops defined in them.
+class MetadataVisitingOopIterateClosure: public ClaimMetadataVisitingOopIterateClosure {
+ public:
+  MetadataVisitingOopIterateClosure(ReferenceDiscoverer* rd = NULL);
+};
+
 // ObjectClosure is used for iterating through an object space
 
 class ObjectClosure : public Closure {

src/hotspot/share/memory/iterator.inline.hpp

@@ -39,13 +39,17 @@
 #include "oops/typeArrayKlass.inline.hpp"
 #include "utilities/debug.hpp"
 
-inline void MetadataVisitingOopIterateClosure::do_cld(ClassLoaderData* cld) {
+// Defaults to strong claiming.
-  cld->oops_do(this, ClassLoaderData::_claim_strong);
+inline MetadataVisitingOopIterateClosure::MetadataVisitingOopIterateClosure(ReferenceDiscoverer* rd) :
+    ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_strong, rd) {}
+
+inline void ClaimMetadataVisitingOopIterateClosure::do_cld(ClassLoaderData* cld) {
+  cld->oops_do(this, _claim);
 }
 
-inline void MetadataVisitingOopIterateClosure::do_klass(Klass* k) {
+inline void ClaimMetadataVisitingOopIterateClosure::do_klass(Klass* k) {
   ClassLoaderData* cld = k->class_loader_data();
-  MetadataVisitingOopIterateClosure::do_cld(cld);
+  ClaimMetadataVisitingOopIterateClosure::do_cld(cld);
 }
 
 #ifdef ASSERT

src/hotspot/share/runtime/vmOperations.hpp

@@ -72,6 +72,7 @@
   template(ZMarkStart)                            \
   template(ZMarkEnd)                              \
   template(ZRelocateStart)                        \
+  template(ZVerify)                               \
   template(HandshakeOneThread)                    \
   template(HandshakeAllThreads)                   \
   template(HandshakeFallback)                     \