6672698: mangle_unused_area() should not remangle the entire heap at each collection

Maintain a high water mark for the allocations in a space and mangle only up to that high water mark.

Reviewed-by: ysr, apetrusenko

hotspot/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp

@@ -42,19 +42,31 @@ MutableNUMASpace::~MutableNUMASpace() {
   delete lgrp_spaces();
 }
 
+#ifndef PRODUCT
 void MutableNUMASpace::mangle_unused_area() {
-  for (int i = 0; i < lgrp_spaces()->length(); i++) {
-    LGRPSpace *ls = lgrp_spaces()->at(i);
-    MutableSpace *s = ls->space();
-    if (!os::numa_has_static_binding()) {
-      HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
-      if (top < s->end()) {
-        ls->add_invalid_region(MemRegion(top, s->end()));
-      }
-    }
-    s->mangle_unused_area();
-  }
+  // This method should do nothing.
+  // It can be called on a numa space during a full compaction.
 }
+void MutableNUMASpace::mangle_unused_area_complete() {
+  // This method should do nothing.
+  // It can be called on a numa space during a full compaction.
+}
+void MutableNUMASpace::mangle_region(MemRegion mr) {
+  // This method should do nothing because numa spaces are not mangled.
+}
+void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
+  assert(false, "Do not mangle MutableNUMASpace's");
+}
+void MutableNUMASpace::set_top_for_allocations() {
+  // This method should do nothing.
+}
+void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
+  // This method should do nothing.
+}
+void MutableNUMASpace::check_mangled_unused_area_complete() {
+  // This method should do nothing.
+}
+#endif  // NOT_PRODUCT
 
 // There may be unallocated holes in the middle chunks
 // that should be filled with dead objects to ensure parseability.
@@ -243,7 +255,10 @@ void MutableNUMASpace::update() {
       s->set_end(s->bottom());
       s->set_top(s->bottom());
     }
-    initialize(region(), true);
+    // A NUMA space is never mangled
+    initialize(region(),
+               SpaceDecorator::Clear,
+               SpaceDecorator::DontMangle);
   } else {
     bool should_initialize = false;
     if (!os::numa_has_static_binding()) {
@@ -257,7 +272,10 @@ void MutableNUMASpace::update() {
 
     if (should_initialize ||
         (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
-      initialize(region(), true);
+      // A NUMA space is never mangled
+      initialize(region(),
+                 SpaceDecorator::Clear,
+                 SpaceDecorator::DontMangle);
     }
   }
 
@@ -448,14 +466,17 @@ void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersecti
         }
 }
 
-void MutableNUMASpace::initialize(MemRegion mr, bool clear_space) {
+void MutableNUMASpace::initialize(MemRegion mr,
+                                  bool clear_space,
+                                  bool mangle_space) {
   assert(clear_space, "Reallocation will destory data!");
   assert(lgrp_spaces()->length() > 0, "There should be at least one space");
 
   MemRegion old_region = region(), new_region;
   set_bottom(mr.start());
   set_end(mr.end());
-  MutableSpace::set_top(bottom());
+  // Must always clear the space
+  clear(SpaceDecorator::DontMangle);
 
   // Compute chunk sizes
   size_t prev_page_size = page_size();
@@ -586,10 +607,8 @@ void MutableNUMASpace::initialize(MemRegion mr, bool clear_space) {
       bias_region(top_region, ls->lgrp_id());
     }
 
-    // If we clear the region, we would mangle it in debug. That would cause page
-    // allocation in a different place. Hence setting the top directly.
-    s->initialize(new_region, false);
-    s->set_top(s->bottom());
+    // Clear space (set top = bottom) but never mangle.
+    s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle);
 
     set_adaptation_cycles(samples_count());
   }
@@ -641,10 +660,12 @@ void MutableNUMASpace::set_top(HeapWord* value) {
   MutableSpace::set_top(value);
 }
 
-void MutableNUMASpace::clear() {
+void MutableNUMASpace::clear(bool mangle_space) {
   MutableSpace::set_top(bottom());
   for (int i = 0; i < lgrp_spaces()->length(); i++) {
-    lgrp_spaces()->at(i)->space()->clear();
+    // Never mangle NUMA spaces because the mangling will
+    // bind the memory to a possibly unwanted lgroup.
+    lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
   }
 }
 

hotspot/src/share/vm/gc_implementation/shared/mutableNUMASpace.hpp

@@ -171,14 +171,21 @@ class MutableNUMASpace : public MutableSpace {
   MutableNUMASpace();
   virtual ~MutableNUMASpace();
   // Space initialization.
-  virtual void initialize(MemRegion mr, bool clear_space);
+  virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space);
   // Update space layout if necessary. Do all adaptive resizing job.
   virtual void update();
   // Update allocation rate averages.
   virtual void accumulate_statistics();
 
-  virtual void clear();
-  virtual void mangle_unused_area();
+  virtual void clear(bool mangle_space);
+  virtual void mangle_unused_area() PRODUCT_RETURN;
+  virtual void mangle_unused_area_complete() PRODUCT_RETURN;
+  virtual void mangle_region(MemRegion mr) PRODUCT_RETURN;
+  virtual void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN;
+  virtual void check_mangled_unused_area_complete() PRODUCT_RETURN;
+  virtual void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN;
+  virtual void set_top_for_allocations() PRODUCT_RETURN;
+
   virtual void ensure_parsability();
   virtual size_t used_in_words() const;
   virtual size_t free_in_words() const;

hotspot/src/share/vm/gc_implementation/shared/mutableSpace.cpp

@@ -25,7 +25,17 @@
 # include "incls/_precompiled.incl"
 # include "incls/_mutableSpace.cpp.incl"
 
-void MutableSpace::initialize(MemRegion mr, bool clear_space) {
+MutableSpace::MutableSpace(): ImmutableSpace(), _top(NULL) {
+  _mangler = new MutableSpaceMangler(this);
+}
+
+MutableSpace::~MutableSpace() {
+  delete _mangler;
+}
+
+void MutableSpace::initialize(MemRegion mr,
+                              bool clear_space,
+                              bool mangle_space) {
   HeapWord* bottom = mr.start();
   HeapWord* end    = mr.end();
 
@@ -34,14 +44,51 @@ void MutableSpace::initialize(MemRegion mr, bool clear_space) {
   set_bottom(bottom);
   set_end(end);
 
-  if (clear_space) clear();
+  if (clear_space) {
+    clear(mangle_space);
+  }
 }
 
-void MutableSpace::clear() {
+void MutableSpace::clear(bool mangle_space) {
   set_top(bottom());
-  if (ZapUnusedHeapArea) mangle_unused_area();
+  if (ZapUnusedHeapArea && mangle_space) {
+    mangle_unused_area();
+  }
 }
 
+#ifndef PRODUCT
+void MutableSpace::check_mangled_unused_area(HeapWord* limit) {
+  mangler()->check_mangled_unused_area(limit);
+}
+
+void MutableSpace::check_mangled_unused_area_complete() {
+  mangler()->check_mangled_unused_area_complete();
+}
+
+// Mangle only the unused space that has not previously
+// been mangled and that has not been allocated since being
+// mangled.
+void MutableSpace::mangle_unused_area() {
+  mangler()->mangle_unused_area();
+}
+
+void MutableSpace::mangle_unused_area_complete() {
+  mangler()->mangle_unused_area_complete();
+}
+
+void MutableSpace::mangle_region(MemRegion mr) {
+  SpaceMangler::mangle_region(mr);
+}
+
+void MutableSpace::set_top_for_allocations(HeapWord* v) {
+  mangler()->set_top_for_allocations(v);
+}
+
+void MutableSpace::set_top_for_allocations() {
+  mangler()->set_top_for_allocations(top());
+}
+#endif
+
 // This version requires locking. */
 HeapWord* MutableSpace::allocate(size_t size) {
   assert(Heap_lock->owned_by_self() ||

hotspot/src/share/vm/gc_implementation/shared/mutableSpace.hpp

@@ -30,14 +30,23 @@
 // Invariant: (ImmutableSpace +) bottom() <= top() <= end()
 // top() is inclusive and end() is exclusive.
 
+class MutableSpaceMangler;
+
 class MutableSpace: public ImmutableSpace {
   friend class VMStructs;
+
+  // Helper for mangling unused space in debug builds
+  MutableSpaceMangler* _mangler;
+
  protected:
   HeapWord* _top;
 
+  MutableSpaceMangler* mangler() { return _mangler; }
+
  public:
-  virtual ~MutableSpace()                  {}
-  MutableSpace()                           { _top = NULL;    }
+  virtual ~MutableSpace();
+  MutableSpace();
+
   // Accessors
   HeapWord* top() const                    { return _top;    }
   virtual void set_top(HeapWord* value)    { _top = value;   }
@@ -52,21 +61,30 @@ class MutableSpace: public ImmutableSpace {
   MemRegion used_region() { return MemRegion(bottom(), top()); }
 
   // Initialization
-  virtual void initialize(MemRegion mr, bool clear_space);
-  virtual void clear();
+  virtual void initialize(MemRegion mr,
+                          bool clear_space,
+                          bool mangle_space);
+  virtual void clear(bool mangle_space);
+  // Does the usual initialization but optionally resets top to bottom.
+#if 0  // MANGLE_SPACE
+  void initialize(MemRegion mr, bool clear_space, bool reset_top);
+#endif
   virtual void update() { }
   virtual void accumulate_statistics() { }
 
-  // Overwrites the unused portion of this space. Note that some collectors
-  // may use this "scratch" space during collections.
-  virtual void mangle_unused_area() {
-    mangle_region(MemRegion(_top, _end));
-  }
+  // Methods used in mangling.  See descriptions under SpaceMangler.
+  virtual void mangle_unused_area() PRODUCT_RETURN;
+  virtual void mangle_unused_area_complete() PRODUCT_RETURN;
+  virtual void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN;
+  virtual void check_mangled_unused_area_complete() PRODUCT_RETURN;
+  virtual void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN;
+
+  // Used to save the space's current top for later use during mangling.
+  virtual void set_top_for_allocations() PRODUCT_RETURN;
+
   virtual void ensure_parsability() { }
 
-  void mangle_region(MemRegion mr) {
-    debug_only(Copy::fill_to_words(mr.start(), mr.word_size(), badHeapWord));
-  }
+  virtual void mangle_region(MemRegion mr) PRODUCT_RETURN;
 
   // Boolean querries.
   bool is_empty() const              { return used_in_words() == 0; }

hotspot/src/share/vm/gc_implementation/shared/spaceDecorator.cpp

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2002-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+# include "incls/_precompiled.incl"
+# include "incls/_spaceDecorator.cpp.incl"
+
+// Catch-all file for utility classes
+
+#ifndef PRODUCT
+
+// Returns true is the location q matches the mangling
+// pattern.
+bool SpaceMangler::is_mangled(HeapWord* q) {
+  // This test loses precision but is good enough
+  return badHeapWord == (max_juint & (uintptr_t) q->value());
+}
+
+
+void SpaceMangler::set_top_for_allocations(HeapWord* v)  {
+  if (v < end()) {
+    assert(is_mangled(v), "The high water mark is not mangled");
+  }
+  _top_for_allocations = v;
+}
+
+// Mangle only the unused space that has not previously
+// been mangled and that has not been allocated since being
+// mangled.
+void SpaceMangler::mangle_unused_area() {
+  assert(ZapUnusedHeapArea, "Mangling should not be in use");
+  // Mangle between top and the high water mark.  Safeguard
+  // against the space changing since top_for_allocations was
+  // set.
+  HeapWord* mangled_end = MIN2(top_for_allocations(), end());
+  if (top() < mangled_end) {
+    MemRegion mangle_mr(top(), mangled_end);
+    SpaceMangler::mangle_region(mangle_mr);
+    // Light weight check of mangling.
+    check_mangled_unused_area(end());
+  }
+  // Complete check of unused area which is functional when
+  // DEBUG_MANGLING is defined.
+  check_mangled_unused_area_complete();
+}
+
+// A complete mangle is expected in the
+// exceptional case where top_for_allocations is not
+// properly tracking the high water mark for mangling.
+// This can be the case when to-space is being used for
+// scratch space during a mark-sweep-compact.  See
+// contribute_scratch() and PSMarkSweep::allocate_stacks().
+void SpaceMangler::mangle_unused_area_complete() {
+  assert(ZapUnusedHeapArea, "Mangling should not be in use");
+  MemRegion mangle_mr(top(), end());
+  SpaceMangler::mangle_region(mangle_mr);
+}
+
+// Simply mangle the MemRegion mr.
+void SpaceMangler::mangle_region(MemRegion mr) {
+  assert(ZapUnusedHeapArea, "Mangling should not be in use");
+#ifdef ASSERT
+  if(TraceZapUnusedHeapArea) {
+    gclog_or_tty->print("Mangling [0x%x to 0x%x)", mr.start(), mr.end());
+  }
+  Copy::fill_to_words(mr.start(), mr.word_size(), badHeapWord);
+  if(TraceZapUnusedHeapArea) {
+    gclog_or_tty->print_cr(" done");
+  }
+#endif
+}
+
+// Check that top, top_for_allocations and the last
+// word of the space are mangled.  In a tight memory
+// situation even this light weight mangling could
+// cause paging by touching the end of the space.
+void  SpaceMangler::check_mangled_unused_area(HeapWord* limit) {
+  if (CheckZapUnusedHeapArea) {
+    // This method can be called while the spaces are
+    // being reshaped so skip the test if the end of the
+    // space is beyond the specified limit;
+    if (end() > limit) return;
+
+    assert(top() == end() ||
+           (is_mangled(top())), "Top not mangled");
+    assert((top_for_allocations() < top()) ||
+           (top_for_allocations() >= end()) ||
+           (is_mangled(top_for_allocations())),
+           "Older unused not mangled");
+    assert(top() == end() ||
+           (is_mangled(end() - 1)), "End not properly mangled");
+    // Only does checking when DEBUG_MANGLING is defined.
+    check_mangled_unused_area_complete();
+  }
+}
+
+#undef DEBUG_MANGLING
+// This should only be used while debugging the mangling
+// because of the high cost of checking the completeness.
+void  SpaceMangler::check_mangled_unused_area_complete() {
+  if (CheckZapUnusedHeapArea) {
+    assert(ZapUnusedHeapArea, "Not mangling unused area");
+#ifdef DEBUG_MANGLING
+    HeapWord* q = top();
+    HeapWord* limit = end();
+
+    bool passed = true;
+    while (q < limit) {
+      if (!is_mangled(q)) {
+        passed = false;
+        break;
+      }
+      q++;
+    }
+    assert(passed, "Mangling is not complete");
+#endif
+  }
+}
+#undef DEBUG_MANGLING
+#endif // not PRODUCT

hotspot/src/share/vm/gc_implementation/shared/spaceDecorator.hpp

@@ -0,0 +1,141 @@
+/*
+ * Copyright 2002-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+class SpaceDecorator: public AllStatic {
+ public:
+  // Initialization flags.
+  static const bool Clear               = true;
+  static const bool DontClear           = false;
+  static const bool Mangle              = true;
+  static const bool DontMangle          = false;
+};
+
+// Functionality for use with class Space and class MutableSpace.
+//   The approach taken with the mangling is to mangle all
+// the space initially and then to mangle areas that have
+// been allocated since the last collection.  Mangling is
+// done in the context of a generation and in the context
+// of a space.
+//   The space in a generation is mangled when it is first
+// initialized and when the generation grows.  The spaces
+// are not necessarily up-to-date when this mangling occurs
+// and the method mangle_region() is used.
+//   After allocations have been done in a space, the space generally
+// need to be remangled.  Remangling is only done on the
+// recently allocated regions in the space.  Typically, that is
+// the region between the new top and the top just before a
+// garbage collection.
+//   An exception to the usual mangling in a space is done when the
+// space is used for an extraordinary purpose.  Specifically, when
+// to-space is used as scratch space for a mark-sweep-compact
+// collection.
+//   Spaces are mangled after a collection.  If the generation
+// grows after a collection, the added space is mangled as part of
+// the growth of the generation.  No additional mangling is needed when the
+// spaces are resized after an expansion.
+//   The class SpaceMangler keeps a pointer to the top of the allocated
+// area and provides the methods for doing the piece meal mangling.
+// Methods for doing sparces and full checking of the mangling are
+// included.  The full checking is done if DEBUG_MANGLING is defined.
+//   GenSpaceMangler is used with the GenCollectedHeap collectors and
+// MutableSpaceMangler is used with the ParallelScavengeHeap collectors.
+// These subclasses abstract the differences in the types of spaces used
+// by each heap.
+
+class SpaceMangler: public CHeapObj {
+  friend class VMStructs;
+
+  // High water mark for allocations.  Typically, the space above
+  // this point have been mangle previously and don't need to be
+  // touched again.  Space belows this point has been allocated
+  // and remangling is needed between the current top and this
+  // high water mark.
+  HeapWord* _top_for_allocations;
+  HeapWord* top_for_allocations() { return _top_for_allocations; }
+
+ public:
+
+  // Setting _top_for_allocations to NULL at initialization
+  // makes it always below top so that mangling done as part
+  // of the initialize() call of a space does nothing (as it
+  // should since the mangling is done as part of the constructor
+  // for the space.
+  SpaceMangler() : _top_for_allocations(NULL) {}
+
+  // Methods for top and end that delegate to the specific
+  // space type.
+  virtual HeapWord* top() const = 0;
+  virtual HeapWord* end() const = 0;
+
+  // Return true if q matches the mangled pattern.
+  static bool is_mangled(HeapWord* q) PRODUCT_RETURN0;
+
+  // Used to save the an address in a space for later use during mangling.
+  void set_top_for_allocations(HeapWord* v);
+
+  // Overwrites the unused portion of this space.
+  // Mangle only the region not previously mangled [top, top_previously_mangled)
+  void mangle_unused_area();
+  // Mangle all the unused region [top, end)
+  void mangle_unused_area_complete();
+  // Do some sparse checking on the area that should have been mangled.
+  void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN;
+  // Do a complete check of the area that should be mangled.
+  void check_mangled_unused_area_complete() PRODUCT_RETURN;
+
+  // Mangle the MemRegion.  This is a non-space specific mangler.  It
+  // is used during the initial mangling of a space before the space
+  // is fully constructed.  Also is used when a generation is expanded
+  // and possibly before the spaces have been reshaped to to the new
+  // size of the generation.
+  static void mangle_region(MemRegion mr);
+};
+
+class ContiguousSpace;
+
+// For use with GenCollectedHeap's
+class GenSpaceMangler: public SpaceMangler {
+  ContiguousSpace* _sp;
+
+  ContiguousSpace* sp() { return _sp; }
+
+  HeapWord* top() const { return _sp->top(); }
+  HeapWord* end() const { return _sp->end(); }
+
+ public:
+  GenSpaceMangler(ContiguousSpace* sp) : SpaceMangler(), _sp(sp) {}
+};
+
+// For use with ParallelScavengeHeap's.
+class MutableSpaceMangler: public SpaceMangler {
+  MutableSpace* _sp;
+
+  MutableSpace* sp() { return _sp; }
+
+  HeapWord* top() const { return _sp->top(); }
+  HeapWord* end() const { return _sp->end(); }
+
+ public:
+  MutableSpaceMangler(MutableSpace* sp) : SpaceMangler(), _sp(sp) {}
+};