6621728: Heap inspection should not crash in the face of C-heap exhaustion

Deal more gracefully with situations where C-heap scratch space cannot be had

Reviewed-by: jmasa

hotspot/src/share/vm/memory/heapInspection.cpp

@@ -65,7 +65,7 @@ void KlassInfoEntry::print_on(outputStream* st) const {
       name = "<no name>";
   }
   // simplify the formatting (ILP32 vs LP64) - always cast the numbers to 64-bit
-  st->print_cr("%13" FORMAT64_MODIFIER "d  %13" FORMAT64_MODIFIER "u  %s",
+  st->print_cr(INT64_FORMAT_W(13) "  " UINT64_FORMAT_W(13) "  %s",
                (jlong)  _instance_count,
                (julong) _instance_words * HeapWordSize,
                name);
@@ -80,7 +80,10 @@ KlassInfoEntry* KlassInfoBucket::lookup(const klassOop k) {
     elt = elt->next();
   }
   elt = new KlassInfoEntry(k, list());
+  // We may be out of space to allocate the new entry.
+  if (elt != NULL) {
     set_list(elt);
+  }
   return elt;
 }
 
@@ -103,22 +106,26 @@ void KlassInfoBucket::empty() {
 }
 
 KlassInfoTable::KlassInfoTable(int size, HeapWord* ref) {
-  _size = size;
+  _size = 0;
   _ref = ref;
-  _buckets = NEW_C_HEAP_ARRAY(KlassInfoBucket, _size);
+  _buckets = NEW_C_HEAP_ARRAY(KlassInfoBucket, size);
-
+  if (_buckets != NULL) {
+    _size = size;
     for (int index = 0; index < _size; index++) {
       _buckets[index].initialize();
     }
   }
+}
 
 KlassInfoTable::~KlassInfoTable() {
+  if (_buckets != NULL) {
     for (int index = 0; index < _size; index++) {
       _buckets[index].empty();
     }
     FREE_C_HEAP_ARRAY(KlassInfoBucket, _buckets);
     _size = 0;
   }
+}
 
 uint KlassInfoTable::hash(klassOop p) {
   assert(Universe::heap()->is_in_permanent((HeapWord*)p), "all klasses in permgen");
@@ -127,19 +134,32 @@ uint KlassInfoTable::hash(klassOop p) {
 
 KlassInfoEntry* KlassInfoTable::lookup(const klassOop k) {
   uint         idx = hash(k) % _size;
+  assert(_buckets != NULL, "Allocation failure should have been caught");
   KlassInfoEntry*  e   = _buckets[idx].lookup(k);
-  assert(k == e->klass(), "must be equal");
+  // Lookup may fail if this is a new klass for which we
+  // could not allocate space for an new entry.
+  assert(e == NULL || k == e->klass(), "must be equal");
   return e;
 }
 
-void KlassInfoTable::record_instance(const oop obj) {
+// Return false if the entry could not be recorded on account
+// of running out of space required to create a new entry.
+bool KlassInfoTable::record_instance(const oop obj) {
   klassOop      k = obj->klass();
   KlassInfoEntry* elt = lookup(k);
+  // elt may be NULL if it's a new klass for which we
+  // could not allocate space for a new entry in the hashtable.
+  if (elt != NULL) {
     elt->set_count(elt->count() + 1);
     elt->set_words(elt->words() + obj->size());
+    return true;
+  } else {
+    return false;
+  }
 }
 
 void KlassInfoTable::iterate(KlassInfoClosure* cic) {
+  assert(_size == 0 || _buckets != NULL, "Allocation failure should have been caught");
   for (int index = 0; index < _size; index++) {
     _buckets[index].iterate(cic);
   }
@@ -176,7 +196,7 @@ void KlassInfoHisto::print_elements(outputStream* st) const {
     total += elements()->at(i)->count();
     totalw += elements()->at(i)->words();
   }
-  st->print_cr("Total %13" FORMAT64_MODIFIER "d  %13" FORMAT64_MODIFIER "u",
+  st->print_cr("Total " INT64_FORMAT_W(13) "  " UINT64_FORMAT_W(13),
                total, totalw * HeapWordSize);
 }
 
@@ -199,12 +219,18 @@ class HistoClosure : public KlassInfoClosure {
 class RecordInstanceClosure : public ObjectClosure {
  private:
   KlassInfoTable* _cit;
+  size_t _missed_count;
  public:
-  RecordInstanceClosure(KlassInfoTable* cit) : _cit(cit) {}
+  RecordInstanceClosure(KlassInfoTable* cit) :
+    _cit(cit), _missed_count(0) {}
 
   void do_object(oop obj) {
-    _cit->record_instance(obj);
+    if (!_cit->record_instance(obj)) {
+      _missed_count++;
     }
+  }
+
+  size_t missed_count() { return _missed_count; }
 };
 
 void HeapInspection::heap_inspection(outputStream* st) {
@@ -230,12 +256,20 @@ void HeapInspection::heap_inspection(outputStream* st) {
       ShouldNotReachHere(); // Unexpected heap kind for this op
   }
   // Collect klass instance info
-
-  // Iterate over objects in the heap
   KlassInfoTable cit(KlassInfoTable::cit_size, ref);
+  if (!cit.allocation_failed()) {
+    // Iterate over objects in the heap
     RecordInstanceClosure ric(&cit);
     Universe::heap()->object_iterate(&ric);
 
+    // Report if certain classes are not counted because of
+    // running out of C-heap for the histogram.
+    size_t missed_count = ric.missed_count();
+    if (missed_count != 0) {
+      st->print_cr("WARNING: Ran out of C-heap; undercounted " SIZE_FORMAT
+                   " total instances in data below",
+                   missed_count);
+    }
     // Sort and print klass instance info
     KlassInfoHisto histo("\n"
                      " num     #instances         #bytes  class name\n"
@@ -245,6 +279,9 @@ void HeapInspection::heap_inspection(outputStream* st) {
     cit.iterate(&hc);
     histo.sort();
     histo.print_on(st);
+  } else {
+    st->print_cr("WARNING: Ran out of C-heap; histogram not generated");
+  }
   st->flush();
 
   if (Universe::heap()->kind() == CollectedHeap::GenCollectedHeap) {

hotspot/src/share/vm/memory/heapInspection.hpp

@@ -98,8 +98,9 @@ class KlassInfoTable: public StackObj {
   };
   KlassInfoTable(int size, HeapWord* ref);
   ~KlassInfoTable();
-  void record_instance(const oop obj);
+  bool record_instance(const oop obj);
   void iterate(KlassInfoClosure* cic);
+  bool allocation_failed() { return _buckets == NULL; }
 };
 
 class KlassInfoHisto : public StackObj {