7197557: NPG: nsk/sysdict/vm/stress/chain/chain004 hangs intermittently

Reviewed-by: johnc, ysr

hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.cpp

@@ -230,15 +230,9 @@ void VM_CollectForMetadataAllocation::doit() {
       // amount of the expansion.
       // This should work unless there really is no more space
       // or a MaxMetaspaceSize has been specified on the command line.
-      MetaspaceGC::set_expand_after_GC(true);
-      size_t before_inc = MetaspaceGC::capacity_until_GC();
-      size_t delta_words = MetaspaceGC::delta_capacity_until_GC(_size);
-      MetaspaceGC::inc_capacity_until_GC(delta_words);
-      if (PrintGCDetails && Verbose) {
-        gclog_or_tty->print_cr("Increase capacity to GC from " SIZE_FORMAT
-          " to " SIZE_FORMAT, before_inc, MetaspaceGC::capacity_until_GC());
-  }
-      _result = _loader_data->metaspace_non_null()->allocate(_size, _mdtype);
+      _result =
+        _loader_data->metaspace_non_null()->expand_and_allocate(_size, _mdtype);
+
       if (do_cms_concurrent && _result == NULL) {
         // Rather than fail with a metaspace out-of-memory, do a full
         // GC for CMS.

hotspot/src/share/vm/memory/collectorPolicy.cpp

@@ -743,6 +743,36 @@ MetaWord* CollectorPolicy::satisfy_failed_metadata_allocation(
   uint full_gc_count = 0;
 
   do {
+    MetaWord* result = NULL;
+    if (GC_locker::is_active_and_needs_gc()) {
+      // If the GC_locker is active, just expand and allocate.
+      // If that does not succeed, wait if this thread is not
+      // in a critical section itself.
+      result =
+        loader_data->metaspace_non_null()->expand_and_allocate(word_size,
+                                                               mdtype);
+      if (result != NULL) {
+        return result;
+      }
+      JavaThread* jthr = JavaThread::current();
+      if (!jthr->in_critical()) {
+        MutexUnlocker mul(Heap_lock);
+        // Wait for JNI critical section to be exited
+        GC_locker::stall_until_clear();
+        // The GC invoked by the last thread leaving the critical
+        // section will be a young collection and a full collection
+        // is (currently) needed for unloading classes so continue
+        // to the next iteration to get a full GC.
+        continue;
+      } else {
+        if (CheckJNICalls) {
+          fatal("Possible deadlock due to allocating while"
+                " in jni critical section");
+        }
+        return NULL;
+      }
+    }
+
     {  // Need lock to get self consistent gc_count's
       MutexLocker ml(Heap_lock);
       gc_count      = Universe::heap()->total_collections();

hotspot/src/share/vm/memory/metaspace.cpp

@@ -2843,6 +2843,21 @@ MetaWord* Metaspace::allocate(size_t word_size, MetadataType mdtype) {
   }
 }
 
+MetaWord* Metaspace::expand_and_allocate(size_t word_size, MetadataType mdtype) {
+  MetaWord* result;
+  MetaspaceGC::set_expand_after_GC(true);
+  size_t before_inc = MetaspaceGC::capacity_until_GC();
+  size_t delta_words = MetaspaceGC::delta_capacity_until_GC(word_size);
+  MetaspaceGC::inc_capacity_until_GC(delta_words);
+  if (PrintGCDetails && Verbose) {
+    gclog_or_tty->print_cr("Increase capacity to GC from " SIZE_FORMAT
+      " to " SIZE_FORMAT, before_inc, MetaspaceGC::capacity_until_GC());
+  }
+  result = allocate(word_size, mdtype);
+
+  return result;
+}
+
 // Space allocated in the Metaspace.  This may
 // be across several metadata virtual spaces.
 char* Metaspace::bottom() const {

hotspot/src/share/vm/memory/metaspace.hpp

@@ -130,9 +130,11 @@ class Metaspace : public CHeapObj<mtClass> {
 
   static MetaWord* allocate(ClassLoaderData* loader_data, size_t size,
                             bool read_only, MetadataType mdtype, TRAPS);
-
   void deallocate(MetaWord* ptr, size_t byte_size, bool is_class);
 
+  MetaWord* expand_and_allocate(size_t size,
+                                MetadataType mdtype);
+
 #ifndef PRODUCT
   bool contains(const void *ptr) const;
   bool contains_class(const void *ptr) const;