8009298: NMT: Special version of class loading/unloading with runThese stresses out NMT

8009777: NMT: add new NMT dcmd to control auto shutdown option

Added diagnostic VM option and DCmd command to allow NMT stay alive under stress situation

Reviewed-by: dcubed, coleenp

hotspot/src/share/vm/runtime/globals.hpp

@@ -869,6 +869,11 @@ class CommandLineFlags {
   diagnostic(bool, PrintNMTStatistics, false,                               \
           "Print native memory tracking summary data if it is on")          \
                                                                             \
+  diagnostic(bool, AutoShutdownNMT, true,                                   \
+          "Automatically shutdown native memory tracking under stress "     \
+          "situation. When set to false, native memory tracking tries to "  \
+          "stay alive at the expense of JVM performance")                   \
+                                                                            \
   diagnostic(bool, LogCompilation, false,                                   \
           "Log compilation activity in detail to hotspot.log or LogFile")   \
                                                                             \

hotspot/src/share/vm/services/memTracker.cpp

@@ -68,6 +68,7 @@ int                             MemTracker::_thread_count = 255;
 volatile jint                   MemTracker::_pooled_recorder_count = 0;
 volatile unsigned long          MemTracker::_processing_generation = 0;
 volatile bool                   MemTracker::_worker_thread_idle = false;
+volatile bool                   MemTracker::_slowdown_calling_thread = false;
 debug_only(intx                 MemTracker::_main_thread_tid = 0;)
 NOT_PRODUCT(volatile jint       MemTracker::_pending_recorder_count = 0;)
 
@@ -364,6 +365,12 @@ void MemTracker::create_memory_record(address addr, MEMFLAGS flags,
     }
 
     if (thread != NULL) {
+      // slow down all calling threads except NMT worker thread, so it
+      // can catch up.
+      if (_slowdown_calling_thread && thread != _worker_thread) {
+        os::yield_all();
+      }
+
       if (thread->is_Java_thread() && ((JavaThread*)thread)->is_safepoint_visible()) {
         JavaThread*      java_thread = (JavaThread*)thread;
         JavaThreadState  state = java_thread->thread_state();
@@ -442,6 +449,7 @@ void MemTracker::enqueue_pending_recorder(MemRecorder* rec) {
 #define MAX_SAFEPOINTS_TO_SKIP     128
 #define SAFE_SEQUENCE_THRESHOLD    30
 #define HIGH_GENERATION_THRESHOLD  60
+#define MAX_RECORDER_THREAD_RATIO  30
 
 void MemTracker::sync() {
   assert(_tracking_level > NMT_off, "NMT is not enabled");
@@ -487,6 +495,13 @@ void MemTracker::sync() {
         pending_recorders = _global_recorder;
         _global_recorder = NULL;
       }
+
+      // see if NMT has too many outstanding recorder instances, it usually
+      // means that worker thread is lagging behind in processing them.
+      if (!AutoShutdownNMT) {
+        _slowdown_calling_thread = (MemRecorder::_instance_count > MAX_RECORDER_THREAD_RATIO * _thread_count);
+      }
+
       // check _worker_thread with lock to avoid racing condition
       if (_worker_thread != NULL) {
         _worker_thread->at_sync_point(pending_recorders, InstanceKlass::number_of_instance_classes());

hotspot/src/share/vm/services/memTracker.hpp

@@ -84,6 +84,7 @@ class MemTracker : AllStatic {
    static inline bool baseline() { return false; }
    static inline bool has_baseline() { return false; }
 
+   static inline void set_autoShutdown(bool value) { }
    static void shutdown(ShutdownReason reason) { }
    static inline bool shutdown_in_progress() {  }
    static bool print_memory_usage(BaselineOutputer& out, size_t unit,
@@ -238,6 +239,16 @@ class MemTracker : AllStatic {
   // if native memory tracking tracks callsite
   static inline bool track_callsite() { return _tracking_level == NMT_detail; }
 
+  // NMT automatically shuts itself down under extreme situation by default.
+  // When the value is set to false,  NMT will try its best to stay alive,
+  // even it has to slow down VM.
+  static inline void set_autoShutdown(bool value) {
+    AutoShutdownNMT = value;
+    if (AutoShutdownNMT && _slowdown_calling_thread) {
+      _slowdown_calling_thread = false;
+    }
+  }
+
   // shutdown native memory tracking capability. Native memory tracking
   // can be shutdown by VM when it encounters low memory scenarios.
   // Memory tracker should gracefully shutdown itself, and preserve the
@@ -507,6 +518,10 @@ class MemTracker : AllStatic {
   // although NMT is still procesing current generation, but
   // there is not more recorder to process, set idle state
   static volatile bool             _worker_thread_idle;
+
+  // if NMT should slow down calling thread to allow
+  // worker thread to catch up
+  static volatile bool             _slowdown_calling_thread;
 };
 
 #endif // !INCLUDE_NMT

hotspot/src/share/vm/services/nmtDCmd.cpp

@@ -49,6 +49,9 @@ NMTDCmd::NMTDCmd(outputStream* output,
   _shutdown("shutdown", "request runtime to shutdown itself and free the " \
             "memory used by runtime.",
             "BOOLEAN", false, "false"),
+  _auto_shutdown("autoShutdown", "automatically shutdown itself under "    \
+            "stress situation",
+            "BOOLEAN", true, "true"),
 #ifndef PRODUCT
   _debug("debug", "print tracker statistics. Debug only, not thread safe", \
             "BOOLEAN", false, "false"),
@@ -61,6 +64,7 @@ NMTDCmd::NMTDCmd(outputStream* output,
   _dcmdparser.add_dcmd_option(&_summary_diff);
   _dcmdparser.add_dcmd_option(&_detail_diff);
   _dcmdparser.add_dcmd_option(&_shutdown);
+  _dcmdparser.add_dcmd_option(&_auto_shutdown);
 #ifndef PRODUCT
   _dcmdparser.add_dcmd_option(&_debug);
 #endif
@@ -84,17 +88,19 @@ void NMTDCmd::execute(TRAPS) {
   }
 
   int nopt = 0;
-  if(_summary.is_set() && _summary.value()) { ++nopt; }
+  if (_summary.is_set() && _summary.value()) { ++nopt; }
-  if(_detail.is_set() && _detail.value()) { ++nopt; }
+  if (_detail.is_set() && _detail.value()) { ++nopt; }
-  if(_baseline.is_set() && _baseline.value()) { ++nopt; }
+  if (_baseline.is_set() && _baseline.value()) { ++nopt; }
-  if(_summary_diff.is_set() && _summary_diff.value()) { ++nopt; }
+  if (_summary_diff.is_set() && _summary_diff.value()) { ++nopt; }
-  if(_detail_diff.is_set() && _detail_diff.value()) { ++nopt; }
+  if (_detail_diff.is_set() && _detail_diff.value()) { ++nopt; }
-  if(_shutdown.is_set() && _shutdown.value()) { ++nopt; }
+  if (_shutdown.is_set() && _shutdown.value()) { ++nopt; }
+  if (_auto_shutdown.is_set()) { ++nopt; }
+
 #ifndef PRODUCT
-  if(_debug.is_set() && _debug.value()) { ++nopt; }
+  if (_debug.is_set() && _debug.value()) { ++nopt; }
 #endif
 
-  if(nopt > 1) {
+  if (nopt > 1) {
       output()->print_cr("At most one of the following option can be specified: " \
         "summary, detail, baseline, summary.diff, detail.diff, shutdown"
 #ifndef PRODUCT
@@ -156,6 +162,8 @@ void NMTDCmd::execute(TRAPS) {
     MemTracker::shutdown(MemTracker::NMT_shutdown_user);
     output()->print_cr("Shutdown is in progress, it will take a few moments to " \
       "completely shutdown");
+  } else if (_auto_shutdown.is_set()) {
+    MemTracker::set_autoShutdown(_auto_shutdown.value());
   } else {
     ShouldNotReachHere();
     output()->print_cr("Unknown command");

hotspot/src/share/vm/services/nmtDCmd.hpp

@@ -39,6 +39,7 @@ class NMTDCmd: public DCmdWithParser {
   DCmdArgument<bool>  _summary_diff;
   DCmdArgument<bool>  _detail_diff;
   DCmdArgument<bool>  _shutdown;
+  DCmdArgument<bool>  _auto_shutdown;
 #ifndef PRODUCT
   DCmdArgument<bool>  _debug;
 #endif