8226416: MonitorUsedDeflationThreshold can cause repeated async deflation requests

Reviewed-by: dholmes, coleenp

src/hotspot/share/runtime/globals.hpp

@@ -733,6 +733,11 @@ const intx ObjectAlignmentInBytes = 8;
           "or AsyncDeflationInterval.")                                     \
           range(0, 100)                                                     \
                                                                             \
+  product(uintx, NoAsyncDeflationProgressMax, 3, DIAGNOSTIC,                \
+          "Max number of no progress async deflation attempts to tolerate " \
+          "before adjusting the in_use_list_ceiling up (0 is off).")        \
+          range(0, max_uintx)                                               \
+                                                                            \
   product(intx, hashCode, 5, EXPERIMENTAL,                                  \
                "(Unstable) select hashCode generation algorithm")           \
                                                                             \

src/hotspot/share/runtime/synchronizer.cpp

@@ -240,23 +240,31 @@ void ObjectSynchronizer::initialize() {
 }
 
 static MonitorList _in_use_list;
+// monitors_used_above_threshold() policy is as follows:
+//
 // The ratio of the current _in_use_list count to the ceiling is used
 // to determine if we are above MonitorUsedDeflationThreshold and need
 // to do an async monitor deflation cycle. The ceiling is increased by
 // AvgMonitorsPerThreadEstimate when a thread is added to the system
 // and is decreased by AvgMonitorsPerThreadEstimate when a thread is
 // removed from the system.
+//
 // Note: If the _in_use_list max exceeds the ceiling, then
 // monitors_used_above_threshold() will use the in_use_list max instead
 // of the thread count derived ceiling because we have used more
 // ObjectMonitors than the estimated average.
 //
+// Note: If deflate_idle_monitors() has NoAsyncDeflationProgressMax
+// no-progress async monitor deflation cycles in a row, then the ceiling
+// is adjusted upwards by monitors_used_above_threshold().
+//
 // Start the ceiling with the estimate for one thread in initialize()
 // which is called after cmd line options are processed.
 static size_t _in_use_list_ceiling = 0;
 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
 bool volatile ObjectSynchronizer::_is_final_audit = false;
 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
+static uintx _no_progress_cnt = 0;
 
 // =====================> Quick functions
 
@@ -1142,21 +1150,35 @@ void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
 }
 
 static bool monitors_used_above_threshold(MonitorList* list) {
+  if (MonitorUsedDeflationThreshold == 0) {  // disabled case is easy
+    return false;
+  }
   // Start with ceiling based on a per-thread estimate:
   size_t ceiling = ObjectSynchronizer::in_use_list_ceiling();
+  size_t old_ceiling = ceiling;
   if (ceiling < list->max()) {
     // The max used by the system has exceeded the ceiling so use that:
     ceiling = list->max();
   }
-  if (ceiling == 0) {
+  size_t monitors_used = list->count();
+  if (monitors_used == 0) {  // empty list is easy
     return false;
   }
-  if (MonitorUsedDeflationThreshold > 0) {
-    size_t monitors_used = list->count();
-    size_t monitor_usage = (monitors_used * 100LL) / ceiling;
-    return int(monitor_usage) > MonitorUsedDeflationThreshold;
+  if (NoAsyncDeflationProgressMax != 0 &&
+      _no_progress_cnt >= NoAsyncDeflationProgressMax) {
+    float remainder = (100.0 - MonitorUsedDeflationThreshold) / 100.0;
+    size_t new_ceiling = ceiling + (ceiling * remainder) + 1;
+    ObjectSynchronizer::set_in_use_list_ceiling(new_ceiling);
+    log_info(monitorinflation)("Too many deflations without progress; "
+                               "bumping in_use_list_ceiling from " SIZE_FORMAT
+                               " to " SIZE_FORMAT, old_ceiling, new_ceiling);
+    _no_progress_cnt = 0;
+    ceiling = new_ceiling;
   }
-  return false;
+
+  // Check if our monitor usage is above the threshold:
+  size_t monitor_usage = (monitors_used * 100LL) / ceiling;
+  return int(monitor_usage) > MonitorUsedDeflationThreshold;
 }
 
 size_t ObjectSynchronizer::in_use_list_ceiling() {
@@ -1583,6 +1605,12 @@ size_t ObjectSynchronizer::deflate_idle_monitors() {
 
   GVars.stw_random = os::random();
 
+  if (deflated_count != 0) {
+    _no_progress_cnt = 0;
+  } else {
+    _no_progress_cnt++;
+  }
+
   return deflated_count;
 }
 

test/hotspot/jtreg/runtime/Monitor/MonitorUsedDeflationThresholdTest.java

@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.Platform;
+import jdk.test.lib.process.ProcessTools;
+
+/*
+ * @test
+ * @bug 8226416
+ * @summary Test the MonitorUsedDeflationThreshold and NoAsyncDeflationProgressMax options.
+ * @modules java.base/jdk.internal.misc
+ * @library /test/lib
+ * @run driver MonitorUsedDeflationThresholdTest
+ */
+
+public class MonitorUsedDeflationThresholdTest {
+    public static final int DELAY_SECS = 10;
+    public static int inflate_count = 0;
+    public static Object[] monitors;
+
+    public static void do_work(int count) {
+        System.out.println("Recursion count=" + count);
+        if (count > inflate_count) {
+            System.out.println("Exceeded inflate_count=" + inflate_count);
+
+            System.out.println("Delaying for " + DELAY_SECS + " secs.");
+            try {
+                Thread.sleep(DELAY_SECS * 1000);
+            } catch (InterruptedException ie) {
+                // ignore InterruptedException
+            }
+            System.out.println("Done delaying for " + DELAY_SECS + " secs.");
+            return;
+        }
+
+        synchronized(monitors[count]) {
+            try {
+                monitors[count].wait(1);  // force inflation
+            } catch (InterruptedException ie) {
+                // ignore InterruptedException
+            }
+            do_work(count + 1);
+        }
+    }
+
+    public static void usage() {
+        System.err.println("Usage: java " +
+                           "MonitorUsedDeflationThresholdTest inflate_count");
+    }
+
+    public static void main(String[] args) throws Exception {
+        if (args.length == 0) {
+            // Without args we invoke the test in a java sub-process:
+            ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+                // Test doesn't need much Java heap:
+                "-Xmx100M",
+                // AvgMonitorsPerThreadEstimate == 1 means we'll start with
+                // an in_use_list_ceiling of <n-threads> plus a couple of
+                // of monitors for threads that call Object.wait().
+                "-XX:+UnlockDiagnosticVMOptions",
+                "-XX:AvgMonitorsPerThreadEstimate=1",
+                // Enable monitorinflation logging so we can see that
+                // MonitorUsedDeflationThreshold and
+                // NoAsyncDeflationProgressMaxoption are working.
+                "-Xlog:monitorinflation=info",
+                // Enable some safepoint logging for diagnostic purposes.
+                "-Xlog:safepoint+cleanup=info",
+                "-Xlog:safepoint+stats=debug",
+                // Run the test with inflate_count == 33 since that
+                // reproduced the bug with JDK13. Anything above the
+                // in_use_list_ceiling will do the trick.
+                "MonitorUsedDeflationThresholdTest", "33");
+
+            OutputAnalyzer output_detail = new OutputAnalyzer(pb.start());
+
+            // This mesg means:
+            // - AvgMonitorsPerThreadEstimate == 1 reduced in_use_list_ceiling
+            //   to a small number.
+            // - and we crossed MonitorUsedDeflationThreshold:
+            output_detail.shouldMatch("begin deflating: .*");
+            System.out.println("Found beginning of a deflation cycle.");
+
+            // This mesg means we hit NoAsyncDeflationProgressMax and
+            // had to adjust the in_use_list_ceiling:
+            String too_many = output_detail.firstMatch("Too many deflations without progress; .*", 0);
+            if (too_many == null) {
+                output_detail.reportDiagnosticSummary();
+                throw new RuntimeException("Did not find too_many string in output.\n");
+            }
+            System.out.println("too_many='" + too_many + "'");
+
+            System.out.println("PASSED.");
+            return;
+        }
+        // else we are the exec'd java subprocess, so run the actual test:
+
+        try {
+            inflate_count = Integer.decode(args[0]);
+        } catch (NumberFormatException nfe) {
+            usage();
+            throw new RuntimeException("ERROR: '" + args[0] +
+                                       "': bad inflate_count.");
+        }
+
+        System.out.println("Hello from MonitorUsedDeflationThresholdTest!");
+        System.out.println("inflate_count=" + inflate_count);
+
+        monitors = new Object[inflate_count + 1];
+        for (int i = 1; i <= inflate_count; i++) {
+            monitors[i] = new Object();
+        }
+        do_work(1);
+    }
+}