8279356: Method linking fails with guarantee(mh->adapter() != NULL) failed: Adapter blob must already exist!

Reviewed-by: chagedorn, kvn, coleenp

src/hotspot/share/oops/method.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1198,7 +1198,7 @@ void Method::unlink_method() {
 void Method::link_method(const methodHandle& h_method, TRAPS) {
   // If the code cache is full, we may reenter this function for the
   // leftover methods that weren't linked.
-  if (_i2i_entry != NULL) {
+  if (adapter() != NULL) {
     return;
   }
   assert( _code == NULL, "nothing compiled yet" );

test/hotspot/jtreg/compiler/codecache/OverflowCodeCacheTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,7 @@
 
 /*
  * @test OverflowCodeCacheTest
- * @bug 8059550
+ * @bug 8059550 8279356
  * @summary testing of code cache segments overflow
  * @library /test/lib
  * @modules java.base/jdk.internal.misc
@@ -33,11 +33,14 @@
  * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions
  *                   -XX:+WhiteBoxAPI -XX:CompileCommand=compileonly,null::*
- *                   -XX:-SegmentedCodeCache
+ *                   -XX:-SegmentedCodeCache -Xmixed
- *                   compiler.codecache.OverflowCodeCacheTest
+ *                   compiler.codecache.OverflowCodeCacheTest CompilationDisabled
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions
  *                   -XX:+WhiteBoxAPI -XX:CompileCommand=compileonly,null::*
- *                   -XX:+SegmentedCodeCache
+ *                   -XX:+SegmentedCodeCache -Xmixed
+ *                   compiler.codecache.OverflowCodeCacheTest CompilationDisabled
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions
+ *                   -XX:+WhiteBoxAPI -XX:-SegmentedCodeCache -Xmixed
  *                   compiler.codecache.OverflowCodeCacheTest
  */
 
@@ -49,13 +52,21 @@ import sun.hotspot.code.BlobType;
 import sun.hotspot.code.CodeBlob;
 
 import java.lang.management.MemoryPoolMXBean;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.EnumSet;
 
+class Helper {
+    // Uncommon signature to prevent sharing and force creation of a new adapter
+    public void method(float a, float b, float c, Object o) { }
+}
+
 public class OverflowCodeCacheTest {
     private static final WhiteBox WHITE_BOX = WhiteBox.getWhiteBox();
+    private static boolean COMPILATION_DISABLED = false;
 
     public static void main(String[] args) {
+        COMPILATION_DISABLED = args.length > 0;
         EnumSet<BlobType> blobTypes = BlobType.getAvailable();
         for (BlobType type : blobTypes) {
             new OverflowCodeCacheTest(type).test();
@@ -74,6 +85,8 @@ public class OverflowCodeCacheTest {
         System.out.println("allocating till possible...");
         ArrayList<Long> blobs = new ArrayList<>();
         int compilationActivityMode = -1;
+        // Lock compilation to be able to better control code cache space
+        WHITE_BOX.lockCompilation();
         try {
             long addr;
             int size = (int) (getHeapSize() >> 7);
@@ -88,16 +101,44 @@ public class OverflowCodeCacheTest {
                 }
             }
             /* now, remember compilationActivityMode to check it later, after freeing, since we
-               possibly have no free cache for futher work */
+               possibly have no free cache for further work */
             compilationActivityMode = WHITE_BOX.getCompilationActivityMode();
+
+            // Use smallest allocation size to make sure all of the available space
+            // is filled up. Don't free these below to put some pressure on the sweeper.
+            while ((addr = WHITE_BOX.allocateCodeBlob(1, type.id)) != 0) { }
         } finally {
+            try {
+                // Trigger creation of a new adapter for Helper::method
+                // which will fail because we are out of code cache space.
+                Helper helper = new Helper();
+            } catch (VirtualMachineError e) {
+                // Expected
+            }
+            // Free code cache space
             for (Long blob : blobs) {
                 WHITE_BOX.freeCodeBlob(blob);
             }
+
+            // Convert some nmethods to zombie and then free them to re-enable compilation
+            WHITE_BOX.unlockCompilation();
+            WHITE_BOX.forceNMethodSweep();
+            WHITE_BOX.forceNMethodSweep();
+
+            // Trigger compilation of Helper::method which will hit an assert because
+            // adapter creation failed above due to a lack of code cache space.
+            Helper helper = new Helper();
+            for (int i = 0; i < 100_000; i++) {
+                helper.method(0, 0, 0, null);
             }
+        }
+        // Only check this if compilation is disabled, otherwise the sweeper might have
+        // freed enough nmethods to allow for re-enabling compilation.
+        if (COMPILATION_DISABLED) {
             Asserts.assertNotEquals(compilationActivityMode, 1 /* run_compilation*/,
                     "Compilation must be disabled when CodeCache(CodeHeap) overflows");
         }
+    }
 
     private long getHeapSize() {
         return bean.getUsage().getMax();