8190797: OSR compilation fails with "assert(__the_thread__->can_call_java()) failed: can not load classes with compiler thread"

Bail out with a meaningful error message in case we cannot throw a Java exception.

Reviewed-by: kvn, dholmes

src/hotspot/share/oops/generateOopMap.cpp

@@ -2146,8 +2146,14 @@ void GenerateOopMap::error_work(const char *format, va_list ap) {
   // Append method name
   char msg_buffer2[512];
   jio_snprintf(msg_buffer2, sizeof(msg_buffer2), "%s in method %s", msg_buffer, method()->name()->as_C_string());
-  _exception = Exceptions::new_exception(Thread::current(),
+  if (Thread::current()->can_call_java()) {
-                vmSymbols::java_lang_LinkageError(), msg_buffer2);
+    _exception = Exceptions::new_exception(Thread::current(),
+                  vmSymbols::java_lang_LinkageError(), msg_buffer2);
+  } else {
+    // We cannot instantiate an exception object from a compiler thread.
+    // Exit the VM with a useful error message.
+    fatal("%s", msg_buffer2);
+  }
 }
 
 void GenerateOopMap::report_error(const char *format, ...) {
@@ -2159,14 +2165,7 @@ void GenerateOopMap::report_error(const char *format, ...) {
 void GenerateOopMap::verify_error(const char *format, ...) {
   // We do not distinguish between different types of errors for verification
   // errors.  Let the verifier give a better message.
-  const char *msg = "Illegal class file encountered. Try running with -Xverify:all";
+  report_error("Illegal class file encountered. Try running with -Xverify:all");
-  _got_error = true;
-  // Append method name
-  char msg_buffer2[512];
-  jio_snprintf(msg_buffer2, sizeof(msg_buffer2), "%s in method %s", msg,
-               method()->name()->as_C_string());
-  _exception = Exceptions::new_exception(Thread::current(),
-                vmSymbols::java_lang_LinkageError(), msg_buffer2);
 }
 
 //

test/hotspot/jtreg/compiler/linkage/OSRWithBadOperandStack.jasm

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+public class compiler/linkage/OSRWithBadOperandStack version 52:0 {
+    public static Method m1:"()I" stack 1 locals 0 {
+        iconst_0;
+        ireturn;
+    }
+
+    public static Method m2:"(Ljava/lang/Object;)V" stack 1 locals 1 {
+        return;
+    }
+
+    public static Method test:"()V" stack 2 locals 1 {
+        iconst_0;
+        istore_0;
+        Loop: stack_frame_type append;
+        locals_map int;
+        iload_0;
+        // This should fail with "java.lang.VerifyError: Bad type on operand stack Exception"
+        // because m1 returns an integer and m2 takes an Object.
+        invokestatic Method compiler/linkage/OSRWithBadOperandStack."m1":"()I";
+        invokestatic Method compiler/linkage/OSRWithBadOperandStack."m2":"(Ljava/lang/Object;)V";
+        iinc 0, 1;
+        ldc 100000;
+        if_icmple Loop;
+        return;
+    }
+}

test/hotspot/jtreg/compiler/linkage/TestLinkageErrorInGenerateOopMap.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8190797
+ * @summary Test OSR compilation with bad operand stack.
+ * @library /test/lib /
+ * @modules java.base/jdk.internal.misc
+ *          java.management
+ * @compile OSRWithBadOperandStack.jasm
+ * @run driver compiler.linkage.TestLinkageErrorInGenerateOopMap
+ */
+
+package compiler.linkage;
+
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+public class TestLinkageErrorInGenerateOopMap {
+
+    public static void main(String args[]) throws Exception {
+        if (args.length == 0) {
+            // Spawn new VM instance to execute test
+            String[] flags = {"-noverify", "-XX:-TieredCompilation",
+                              "-XX:CompileCommand=dontinline,compiler/linkage/OSRWithBadOperandStack.m*",
+                              "compiler.linkage.TestLinkageErrorInGenerateOopMap", "run"};
+            ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(flags);
+            OutputAnalyzer out = new OutputAnalyzer(pb.start());
+            if (out.getExitValue() != 0) {
+                // OSR compilation should exit with an error during OopMap verification
+                // because a LinkageError cannot be thrown from a compiler thread.
+                out.shouldContain("fatal error: Illegal class file encountered");
+            }
+        } else {
+            // Execute test
+            OSRWithBadOperandStack.test();
+        }
+    }
+}