8146416: java.lang.OutOfMemoryError triggers: assert(current_bci == 0) failed: bci isn't zero for do_not_unlock_if_synchronized

Handle realloc failure pending exception.

Reviewed-by: roland

hotspot/src/share/vm/runtime/deoptimization.cpp

@@ -498,6 +498,19 @@ Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread
   }
 #endif
 
+  if (thread->frames_to_pop_failed_realloc() > 0 && exec_mode != Unpack_uncommon_trap) {
+    assert(thread->has_pending_exception(), "should have thrown OOME");
+    thread->set_exception_oop(thread->pending_exception());
+    thread->clear_pending_exception();
+    exec_mode = Unpack_exception;
+  }
+
+#if INCLUDE_JVMCI
+  if (thread->frames_to_pop_failed_realloc() > 0) {
+    thread->set_pending_monitorenter(false);
+  }
+#endif
+
   UnrollBlock* info = new UnrollBlock(array->frame_size() * BytesPerWord,
                                       caller_adjustment * BytesPerWord,
                                       caller_was_method_handle ? 0 : callee_parameters,

hotspot/src/share/vm/runtime/vframeArray.cpp

@@ -171,6 +171,8 @@ void vframeArrayElement::unpack_on_stack(int caller_actual_parameters,
                                          int exec_mode) {
   JavaThread* thread = (JavaThread*) Thread::current();
 
+  bool realloc_failure_exception = thread->frames_to_pop_failed_realloc() > 0;
+
   // Look at bci and decide on bcp and continuation pc
   address bcp;
   // C++ interpreter doesn't need a pc since it will figure out what to do when it
@@ -204,10 +206,12 @@ void vframeArrayElement::unpack_on_stack(int caller_actual_parameters,
   //
   // For Compiler1, deoptimization can occur while throwing a NullPointerException at monitorenter,
   // in which case bcp should point to the monitorenter since it is within the exception's range.
+  //
+  // For realloc failure exception we just pop frames, skip the guarantee.
 
   assert(*bcp != Bytecodes::_monitorenter || is_top_frame, "a _monitorenter must be a top frame");
   assert(thread->deopt_compiled_method() != NULL, "compiled method should be known");
-  guarantee(!(thread->deopt_compiled_method()->is_compiled_by_c2() &&
+  guarantee(realloc_failure_exception || !(thread->deopt_compiled_method()->is_compiled_by_c2() &&
               *bcp == Bytecodes::_monitorenter             &&
               exec_mode == Deoptimization::Unpack_exception),
             "shouldn't get exception during monitorenter");
@@ -237,12 +241,17 @@ void vframeArrayElement::unpack_on_stack(int caller_actual_parameters,
         // Deoptimization::fetch_unroll_info_helper
         popframe_preserved_args_size_in_words = in_words(thread->popframe_preserved_args_size_in_words());
       }
-    } else if (JvmtiExport::can_force_early_return() && state != NULL && state->is_earlyret_pending()) {
+    } else if (!realloc_failure_exception && JvmtiExport::can_force_early_return() && state != NULL && state->is_earlyret_pending()) {
       // Force early return from top frame after deoptimization
 #ifndef CC_INTERP
       pc = Interpreter::remove_activation_early_entry(state->earlyret_tos());
 #endif
     } else {
+      if (realloc_failure_exception && JvmtiExport::can_force_early_return() && state != NULL && state->is_earlyret_pending()) {
+        state->clr_earlyret_pending();
+        state->set_earlyret_oop(NULL);
+        state->clr_earlyret_value();
+      }
       // Possibly override the previous pc computation of the top (youngest) frame
       switch (exec_mode) {
       case Deoptimization::Unpack_deopt:

hotspot/test/compiler/uncommontrap/DeoptReallocFailure.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8146416
+ * @library /test/lib /testlibrary /
+ * @build sun.hotspot.WhiteBox
+ * @run main ClassFileInstaller sun.hotspot.WhiteBox
+ *                              sun.hotspot.WhiteBox$WhiteBoxPermission
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -Xbatch -XX:CompileCommand=exclude,DeoptReallocFailure::main -Xmx100m DeoptReallocFailure
+ *
+ */
+import java.lang.reflect.Method;
+import sun.hotspot.WhiteBox;
+
+class MemoryChunk {
+    MemoryChunk other;
+    Object[][] array;
+
+    MemoryChunk(MemoryChunk other) {
+        this.other = other;
+        array = new Object[1024 * 256][];
+    }
+}
+
+class NoEscape {
+    long f1;
+}
+
+public class DeoptReallocFailure {
+
+    static MemoryChunk root;
+    private static final WhiteBox WB = WhiteBox.getWhiteBox();
+
+    public static synchronized long  test() {
+        NoEscape[] noEscape = new NoEscape[45];
+        noEscape[0] = new NoEscape();
+        for (int i=0;i<1024*256;i++) {
+           root.array[i]= new Object[45];
+        }
+        return noEscape[0].f1;
+    }
+
+    public static void main(String[] args) throws Throwable {
+
+        //Exhaust Memory
+        root = null;
+        try {
+            while (true) {
+                root = new MemoryChunk(root);
+            }
+        } catch (OutOfMemoryError oom) {
+        }
+
+        if (root == null) {
+          return;
+        }
+
+        try {
+            NoEscape dummy = new NoEscape();
+            Method m = DeoptReallocFailure.class.getMethod("test");
+            WB.enqueueMethodForCompilation(m, 4);
+            test();
+        } catch (OutOfMemoryError oom) {
+            root = null;
+            oom.printStackTrace();
+        }
+        System.out.println("TEST PASSED");
+    }
+}