6898462: The escape analysis with G1 cause crash assertion src/share/vm/runtime/vframeArray.cpp:94

OOM during reallocation of scalar replaced objects in deoptimization causes crashes

Reviewed-by: kvn, jrose

hotspot/src/share/vm/interpreter/interpreterRuntime.cpp

@@ -385,6 +385,18 @@ IRT_ENTRY(address, InterpreterRuntime::exception_handler_for_exception(JavaThrea
   int                handler_bci;
   int                current_bci = bci(thread);
 
+  if (thread->frames_to_pop_failed_realloc() > 0) {
+    // Allocation of scalar replaced object used in this frame
+    // failed. Unconditionally pop the frame.
+    thread->dec_frames_to_pop_failed_realloc();
+    thread->set_vm_result(h_exception());
+    // If the method is synchronized we already unlocked the monitor
+    // during deoptimization so the interpreter needs to skip it when
+    // the frame is popped.
+    thread->set_do_not_unlock_if_synchronized(true);
+    return Interpreter::remove_activation_entry();
+  }
+
   // Need to do this check first since when _do_not_unlock_if_synchronized
   // is set, we don't want to trigger any classloading which may make calls
   // into java, or surprisingly find a matching exception handler for bci 0

hotspot/src/share/vm/memory/universe.cpp

@@ -120,6 +120,7 @@ oop Universe::_out_of_memory_error_metaspace          = NULL;
 oop Universe::_out_of_memory_error_class_metaspace    = NULL;
 oop Universe::_out_of_memory_error_array_size         = NULL;
 oop Universe::_out_of_memory_error_gc_overhead_limit  = NULL;
+oop Universe::_out_of_memory_error_realloc_objects    = NULL;
 objArrayOop Universe::_preallocated_out_of_memory_error_array = NULL;
 volatile jint Universe::_preallocated_out_of_memory_error_avail_count = 0;
 bool Universe::_verify_in_progress                    = false;
@@ -191,6 +192,7 @@ void Universe::oops_do(OopClosure* f, bool do_all) {
   f->do_oop((oop*)&_out_of_memory_error_class_metaspace);
   f->do_oop((oop*)&_out_of_memory_error_array_size);
   f->do_oop((oop*)&_out_of_memory_error_gc_overhead_limit);
+  f->do_oop((oop*)&_out_of_memory_error_realloc_objects);
     f->do_oop((oop*)&_preallocated_out_of_memory_error_array);
   f->do_oop((oop*)&_null_ptr_exception_instance);
   f->do_oop((oop*)&_arithmetic_exception_instance);
@@ -575,7 +577,8 @@ bool Universe::should_fill_in_stack_trace(Handle throwable) {
           (throwable() != Universe::_out_of_memory_error_metaspace)  &&
           (throwable() != Universe::_out_of_memory_error_class_metaspace)  &&
           (throwable() != Universe::_out_of_memory_error_array_size) &&
-          (throwable() != Universe::_out_of_memory_error_gc_overhead_limit));
+          (throwable() != Universe::_out_of_memory_error_gc_overhead_limit) &&
+          (throwable() != Universe::_out_of_memory_error_realloc_objects));
 }
 
 
@@ -1039,6 +1042,7 @@ bool universe_post_init() {
     Universe::_out_of_memory_error_array_size = k_h->allocate_instance(CHECK_false);
     Universe::_out_of_memory_error_gc_overhead_limit =
       k_h->allocate_instance(CHECK_false);
+    Universe::_out_of_memory_error_realloc_objects = k_h->allocate_instance(CHECK_false);
 
     // Setup preallocated NullPointerException
     // (this is currently used for a cheap & dirty solution in compiler exception handling)
@@ -1078,6 +1082,9 @@ bool universe_post_init() {
     msg = java_lang_String::create_from_str("GC overhead limit exceeded", CHECK_false);
     java_lang_Throwable::set_message(Universe::_out_of_memory_error_gc_overhead_limit, msg());
 
+    msg = java_lang_String::create_from_str("Java heap space: failed reallocation of scalar replaced objects", CHECK_false);
+    java_lang_Throwable::set_message(Universe::_out_of_memory_error_realloc_objects, msg());
+
     msg = java_lang_String::create_from_str("/ by zero", CHECK_false);
     java_lang_Throwable::set_message(Universe::_arithmetic_exception_instance, msg());
 

hotspot/src/share/vm/memory/universe.hpp

@@ -157,6 +157,7 @@ class Universe: AllStatic {
   static oop          _out_of_memory_error_class_metaspace;
   static oop          _out_of_memory_error_array_size;
   static oop          _out_of_memory_error_gc_overhead_limit;
+  static oop          _out_of_memory_error_realloc_objects;
 
   static Array<int>*       _the_empty_int_array;    // Canonicalized int array
   static Array<u2>*        _the_empty_short_array;  // Canonicalized short array
@@ -328,6 +329,7 @@ class Universe: AllStatic {
   static oop out_of_memory_error_class_metaspace()    { return gen_out_of_memory_error(_out_of_memory_error_class_metaspace);   }
   static oop out_of_memory_error_array_size()         { return gen_out_of_memory_error(_out_of_memory_error_array_size); }
   static oop out_of_memory_error_gc_overhead_limit()  { return gen_out_of_memory_error(_out_of_memory_error_gc_overhead_limit);  }
+  static oop out_of_memory_error_realloc_objects()    { return gen_out_of_memory_error(_out_of_memory_error_realloc_objects);  }
 
   // Accessors needed for fast allocation
   static Klass** boolArrayKlassObj_addr()           { return &_boolArrayKlassObj;   }

hotspot/src/share/vm/opto/macro.cpp

@@ -971,7 +971,11 @@ void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
 }
 
 bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) {
-  if (!EliminateAllocations || !alloc->_is_non_escaping) {
+  // Don't do scalar replacement if the frame can be popped by JVMTI:
+  // if reallocation fails during deoptimization we'll pop all
+  // interpreter frames for this compiled frame and that won't play
+  // nice with JVMTI popframe.
+  if (!EliminateAllocations || JvmtiExport::can_pop_frame() || !alloc->_is_non_escaping) {
     return false;
   }
   Node* klass = alloc->in(AllocateNode::KlassNode);

hotspot/src/share/vm/runtime/deoptimization.cpp

@@ -176,6 +176,8 @@ Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread
   assert(vf->is_compiled_frame(), "Wrong frame type");
   chunk->push(compiledVFrame::cast(vf));
 
+  bool realloc_failures = false;
+
 #ifdef COMPILER2
   // Reallocate the non-escaping objects and restore their fields. Then
   // relock objects if synchronization on them was eliminated.
@@ -206,22 +208,19 @@ Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread
           tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, (void *)result, thread);
         }
       }
-      bool reallocated = false;
       if (objects != NULL) {
         JRT_BLOCK
-          reallocated = realloc_objects(thread, &deoptee, objects, THREAD);
+          realloc_failures = realloc_objects(thread, &deoptee, objects, THREAD);
         JRT_END
+        reassign_fields(&deoptee, &map, objects, realloc_failures);
       }
-      if (reallocated) {
-        reassign_fields(&deoptee, &map, objects);
 #ifndef PRODUCT
       if (TraceDeoptimization) {
         ttyLocker ttyl;
         tty->print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, thread);
-          print_objects(objects);
+        print_objects(objects, realloc_failures);
       }
 #endif
-      }
       if (save_oop_result) {
         // Restore result.
         deoptee.set_saved_oop_result(&map, return_value());
@@ -236,7 +235,7 @@ Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread
         assert (cvf->scope() != NULL,"expect only compiled java frames");
         GrowableArray<MonitorInfo*>* monitors = cvf->monitors();
         if (monitors->is_nonempty()) {
-          relock_objects(monitors, thread);
+          relock_objects(monitors, thread, realloc_failures);
 #ifndef PRODUCT
           if (TraceDeoptimization) {
             ttyLocker ttyl;
@@ -247,10 +246,15 @@ Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread
                   first = false;
                   tty->print_cr("RELOCK OBJECTS in thread " INTPTR_FORMAT, thread);
                 }
+                if (mi->owner_is_scalar_replaced()) {
+                  Klass* k = java_lang_Class::as_Klass(mi->owner_klass());
+                  tty->print_cr("     failed reallocation for klass %s", k->external_name());
+                } else {
                   tty->print_cr("     object <" INTPTR_FORMAT "> locked", (void *)mi->owner());
                 }
               }
             }
+          }
 #endif
         }
       }
@@ -262,9 +266,14 @@ Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread
   // out the java state residing in the vframeArray will be missed.
   No_Safepoint_Verifier no_safepoint;
 
-  vframeArray* array = create_vframeArray(thread, deoptee, &map, chunk);
+  vframeArray* array = create_vframeArray(thread, deoptee, &map, chunk, realloc_failures);
+#ifdef COMPILER2
+  if (realloc_failures) {
+    pop_frames_failed_reallocs(thread, array);
+  }
+#endif
 
-  assert(thread->vframe_array_head() == NULL, "Pending deopt!");;
+  assert(thread->vframe_array_head() == NULL, "Pending deopt!");
   thread->set_vframe_array_head(array);
 
   // Now that the vframeArray has been created if we have any deferred local writes
@@ -718,6 +727,8 @@ bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, GrowableArra
   int exception_line = thread->exception_line();
   thread->clear_pending_exception();
 
+  bool failures = false;
+
   for (int i = 0; i < objects->length(); i++) {
     assert(objects->at(i)->is_object(), "invalid debug information");
     ObjectValue* sv = (ObjectValue*) objects->at(i);
@@ -727,27 +738,34 @@ bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, GrowableArra
 
     if (k->oop_is_instance()) {
       InstanceKlass* ik = InstanceKlass::cast(k());
-      obj = ik->allocate_instance(CHECK_(false));
+      obj = ik->allocate_instance(THREAD);
     } else if (k->oop_is_typeArray()) {
       TypeArrayKlass* ak = TypeArrayKlass::cast(k());
       assert(sv->field_size() % type2size[ak->element_type()] == 0, "non-integral array length");
       int len = sv->field_size() / type2size[ak->element_type()];
-      obj = ak->allocate(len, CHECK_(false));
+      obj = ak->allocate(len, THREAD);
     } else if (k->oop_is_objArray()) {
       ObjArrayKlass* ak = ObjArrayKlass::cast(k());
-      obj = ak->allocate(sv->field_size(), CHECK_(false));
+      obj = ak->allocate(sv->field_size(), THREAD);
+    }
+
+    if (obj == NULL) {
+      failures = true;
     }
 
-    assert(obj != NULL, "allocation failed");
     assert(sv->value().is_null(), "redundant reallocation");
+    assert(obj != NULL || HAS_PENDING_EXCEPTION, "allocation should succeed or we should get an exception");
+    CLEAR_PENDING_EXCEPTION;
     sv->set_value(obj);
   }
 
-  if (pending_exception.not_null()) {
+  if (failures) {
+    THROW_OOP_(Universe::out_of_memory_error_realloc_objects(), failures);
+  } else if (pending_exception.not_null()) {
     thread->set_pending_exception(pending_exception(), exception_file, exception_line);
   }
 
-  return true;
+  return failures;
 }
 
 // This assumes that the fields are stored in ObjectValue in the same order
@@ -885,12 +903,15 @@ void Deoptimization::reassign_object_array_elements(frame* fr, RegisterMap* reg_
 
 
 // restore fields of all eliminated objects and arrays
-void Deoptimization::reassign_fields(frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects) {
+void Deoptimization::reassign_fields(frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, bool realloc_failures) {
   for (int i = 0; i < objects->length(); i++) {
     ObjectValue* sv = (ObjectValue*) objects->at(i);
     KlassHandle k(java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()()));
     Handle obj = sv->value();
-    assert(obj.not_null(), "reallocation was missed");
+    assert(obj.not_null() || realloc_failures, "reallocation was missed");
+    if (obj.is_null()) {
+      continue;
+    }
 
     if (k->oop_is_instance()) {
       InstanceKlass* ik = InstanceKlass::cast(k());
@@ -907,11 +928,12 @@ void Deoptimization::reassign_fields(frame* fr, RegisterMap* reg_map, GrowableAr
 
 
 // relock objects for which synchronization was eliminated
-void Deoptimization::relock_objects(GrowableArray<MonitorInfo*>* monitors, JavaThread* thread) {
+void Deoptimization::relock_objects(GrowableArray<MonitorInfo*>* monitors, JavaThread* thread, bool realloc_failures) {
   for (int i = 0; i < monitors->length(); i++) {
     MonitorInfo* mon_info = monitors->at(i);
     if (mon_info->eliminated()) {
-      assert(mon_info->owner() != NULL, "reallocation was missed");
+      assert(!mon_info->owner_is_scalar_replaced() || realloc_failures, "reallocation was missed");
+      if (!mon_info->owner_is_scalar_replaced()) {
         Handle obj = Handle(mon_info->owner());
         markOop mark = obj->mark();
         if (UseBiasedLocking && mark->has_bias_pattern()) {
@@ -926,15 +948,16 @@ void Deoptimization::relock_objects(GrowableArray<MonitorInfo*>* monitors, JavaT
         }
         BasicLock* lock = mon_info->lock();
         ObjectSynchronizer::slow_enter(obj, lock, thread);
-    }
         assert(mon_info->owner()->is_locked(), "object must be locked now");
       }
+    }
+  }
 }
 
 
 #ifndef PRODUCT
 // print information about reallocated objects
-void Deoptimization::print_objects(GrowableArray<ScopeValue*>* objects) {
+void Deoptimization::print_objects(GrowableArray<ScopeValue*>* objects, bool realloc_failures) {
   fieldDescriptor fd;
 
   for (int i = 0; i < objects->length(); i++) {
@@ -944,10 +967,15 @@ void Deoptimization::print_objects(GrowableArray<ScopeValue*>* objects) {
 
     tty->print("     object <" INTPTR_FORMAT "> of type ", (void *)sv->value()());
     k->print_value();
+    assert(obj.not_null() || realloc_failures, "reallocation was missed");
+    if (obj.is_null()) {
+      tty->print(" allocation failed");
+    } else {
       tty->print(" allocated (%d bytes)", obj->size() * HeapWordSize);
+    }
     tty->cr();
 
-    if (Verbose) {
+    if (Verbose && !obj.is_null()) {
       k->oop_print_on(obj(), tty);
     }
   }
@@ -955,7 +983,7 @@ void Deoptimization::print_objects(GrowableArray<ScopeValue*>* objects) {
 #endif
 #endif // COMPILER2
 
-vframeArray* Deoptimization::create_vframeArray(JavaThread* thread, frame fr, RegisterMap *reg_map, GrowableArray<compiledVFrame*>* chunk) {
+vframeArray* Deoptimization::create_vframeArray(JavaThread* thread, frame fr, RegisterMap *reg_map, GrowableArray<compiledVFrame*>* chunk, bool realloc_failures) {
   Events::log(thread, "DEOPT PACKING pc=" INTPTR_FORMAT " sp=" INTPTR_FORMAT, fr.pc(), fr.sp());
 
 #ifndef PRODUCT
@@ -998,7 +1026,7 @@ vframeArray* Deoptimization::create_vframeArray(JavaThread* thread, frame fr, Re
   // Since the Java thread being deoptimized will eventually adjust it's own stack,
   // the vframeArray containing the unpacking information is allocated in the C heap.
   // For Compiler1, the caller of the deoptimized frame is saved for use by unpack_frames().
-  vframeArray* array = vframeArray::allocate(thread, frame_size, chunk, reg_map, sender, caller, fr);
+  vframeArray* array = vframeArray::allocate(thread, frame_size, chunk, reg_map, sender, caller, fr, realloc_failures);
 
   // Compare the vframeArray to the collected vframes
   assert(array->structural_compare(thread, chunk), "just checking");
@@ -1013,6 +1041,33 @@ vframeArray* Deoptimization::create_vframeArray(JavaThread* thread, frame fr, Re
   return array;
 }
 
+#ifdef COMPILER2
+void Deoptimization::pop_frames_failed_reallocs(JavaThread* thread, vframeArray* array) {
+  // Reallocation of some scalar replaced objects failed. Record
+  // that we need to pop all the interpreter frames for the
+  // deoptimized compiled frame.
+  assert(thread->frames_to_pop_failed_realloc() == 0, "missed frames to pop?");
+  thread->set_frames_to_pop_failed_realloc(array->frames());
+  // Unlock all monitors here otherwise the interpreter will see a
+  // mix of locked and unlocked monitors (because of failed
+  // reallocations of synchronized objects) and be confused.
+  for (int i = 0; i < array->frames(); i++) {
+    MonitorChunk* monitors = array->element(i)->monitors();
+    if (monitors != NULL) {
+      for (int j = 0; j < monitors->number_of_monitors(); j++) {
+        BasicObjectLock* src = monitors->at(j);
+        if (src->obj() != NULL) {
+          ObjectSynchronizer::fast_exit(src->obj(), src->lock(), thread);
+        }
+      }
+      array->element(i)->free_monitors(thread);
+#ifdef ASSERT
+      array->element(i)->set_removed_monitors();
+#endif
+    }
+  }
+}
+#endif
 
 static void collect_monitors(compiledVFrame* cvf, GrowableArray<Handle>* objects_to_revoke) {
   GrowableArray<MonitorInfo*>* monitors = cvf->monitors();

hotspot/src/share/vm/runtime/deoptimization.hpp

@@ -125,13 +125,14 @@ class Deoptimization : AllStatic {
   static bool realloc_objects(JavaThread* thread, frame* fr, GrowableArray<ScopeValue*>* objects, TRAPS);
   static void reassign_type_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, typeArrayOop obj, BasicType type);
   static void reassign_object_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, objArrayOop obj);
-  static void reassign_fields(frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects);
+  static void reassign_fields(frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, bool realloc_failures);
-  static void relock_objects(GrowableArray<MonitorInfo*>* monitors, JavaThread* thread);
+  static void relock_objects(GrowableArray<MonitorInfo*>* monitors, JavaThread* thread, bool realloc_failures);
-  NOT_PRODUCT(static void print_objects(GrowableArray<ScopeValue*>* objects);)
+  static void pop_frames_failed_reallocs(JavaThread* thread, vframeArray* array);
+  NOT_PRODUCT(static void print_objects(GrowableArray<ScopeValue*>* objects, bool realloc_failures);)
 #endif // COMPILER2
 
   public:
-  static vframeArray* create_vframeArray(JavaThread* thread, frame fr, RegisterMap *reg_map, GrowableArray<compiledVFrame*>* chunk);
+  static vframeArray* create_vframeArray(JavaThread* thread, frame fr, RegisterMap *reg_map, GrowableArray<compiledVFrame*>* chunk, bool realloc_failures);
 
   // Interface used for unpacking deoptimized frames
 

hotspot/src/share/vm/runtime/sharedRuntime.cpp

@@ -456,6 +456,7 @@ JRT_END
 
 address SharedRuntime::raw_exception_handler_for_return_address(JavaThread* thread, address return_address) {
   assert(frame::verify_return_pc(return_address), err_msg("must be a return address: " INTPTR_FORMAT, return_address));
+  assert(thread->frames_to_pop_failed_realloc() == 0 || Interpreter::contains(return_address), "missed frames to pop?");
 
   // Reset method handle flag.
   thread->set_is_method_handle_return(false);

hotspot/src/share/vm/runtime/thread.cpp

@@ -1448,6 +1448,7 @@ void JavaThread::initialize() {
   _popframe_condition = popframe_inactive;
   _popframe_preserved_args = NULL;
   _popframe_preserved_args_size = 0;
+  _frames_to_pop_failed_realloc = 0;
 
   pd_initialize();
 }

hotspot/src/share/vm/runtime/thread.hpp

@@ -908,6 +908,12 @@ class JavaThread: public Thread {
   // This is set to popframe_pending to signal that top Java frame should be popped immediately
   int _popframe_condition;
 
+  // If reallocation of scalar replaced objects fails, we throw OOM
+  // and during exception propagation, pop the top
+  // _frames_to_pop_failed_realloc frames, the ones that reference
+  // failed reallocations.
+  int _frames_to_pop_failed_realloc;
+
 #ifndef PRODUCT
   int _jmp_ring_index;
   struct {
@@ -1567,6 +1573,10 @@ class JavaThread: public Thread {
   void clr_pop_frame_in_process(void)                 { _popframe_condition &= ~popframe_processing_bit; }
 #endif
 
+  int frames_to_pop_failed_realloc() const            { return _frames_to_pop_failed_realloc; }
+  void set_frames_to_pop_failed_realloc(int nb)       { _frames_to_pop_failed_realloc = nb; }
+  void dec_frames_to_pop_failed_realloc()             { _frames_to_pop_failed_realloc--; }
+
  private:
   // Saved incoming arguments to popped frame.
   // Used only when popped interpreted frame returns to deoptimized frame.

hotspot/src/share/vm/runtime/vframeArray.cpp

@@ -57,7 +57,7 @@ void vframeArrayElement::free_monitors(JavaThread* jt) {
   }
 }
 
-void vframeArrayElement::fill_in(compiledVFrame* vf) {
+void vframeArrayElement::fill_in(compiledVFrame* vf, bool realloc_failures) {
 
 // Copy the information from the compiled vframe to the
 // interpreter frame we will be creating to replace vf
@@ -65,6 +65,9 @@ void vframeArrayElement::fill_in(compiledVFrame* vf) {
   _method = vf->method();
   _bci    = vf->raw_bci();
   _reexecute = vf->should_reexecute();
+#ifdef ASSERT
+  _removed_monitors = false;
+#endif
 
   int index;
 
@@ -82,13 +85,17 @@ void vframeArrayElement::fill_in(compiledVFrame* vf) {
     // Migrate the BasicLocks from the stack to the monitor chunk
     for (index = 0; index < list->length(); index++) {
       MonitorInfo* monitor = list->at(index);
-      assert(!monitor->owner_is_scalar_replaced(), "object should be reallocated already");
+      assert(!monitor->owner_is_scalar_replaced() || realloc_failures, "object should be reallocated already");
-      assert(monitor->owner() == NULL || (!monitor->owner()->is_unlocked() && !monitor->owner()->has_bias_pattern()), "object must be null or locked, and unbiased");
       BasicObjectLock* dest = _monitors->at(index);
+      if (monitor->owner_is_scalar_replaced()) {
+        dest->set_obj(NULL);
+      } else {
+        assert(monitor->owner() == NULL || (!monitor->owner()->is_unlocked() && !monitor->owner()->has_bias_pattern()), "object must be null or locked, and unbiased");
         dest->set_obj(monitor->owner());
         monitor->lock()->move_to(monitor->owner(), dest->lock());
       }
     }
+  }
 
   // Convert the vframe locals and expressions to off stack
   // values. Because we will not gc all oops can be converted to
@@ -111,7 +118,7 @@ void vframeArrayElement::fill_in(compiledVFrame* vf) {
     StackValue* value = locs->at(index);
     switch(value->type()) {
       case T_OBJECT:
-        assert(!value->obj_is_scalar_replaced(), "object should be reallocated already");
+        assert(!value->obj_is_scalar_replaced() || realloc_failures, "object should be reallocated already");
         // preserve object type
         _locals->add( new StackValue(cast_from_oop<intptr_t>((value->get_obj()())), T_OBJECT ));
         break;
@@ -136,7 +143,7 @@ void vframeArrayElement::fill_in(compiledVFrame* vf) {
     StackValue* value = exprs->at(index);
     switch(value->type()) {
       case T_OBJECT:
-        assert(!value->obj_is_scalar_replaced(), "object should be reallocated already");
+        assert(!value->obj_is_scalar_replaced() || realloc_failures, "object should be reallocated already");
         // preserve object type
         _expressions->add( new StackValue(cast_from_oop<intptr_t>((value->get_obj()())), T_OBJECT ));
         break;
@@ -287,7 +294,7 @@ void vframeArrayElement::unpack_on_stack(int caller_actual_parameters,
 
   _frame.patch_pc(thread, pc);
 
-  assert (!method()->is_synchronized() || locks > 0, "synchronized methods must have monitors");
+  assert (!method()->is_synchronized() || locks > 0 || _removed_monitors, "synchronized methods must have monitors");
 
   BasicObjectLock* top = iframe()->interpreter_frame_monitor_begin();
   for (int index = 0; index < locks; index++) {
@@ -439,7 +446,8 @@ int vframeArrayElement::on_stack_size(int callee_parameters,
 
 
 vframeArray* vframeArray::allocate(JavaThread* thread, int frame_size, GrowableArray<compiledVFrame*>* chunk,
-                                   RegisterMap *reg_map, frame sender, frame caller, frame self) {
+                                   RegisterMap *reg_map, frame sender, frame caller, frame self,
+                                   bool realloc_failures) {
 
   // Allocate the vframeArray
   vframeArray * result = (vframeArray*) AllocateHeap(sizeof(vframeArray) + // fixed part
@@ -451,19 +459,20 @@ vframeArray* vframeArray::allocate(JavaThread* thread, int frame_size, GrowableA
   result->_caller = caller;
   result->_original = self;
   result->set_unroll_block(NULL); // initialize it
-  result->fill_in(thread, frame_size, chunk, reg_map);
+  result->fill_in(thread, frame_size, chunk, reg_map, realloc_failures);
   return result;
 }
 
 void vframeArray::fill_in(JavaThread* thread,
                           int frame_size,
                           GrowableArray<compiledVFrame*>* chunk,
-                          const RegisterMap *reg_map) {
+                          const RegisterMap *reg_map,
+                          bool realloc_failures) {
   // Set owner first, it is used when adding monitor chunks
 
   _frame_size = frame_size;
   for(int i = 0; i < chunk->length(); i++) {
-    element(i)->fill_in(chunk->at(i));
+    element(i)->fill_in(chunk->at(i), realloc_failures);
   }
 
   // Copy registers for callee-saved registers

hotspot/src/share/vm/runtime/vframeArray.hpp

@@ -58,6 +58,9 @@ class vframeArrayElement : public _ValueObj {
     MonitorChunk* _monitors;                                     // active monitors for this vframe
     StackValueCollection* _locals;
     StackValueCollection* _expressions;
+#ifdef ASSERT
+    bool _removed_monitors;
+#endif
 
   public:
 
@@ -78,7 +81,7 @@ class vframeArrayElement : public _ValueObj {
 
   StackValueCollection* expressions(void) const        { return _expressions; }
 
-  void fill_in(compiledVFrame* vf);
+  void fill_in(compiledVFrame* vf, bool realloc_failures);
 
   // Formerly part of deoptimizedVFrame
 
@@ -99,6 +102,12 @@ class vframeArrayElement : public _ValueObj {
                        bool is_bottom_frame,
                        int exec_mode);
 
+#ifdef ASSERT
+  void set_removed_monitors() {
+    _removed_monitors = true;
+  }
+#endif
+
 #ifndef PRODUCT
   void print(outputStream* st);
 #endif /* PRODUCT */
@@ -160,13 +169,14 @@ class vframeArray: public CHeapObj<mtCompiler> {
   int frames() const                            { return _frames;   }
 
   static vframeArray* allocate(JavaThread* thread, int frame_size, GrowableArray<compiledVFrame*>* chunk,
-                               RegisterMap* reg_map, frame sender, frame caller, frame self);
+                               RegisterMap* reg_map, frame sender, frame caller, frame self,
+                               bool realloc_failures);
 
 
   vframeArrayElement* element(int index)        { assert(is_within_bounds(index), "Bad index"); return &_elements[index]; }
 
   // Allocates a new vframe in the array and fills the array with vframe information in chunk
-  void fill_in(JavaThread* thread, int frame_size, GrowableArray<compiledVFrame*>* chunk, const RegisterMap *reg_map);
+  void fill_in(JavaThread* thread, int frame_size, GrowableArray<compiledVFrame*>* chunk, const RegisterMap *reg_map, bool realloc_failures);
 
   // Returns the owner of this vframeArray
   JavaThread* owner_thread() const           { return _owner_thread; }

hotspot/test/compiler/uncommontrap/TestDeoptOOM.java

@@ -0,0 +1,426 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 6898462
+ * @summary failed reallocations of scalar replaced objects during deoptimization causes crash
+ * @run main/othervm -XX:-BackgroundCompilation -XX:CompileCommand=exclude,TestDeoptOOM::main -XX:CompileCommand=exclude,TestDeoptOOM::m9_1 -Xmx128M TestDeoptOOM
+ *
+ */
+
+public class TestDeoptOOM {
+
+    long f1;
+    long f2;
+    long f3;
+    long f4;
+    long f5;
+
+    static class LinkedList {
+        LinkedList l;
+        long[] array;
+        LinkedList(LinkedList l, int size) {
+            array = new long[size];
+            this.l = l;
+        }
+    }
+
+    static LinkedList ll;
+
+    static void consume_all_memory() {
+        int size = 128 * 1024 * 1024;
+        while(size > 0) {
+            try {
+                while(true) {
+                    ll = new LinkedList(ll, size);
+                }
+            } catch(OutOfMemoryError oom) {
+            }
+            size = size / 2;
+        }
+    }
+
+    static void free_memory() {
+        ll = null;
+    }
+
+    static TestDeoptOOM m1(boolean deopt) {
+        try {
+            TestDeoptOOM tdoom = new TestDeoptOOM();
+            if (deopt) {
+                return tdoom;
+            }
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m1");
+        }
+        return null;
+    }
+
+    static TestDeoptOOM m2_1(boolean deopt) {
+        try {
+            TestDeoptOOM tdoom = new TestDeoptOOM();
+            if (deopt) {
+                return tdoom;
+            }
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m2_1");
+        }
+        return null;
+    }
+
+    static TestDeoptOOM m2(boolean deopt) {
+        try {
+            return m2_1(deopt);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m2");
+        }
+        return null;
+    }
+
+    static TestDeoptOOM m3_3(boolean deopt) {
+        try {
+            TestDeoptOOM tdoom = new TestDeoptOOM();
+            if (deopt) {
+                return tdoom;
+            }
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m3_3");
+        }
+        return null;
+    }
+
+    static boolean m3_2(boolean deopt) {
+        try {
+            return m3_3(deopt) != null;
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m3_2");
+        }
+        return false;
+    }
+
+    static TestDeoptOOM m3_1(boolean deopt) {
+        try {
+            TestDeoptOOM tdoom = new TestDeoptOOM();
+            if (m3_2(deopt)) {
+                return tdoom;
+            }
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m3_1");
+        }
+        return null;
+    }
+
+    static TestDeoptOOM m3(boolean deopt) {
+        try {
+            return m3_1(deopt);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m3");
+        }
+        return null;
+    }
+
+    static TestDeoptOOM m4(boolean deopt) {
+        try {
+            TestDeoptOOM tdoom = new TestDeoptOOM();
+            if (deopt) {
+                tdoom.f1 = 1l;
+                tdoom.f2 = 2l;
+                tdoom.f3 = 3l;
+                return tdoom;
+            }
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m4");
+        }
+        return null;
+    }
+
+    static TestDeoptOOM m5(boolean deopt) {
+        try {
+            TestDeoptOOM tdoom = new TestDeoptOOM();
+            synchronized(tdoom) {
+                if (deopt) {
+                    return tdoom;
+                }
+            }
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m5");
+        }
+        return null;
+    }
+
+    synchronized TestDeoptOOM m6_1(boolean deopt) {
+        if (deopt) {
+            return this;
+        }
+        return null;
+    }
+
+    static TestDeoptOOM m6(boolean deopt) {
+        try {
+            TestDeoptOOM tdoom = new TestDeoptOOM();
+            return tdoom.m6_1(deopt);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m6");
+        }
+        return null;
+    }
+
+    static TestDeoptOOM m7_1(boolean deopt, Object lock) {
+        try {
+            synchronized(lock) {
+                TestDeoptOOM tdoom = new TestDeoptOOM();
+                if (deopt) {
+                    return tdoom;
+                }
+            }
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m7_1");
+        }
+        return null;
+    }
+
+    static TestDeoptOOM m7(boolean deopt, Object lock) {
+        try {
+            return m7_1(deopt, lock);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m7");
+        }
+        return null;
+    }
+
+    static class A {
+        long f1;
+        long f2;
+        long f3;
+        long f4;
+        long f5;
+    }
+
+    static class B {
+        long f1;
+        long f2;
+        long f3;
+        long f4;
+        long f5;
+
+        A a;
+    }
+
+    static B m8(boolean deopt) {
+        try {
+            A a = new A();
+            B b = new B();
+            b.a = a;
+            if (deopt) {
+                return b;
+            }
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m8");
+        }
+        return null;
+    }
+
+    static void m9_1(int i) {
+        if (i > 90000) {
+            consume_all_memory();
+        }
+    }
+
+    static TestDeoptOOM m9() {
+        try {
+            for (int i = 0; i < 100000; i++) {
+                TestDeoptOOM tdoom = new TestDeoptOOM();
+                m9_1(i);
+                if (i > 90000) {
+                    return tdoom;
+                }
+            }
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in m1");
+        }
+        return null;
+    }
+
+    public static void main(String[] args) {
+        for (int i = 0; i < 20000; i++) {
+            m1(false);
+        }
+
+        consume_all_memory();
+
+        try {
+            m1(true);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in main " + oom.getMessage());
+        }
+
+        free_memory();
+
+        for (int i = 0; i < 20000; i++) {
+            m2(false);
+        }
+
+        consume_all_memory();
+
+        try {
+            m2(true);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in main");
+        }
+
+        free_memory();
+
+        for (int i = 0; i < 20000; i++) {
+            m3(false);
+        }
+
+        consume_all_memory();
+
+        try {
+            m3(true);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in main");
+        }
+
+        free_memory();
+
+        for (int i = 0; i < 20000; i++) {
+            m4(false);
+        }
+
+        consume_all_memory();
+
+        try {
+            m4(true);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in main");
+        }
+
+        free_memory();
+
+        for (int i = 0; i < 20000; i++) {
+            m5(false);
+        }
+
+        consume_all_memory();
+
+        try {
+            m5(true);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in main");
+        }
+
+        free_memory();
+
+        for (int i = 0; i < 20000; i++) {
+            m6(false);
+        }
+
+        consume_all_memory();
+
+        try {
+            m6(true);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in main");
+        }
+
+        free_memory();
+
+        final Object lock = new Object();
+
+        for (int i = 0; i < 20000; i++) {
+            m7(false, lock);
+        }
+
+        consume_all_memory();
+
+        try {
+            m7(true, lock);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in main");
+        }
+
+        free_memory();
+
+        Thread thread = new Thread() {
+                public void run() {
+                    System.out.println("Acquiring lock");
+                    synchronized(lock) {
+                        System.out.println("Lock acquired");
+                    }
+                    System.out.println("Lock released");
+                }
+            };
+        thread.start();
+        try {
+            thread.join();
+        } catch(InterruptedException ie) {
+        }
+
+        for (int i = 0; i < 20000; i++) {
+            m8(false);
+        }
+
+        consume_all_memory();
+
+        try {
+            m8(true);
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in main");
+        }
+
+        free_memory();
+
+        try {
+            m9();
+        } catch(OutOfMemoryError oom) {
+            free_memory();
+            System.out.println("OOM caught in main");
+        }
+
+        free_memory();
+    }
+}