8268347: C2: nested locks optimization may create unbalanced monitor enter/exit code

Reviewed-by: roland, vlivanov, dcubed

src/hotspot/share/opto/c2compiler.cpp

@@ -45,6 +45,9 @@ const char* C2Compiler::retry_no_subsuming_loads() {
 const char* C2Compiler::retry_no_escape_analysis() {
   return "retry without escape analysis";
 }
+const char* C2Compiler::retry_no_locks_coarsening() {
+  return "retry without locks coarsening";
+}
 const char* C2Compiler::retry_class_loading_during_parsing() {
   return "retry class loading during parsing";
 }
@@ -97,10 +100,11 @@ void C2Compiler::compile_method(ciEnv* env, ciMethod* target, int entry_bci, boo
   bool subsume_loads = SubsumeLoads;
   bool do_escape_analysis = DoEscapeAnalysis;
   bool eliminate_boxing = EliminateAutoBox;
+  bool do_locks_coarsening = EliminateLocks;
 
   while (!env->failing()) {
     // Attempt to compile while subsuming loads into machine instructions.
-    Compile C(env, target, entry_bci, subsume_loads, do_escape_analysis, eliminate_boxing, install_code, directive);
+    Compile C(env, target, entry_bci, subsume_loads, do_escape_analysis, eliminate_boxing, do_locks_coarsening, install_code, directive);
 
     // Check result and retry if appropriate.
     if (C.failure_reason() != NULL) {
@@ -120,6 +124,12 @@ void C2Compiler::compile_method(ciEnv* env, ciMethod* target, int entry_bci, boo
         env->report_failure(C.failure_reason());
         continue;  // retry
       }
+      if (C.failure_reason_is(retry_no_locks_coarsening())) {
+        assert(do_locks_coarsening, "must make progress");
+        do_locks_coarsening = false;
+        env->report_failure(C.failure_reason());
+        continue;  // retry
+      }
       if (C.has_boxed_value()) {
         // Recompile without boxing elimination regardless failure reason.
         assert(eliminate_boxing, "must make progress");
@@ -141,6 +151,10 @@ void C2Compiler::compile_method(ciEnv* env, ciMethod* target, int entry_bci, boo
         do_escape_analysis = false;
         continue;  // retry
       }
+      if (do_locks_coarsening) {
+        do_locks_coarsening = false;
+        continue;  // retry
+      }
     }
     // print inlining for last compilation only
     C.dump_print_inlining();

src/hotspot/share/opto/c2compiler.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -49,6 +49,7 @@ public:
   // sentinel value used to trigger backtracking in compile_method().
   static const char* retry_no_subsuming_loads();
   static const char* retry_no_escape_analysis();
+  static const char* retry_no_locks_coarsening();
   static const char* retry_class_loading_during_parsing();
 
   // Print compilation timers and statistics

src/hotspot/share/opto/callnode.cpp

@@ -2054,6 +2054,12 @@ bool AbstractLockNode::find_unlocks_for_region(const RegionNode* region, LockNod
 
 }
 
+const char* AbstractLockNode::_kind_names[] = {"Regular", "NonEscObj", "Coarsened", "Nested"};
+
+const char * AbstractLockNode::kind_as_string() const {
+  return _kind_names[_kind];
+}
+
 #ifndef PRODUCT
 //
 // Create a counter which counts the number of times this lock is acquired
@@ -2071,8 +2077,6 @@ void AbstractLockNode::set_eliminated_lock_counter() {
   }
 }
 
-const char* AbstractLockNode::_kind_names[] = {"Regular", "NonEscObj", "Coarsened", "Nested"};
-
 void AbstractLockNode::dump_spec(outputStream* st) const {
   st->print("%s ", _kind_names[_kind]);
   CallNode::dump_spec(st);
@@ -2124,6 +2128,9 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
       return result;
     }
 
+    if (!phase->C->do_locks_coarsening()) {
+      return result; // Compiling without locks coarsening
+    }
     //
     // Try lock coarsening
     //
@@ -2161,6 +2168,9 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
         if (PrintEliminateLocks) {
           int locks = 0;
           int unlocks = 0;
+          if (Verbose) {
+            tty->print_cr("=== Locks coarsening ===");
+          }
           for (int i = 0; i < lock_ops.length(); i++) {
             AbstractLockNode* lock = lock_ops.at(i);
             if (lock->Opcode() == Op_Lock)
@@ -2168,10 +2178,11 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
             else
               unlocks++;
             if (Verbose) {
-              lock->dump(1);
+              tty->print(" %d: ", i);
+              lock->dump();
             }
           }
-          tty->print_cr("***Eliminated %d unlocks and %d locks", unlocks, locks);
+          tty->print_cr("=== Coarsened %d unlocks and %d locks", unlocks, locks);
         }
   #endif
 
@@ -2186,6 +2197,8 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 #endif
           lock->set_coarsened();
         }
+        // Record this coarsened group.
+        phase->C->add_coarsened_locks(lock_ops);
       } else if (ctrl->is_Region() &&
                  iter->_worklist.member(ctrl)) {
         // We weren't able to find any opportunities but the region this
@@ -2219,15 +2232,34 @@ bool LockNode::is_nested_lock_region(Compile * c) {
   // Ignore complex cases: merged locks or multiple locks.
   Node* obj = obj_node();
   LockNode* unique_lock = NULL;
-  if (!box->is_simple_lock_region(&unique_lock, obj)) {
+  Node* bad_lock = NULL;
+  if (!box->is_simple_lock_region(&unique_lock, obj, &bad_lock)) {
 #ifdef ASSERT
-    this->log_lock_optimization(c, "eliminate_lock_INLR_2a");
+    this->log_lock_optimization(c, "eliminate_lock_INLR_2a", bad_lock);
 #endif
     return false;
   }
   if (unique_lock != this) {
 #ifdef ASSERT
-    this->log_lock_optimization(c, "eliminate_lock_INLR_2b");
+    this->log_lock_optimization(c, "eliminate_lock_INLR_2b", (unique_lock != NULL ? unique_lock : bad_lock));
+    if (PrintEliminateLocks && Verbose) {
+      tty->print_cr("=============== unique_lock != this ============");
+      tty->print(" this: ");
+      this->dump();
+      tty->print(" box: ");
+      box->dump();
+      tty->print(" obj: ");
+      obj->dump();
+      if (unique_lock != NULL) {
+        tty->print(" unique_lock: ");
+        unique_lock->dump();
+      }
+      if (bad_lock != NULL) {
+        tty->print(" bad_lock: ");
+        bad_lock->dump();
+      }
+      tty->print_cr("===============");
+    }
 #endif
     return false;
   }
@@ -2294,23 +2326,21 @@ Node *UnlockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   return result;
 }
 
-const char * AbstractLockNode::kind_as_string() const {
+void AbstractLockNode::log_lock_optimization(Compile *C, const char * tag, Node* bad_lock)  const {
-  return is_coarsened()   ? "coarsened" :
-         is_nested()      ? "nested" :
-         is_non_esc_obj() ? "non_escaping" :
-         "?";
-}
-
-void AbstractLockNode::log_lock_optimization(Compile *C, const char * tag)  const {
   if (C == NULL) {
     return;
   }
   CompileLog* log = C->log();
   if (log != NULL) {
-    log->begin_head("%s lock='%d' compile_id='%d' class_id='%s' kind='%s'",
+    Node* box = box_node();
-          tag, is_Lock(), C->compile_id(),
+    Node* obj = obj_node();
+    int box_id = box != NULL ? box->_idx : -1;
+    int obj_id = obj != NULL ? obj->_idx : -1;
+
+    log->begin_head("%s compile_id='%d' lock_id='%d' class='%s' kind='%s' box_id='%d' obj_id='%d' bad_id='%d'",
+          tag, C->compile_id(), this->_idx,
           is_Unlock() ? "unlock" : is_Lock() ? "lock" : "?",
-          kind_as_string());
+          kind_as_string(), box_id, obj_id, (bad_lock != NULL ? bad_lock->_idx : -1));
     log->stamp();
     log->end_head();
     JVMState* p = is_Unlock() ? (as_Unlock()->dbg_jvms()) : jvms();

src/hotspot/share/opto/callnode.hpp

@@ -1056,9 +1056,11 @@ private:
     Coarsened,    // Lock was coarsened
     Nested        // Nested lock
   } _kind;
+
+  static const char* _kind_names[Nested+1];
+
 #ifndef PRODUCT
   NamedCounter* _counter;
-  static const char* _kind_names[Nested+1];
 #endif
 
 protected:
@@ -1101,7 +1103,7 @@ public:
   bool is_nested()      const { return (_kind == Nested); }
 
   const char * kind_as_string() const;
-  void log_lock_optimization(Compile* c, const char * tag) const;
+  void log_lock_optimization(Compile* c, const char * tag, Node* bad_lock = NULL) const;
 
   void set_non_esc_obj() { _kind = NonEscObj; set_eliminated_lock_counter(); }
   void set_coarsened()   { _kind = Coarsened; set_eliminated_lock_counter(); }

src/hotspot/share/opto/compile.cpp

@@ -432,6 +432,7 @@ void Compile::remove_useless_nodes(Unique_Node_List &useful) {
   remove_useless_nodes(_skeleton_predicate_opaqs, useful);
   remove_useless_nodes(_expensive_nodes,    useful); // remove useless expensive nodes
   remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
+  remove_useless_coarsened_locks(useful);            // remove useless coarsened locks nodes
 
   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
   bs->eliminate_useless_gc_barriers(useful, this);
@@ -501,6 +502,12 @@ void Compile::print_compile_messages() {
     tty->print_cr("** Bailout: Recompile without boxing elimination       **");
     tty->print_cr("*********************************************************");
   }
+  if ((_do_locks_coarsening != EliminateLocks) && PrintOpto) {
+    // Recompiling without locks coarsening
+    tty->print_cr("*********************************************************");
+    tty->print_cr("** Bailout: Recompile without locks coarsening         **");
+    tty->print_cr("*********************************************************");
+  }
   if (env()->break_at_compile()) {
     // Open the debugger when compiling this method.
     tty->print("### Breaking when compiling: ");
@@ -528,13 +535,15 @@ debug_only( int Compile::_debug_idx = 100000; )
 
 
 Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci,
-                  bool subsume_loads, bool do_escape_analysis, bool eliminate_boxing, bool install_code, DirectiveSet* directive)
+                  bool subsume_loads, bool do_escape_analysis, bool eliminate_boxing,
+                  bool do_locks_coarsening, bool install_code, DirectiveSet* directive)
                 : Phase(Compiler),
                   _compile_id(ci_env->compile_id()),
                   _subsume_loads(subsume_loads),
                   _do_escape_analysis(do_escape_analysis),
                   _install_code(install_code),
                   _eliminate_boxing(eliminate_boxing),
+                  _do_locks_coarsening(do_locks_coarsening),
                   _method(target),
                   _entry_bci(osr_bci),
                   _stub_function(NULL),
@@ -566,6 +575,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci,
                   _skeleton_predicate_opaqs (comp_arena(), 8, 0, NULL),
                   _expensive_nodes   (comp_arena(), 8, 0, NULL),
                   _for_post_loop_igvn(comp_arena(), 8, 0, NULL),
+                  _coarsened_locks   (comp_arena(), 8, 0, NULL),
                   _congraph(NULL),
                   NOT_PRODUCT(_printer(NULL) COMMA)
                   _dead_node_list(comp_arena()),
@@ -832,6 +842,7 @@ Compile::Compile( ciEnv* ci_env,
     _do_escape_analysis(false),
     _install_code(true),
     _eliminate_boxing(false),
+    _do_locks_coarsening(false),
     _method(NULL),
     _entry_bci(InvocationEntryBci),
     _stub_function(stub_function),
@@ -4447,6 +4458,101 @@ void Compile::add_expensive_node(Node * n) {
   }
 }
 
+/**
+ * Track coarsened Lock and Unlock nodes.
+ */
+
+class Lock_List : public Node_List {
+  uint _origin_cnt;
+public:
+  Lock_List(Arena *a, uint cnt) : Node_List(a), _origin_cnt(cnt) {}
+  uint origin_cnt() const { return _origin_cnt; }
+};
+
+void Compile::add_coarsened_locks(GrowableArray<AbstractLockNode*>& locks) {
+  int length = locks.length();
+  if (length > 0) {
+    // Have to keep this list until locks elimination during Macro nodes elimination.
+    Lock_List* locks_list = new (comp_arena()) Lock_List(comp_arena(), length);
+    for (int i = 0; i < length; i++) {
+      AbstractLockNode* lock = locks.at(i);
+      assert(lock->is_coarsened(), "expecting only coarsened AbstractLock nodes, but got '%s'[%d] node", lock->Name(), lock->_idx);
+      locks_list->push(lock);
+    }
+    _coarsened_locks.append(locks_list);
+  }
+}
+
+void Compile::remove_useless_coarsened_locks(Unique_Node_List& useful) {
+  int count = coarsened_count();
+  for (int i = 0; i < count; i++) {
+    Node_List* locks_list = _coarsened_locks.at(i);
+    for (uint j = 0; j < locks_list->size(); j++) {
+      Node* lock = locks_list->at(j);
+      assert(lock->is_AbstractLock(), "sanity");
+      if (!useful.member(lock)) {
+        locks_list->yank(lock);
+      }
+    }
+  }
+}
+
+void Compile::remove_coarsened_lock(Node* n) {
+  if (n->is_AbstractLock()) {
+    int count = coarsened_count();
+    for (int i = 0; i < count; i++) {
+      Node_List* locks_list = _coarsened_locks.at(i);
+      locks_list->yank(n);
+    }
+  }
+}
+
+bool Compile::coarsened_locks_consistent() {
+  int count = coarsened_count();
+  for (int i = 0; i < count; i++) {
+    bool unbalanced = false;
+    bool modified = false; // track locks kind modifications
+    Lock_List* locks_list = (Lock_List*)_coarsened_locks.at(i);
+    uint size = locks_list->size();
+    if (size != locks_list->origin_cnt()) {
+      unbalanced = true; // Some locks were removed from list
+    } else {
+      for (uint j = 0; j < size; j++) {
+        Node* lock = locks_list->at(j);
+        // All nodes in group should have the same state (modified or not)
+        if (!lock->as_AbstractLock()->is_coarsened()) {
+          if (j == 0) {
+            // first on list was modified, the rest should be too for consistency
+            modified = true;
+          } else if (!modified) {
+            // this lock was modified but previous locks on the list were not
+            unbalanced = true;
+            break;
+          }
+        } else if (modified) {
+          // previous locks on list were modified but not this lock
+          unbalanced = true;
+          break;
+        }
+      }
+    }
+    if (unbalanced) {
+      // unbalanced monitor enter/exit - only some [un]lock nodes were removed or modified
+#ifdef ASSERT
+      if (PrintEliminateLocks) {
+        tty->print_cr("=== unbalanced coarsened locks ===");
+        for (uint l = 0; l < size; l++) {
+          locks_list->at(l)->dump();
+        }
+      }
+#endif
+      record_failure(C2Compiler::retry_no_locks_coarsening());
+      return false;
+    }
+  }
+  return true;
+}
+
 /**
  * Remove the speculative part of types and clean up the graph
  */

src/hotspot/share/opto/compile.hpp

@@ -46,6 +46,7 @@
 #include "runtime/vmThread.hpp"
 #include "utilities/ticks.hpp"
 
+class AbstractLockNode;
 class AddPNode;
 class Block;
 class Bundle;
@@ -63,6 +64,7 @@ class MachOper;
 class MachSafePointNode;
 class Node;
 class Node_Array;
+class Node_List;
 class Node_Notes;
 class NodeCloneInfo;
 class OptoReg;
@@ -248,6 +250,7 @@ class Compile : public Phase {
   const bool            _do_escape_analysis;    // Do escape analysis.
   const bool            _install_code;          // Install the code that was compiled
   const bool            _eliminate_boxing;      // Do boxing elimination.
+  const bool            _do_locks_coarsening;   // Do locks coarsening
   ciMethod*             _method;                // The method being compiled.
   int                   _entry_bci;             // entry bci for osr methods.
   const TypeFunc*       _tf;                    // My kind of signature
@@ -317,6 +320,7 @@ class Compile : public Phase {
   GrowableArray<Node*>  _skeleton_predicate_opaqs; // List of Opaque4 nodes for the loop skeleton predicates.
   GrowableArray<Node*>  _expensive_nodes;       // List of nodes that are expensive to compute and that we'd better not let the GVN freely common
   GrowableArray<Node*>  _for_post_loop_igvn;    // List of nodes for IGVN after loop opts are over
+  GrowableArray<Node_List*> _coarsened_locks;   // List of coarsened Lock and Unlock nodes
   ConnectionGraph*      _congraph;
 #ifndef PRODUCT
   IdealGraphPrinter*    _printer;
@@ -508,6 +512,8 @@ class Compile : public Phase {
   /** Do aggressive boxing elimination. */
   bool              aggressive_unboxing() const { return _eliminate_boxing && AggressiveUnboxing; }
   bool              should_install_code() const { return _install_code; }
+  /** Do locks coarsening. */
+  bool              do_locks_coarsening() const { return _do_locks_coarsening; }
 
   // Other fixed compilation parameters.
   ciMethod*         method() const              { return _method; }
@@ -656,6 +662,7 @@ class Compile : public Phase {
   int           predicate_count()         const { return _predicate_opaqs.length(); }
   int           skeleton_predicate_count() const { return _skeleton_predicate_opaqs.length(); }
   int           expensive_count()         const { return _expensive_nodes.length(); }
+  int           coarsened_count()         const { return _coarsened_locks.length(); }
 
   Node*         macro_node(int idx)       const { return _macro_nodes.at(idx); }
   Node*         predicate_opaque1_node(int idx) const { return _predicate_opaqs.at(idx); }
@@ -677,6 +684,10 @@ class Compile : public Phase {
     if (predicate_count() > 0) {
       _predicate_opaqs.remove_if_existing(n);
     }
+    // Remove from coarsened locks list if present
+    if (coarsened_count() > 0) {
+      remove_coarsened_lock(n);
+    }
   }
   void add_expensive_node(Node* n);
   void remove_expensive_node(Node* n) {
@@ -696,6 +707,10 @@ class Compile : public Phase {
       _skeleton_predicate_opaqs.remove_if_existing(n);
     }
   }
+  void add_coarsened_locks(GrowableArray<AbstractLockNode*>& locks);
+  void remove_coarsened_lock(Node* n);
+  bool coarsened_locks_consistent();
+
   bool       post_loop_opts_phase() { return _post_loop_opts_phase;  }
   void   set_post_loop_opts_phase() { _post_loop_opts_phase = true;  }
   void reset_post_loop_opts_phase() { _post_loop_opts_phase = false; }
@@ -952,6 +967,8 @@ class Compile : public Phase {
   void remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Unique_Node_List &useful);
   void remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Node* dead);
 
+  void remove_useless_coarsened_locks(Unique_Node_List& useful);
+
   void process_print_inlining();
   void dump_print_inlining();
 
@@ -1018,7 +1035,8 @@ class Compile : public Phase {
   // continuation.
   Compile(ciEnv* ci_env, ciMethod* target,
           int entry_bci, bool subsume_loads, bool do_escape_analysis,
-          bool eliminate_boxing, bool install_code, DirectiveSet* directive);
+          bool eliminate_boxing, bool do_locks_coarsening,
+          bool install_code, DirectiveSet* directive);
 
   // Second major entry point.  From the TypeFunc signature, generate code
   // to pass arguments from the Java calling convention to the C calling

src/hotspot/share/opto/locknode.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -85,7 +85,7 @@ OptoReg::Name BoxLockNode::reg(Node* box) {
 }
 
 // Is BoxLock node used for one simple lock region (same box and obj)?
-bool BoxLockNode::is_simple_lock_region(LockNode** unique_lock, Node* obj) {
+bool BoxLockNode::is_simple_lock_region(LockNode** unique_lock, Node* obj, Node** bad_lock) {
   LockNode* lock = NULL;
   bool has_one_lock = false;
   for (uint i = 0; i < this->outcnt(); i++) {
@@ -102,9 +102,15 @@ bool BoxLockNode::is_simple_lock_region(LockNode** unique_lock, Node* obj) {
               has_one_lock = true;
             } else if (lock != alock->as_Lock()) {
               has_one_lock = false;
+              if (bad_lock != NULL) {
+                *bad_lock = alock;
+              }
             }
           }
         } else {
+          if (bad_lock != NULL) {
+            *bad_lock = alock;
+          }
           return false; // Different objects
         }
       }

src/hotspot/share/opto/locknode.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -63,7 +63,7 @@ public:
   void set_eliminated()      { _is_eliminated = true; }
 
   // Is BoxLock node used for one simple lock region?
-  bool is_simple_lock_region(LockNode** unique_lock, Node* obj);
+  bool is_simple_lock_region(LockNode** unique_lock, Node* obj, Node** bad_lock);
 
 #ifndef PRODUCT
   virtual void format( PhaseRegAlloc *, outputStream *st ) const;

src/hotspot/share/opto/macro.cpp

@@ -1909,15 +1909,15 @@ void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) {
 // Mark all associated (same box and obj) lock and unlock nodes for
 // elimination if some of them marked already.
 void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) {
-  if (oldbox->as_BoxLock()->is_eliminated())
+  if (oldbox->as_BoxLock()->is_eliminated()) {
     return; // This BoxLock node was processed already.
-
+  }
   // New implementation (EliminateNestedLocks) has separate BoxLock
   // node for each locked region so mark all associated locks/unlocks as
   // eliminated even if different objects are referenced in one locked region
   // (for example, OSR compilation of nested loop inside locked scope).
   if (EliminateNestedLocks ||
-      oldbox->as_BoxLock()->is_simple_lock_region(NULL, obj)) {
+      oldbox->as_BoxLock()->is_simple_lock_region(NULL, obj, NULL)) {
     // Box is used only in one lock region. Mark this box as eliminated.
     _igvn.hash_delete(oldbox);
     oldbox->as_BoxLock()->set_eliminated(); // This changes box's hash value
@@ -2089,11 +2089,7 @@ bool PhaseMacroExpand::eliminate_locking_node(AbstractLockNode *alock) {
 
 #ifndef PRODUCT
   if (PrintEliminateLocks) {
-    if (alock->is_Lock()) {
+    tty->print_cr("++++ Eliminated: %d %s '%s'", alock->_idx, (alock->is_Lock() ? "Lock" : "Unlock"), alock->kind_as_string());
-      tty->print_cr("++++ Eliminated: %d Lock", alock->_idx);
-    } else {
-      tty->print_cr("++++ Eliminated: %d Unlock", alock->_idx);
-    }
   }
 #endif
 
@@ -2502,16 +2498,21 @@ void PhaseMacroExpand::eliminate_macro_nodes() {
   if (C->macro_count() == 0)
     return;
 
-  // First, attempt to eliminate locks
+  // Before elimination may re-mark (change to Nested or NonEscObj)
+  // all associated (same box and obj) lock and unlock nodes.
   int cnt = C->macro_count();
   for (int i=0; i < cnt; i++) {
     Node *n = C->macro_node(i);
     if (n->is_AbstractLock()) { // Lock and Unlock nodes
-      // Before elimination mark all associated (same box and obj)
-      // lock and unlock nodes.
       mark_eliminated_locking_nodes(n->as_AbstractLock());
     }
   }
+  // Re-marking may break consistency of Coarsened locks.
+  if (!C->coarsened_locks_consistent()) {
+    return; // recompile without Coarsened locks if broken
+  }
+
+  // First, attempt to eliminate locks
   bool progress = true;
   while (progress) {
     progress = false;
@@ -2574,6 +2575,7 @@ void PhaseMacroExpand::eliminate_macro_nodes() {
 bool PhaseMacroExpand::expand_macro_nodes() {
   // Last attempt to eliminate macro nodes.
   eliminate_macro_nodes();
+  if (C->failing())  return true;
 
   // Eliminate Opaque and LoopLimit nodes. Do it after all loop optimizations.
   bool progress = true;

test/hotspot/jtreg/TEST.groups

@@ -129,6 +129,7 @@ tier1_compiler_2 = \
 tier1_compiler_3 = \
   compiler/intrinsics/ \
   compiler/jsr292/ \
+  compiler/locks/ \
   compiler/loopopts/ \
   compiler/macronodes/ \
   compiler/memoryinitialization/ \

test/hotspot/jtreg/compiler/locks/TestNestedLocksElimination.java

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @bug 8268347
+ * @summary Nested locks optimization may create unbalanced monitor enter/exit code
+ *
+ * @run main/othervm -XX:-BackgroundCompilation
+ *                   -XX:CompileCommand=dontinline,TestNestedLocksElimination::foo
+ *                   -XX:CompileCommand=dontinline,TestNestedLocksElimination::getNext
+ *                   -XX:CompileCommand=dontinline,TestNestedLocksElimination::getHolder
+ *                   TestNestedLocksElimination
+ */
+
+import java.util.LinkedList;
+
+public class TestNestedLocksElimination {
+
+    private LinkedList<char[]> buffers = new LinkedList<>();
+    private boolean complete = false;
+    private int bufferSize;
+
+    void foo(char[] ca) {
+        // Don't inline dummy method
+    }
+
+    // Don't inline
+    char[] getNext(int length, int count) {
+        if (this.buffers.isEmpty()) {
+            return new char[100];
+        }
+        char[] b = (char[]) this.buffers.getFirst();
+        if (count >= 100) {
+            this.complete = true;
+            this.buffers.clear(); // empty
+        }
+        return b;
+    }
+
+    synchronized boolean isComplete() {
+        return this.complete;
+    }
+
+    synchronized boolean availableSegment() {
+        return (buffers.isEmpty() == false);
+    }
+
+    // Don't inline
+    TestNestedLocksElimination getHolder(TestNestedLocksElimination s1, TestNestedLocksElimination s2, int count) {
+        return (count & 7) == 0 ? s2 : s1;
+    }
+
+    int test(TestNestedLocksElimination s1, TestNestedLocksElimination s2, int maxToSend) {
+        boolean isComplete = true;
+        boolean availableSegment = false;
+        int size = 0;
+        int count = 0;
+        do {
+            TestNestedLocksElimination s = getHolder(s1, s2, count++);
+
+            synchronized(s) {
+                isComplete = s.isComplete();
+                availableSegment = s.availableSegment();
+            }
+
+            synchronized (this) {
+                size = 0;
+                while (size < maxToSend) {
+                    char[] b = null;
+                    // This is outer Lock region for object 's'.
+                    // Locks from following inlined methods are "nested"
+                    // because they reference the same object.
+                    synchronized(s) {
+                        b = s.getNext(maxToSend - size, count);
+
+                        // The next is bi-morphic call with both calls inlined.
+                        // But one is synchronized and the other is not.
+                        // Class check for bi-morphic call is loop invariant
+                        // and will trigger loop unswitching.
+                        // Loop unswitching will create two versions of loop
+                        // with gollowing calls inlinined in both versions.
+
+                        isComplete = s.isComplete();
+
+                        // The next synchronized method availableSegment() is
+                        // inlined and its Lock will be "coarsened" with Unlock
+                        // in version of loop with inlined synchronized method
+                        // isComplete().
+                        // Nested Lock Optimization will mark only this Unlock
+                        // as nested (as part of "nested" pair lock/unlock).
+                        // Locks elimination will remove "coarsened" Lock from
+                        // availableSegment() method leaving unmatched unlock.
+
+                        availableSegment = s.availableSegment();
+                    }
+                    foo(b);
+                    size += b.length;
+                }
+            }
+      } while (availableSegment == true || isComplete == false);
+        return size;
+    }
+
+    public static void main(String[] args) {
+        int count = 0;
+        int n = 0;
+
+        TestNestedLocksElimination t = new TestNestedLocksElimination();
+        TestNestedLocksElimination s1 = new TestNestedLocksElimination();
+        TestNestedLocksElimination s2 = new TestNestedLocksEliminationSub();
+
+        char[] c = new char[100];
+        while (n++ < 20_000) {
+            s1.buffers.add(c);
+            s2.buffers.add(c);
+            count += t.test(s1, s2, 10000);
+        }
+
+        System.out.println(" count: " + count);
+    }
+}
+
+class TestNestedLocksEliminationSub extends TestNestedLocksElimination {
+    public boolean isComplete() {
+        return true;
+    }
+}
+