8291555: Implement alternative fast-locking scheme

Co-authored-by: Fei Yang <fyang@openjdk.org>
Co-authored-by: Thomas Stuefe <stuefe@openjdk.org>
Reviewed-by: dcubed, stuefe, shade, dholmes, dlong

src/hotspot/cpu/aarch64/aarch64.ad

@@ -3822,7 +3822,7 @@ encode %{
     Register tmp = as_Register($tmp2$$reg);
     Label cont;
     Label object_has_monitor;
-    Label no_count;
+    Label count, no_count;
 
     assert_different_registers(oop, box, tmp, disp_hdr);
 
@@ -3839,7 +3839,10 @@ encode %{
     // Check for existing monitor
     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
+      __ b(cont);
+    } else if (LockingMode == LM_LEGACY) {
       // Set tmp to be (markWord of object | UNLOCK_VALUE).
       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 
@@ -3867,10 +3870,12 @@ encode %{
       // displaced header in the box, which indicates that it is a recursive lock.
       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
-    } else {
-      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
-    }
       __ b(cont);
+    } else {
+      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+      __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count);
+      __ b(count);
+    }
 
     // Handle existing monitor.
     __ bind(object_has_monitor);
@@ -3883,13 +3888,14 @@ encode %{
     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 
+    if (LockingMode != LM_LIGHTWEIGHT) {
       // Store a non-null value into the box to avoid looking like a re-entrant
       // lock. The fast-path monitor unlock code checks for
       // markWord::monitor_value so use markWord::unused_mark which has the
       // relevant bit set, and also matches ObjectSynchronizer::enter.
       __ mov(tmp, (address)markWord::unused_mark().value());
       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
-
+    }
     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 
     __ cmp(rscratch1, rthread);
@@ -3904,6 +3910,7 @@ encode %{
     // flag == NE indicates failure
     __ br(Assembler::NE, no_count);
 
+    __ bind(count);
     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 
     __ bind(no_count);
@@ -3917,11 +3924,11 @@ encode %{
     Register tmp = as_Register($tmp2$$reg);
     Label cont;
     Label object_has_monitor;
-    Label no_count;
+    Label count, no_count;
 
     assert_different_registers(oop, box, tmp, disp_hdr);
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_LEGACY) {
       // Find the lock address and load the displaced header from the stack.
       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 
@@ -3934,17 +3941,22 @@ encode %{
     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
+      __ b(cont);
+    } else if (LockingMode == LM_LEGACY) {
       // Check if it is still a light weight lock, this is is true if we
       // see the stack address of the basicLock in the markWord of the
       // object.
 
       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
                  /*release*/ true, /*weak*/ false, tmp);
-    } else {
-      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
-    }
       __ b(cont);
+    } else {
+      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+      __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
+      __ b(count);
+    }
 
     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 
@@ -3952,6 +3964,20 @@ encode %{
     __ bind(object_has_monitor);
     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
+
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      // If the owner is anonymous, we need to fix it -- in an outline stub.
+      Register tmp2 = disp_hdr;
+      __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
+      // We cannot use tbnz here, the target might be too far away and cannot
+      // be encoded.
+      __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
+      C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
+      Compile::current()->output()->add_stub(stub);
+      __ br(Assembler::NE, stub->entry());
+      __ bind(stub->continuation());
+    }
+
     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 
     Label notRecursive;
@@ -3978,6 +4004,7 @@ encode %{
     // flag == NE indicates failure
     __ br(Assembler::NE, no_count);
 
+    __ bind(count);
     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 
     __ bind(no_count);

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -431,7 +431,7 @@ int LIR_Assembler::emit_unwind_handler() {
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::r0_opr);
     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
-    if (UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
       __ b(*stub->entry());
     } else {
       __ unlock_object(r5, r4, r0, *stub->entry());
@@ -2558,7 +2558,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register obj = op->obj_opr()->as_register();  // may not be an oop
   Register hdr = op->hdr_opr()->as_register();
   Register lock = op->lock_opr()->as_register();
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     if (op->info() != nullptr) {
       add_debug_info_for_null_check_here(op->info());
       __ null_check(obj, -1);

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

@@ -63,8 +63,7 @@ void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   const int aligned_mask = BytesPerWord -1;
   const int hdr_offset = oopDesc::mark_offset_in_bytes();
-  assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
+  assert_different_registers(hdr, obj, disp_hdr);
-  Label done;
   int null_check_offset = -1;
 
   verify_oop(obj);
@@ -83,6 +82,10 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
 
   // Load object header
   ldr(hdr, Address(obj, hdr_offset));
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    fast_lock(obj, hdr, rscratch1, rscratch2, slow_case);
+  } else if (LockingMode == LM_LEGACY) {
+    Label done;
     // and mark it as unlocked
     orr(hdr, hdr, markWord::unlocked_value);
     // save unlocked object header into the displaced header location on the stack
@@ -116,6 +119,7 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     cbnz(hdr, slow_case);
     // done
     bind(done);
+  }
   increment(Address(rthread, JavaThread::held_monitor_count_offset()));
   return null_check_offset;
 }
@@ -127,14 +131,26 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
   Label done;
 
+  if (LockingMode != LM_LIGHTWEIGHT) {
     // load displaced header
     ldr(hdr, Address(disp_hdr, 0));
     // if the loaded hdr is null we had recursive locking
     // if we had recursive locking, we are done
     cbz(hdr, done);
+  }
+
   // load object
   ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
   verify_oop(obj);
+
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
+    // We cannot use tbnz here, the target might be too far away and cannot
+    // be encoded.
+    tst(hdr, markWord::monitor_value);
+    br(Assembler::NE, slow_case);
+    fast_unlock(obj, hdr, rscratch1, rscratch2, slow_case);
+  } else if (LockingMode == LM_LEGACY) {
     // test if object header is pointing to the displaced header, and if so, restore
     // the displaced header in the object - if the object header is not pointing to
     // the displaced header, get the object header instead
@@ -148,6 +164,7 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
     }
     // done
     bind(done);
+  }
   decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 }
 

src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp

@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "opto/c2_MacroAssembler.hpp"
 #include "opto/c2_CodeStubs.hpp"
+#include "runtime/objectMonitor.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 
@@ -63,4 +64,31 @@ void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
   __ emit_int32(0);   // nmethod guard value
 }
 
+int C2HandleAnonOMOwnerStub::max_size() const {
+  // Max size of stub has been determined by testing with 0, in which case
+  // C2CodeStubList::emit() will throw an assertion and report the actual size that
+  // is needed.
+  return 24;
+}
+
+void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
+  __ bind(entry());
+  Register mon = monitor();
+  Register t = tmp();
+  assert(t != noreg, "need tmp register");
+
+  // Fix owner to be the current thread.
+  __ str(rthread, Address(mon, ObjectMonitor::owner_offset_in_bytes()));
+
+  // Pop owner object from lock-stack.
+  __ ldrw(t, Address(rthread, JavaThread::lock_stack_top_offset()));
+  __ subw(t, t, oopSize);
+#ifdef ASSERT
+  __ str(zr, Address(rthread, t));
+#endif
+  __ strw(t, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+  __ b(continuation());
+}
+
 #undef __

src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp

@@ -730,7 +730,7 @@ void InterpreterMacroAssembler::remove_activation(
 void InterpreterMacroAssembler::lock_object(Register lock_reg)
 {
   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     call_VM(noreg,
             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
             lock_reg);
@@ -758,6 +758,11 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
       br(Assembler::NE, slow_case);
     }
 
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      ldr(tmp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      fast_lock(obj_reg, tmp, rscratch1, rscratch2, slow_case);
+      b(count);
+    } else if (LockingMode == LM_LEGACY) {
       // Load (object->mark() | 1) into swap_reg
       ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
       orr(swap_reg, rscratch1, 1);
@@ -807,13 +812,19 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
       // Save the test result, for recursive case, the result is zero
       str(swap_reg, Address(lock_reg, mark_offset));
       br(Assembler::EQ, count);
-
+    }
     bind(slow_case);
 
     // Call the runtime routine for slow case
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      call_VM(noreg,
+              CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj),
+              obj_reg);
+    } else {
       call_VM(noreg,
               CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
               lock_reg);
+    }
     b(done);
 
     bind(count);
@@ -839,7 +850,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
 {
   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
 
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
   } else {
     Label count, done;
@@ -850,9 +861,11 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
 
     save_bcp(); // Save in case of exception
 
+    if (LockingMode != LM_LIGHTWEIGHT) {
       // Convert from BasicObjectLock structure to object and BasicLock
       // structure Store the BasicLock address into %r0
       lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
+    }
 
     // Load oop into obj_reg(%c_rarg3)
     ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
@@ -860,6 +873,28 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
     // Free entry
     str(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
 
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      Label slow_case;
+
+      // Check for non-symmetric locking. This is allowed by the spec and the interpreter
+      // must handle it.
+      Register tmp = rscratch1;
+      // First check for lock-stack underflow.
+      ldrw(tmp, Address(rthread, JavaThread::lock_stack_top_offset()));
+      cmpw(tmp, (unsigned)LockStack::start_offset());
+      br(Assembler::LE, slow_case);
+      // Then check if the top of the lock-stack matches the unlocked object.
+      subw(tmp, tmp, oopSize);
+      ldr(tmp, Address(rthread, tmp));
+      cmpoop(tmp, obj_reg);
+      br(Assembler::NE, slow_case);
+
+      ldr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      tbnz(header_reg, exact_log2(markWord::monitor_value), slow_case);
+      fast_unlock(obj_reg, header_reg, swap_reg, rscratch1, slow_case);
+      b(count);
+      bind(slow_case);
+    } else if (LockingMode == LM_LEGACY) {
       // Load the old header from BasicLock structure
       ldr(header_reg, Address(swap_reg,
                               BasicLock::displaced_header_offset_in_bytes()));
@@ -869,7 +904,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
 
       // Atomic swap back the old header
       cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
-
+    }
     // Call the runtime routine for slow case.
     str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj
     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -6206,3 +6206,97 @@ void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) {
       strd(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first())));
   }
 }
+
+// Implements fast-locking.
+// Branches to slow upon failure to lock the object, with ZF cleared.
+// Falls through upon success with ZF set.
+//
+//  - obj: the object to be locked
+//  - hdr: the header, already loaded from obj, will be destroyed
+//  - t1, t2: temporary registers, will be destroyed
+void MacroAssembler::fast_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
+  assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
+  assert_different_registers(obj, hdr, t1, t2);
+
+  // Check if we would have space on lock-stack for the object.
+  ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+  cmpw(t1, (unsigned)LockStack::end_offset() - 1);
+  br(Assembler::GT, slow);
+
+  // Load (object->mark() | 1) into hdr
+  orr(hdr, hdr, markWord::unlocked_value);
+  // Clear lock-bits, into t2
+  eor(t2, hdr, markWord::unlocked_value);
+  // Try to swing header from unlocked to locked
+  cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
+          /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
+  br(Assembler::NE, slow);
+
+  // After successful lock, push object on lock-stack
+  ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+  str(obj, Address(rthread, t1));
+  addw(t1, t1, oopSize);
+  strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+}
+
+// Implements fast-unlocking.
+// Branches to slow upon failure, with ZF cleared.
+// Falls through upon success, with ZF set.
+//
+// - obj: the object to be unlocked
+// - hdr: the (pre-loaded) header of the object
+// - t1, t2: temporary registers
+void MacroAssembler::fast_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
+  assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
+  assert_different_registers(obj, hdr, t1, t2);
+
+#ifdef ASSERT
+  {
+    // The following checks rely on the fact that LockStack is only ever modified by
+    // its owning thread, even if the lock got inflated concurrently; removal of LockStack
+    // entries after inflation will happen delayed in that case.
+
+    // Check for lock-stack underflow.
+    Label stack_ok;
+    ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+    cmpw(t1, (unsigned)LockStack::start_offset());
+    br(Assembler::GT, stack_ok);
+    STOP("Lock-stack underflow");
+    bind(stack_ok);
+  }
+  {
+    // Check if the top of the lock-stack matches the unlocked object.
+    Label tos_ok;
+    subw(t1, t1, oopSize);
+    ldr(t1, Address(rthread, t1));
+    cmpoop(t1, obj);
+    br(Assembler::EQ, tos_ok);
+    STOP("Top of lock-stack does not match the unlocked object");
+    bind(tos_ok);
+  }
+  {
+    // Check that hdr is fast-locked.
+    Label hdr_ok;
+    tst(hdr, markWord::lock_mask_in_place);
+    br(Assembler::EQ, hdr_ok);
+    STOP("Header is not fast-locked");
+    bind(hdr_ok);
+  }
+#endif
+
+  // Load the new header (unlocked) into t1
+  orr(t1, hdr, markWord::unlocked_value);
+
+  // Try to swing header from locked to unlocked
+  cmpxchg(obj, hdr, t1, Assembler::xword,
+          /*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
+  br(Assembler::NE, slow);
+
+  // After successful unlock, pop object from lock-stack
+  ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+  subw(t1, t1, oopSize);
+#ifdef ASSERT
+  str(zr, Address(rthread, t1));
+#endif
+  strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+}

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -1580,6 +1580,9 @@ public:
   // Code for java.lang.Thread::onSpinWait() intrinsic.
   void spin_wait();
 
+  void fast_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow);
+  void fast_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow);
+
 private:
   // Check the current thread doesn't need a cross modify fence.
   void verify_cross_modify_fence_not_required() PRODUCT_RETURN;

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -1778,7 +1778,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Load the oop from the handle
     __ ldr(obj_reg, Address(oop_handle_reg, 0));
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ b(slow_path_lock);
+    } else if (LockingMode == LM_LEGACY) {
       // Load (object->mark() | 1) into swap_reg %r0
       __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
       __ orr(swap_reg, rscratch1, 1);
@@ -1808,7 +1810,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ str(swap_reg, Address(lock_reg, mark_word_offset));
       __ br(Assembler::NE, slow_path_lock);
     } else {
-      __ b(slow_path_lock);
+      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+      __ ldr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      __ fast_lock(obj_reg, swap_reg, tmp, rscratch1, slow_path_lock);
     }
     __ bind(count);
     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
@@ -1917,7 +1921,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
     Label done, not_recursive;
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_LEGACY) {
       // Simple recursive lock?
       __ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
       __ cbnz(rscratch1, not_recursive);
@@ -1932,7 +1936,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       save_native_result(masm, ret_type, stack_slots);
     }
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ b(slow_path_unlock);
+    } else if (LockingMode == LM_LEGACY) {
       // get address of the stack lock
       __ lea(r0, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
       //  get old displaced header
@@ -1944,7 +1950,11 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ bind(count);
       __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
     } else {
-      __ b(slow_path_unlock);
+      assert(LockingMode == LM_LIGHTWEIGHT, "");
+      __ ldr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      __ tbnz(old_hdr, exact_log2(markWord::monitor_value), slow_path_unlock);
+      __ fast_unlock(obj_reg, old_hdr, swap_reg, rscratch1, slow_path_unlock);
+      __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
     }
 
     // slow path re-enters here

src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp

@@ -2431,7 +2431,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register hdr = op->hdr_opr()->as_pointer_register();
   Register lock = op->lock_opr()->as_pointer_register();
 
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     if (op->info() != nullptr) {
       add_debug_info_for_null_check_here(op->info());
       __ null_check(obj);

src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp

@@ -30,6 +30,7 @@
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/tlab_globals.hpp"
 #include "interpreter/interpreter.hpp"
+#include "logging/log.hpp"
 #include "oops/arrayOop.hpp"
 #include "oops/markWord.hpp"
 #include "runtime/basicLock.hpp"
@@ -199,6 +200,7 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
   const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
   const int mark_offset = BasicLock::displaced_header_offset_in_bytes();
 
+  // save object being locked into the BasicObjectLock
   str(obj, Address(disp_hdr, obj_offset));
 
   null_check_offset = offset();
@@ -212,6 +214,18 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
 
   assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
 
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    log_trace(fastlock)("C1_MacroAssembler::lock fast");
+
+    Register t1 = disp_hdr; // Needs saving, probably
+    Register t2 = hdr;      // blow
+    Register t3 = Rtemp;    // blow
+
+    fast_lock_2(obj /* obj */, t1, t2, t3, 1 /* savemask - save t1 */, slow_case);
+    // Success: fall through
+
+  } else if (LockingMode == LM_LEGACY) {
+
     // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
     // That would be acceptable as ether CAS or slow case path is taken in that case.
 
@@ -244,6 +258,7 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     cas_for_lock_acquire(hdr, disp_hdr, obj, tmp2, slow_case);
 
     bind(fast_lock_done);
+  }
   bind(done);
 
   return null_check_offset;
@@ -261,6 +276,21 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
 
   assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
 
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    log_trace(fastlock)("C1_MacroAssembler::unlock fast");
+
+    ldr(obj, Address(disp_hdr, obj_offset));
+
+    Register t1 = disp_hdr; // Needs saving, probably
+    Register t2 = hdr;      // blow
+    Register t3 = Rtemp;    // blow
+
+    fast_unlock_2(obj /* object */, t1, t2, t3, 1 /* savemask (save t1) */,
+                    slow_case);
+    // Success: Fall through
+
+  } else if (LockingMode == LM_LEGACY) {
+
     // Load displaced header and object from the lock
     ldr(hdr, Address(disp_hdr, mark_offset));
     // If hdr is null, we've got recursive locking and there's nothing more to do
@@ -271,11 +301,10 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
 
     // Restore the object header
     cas_for_lock_release(disp_hdr, hdr, obj, tmp2, slow_case);
-
+  }
   bind(done);
 }
 
-
 #ifndef PRODUCT
 
 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {

src/hotspot/cpu/arm/c2_MacroAssembler_arm.cpp

@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "asm/assembler.hpp"
 #include "asm/assembler.inline.hpp"
+#include "logging/log.hpp"
 #include "opto/c2_MacroAssembler.hpp"
 #include "runtime/basicLock.hpp"
 
@@ -80,13 +81,7 @@ void C2_MacroAssembler::char_arrays_equals(Register ary1, Register ary2,
 
 void C2_MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2) {
   assert(VM_Version::supports_ldrex(), "unsupported, yet?");
-
+  assert_different_registers(Roop, Rbox, Rscratch, Rscratch2);
-  Register Rmark      = Rscratch2;
-
-  assert(Roop != Rscratch, "");
-  assert(Roop != Rmark, "");
-  assert(Rbox != Rscratch, "");
-  assert(Rbox != Rmark, "");
 
   Label fast_lock, done;
 
@@ -97,6 +92,19 @@ void C2_MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratc
     b(done, ne);
   }
 
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    log_trace(fastlock)("C2_MacroAssembler::lock fast");
+
+    fast_lock_2(Roop /* obj */, Rbox /* t1 */, Rscratch /* t2 */, Rscratch2 /* t3 */,
+                1 /* savemask (save t1) */, done);
+
+    // Success: set Z
+    cmp(Roop, Roop);
+
+  } else if (LockingMode == LM_LEGACY) {
+
+    Register Rmark      = Rscratch2;
+
     ldr(Rmark, Address(Roop, oopDesc::mark_offset_in_bytes()));
     tst(Rmark, markWord::unlocked_value);
     b(fast_lock, ne);
@@ -120,6 +128,7 @@ void C2_MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratc
     bool allow_fallthrough_on_failure = true;
     bool one_shot = true;
     cas_for_lock_acquire(Rmark, Rbox, Roop, Rscratch, done, allow_fallthrough_on_failure, one_shot);
+  }
 
   bind(done);
 
@@ -130,16 +139,24 @@ void C2_MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratc
 
 void C2_MacroAssembler::fast_unlock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2) {
   assert(VM_Version::supports_ldrex(), "unsupported, yet?");
-
+  assert_different_registers(Roop, Rbox, Rscratch, Rscratch2);
-  Register Rmark      = Rscratch2;
-
-  assert(Roop != Rscratch, "");
-  assert(Roop != Rmark, "");
-  assert(Rbox != Rscratch, "");
-  assert(Rbox != Rmark, "");
 
   Label done;
 
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    log_trace(fastlock)("C2_MacroAssembler::unlock fast");
+
+    fast_unlock_2(Roop /* obj */, Rbox /* t1 */, Rscratch /* t2 */, Rscratch2 /* t3 */,
+                  1 /* savemask (save t1) */, done);
+
+    cmp(Roop, Roop); // Success: Set Z
+    // Fall through
+
+  } else if (LockingMode == LM_LEGACY) {
+
+    Register Rmark      = Rscratch2;
+
+    // Find the lock address and load the displaced header from the stack.
     ldr(Rmark, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
     // If hdr is null, we've got recursive locking and there's nothing more to do
     cmp(Rmark, 0);
@@ -149,7 +166,10 @@ void C2_MacroAssembler::fast_unlock(Register Roop, Register Rbox, Register Rscra
     bool allow_fallthrough_on_failure = true;
     bool one_shot = true;
     cas_for_lock_release(Rbox, Rmark, Roop, Rscratch, done, allow_fallthrough_on_failure, one_shot);
-
+  }
   bind(done);
-}
 
+  // At this point flags are set as follows:
+  //  EQ -> Success
+  //  NE -> Failure, branch to slow path
+}

src/hotspot/cpu/arm/interp_masm_arm.cpp

@@ -885,7 +885,7 @@ void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Reg
 void InterpreterMacroAssembler::lock_object(Register Rlock) {
   assert(Rlock == R1, "the second argument");
 
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
   } else {
     Label done;
@@ -910,6 +910,11 @@ void InterpreterMacroAssembler::lock_object(Register Rlock) {
       b(slow_case, ne);
     }
 
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      log_trace(fastlock)("InterpreterMacroAssembler lock fast");
+      fast_lock_2(Robj, R0 /* t1 */, Rmark /* t2 */, Rtemp /* t3 */, 0 /* savemask */, slow_case);
+      b(done);
+    } else if (LockingMode == LM_LEGACY) {
       // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
       // That would be acceptable as ether CAS or slow case path is taken in that case.
       // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as
@@ -972,17 +977,24 @@ void InterpreterMacroAssembler::lock_object(Register Rlock) {
       str(R0, Address(Rlock, mark_offset), eq);
 
       b(done, eq);
+    }
 
     bind(slow_case);
 
     // Call the runtime routine for slow case
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      // Pass oop, not lock, in fast lock case. call_VM wants R1 though.
+      push(R1);
+      mov(R1, Robj);
+      call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj), R1);
+      pop(R1);
+    } else {
       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
-
+    }
     bind(done);
   }
 }
 
-
 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
 //
 // Argument: R0: Points to BasicObjectLock structure for lock
@@ -991,7 +1003,7 @@ void InterpreterMacroAssembler::lock_object(Register Rlock) {
 void InterpreterMacroAssembler::unlock_object(Register Rlock) {
   assert(Rlock == R0, "the first argument");
 
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock);
   } else {
     Label done, slow_case;
@@ -1012,6 +1024,25 @@ void InterpreterMacroAssembler::unlock_object(Register Rlock) {
     // Free entry
     str(Rzero, Address(Rlock, obj_offset));
 
+    if (LockingMode == LM_LIGHTWEIGHT) {
+
+      log_trace(fastlock)("InterpreterMacroAssembler unlock fast");
+
+      // Check for non-symmetric locking. This is allowed by the spec and the interpreter
+      // must handle it.
+      ldr(Rtemp, Address(Rthread, JavaThread::lock_stack_top_offset()));
+      sub(Rtemp, Rtemp, oopSize);
+      ldr(Rtemp, Address(Rthread, Rtemp));
+      cmpoop(Rtemp, Robj);
+      b(slow_case, ne);
+
+      fast_unlock_2(Robj /* obj */, Rlock /* t1 */, Rmark /* t2 */, Rtemp /* t3 */,
+                    1 /* savemask (save t1) */, slow_case);
+
+      b(done);
+
+    } else if (LockingMode == LM_LEGACY) {
+
       // Load the old header from BasicLock structure
       ldr(Rmark, Address(Rlock, mark_offset));
 
@@ -1024,6 +1055,7 @@ void InterpreterMacroAssembler::unlock_object(Register Rlock) {
 
       b(done, eq);
 
+    }
     bind(slow_case);
 
     // Call the runtime routine for slow case.
@@ -1034,7 +1066,6 @@ void InterpreterMacroAssembler::unlock_object(Register Rlock) {
   }
 }
 
-
 // Test ImethodDataPtr.  If it is null, continue at the specified label
 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) {
   assert(ProfileInterpreter, "must be profiling interpreter");

src/hotspot/cpu/arm/macroAssembler_arm.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, Red Hat, Inc.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -42,6 +43,7 @@
 #include "oops/klass.inline.hpp"
 #include "prims/methodHandles.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
+#include "runtime/javaThread.hpp"
 #include "runtime/jniHandles.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/os.hpp"
@@ -1194,11 +1196,15 @@ void MacroAssembler::cas_for_lock_acquire(Register oldval, Register newval,
     atomic_cas_bool(oldval, newval, base, oopDesc::mark_offset_in_bytes(), tmp);
   }
 
+  // Here, on success, EQ is set, NE otherwise
+
   // MemBarAcquireLock barrier
   // According to JSR-133 Cookbook, this should be LoadLoad | LoadStore,
   // but that doesn't prevent a load or store from floating up between
   // the load and store in the CAS sequence, so play it safe and
   // do a full fence.
+  // Note: we preserve flags here.
+  // Todo: Do we really need this also for the CAS fail case?
   membar(Membar_mask_bits(LoadLoad | LoadStore | StoreStore | StoreLoad), noreg);
   if (!fallthrough_is_success && !allow_fallthrough_on_failure) {
     b(slow_case, ne);
@@ -1209,7 +1215,6 @@ void MacroAssembler::cas_for_lock_release(Register oldval, Register newval,
   Register base, Register tmp, Label &slow_case,
   bool allow_fallthrough_on_failure, bool one_shot)
 {
-
   bool fallthrough_is_success = false;
 
   assert_different_registers(oldval,newval,base,tmp);
@@ -1713,3 +1718,145 @@ void MacroAssembler::read_polling_page(Register dest, relocInfo::relocType rtype
   ldr(dest, Address(dest));
 }
 
+#define PUSH_REG(mask, bit, Reg)      \
+  if (mask & ((unsigned)1 << bit)) {  \
+    push(Reg);                        \
+  }
+
+#define POP_REG(mask, bit, Reg, condition)   \
+  if (mask & ((unsigned)1 << bit)) {         \
+    pop(Reg, condition);                     \
+  }
+
+#define PUSH_REGS(mask, R1, R2, R3) \
+  PUSH_REG(mask, 0, R1)             \
+  PUSH_REG(mask, 1, R2)             \
+  PUSH_REG(mask, 2, R3)
+
+#define POP_REGS(mask, R1, R2, R3, condition)   \
+  POP_REG(mask, 0, R1, condition)               \
+  POP_REG(mask, 1, R2, condition)               \
+  POP_REG(mask, 2, R3, condition)
+
+#define POISON_REG(mask, bit, Reg, poison)      \
+  if (mask & ((unsigned)1 << bit)) {            \
+    mov(Reg, poison);                           \
+  }
+
+#define POISON_REGS(mask, R1, R2, R3, poison)   \
+  POISON_REG(mask, 0, R1, poison)               \
+  POISON_REG(mask, 1, R2, poison)               \
+  POISON_REG(mask, 2, R3, poison)
+
+// Attempt to fast-lock an object
+// Registers:
+//  - obj: the object to be locked
+//  - t1, t2, t3: temp registers. If corresponding bit in savemask is set, they get saved, otherwise blown.
+// Result:
+//  - Success: fallthrough
+//  - Error:   break to slow, Z cleared.
+void MacroAssembler::fast_lock_2(Register obj, Register t1, Register t2, Register t3, unsigned savemask, Label& slow) {
+  assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
+  assert_different_registers(obj, t1, t2, t3);
+
+#ifdef ASSERT
+  // Poison scratch regs
+  POISON_REGS((~savemask), t1, t2, t3, 0x10000001);
+#endif
+
+  PUSH_REGS(savemask, t1, t2, t3);
+
+  // Check if we would have space on lock-stack for the object.
+  ldr(t1, Address(Rthread, JavaThread::lock_stack_top_offset()));
+  // cmp(t1, (unsigned)LockStack::end_offset()); //  too complicated constant: 1132 (46c)
+  movw(t2, LockStack::end_offset() - 1);
+  cmp(t1, t2);
+  POP_REGS(savemask, t1, t2, t3, gt);
+  b(slow, gt); // Z is cleared
+
+  // Prepare old, new header
+  Register old_hdr = t1;
+  Register new_hdr = t2;
+  ldr(new_hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
+  bic(new_hdr, new_hdr, markWord::lock_mask_in_place);  // new header (00)
+  orr(old_hdr, new_hdr, markWord::unlocked_value);      // old header (01)
+
+  Label dummy;
+
+  cas_for_lock_acquire(old_hdr /* old */, new_hdr /* new */,
+      obj /* location */, t3 /* scratch */, dummy,
+      true /* allow_fallthrough_on_failure */, true /* one_shot */);
+
+  POP_REGS(savemask, t1, t2, t3, ne); // Cas failed -> slow
+  b(slow, ne);                        // Cas failed -> slow
+
+  // After successful lock, push object onto lock-stack
+  ldr(t1, Address(Rthread, JavaThread::lock_stack_top_offset()));
+  str(obj, Address(Rthread, t1));
+  add(t1, t1, oopSize);
+  str(t1, Address(Rthread, JavaThread::lock_stack_top_offset()));
+
+  POP_REGS(savemask, t1, t2, t3, al);
+
+#ifdef ASSERT
+  // Poison scratch regs
+  POISON_REGS((~savemask), t1, t2, t3, 0x20000002);
+#endif
+
+  // Success: fall through
+}
+
+// Attempt to fast-unlock an object
+// Registers:
+//  - obj: the object to be unlocked
+//  - t1, t2, t3: temp registers. If corresponding bit in savemask is set, they get saved, otherwise blown.
+// Result:
+//  - Success: fallthrough
+//  - Error:   break to slow, Z cleared.
+void MacroAssembler::fast_unlock_2(Register obj, Register t1, Register t2, Register t3, unsigned savemask, Label& slow) {
+  assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
+  assert_different_registers(obj, t1, t2, t3);
+
+#ifdef ASSERT
+  // Poison scratch regs
+  POISON_REGS((~savemask), t1, t2, t3, 0x30000003);
+#endif
+
+  PUSH_REGS(savemask, t1, t2, t3);
+
+  // Prepare old, new header
+  Register old_hdr = t1;
+  Register new_hdr = t2;
+  ldr(old_hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
+  bic(old_hdr, old_hdr, markWord::lock_mask_in_place);    // old header (00)
+  orr(new_hdr, old_hdr, markWord::unlocked_value);        // new header (01)
+
+  // Try to swing header from locked to unlocked
+  Label dummy;
+  cas_for_lock_release(old_hdr /* old */, new_hdr /* new */,
+      obj /* location */, t3 /* scratch */, dummy,
+      true /* allow_fallthrough_on_failure */, true /* one_shot */);
+
+  POP_REGS(savemask, t1, t2, t3, ne); // Cas failed -> slow
+  b(slow, ne);                        // Cas failed -> slow
+
+  // After successful unlock, pop object from lock-stack
+  ldr(t1, Address(Rthread, JavaThread::lock_stack_top_offset()));
+  sub(t1, t1, oopSize);
+  str(t1, Address(Rthread, JavaThread::lock_stack_top_offset()));
+
+#ifdef ASSERT
+  // zero out popped slot
+  mov(t2, 0);
+  str(t2, Address(Rthread, t1));
+#endif
+
+  POP_REGS(savemask, t1, t2, t3, al);
+
+#ifdef ASSERT
+  // Poison scratch regs
+  POISON_REGS((~savemask), t1, t2, t3, 0x40000004);
+#endif
+
+  // Fallthrough: success
+}

src/hotspot/cpu/arm/macroAssembler_arm.hpp

@@ -1009,6 +1009,24 @@ public:
   void cas_for_lock_acquire(Register oldval, Register newval, Register base, Register tmp, Label &slow_case, bool allow_fallthrough_on_failure = false, bool one_shot = false);
   void cas_for_lock_release(Register oldval, Register newval, Register base, Register tmp, Label &slow_case, bool allow_fallthrough_on_failure = false, bool one_shot = false);
 
+  // Attempt to fast-lock an object
+  // Registers:
+  //  - obj: the object to be locked
+  //  - t1, t2, t3: temp registers. If corresponding bit in savemask is set, they get saved, otherwise blown.
+  // Result:
+  //  - Success: fallthrough
+  //  - Error:   break to slow, Z cleared.
+  void fast_lock_2(Register obj, Register t1, Register t2, Register t3, unsigned savemask, Label& slow);
+
+  // Attempt to fast-unlock an object
+  // Registers:
+  //  - obj: the object to be unlocked
+  //  - t1, t2, t3: temp registers. If corresponding bit in savemask is set, they get saved, otherwise blown.
+  // Result:
+  //  - Success: fallthrough
+  //  - Error:   break to slow, Z cleared.
+  void fast_unlock_2(Register obj, Register t1, Register t2, Register t3, unsigned savemask, Label& slow);
+
 #ifndef PRODUCT
   // Preserves flags and all registers.
   // On SMP the updated value might not be visible to external observers without a synchronization barrier

src/hotspot/cpu/arm/sharedRuntime_arm.cpp

@@ -1153,6 +1153,12 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Remember the handle for the unlocking code
     __ mov(sync_handle, R1);
 
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      log_trace(fastlock)("SharedRuntime lock fast");
+      __ fast_lock_2(sync_obj /* object */, disp_hdr /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
+                     0x7 /* savemask */, slow_lock);
+      // Fall through to lock_done
+    } else if (LockingMode == LM_LEGACY) {
       const Register mark = tmp;
       // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
       // That would be acceptable as either CAS or slow case path is taken in that case
@@ -1181,7 +1187,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ str(mark, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
 
       __ cas_for_lock_acquire(mark, disp_hdr, sync_obj, Rtemp, slow_lock);
-
+    }
     __ bind(lock_done);
   }
 
@@ -1234,6 +1240,13 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
   Label slow_unlock, unlock_done;
   if (method->is_synchronized()) {
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      log_trace(fastlock)("SharedRuntime unlock fast");
+      __ fast_unlock_2(sync_obj, R2 /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
+                       7 /* savemask */, slow_unlock);
+      // Fall through
+    } else if (LockingMode == LM_LEGACY) {
+      // See C1_MacroAssembler::unlock_object() for more comments
       __ ldr(sync_obj, Address(sync_handle));
 
       // See C1_MacroAssembler::unlock_object() for more comments
@@ -1241,7 +1254,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ cbz(R2, unlock_done);
 
       __ cas_for_lock_release(disp_hdr, R2, sync_obj, Rtemp, slow_unlock);
-
+    }
     __ bind(unlock_done);
   }
 

src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp

@@ -360,7 +360,7 @@ int LIR_Assembler::emit_unwind_handler() {
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::r10_opr);
     stub = new MonitorExitStub(FrameMap::r10_opr, true, 0);
-    if (UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
       __ j(*stub->entry());
     } else {
       __ unlock_object(x15, x14, x10, *stub->entry());
@@ -1499,7 +1499,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register obj = op->obj_opr()->as_register();  // may not be an oop
   Register hdr = op->hdr_opr()->as_register();
   Register lock = op->lock_opr()->as_register();
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     if (op->info() != nullptr) {
       add_debug_info_for_null_check_here(op->info());
       __ null_check(obj);

src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp

@@ -52,8 +52,7 @@ void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   const int aligned_mask = BytesPerWord - 1;
   const int hdr_offset = oopDesc::mark_offset_in_bytes();
-  assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
+  assert_different_registers(hdr, obj, disp_hdr);
-  Label done;
   int null_check_offset = -1;
 
   verify_oop(obj);
@@ -72,6 +71,11 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
 
   // Load object header
   ld(hdr, Address(obj, hdr_offset));
+
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    fast_lock(obj, hdr, t0, t1, slow_case);
+  } else if (LockingMode == LM_LEGACY) {
+    Label done;
     // and mark it as unlocked
     ori(hdr, hdr, markWord::unlocked_value);
     // save unlocked object header into the displaced header location on the stack
@@ -105,6 +109,8 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     bnez(hdr, slow_case, /* is_far */ true);
     // done
     bind(done);
+  }
+
   increment(Address(xthread, JavaThread::held_monitor_count_offset()));
   return null_check_offset;
 }
@@ -115,14 +121,24 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
   Label done;
 
+  if (LockingMode != LM_LIGHTWEIGHT) {
     // load displaced header
     ld(hdr, Address(disp_hdr, 0));
     // if the loaded hdr is null we had recursive locking
     // if we had recursive locking, we are done
     beqz(hdr, done);
+  }
+
   // load object
   ld(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
   verify_oop(obj);
+
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    ld(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
+    andi(t0, hdr, markWord::monitor_value);
+    bnez(t0, slow_case, /* is_far */ true);
+    fast_unlock(obj, hdr, t0, t1, slow_case);
+  } else if (LockingMode == LM_LEGACY) {
     // test if object header is pointing to the displaced header, and if so, restore
     // the displaced header in the object - if the object header is not pointing to
     // the displaced header, get the object header instead
@@ -136,6 +152,8 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
     }
     // done
     bind(done);
+  }
+
   decrement(Address(xthread, JavaThread::held_monitor_count_offset()));
 }
 

src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp

@@ -26,6 +26,7 @@
 #include "precompiled.hpp"
 #include "opto/c2_CodeStubs.hpp"
 #include "opto/c2_MacroAssembler.hpp"
+#include "runtime/objectMonitor.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 
@@ -72,4 +73,32 @@ void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
   __ emit_int32(0);  // nmethod guard value
 }
 
+int C2HandleAnonOMOwnerStub::max_size() const {
+  // Max size of stub has been determined by testing with 0 without using RISC-V compressed
+  // instruction-set extension, in which case C2CodeStubList::emit() will throw an assertion
+  // and report the actual size that is needed.
+  return 20 DEBUG_ONLY(+8);
+}
+
+void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
+  __ bind(entry());
+  Register mon = monitor();
+  Register t = tmp();
+  assert(t != noreg, "need tmp register");
+
+  // Fix owner to be the current thread.
+  __ sd(xthread, Address(mon, ObjectMonitor::owner_offset_in_bytes()));
+
+  // Pop owner object from lock-stack.
+  __ lwu(t, Address(xthread, JavaThread::lock_stack_top_offset()));
+  __ subw(t, t, oopSize);
+#ifdef ASSERT
+  __ add(t0, xthread, t);
+  __ sd(zr, Address(t0, 0));
+#endif
+  __ sw(t, Address(xthread, JavaThread::lock_stack_top_offset()));
+
+  __ j(continuation());
+}
+
 #undef __

src/hotspot/cpu/riscv/interp_masm_riscv.cpp

@@ -781,7 +781,7 @@ void InterpreterMacroAssembler::remove_activation(
 void InterpreterMacroAssembler::lock_object(Register lock_reg)
 {
   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     call_VM(noreg,
             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
             lock_reg);
@@ -809,6 +809,11 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
       bnez(tmp, slow_case);
     }
 
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      ld(tmp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      fast_lock(obj_reg, tmp, t0, t1, slow_case);
+      j(count);
+    } else if (LockingMode == LM_LEGACY) {
       // Load (object->mark() | 1) into swap_reg
       ld(t0, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
       ori(swap_reg, t0, 1);
@@ -837,14 +842,20 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
       // Save the test result, for recursive case, the result is zero
       sd(swap_reg, Address(lock_reg, mark_offset));
       beqz(swap_reg, count);
+    }
 
     bind(slow_case);
 
     // Call the runtime routine for slow case
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      call_VM(noreg,
+              CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj),
+              obj_reg);
+    } else {
       call_VM(noreg,
               CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
               lock_reg);
-
+    }
     j(done);
 
     bind(count);
@@ -870,7 +881,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
 {
   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
 
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
   } else {
     Label count, done;
@@ -881,9 +892,11 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
 
     save_bcp(); // Save in case of exception
 
+    if (LockingMode != LM_LIGHTWEIGHT) {
       // Convert from BasicObjectLock structure to object and BasicLock
       // structure Store the BasicLock address into x10
       la(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
+    }
 
     // Load oop into obj_reg(c_rarg3)
     ld(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
@@ -891,6 +904,31 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
     // Free entry
     sd(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
 
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      Label slow_case;
+
+      // Check for non-symmetric locking. This is allowed by the spec and the interpreter
+      // must handle it.
+      Register tmp1 = t0;
+      Register tmp2 = header_reg;
+      // First check for lock-stack underflow.
+      lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
+      mv(tmp2, (unsigned)LockStack::start_offset());
+      ble(tmp1, tmp2, slow_case);
+      // Then check if the top of the lock-stack matches the unlocked object.
+      subw(tmp1, tmp1, oopSize);
+      add(tmp1, xthread, tmp1);
+      ld(tmp1, Address(tmp1, 0));
+      bne(tmp1, obj_reg, slow_case);
+
+      ld(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      andi(t0, header_reg, markWord::monitor_value);
+      bnez(t0, slow_case);
+      fast_unlock(obj_reg, header_reg, swap_reg, t0, slow_case);
+      j(count);
+
+      bind(slow_case);
+    } else if (LockingMode == LM_LEGACY) {
       // Load the old header from BasicLock structure
       ld(header_reg, Address(swap_reg,
                              BasicLock::displaced_header_offset_in_bytes()));
@@ -900,6 +938,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
 
       // Atomic swap back the old header
       cmpxchg_obj_header(swap_reg, header_reg, obj_reg, t0, count, /*fallthrough*/nullptr);
+    }
 
     // Call the runtime routine for slow case.
     sd(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj

src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

@@ -59,6 +59,7 @@
 #else
 #define BLOCK_COMMENT(str) block_comment(str)
 #endif
+#define STOP(str) stop(str);
 #define BIND(label) bind(label); __ BLOCK_COMMENT(#label ":")
 
 static void pass_arg0(MacroAssembler* masm, Register arg) {
@@ -2416,7 +2417,7 @@ void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acqui
     membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
   }
   if (at_return) {
-    bgtu(in_nmethod ? sp : fp, t0, slow_path, true /* is_far */);
+    bgtu(in_nmethod ? sp : fp, t0, slow_path, /* is_far */ true);
   } else {
     test_bit(t0, t0, exact_log2(SafepointMechanism::poll_bit()));
     bnez(t0, slow_path, true /* is_far */);
@@ -4486,3 +4487,100 @@ void MacroAssembler::test_bit(Register Rd, Register Rs, uint32_t bit_pos, Regist
   }
   andi(Rd, Rs, 1UL << bit_pos, tmp);
 }
+
+// Implements fast-locking.
+// Branches to slow upon failure to lock the object.
+// Falls through upon success.
+//
+//  - obj: the object to be locked
+//  - hdr: the header, already loaded from obj, will be destroyed
+//  - tmp1, tmp2: temporary registers, will be destroyed
+void MacroAssembler::fast_lock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow) {
+  assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
+  assert_different_registers(obj, hdr, tmp1, tmp2);
+
+  // Check if we would have space on lock-stack for the object.
+  lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
+  mv(tmp2, (unsigned)LockStack::end_offset());
+  bge(tmp1, tmp2, slow, /* is_far */ true);
+
+  // Load (object->mark() | 1) into hdr
+  ori(hdr, hdr, markWord::unlocked_value);
+  // Clear lock-bits, into tmp2
+  xori(tmp2, hdr, markWord::unlocked_value);
+
+  // Try to swing header from unlocked to locked
+  Label success;
+  cmpxchgptr(hdr, tmp2, obj, tmp1, success, &slow);
+  bind(success);
+
+  // After successful lock, push object on lock-stack
+  lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
+  add(tmp2, xthread, tmp1);
+  sd(obj, Address(tmp2, 0));
+  addw(tmp1, tmp1, oopSize);
+  sw(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
+}
+
+// Implements fast-unlocking.
+// Branches to slow upon failure.
+// Falls through upon success.
+//
+// - obj: the object to be unlocked
+// - hdr: the (pre-loaded) header of the object
+// - tmp1, tmp2: temporary registers
+void MacroAssembler::fast_unlock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow) {
+  assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
+  assert_different_registers(obj, hdr, tmp1, tmp2);
+
+#ifdef ASSERT
+  {
+    // The following checks rely on the fact that LockStack is only ever modified by
+    // its owning thread, even if the lock got inflated concurrently; removal of LockStack
+    // entries after inflation will happen delayed in that case.
+
+    // Check for lock-stack underflow.
+    Label stack_ok;
+    lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
+    mv(tmp2, (unsigned)LockStack::start_offset());
+    bgt(tmp1, tmp2, stack_ok);
+    STOP("Lock-stack underflow");
+    bind(stack_ok);
+  }
+  {
+    // Check if the top of the lock-stack matches the unlocked object.
+    Label tos_ok;
+    subw(tmp1, tmp1, oopSize);
+    add(tmp1, xthread, tmp1);
+    ld(tmp1, Address(tmp1, 0));
+    beq(tmp1, obj, tos_ok);
+    STOP("Top of lock-stack does not match the unlocked object");
+    bind(tos_ok);
+  }
+  {
+    // Check that hdr is fast-locked.
+   Label hdr_ok;
+    andi(tmp1, hdr, markWord::lock_mask_in_place);
+    beqz(tmp1, hdr_ok);
+    STOP("Header is not fast-locked");
+    bind(hdr_ok);
+  }
+#endif
+
+  // Load the new header (unlocked) into tmp1
+  ori(tmp1, hdr, markWord::unlocked_value);
+
+  // Try to swing header from locked to unlocked
+  Label success;
+  cmpxchgptr(hdr, tmp1, obj, tmp2, success, &slow);
+  bind(success);
+
+  // After successful unlock, pop object from lock-stack
+  lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
+  subw(tmp1, tmp1, oopSize);
+#ifdef ASSERT
+  add(tmp2, xthread, tmp1);
+  sd(zr, Address(tmp2, 0));
+#endif
+  sw(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
+}

src/hotspot/cpu/riscv/macroAssembler_riscv.hpp

@@ -1418,6 +1418,10 @@ private:
 
   void load_reserved(Register addr, enum operand_size size, Assembler::Aqrl acquire);
   void store_conditional(Register addr, Register new_val, enum operand_size size, Assembler::Aqrl release);
+
+public:
+  void fast_lock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow);
+  void fast_unlock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow);
 };
 
 #ifdef ASSERT

src/hotspot/cpu/riscv/riscv.ad

@@ -2485,7 +2485,7 @@ encode %{
     }
   %}
 
-  // using the cr register as the bool result: 0 for success; others failed.
+  // Use cr register to indicate the fast_lock result: zero for success; non-zero for failure.
   enc_class riscv_enc_fast_lock(iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) %{
     C2_MacroAssembler _masm(&cbuf);
     Register flag = t1;
@@ -2495,7 +2495,7 @@ encode %{
     Register tmp = as_Register($tmp2$$reg);
     Label cont;
     Label object_has_monitor;
-    Label no_count;
+    Label count, no_count;
 
     assert_different_registers(oop, box, tmp, disp_hdr, t0);
 
@@ -2513,7 +2513,10 @@ encode %{
     __ test_bit(t0, disp_hdr, exact_log2(markWord::monitor_value));
     __ bnez(t0, object_has_monitor);
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path
+      __ j(cont);
+    } else if (LockingMode == LM_LEGACY) {
       // Set tmp to be (markWord of object | UNLOCK_VALUE).
       __ ori(tmp, disp_hdr, markWord::unlocked_value);
 
@@ -2544,11 +2547,19 @@ encode %{
       __ andr(tmp/*==0?*/, disp_hdr, tmp);
       __ sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
       __ mv(flag, tmp); // we can use the value of tmp as the result here
-    } else {
-      __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path
-    }
-
       __ j(cont);
+    } else {
+      assert(LockingMode == LM_LIGHTWEIGHT, "");
+      Label slow;
+      __ fast_lock(oop, disp_hdr, tmp, t0, slow);
+
+      // Indicate success on completion.
+      __ mv(flag, zr);
+      __ j(count);
+      __ bind(slow);
+      __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path
+      __ j(no_count);
+    }
 
     // Handle existing monitor.
     __ bind(object_has_monitor);
@@ -2560,12 +2571,14 @@ encode %{
     __ cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64, Assembler::aq,
              Assembler::rl, /*result*/flag); // cas succeeds if flag == zr(expected)
 
+    if (LockingMode != LM_LIGHTWEIGHT) {
       // Store a non-null value into the box to avoid looking like a re-entrant
       // lock. The fast-path monitor unlock code checks for
       // markWord::monitor_value so use markWord::unused_mark which has the
       // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
       __ mv(tmp, (address)markWord::unused_mark().value());
       __ sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
+    }
 
     __ beqz(flag, cont); // CAS success means locking succeeded
 
@@ -2576,15 +2589,17 @@ encode %{
     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1, t0, tmp);
 
     __ bind(cont);
-
+    // zero flag indicates success
+    // non-zero flag indicates failure
     __ bnez(flag, no_count);
 
+    __ bind(count);
     __ increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp);
 
     __ bind(no_count);
   %}
 
-  // using cr flag to indicate the fast_unlock result: 0 for success; others failed.
+  // Use cr register to indicate the fast_unlock result: zero for success; non-zero for failure.
   enc_class riscv_enc_fast_unlock(iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) %{
     C2_MacroAssembler _masm(&cbuf);
     Register flag = t1;
@@ -2594,11 +2609,11 @@ encode %{
     Register tmp = as_Register($tmp2$$reg);
     Label cont;
     Label object_has_monitor;
-    Label no_count;
+    Label count, no_count;
 
     assert_different_registers(oop, box, tmp, disp_hdr, flag);
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_LEGACY) {
       // Find the lock address and load the displaced header from the stack.
       __ ld(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 
@@ -2612,7 +2627,10 @@ encode %{
     __ test_bit(t0, tmp, exact_log2(markWord::monitor_value));
     __ bnez(t0, object_has_monitor);
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path
+      __ j(cont);
+    } else if (LockingMode == LM_LEGACY) {
       // Check if it is still a light weight lock, this is true if we
       // see the stack address of the basicLock in the markWord of the
       // object.
@@ -2620,10 +2638,19 @@ encode %{
       __ cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64, Assembler::relaxed,
                  Assembler::rl, /*result*/tmp);
       __ xorr(flag, box, tmp); // box == tmp if cas succeeds
-    } else {
-      __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path
-    }
       __ j(cont);
+    } else {
+      assert(LockingMode == LM_LIGHTWEIGHT, "");
+      Label slow;
+      __ fast_unlock(oop, tmp, box, disp_hdr, slow);
+
+      // Indicate success on completion.
+      __ mv(flag, zr);
+      __ j(count);
+      __ bind(slow);
+      __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path
+      __ j(no_count);
+    }
 
     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 
@@ -2631,6 +2658,18 @@ encode %{
     __ bind(object_has_monitor);
     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
+
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      // If the owner is anonymous, we need to fix it -- in an outline stub.
+      Register tmp2 = disp_hdr;
+      __ ld(tmp2, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
+      __ andi(t0, tmp2, (int64_t)ObjectMonitor::ANONYMOUS_OWNER);
+      C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
+      Compile::current()->output()->add_stub(stub);
+      __ bnez(t0, stub->entry(), /* is_far */ true);
+      __ bind(stub->continuation());
+    }
+
     __ ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 
     Label notRecursive;
@@ -2653,9 +2692,11 @@ encode %{
     __ sd(zr, Address(tmp)); // set unowned
 
     __ bind(cont);
-
+    // zero flag indicates success
+    // non-zero flag indicates failure
     __ bnez(flag, no_count);
 
+    __ bind(count);
     __ decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp);
 
     __ bind(no_count);

src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp

@@ -1671,7 +1671,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Load the oop from the handle
     __ ld(obj_reg, Address(oop_handle_reg, 0));
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ j(slow_path_lock);
+    } else if (LockingMode == LM_LEGACY) {
       // Load (object->mark() | 1) into swap_reg % x10
       __ ld(t0, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
       __ ori(swap_reg, t0, 1);
@@ -1698,7 +1700,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ sd(swap_reg, Address(lock_reg, mark_word_offset));
       __ bnez(swap_reg, slow_path_lock);
     } else {
-      __ j(slow_path_lock);
+      assert(LockingMode == LM_LIGHTWEIGHT, "");
+      __ ld(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      __ fast_lock(obj_reg, swap_reg, tmp, t0, slow_path_lock);
     }
 
     __ bind(count);
@@ -1793,7 +1797,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
     Label done, not_recursive;
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_LEGACY) {
       // Simple recursive lock?
       __ ld(t0, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
       __ bnez(t0, not_recursive);
@@ -1808,7 +1812,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       save_native_result(masm, ret_type, stack_slots);
     }
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ j(slow_path_unlock);
+    } else if (LockingMode == LM_LEGACY) {
       // get address of the stack lock
       __ la(x10, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
       //  get old displaced header
@@ -1820,7 +1826,12 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ bind(count);
       __ decrement(Address(xthread, JavaThread::held_monitor_count_offset()));
     } else {
-      __ j(slow_path_unlock);
+      assert(LockingMode == LM_LIGHTWEIGHT, "");
+      __ ld(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      __ andi(t0, old_hdr, markWord::monitor_value);
+      __ bnez(t0, slow_path_unlock);
+      __ fast_unlock(obj_reg, old_hdr, swap_reg, t0, slow_path_unlock);
+      __ decrement(Address(xthread, JavaThread::held_monitor_count_offset()));
     }
 
     // slow path re-enters here

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -454,7 +454,7 @@ int LIR_Assembler::emit_unwind_handler() {
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::rax_opr);
     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
-    if (UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
       __ jmp(*stub->entry());
     } else {
       __ unlock_object(rdi, rsi, rax, *stub->entry());
@@ -3500,7 +3500,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register obj = op->obj_opr()->as_register();  // may not be an oop
   Register hdr = op->hdr_opr()->as_register();
   Register lock = op->lock_opr()->as_register();
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     if (op->info() != nullptr) {
       add_debug_info_for_null_check_here(op->info());
       __ null_check(obj);
@@ -3508,8 +3508,9 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
     __ jmp(*op->stub()->entry());
   } else if (op->code() == lir_lock) {
     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
+    Register tmp = LockingMode == LM_LIGHTWEIGHT ? op->scratch_opr()->as_register() : noreg;
     // add debug info for NullPointerException only if one is possible
-    int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
+    int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
     if (op->info() != nullptr) {
       add_debug_info_for_null_check(null_check_offset, op->info());
     }

src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp

@@ -319,7 +319,8 @@ void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {
   // this CodeEmitInfo must not have the xhandlers because here the
   // object is already locked (xhandlers expect object to be unlocked)
   CodeEmitInfo* info = state_for(x, x->state(), true);
-  monitor_enter(obj.result(), lock, syncTempOpr(), LIR_OprFact::illegalOpr,
+  LIR_Opr tmp = LockingMode == LM_LIGHTWEIGHT ? new_register(T_ADDRESS) : LIR_OprFact::illegalOpr;
+  monitor_enter(obj.result(), lock, syncTempOpr(), tmp,
                         x->monitor_no(), info_for_exception, info);
 }
 

src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp

@@ -38,12 +38,11 @@
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
   const int aligned_mask = BytesPerWord -1;
   const int hdr_offset = oopDesc::mark_offset_in_bytes();
   assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
-  assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
+  assert_different_registers(hdr, obj, disp_hdr, tmp);
-  Label done;
   int null_check_offset = -1;
 
   verify_oop(obj);
@@ -62,6 +61,17 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
 
   // Load object header
   movptr(hdr, Address(obj, hdr_offset));
+
+  if (LockingMode == LM_LIGHTWEIGHT) {
+#ifdef _LP64
+    const Register thread = r15_thread;
+#else
+    const Register thread = disp_hdr;
+    get_thread(thread);
+#endif
+    fast_lock_impl(obj, hdr, thread, tmp, slow_case);
+  } else  if (LockingMode == LM_LEGACY) {
+    Label done;
     // and mark it as unlocked
     orptr(hdr, markWord::unlocked_value);
     // save unlocked object header into the displaced header location on the stack
@@ -95,6 +105,7 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     jcc(Assembler::notZero, slow_case);
     // done
     bind(done);
+  }
 
   inc_held_monitor_count();
 
@@ -108,16 +119,24 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
   Label done;
 
+  if (LockingMode != LM_LIGHTWEIGHT) {
     // load displaced header
     movptr(hdr, Address(disp_hdr, 0));
     // if the loaded hdr is null we had recursive locking
     testptr(hdr, hdr);
     // if we had recursive locking, we are done
     jcc(Assembler::zero, done);
+  }
+
   // load object
   movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
-
   verify_oop(obj);
+
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    movptr(disp_hdr, Address(obj, hdr_offset));
+    andptr(disp_hdr, ~(int32_t)markWord::lock_mask_in_place);
+    fast_unlock_impl(obj, disp_hdr, hdr, slow_case);
+  } else if (LockingMode == LM_LEGACY) {
     // test if object header is pointing to the displaced header, and if so, restore
     // the displaced header in the object - if the object header is not pointing to
     // the displaced header, get the object header instead
@@ -127,8 +146,8 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
     // we do unlocking via runtime call
     jcc(Assembler::notEqual, slow_case);
     // done
+  }
   bind(done);
-
   dec_held_monitor_count();
 }
 

src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp

@@ -50,7 +50,7 @@
   // obj     : must point to the object to lock, contents preserved
   // disp_hdr: must point to the displaced header location, contents preserved
   // returns code offset at which to add null check debug information
-  int lock_object  (Register swap, Register obj, Register disp_hdr, Label& slow_case);
+  int lock_object  (Register swap, Register obj, Register disp_hdr, Register tmp, Label& slow_case);
 
   // unlocking
   // hdr     : contents destroyed

src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp

@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "opto/c2_MacroAssembler.hpp"
 #include "opto/c2_CodeStubs.hpp"
+#include "runtime/objectMonitor.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 
@@ -72,4 +73,26 @@ void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
   __ jmp(continuation(), false /* maybe_short */);
 }
 
+#ifdef _LP64
+int C2HandleAnonOMOwnerStub::max_size() const {
+  // Max size of stub has been determined by testing with 0, in which case
+  // C2CodeStubList::emit() will throw an assertion and report the actual size that
+  // is needed.
+  return DEBUG_ONLY(36) NOT_DEBUG(21);
+}
+
+void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
+  __ bind(entry());
+  Register mon = monitor();
+  Register t = tmp();
+  __ movptr(Address(mon, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), r15_thread);
+  __ subl(Address(r15_thread, JavaThread::lock_stack_top_offset()), oopSize);
+#ifdef ASSERT
+  __ movl(t, Address(r15_thread, JavaThread::lock_stack_top_offset()));
+  __ movptr(Address(r15_thread, t), 0);
+#endif
+  __ jmp(continuation());
+}
+#endif
+
 #undef __

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@@ -548,7 +548,7 @@ void C2_MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, R
 // rax,: tmp -- KILLED
 // scr: tmp -- KILLED
 void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
-                                 Register scrReg, Register cx1Reg, Register cx2Reg,
+                                 Register scrReg, Register cx1Reg, Register cx2Reg, Register thread,
                                  RTMLockingCounters* rtm_counters,
                                  RTMLockingCounters* stack_rtm_counters,
                                  Metadata* method_data,
@@ -590,7 +590,7 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
 
 #if INCLUDE_RTM_OPT
   if (UseRTMForStackLocks && use_rtm) {
-    assert(!UseHeavyMonitors, "+UseHeavyMonitors and +UseRTMForStackLocks are mutually exclusive");
+    assert(LockingMode != LM_MONITOR, "LockingMode == 0 (LM_MONITOR) and +UseRTMForStackLocks are mutually exclusive");
     rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
                       stack_rtm_counters, method_data, profile_rtm,
                       DONE_LABEL, IsInflated);
@@ -599,9 +599,12 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
 
   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
   testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral
-  jccb(Assembler::notZero, IsInflated);
+  jcc(Assembler::notZero, IsInflated);
 
-  if (!UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
+    // Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
+    testptr(objReg, objReg);
+  } else if (LockingMode == LM_LEGACY) {
     // Attempt stack-locking ...
     orptr (tmpReg, markWord::unlocked_value);
     movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
@@ -617,8 +620,9 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
     andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - (int)os::vm_page_size())) );
     movptr(Address(boxReg, 0), tmpReg);
   } else {
-    // Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
+    assert(LockingMode == LM_LIGHTWEIGHT, "");
-    testptr(objReg, objReg);
+    fast_lock_impl(objReg, tmpReg, thread, scrReg, NO_COUNT);
+    jmp(COUNT);
   }
   jmp(DONE_LABEL);
 
@@ -681,14 +685,14 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
   movq(scrReg, tmpReg);
   xorq(tmpReg, tmpReg);
   lock();
-  cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+  cmpxchgptr(thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
   // Unconditionally set box->_displaced_header = markWord::unused_mark().
   // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
   movptr(Address(boxReg, 0), checked_cast<int32_t>(markWord::unused_mark().value()));
   // Propagate ICC.ZF from CAS above into DONE_LABEL.
   jccb(Assembler::equal, COUNT);          // CAS above succeeded; propagate ZF = 1 (success)
 
-  cmpptr(r15_thread, rax);                // Check if we are already the owner (recursive lock)
+  cmpptr(thread, rax);                // Check if we are already the owner (recursive lock)
   jccb(Assembler::notEqual, NO_COUNT);    // If not recursive, ZF = 0 at this point (fail)
   incq(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
   xorq(rax, rax); // Set ZF = 1 (success) for recursive lock, denoting locking success
@@ -704,12 +708,7 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
 
   bind(COUNT);
   // Count monitors in fast path
-#ifndef _LP64
+  increment(Address(thread, JavaThread::held_monitor_count_offset()));
-  get_thread(tmpReg);
-  incrementl(Address(tmpReg, JavaThread::held_monitor_count_offset()));
-#else // _LP64
-  incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
-#endif
 
   xorl(tmpReg, tmpReg); // Set ZF == 1
 
@@ -761,7 +760,7 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
 
 #if INCLUDE_RTM_OPT
   if (UseRTMForStackLocks && use_rtm) {
-    assert(!UseHeavyMonitors, "+UseHeavyMonitors and +UseRTMForStackLocks are mutually exclusive");
+    assert(LockingMode != LM_MONITOR, "LockingMode == 0 (LM_MONITOR) and +UseRTMForStackLocks are mutually exclusive");
     Label L_regular_unlock;
     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // fetch markword
     andptr(tmpReg, markWord::lock_mask_in_place);                     // look at 2 lock bits
@@ -773,17 +772,35 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
   }
 #endif
 
-  if (!UseHeavyMonitors) {
+  if (LockingMode == LM_LEGACY) {
     cmpptr(Address(boxReg, 0), NULL_WORD);                            // Examine the displaced header
     jcc   (Assembler::zero, COUNT);                                   // 0 indicates recursive stack-lock
   }
   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));   // Examine the object's markword
-  if (!UseHeavyMonitors) {
+  if (LockingMode != LM_MONITOR) {
     testptr(tmpReg, markWord::monitor_value);                         // Inflated?
-    jccb   (Assembler::zero, Stacked);
+    jcc(Assembler::zero, Stacked);
   }
 
   // It's inflated.
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    // If the owner is ANONYMOUS, we need to fix it -  in an outline stub.
+    testb(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t) ObjectMonitor::ANONYMOUS_OWNER);
+#ifdef _LP64
+    if (!Compile::current()->output()->in_scratch_emit_size()) {
+      C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmpReg, boxReg);
+      Compile::current()->output()->add_stub(stub);
+      jcc(Assembler::notEqual, stub->entry());
+      bind(stub->continuation());
+    } else
+#endif
+    {
+      // We can't easily implement this optimization on 32 bit because we don't have a thread register.
+      // Call the slow-path instead.
+      jcc(Assembler::notEqual, NO_COUNT);
+    }
+  }
+
 #if INCLUDE_RTM_OPT
   if (use_rtm) {
     Label L_regular_inflated_unlock;
@@ -792,7 +809,7 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
     testptr(boxReg, boxReg);
     jccb(Assembler::notZero, L_regular_inflated_unlock);
     xend();
-    jmpb(DONE_LABEL);
+    jmp(DONE_LABEL);
     bind(L_regular_inflated_unlock);
   }
 #endif
@@ -904,11 +921,17 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
   jmpb  (DONE_LABEL);
 
 #endif
-  if (!UseHeavyMonitors) {
+  if (LockingMode != LM_MONITOR) {
     bind  (Stacked);
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      mov(boxReg, tmpReg);
+      fast_unlock_impl(objReg, boxReg, tmpReg, NO_COUNT);
+      jmp(COUNT);
+    } else if (LockingMode == LM_LEGACY) {
       movptr(tmpReg, Address (boxReg, 0));      // re-fetch
       lock();
       cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
+    }
     // Intentional fall-thru into DONE_LABEL
   }
   bind(DONE_LABEL);

src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp

@@ -36,7 +36,7 @@ public:
   // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
   // See full description in macroAssembler_x86.cpp.
   void fast_lock(Register obj, Register box, Register tmp,
-                 Register scr, Register cx1, Register cx2,
+                 Register scr, Register cx1, Register cx2, Register thread,
                  RTMLockingCounters* rtm_counters,
                  RTMLockingCounters* stack_rtm_counters,
                  Metadata* method_data,

src/hotspot/cpu/x86/interp_masm_x86.cpp

@@ -1196,7 +1196,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
   assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
          "The argument is only for looks. It must be c_rarg1");
 
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     call_VM(noreg,
             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
             lock_reg);
@@ -1223,6 +1223,17 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
       jcc(Assembler::notZero, slow_case);
     }
 
+    if (LockingMode == LM_LIGHTWEIGHT) {
+#ifdef _LP64
+      const Register thread = r15_thread;
+#else
+      const Register thread = lock_reg;
+      get_thread(thread);
+#endif
+      // Load object header, prepare for CAS from unlocked to locked.
+      movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      fast_lock_impl(obj_reg, swap_reg, thread, tmp_reg, slow_case);
+    } else if (LockingMode == LM_LEGACY) {
       // Load immediate 1 into swap_reg %rax
       movl(swap_reg, 1);
 
@@ -1276,16 +1287,22 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
       jcc(Assembler::notZero, slow_case);
 
       bind(count_locking);
+    }
     inc_held_monitor_count();
     jmp(done);
 
     bind(slow_case);
 
     // Call the runtime routine for slow case
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      call_VM(noreg,
+              CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj),
+              obj_reg);
+    } else {
       call_VM(noreg,
               CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
               lock_reg);
-
+    }
     bind(done);
   }
 }
@@ -1307,7 +1324,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
   assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
          "The argument is only for looks. It must be c_rarg1");
 
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
   } else {
     Label count_locking, done, slow_case;
@@ -1318,9 +1335,11 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
 
     save_bcp(); // Save in case of exception
 
+    if (LockingMode != LM_LIGHTWEIGHT) {
       // Convert from BasicObjectLock structure to object and BasicLock
       // structure Store the BasicLock address into %rax
       lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
+    }
 
     // Load oop into obj_reg(%c_rarg3)
     movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
@@ -1328,6 +1347,23 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
     // Free entry
     movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), NULL_WORD);
 
+    if (LockingMode == LM_LIGHTWEIGHT) {
+#ifdef _LP64
+      const Register thread = r15_thread;
+#else
+      const Register thread = header_reg;
+      get_thread(thread);
+#endif
+      // Handle unstructured locking.
+      Register tmp = swap_reg;
+      movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
+      cmpptr(obj_reg, Address(thread, tmp, Address::times_1,  -oopSize));
+      jcc(Assembler::notEqual, slow_case);
+      // Try to swing header from locked to unlocked.
+      movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
+      fast_unlock_impl(obj_reg, swap_reg, header_reg, slow_case);
+    } else if (LockingMode == LM_LEGACY) {
       // Load the old header from BasicLock structure
       movptr(header_reg, Address(swap_reg,
                                  BasicLock::displaced_header_offset_in_bytes()));
@@ -1346,6 +1382,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
       jcc(Assembler::notZero, slow_case);
 
       bind(count_locking);
+    }
     dec_held_monitor_count();
     jmp(done);
 

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -9670,3 +9670,70 @@ void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigne
   stop(msg);
   bind(L_stack_ok);
 }
+
+// Implements fast-locking.
+// Branches to slow upon failure to lock the object, with ZF cleared.
+// Falls through upon success with unspecified ZF.
+//
+// obj: the object to be locked
+// hdr: the (pre-loaded) header of the object, must be rax
+// thread: the thread which attempts to lock obj
+// tmp: a temporary register
+void MacroAssembler::fast_lock_impl(Register obj, Register hdr, Register thread, Register tmp, Label& slow) {
+  assert(hdr == rax, "header must be in rax for cmpxchg");
+  assert_different_registers(obj, hdr, thread, tmp);
+
+  // First we need to check if the lock-stack has room for pushing the object reference.
+  // Note: we subtract 1 from the end-offset so that we can do a 'greater' comparison, instead
+  // of 'greaterEqual' below, which readily clears the ZF. This makes C2 code a little simpler and
+  // avoids one branch.
+  cmpl(Address(thread, JavaThread::lock_stack_top_offset()), LockStack::end_offset() - 1);
+  jcc(Assembler::greater, slow);
+
+  // Now we attempt to take the fast-lock.
+  // Clear lock_mask bits (locked state).
+  andptr(hdr, ~(int32_t)markWord::lock_mask_in_place);
+  movptr(tmp, hdr);
+  // Set unlocked_value bit.
+  orptr(hdr, markWord::unlocked_value);
+  lock();
+  cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
+  jcc(Assembler::notEqual, slow);
+
+  // If successful, push object to lock-stack.
+  movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
+  movptr(Address(thread, tmp), obj);
+  incrementl(tmp, oopSize);
+  movl(Address(thread, JavaThread::lock_stack_top_offset()), tmp);
+}
+
+// Implements fast-unlocking.
+// Branches to slow upon failure, with ZF cleared.
+// Falls through upon success, with unspecified ZF.
+//
+// obj: the object to be unlocked
+// hdr: the (pre-loaded) header of the object, must be rax
+// tmp: a temporary register
+void MacroAssembler::fast_unlock_impl(Register obj, Register hdr, Register tmp, Label& slow) {
+  assert(hdr == rax, "header must be in rax for cmpxchg");
+  assert_different_registers(obj, hdr, tmp);
+
+  // Mark-word must be lock_mask now, try to swing it back to unlocked_value.
+  movptr(tmp, hdr); // The expected old value
+  orptr(tmp, markWord::unlocked_value);
+  lock();
+  cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
+  jcc(Assembler::notEqual, slow);
+  // Pop the lock object from the lock-stack.
+#ifdef _LP64
+  const Register thread = r15_thread;
+#else
+  const Register thread = rax;
+  get_thread(thread);
+#endif
+  subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+#ifdef ASSERT
+  movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
+  movptr(Address(thread, tmp), 0);
+#endif
+}

src/hotspot/cpu/x86/macroAssembler_x86.hpp

@@ -149,6 +149,8 @@ class MacroAssembler: public Assembler {
 
   void increment(Register reg, int value = 1) { LP64_ONLY(incrementq(reg, value)) NOT_LP64(incrementl(reg, value)) ; }
   void decrement(Register reg, int value = 1) { LP64_ONLY(decrementq(reg, value)) NOT_LP64(decrementl(reg, value)) ; }
+  void increment(Address dst, int value = 1)  { LP64_ONLY(incrementq(dst, value)) NOT_LP64(incrementl(dst, value)) ; }
+  void decrement(Address dst, int value = 1)  { LP64_ONLY(decrementq(dst, value)) NOT_LP64(decrementl(dst, value)) ; }
 
   void decrementl(Address dst, int value = 1);
   void decrementl(Register reg, int value = 1);
@@ -2007,6 +2009,8 @@ public:
 
   void check_stack_alignment(Register sp, const char* msg, unsigned bias = 0, Register tmp = noreg);
 
+  void fast_lock_impl(Register obj, Register hdr, Register thread, Register tmp, Label& slow);
+  void fast_unlock_impl(Register obj, Register hdr, Register tmp, Label& slow);
 };
 
 /**

src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp

@@ -1680,7 +1680,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Load the oop from the handle
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ jmp(slow_path_lock);
+    } else if (LockingMode == LM_LEGACY) {
       // Load immediate 1 into swap_reg %rax,
       __ movptr(swap_reg, 1);
 
@@ -1712,7 +1714,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
       __ jcc(Assembler::notEqual, slow_path_lock);
     } else {
-      __ jmp(slow_path_lock);
+      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+      // Load object header
+      __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      __ fast_lock_impl(obj_reg, swap_reg, thread, lock_reg, slow_path_lock);
     }
     __ bind(count_mon);
     __ inc_held_monitor_count();
@@ -1836,7 +1841,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Get locked oop from the handle we passed to jni
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_LEGACY) {
       Label not_recur;
       // Simple recursive lock?
       __ cmpptr(Address(rbp, lock_slot_rbp_offset), NULL_WORD);
@@ -1851,7 +1856,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       save_native_result(masm, ret_type, stack_slots);
     }
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ jmp(slow_path_unlock);
+    } else if (LockingMode == LM_LEGACY) {
       //  get old displaced header
       __ movptr(rbx, Address(rbp, lock_slot_rbp_offset));
 
@@ -1866,7 +1873,11 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ jcc(Assembler::notEqual, slow_path_unlock);
       __ dec_held_monitor_count();
     } else {
-      __ jmp(slow_path_unlock);
+      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+      __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      __ andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
+      __ fast_unlock_impl(obj_reg, swap_reg, lock_reg, slow_path_unlock);
+      __ dec_held_monitor_count();
     }
 
     // slow path re-enters here

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

@@ -2149,8 +2149,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Load the oop from the handle
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
-
+      __ jmp(slow_path_lock);
+    } else if (LockingMode == LM_LEGACY) {
       // Load immediate 1 into swap_reg %rax
       __ movl(swap_reg, 1);
 
@@ -2183,7 +2184,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
       __ jcc(Assembler::notEqual, slow_path_lock);
     } else {
-      __ jmp(slow_path_lock);
+      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+      // Load object header
+      __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      __ fast_lock_impl(obj_reg, swap_reg, r15_thread, rscratch1, slow_path_lock);
     }
     __ bind(count_mon);
     __ inc_held_monitor_count();
@@ -2295,7 +2299,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Get locked oop from the handle we passed to jni
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_LEGACY) {
       Label not_recur;
       // Simple recursive lock?
       __ cmpptr(Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size), NULL_WORD);
@@ -2310,7 +2314,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       save_native_result(masm, ret_type, stack_slots);
     }
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ jmp(slow_path_unlock);
+    } else if (LockingMode == LM_LEGACY) {
       // get address of the stack lock
       __ lea(rax, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
       //  get old displaced header
@@ -2322,7 +2328,11 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ jcc(Assembler::notEqual, slow_path_unlock);
       __ dec_held_monitor_count();
     } else {
-      __ jmp(slow_path_unlock);
+      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+      __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+      __ andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
+      __ fast_unlock_impl(obj_reg, swap_reg, lock_reg, slow_path_unlock);
+      __ dec_held_monitor_count();
     }
 
     // slow path re-enters here

src/hotspot/cpu/x86/x86_32.ad

@@ -13760,15 +13760,16 @@ instruct RethrowException()
 
 // inlined locking and unlocking
 
-instruct cmpFastLockRTM(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eDXRegI scr, rRegI cx1, rRegI cx2) %{
+instruct cmpFastLockRTM(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eDXRegI scr, rRegI cx1, rRegI cx2, eRegP thread) %{
   predicate(Compile::current()->use_rtm());
   match(Set cr (FastLock object box));
-  effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
+  effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box, TEMP thread);
   ins_cost(300);
   format %{ "FASTLOCK $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
   ins_encode %{
+    __ get_thread($thread$$Register);
     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
-                 $scr$$Register, $cx1$$Register, $cx2$$Register,
+                 $scr$$Register, $cx1$$Register, $cx2$$Register, $thread$$Register,
                  _rtm_counters, _stack_rtm_counters,
                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
                  true, ra_->C->profile_rtm());
@@ -13776,15 +13777,16 @@ instruct cmpFastLockRTM(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eD
   ins_pipe(pipe_slow);
 %}
 
-instruct cmpFastLock(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP scr) %{
+instruct cmpFastLock(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP scr, eRegP thread) %{
   predicate(!Compile::current()->use_rtm());
   match(Set cr (FastLock object box));
-  effect(TEMP tmp, TEMP scr, USE_KILL box);
+  effect(TEMP tmp, TEMP scr, USE_KILL box, TEMP thread);
   ins_cost(300);
   format %{ "FASTLOCK $object,$box\t! kills $box,$tmp,$scr" %}
   ins_encode %{
+    __ get_thread($thread$$Register);
     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
-                 $scr$$Register, noreg, noreg, NULL, NULL, NULL, false, false);
+                 $scr$$Register, noreg, noreg, $thread$$Register, nullptr, nullptr, nullptr, false, false);
   %}
   ins_pipe(pipe_slow);
 %}

src/hotspot/cpu/x86/x86_64.ad

@@ -13291,7 +13291,7 @@ instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp,
   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
   ins_encode %{
     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
-                 $scr$$Register, $cx1$$Register, $cx2$$Register,
+                 $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
                  _rtm_counters, _stack_rtm_counters,
                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
                  true, ra_->C->profile_rtm());
@@ -13307,7 +13307,7 @@ instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRe
   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
   ins_encode %{
     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
-                 $scr$$Register, noreg, noreg, NULL, NULL, NULL, false, false);
+                 $scr$$Register, noreg, noreg, r15_thread, nullptr, nullptr, nullptr, false, false);
   %}
   ins_pipe(pipe_slow);
 %}

src/hotspot/cpu/zero/vm_version_zero.cpp

@@ -116,6 +116,11 @@ void VM_Version::initialize() {
     FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false);
   }
 
+  if ((LockingMode != LM_LEGACY) && (LockingMode != LM_MONITOR)) {
+    warning("Unsupported locking mode for this CPU.");
+    FLAG_SET_DEFAULT(LockingMode, LM_LEGACY);
+  }
+
   // Enable error context decoding on known platforms
 #if defined(IA32) || defined(AMD64) || defined(ARM) || \
     defined(AARCH64) || defined(PPC) || defined(RISCV) || \

src/hotspot/cpu/zero/zeroInterpreter_zero.cpp

@@ -333,7 +333,7 @@ int ZeroInterpreter::native_entry(Method* method, intptr_t UNUSED, TRAPS) {
     oop lockee = monitor->obj();
     markWord disp = lockee->mark().set_unlocked();
     monitor->lock()->set_displaced_header(disp);
-    bool call_vm = UseHeavyMonitors;
+    bool call_vm = (LockingMode == LM_MONITOR);
     bool inc_monitor_count = true;
     if (call_vm || lockee->cas_set_mark(markWord::from_pointer(monitor), disp) != disp) {
       // Is it simple recursive case?

src/hotspot/share/c1/c1_LIRGenerator.cpp

@@ -621,7 +621,7 @@ void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, L
   // setup registers
   LIR_Opr hdr = lock;
   lock = new_hdr;
-  CodeStub* slow_path = new MonitorExitStub(lock, !UseHeavyMonitors, monitor_no);
+  CodeStub* slow_path = new MonitorExitStub(lock, LockingMode != LM_MONITOR, monitor_no);
   __ load_stack_address_monitor(monitor_no, lock);
   __ unlock_object(hdr, object, lock, scratch, slow_path);
 }

src/hotspot/share/c1/c1_Runtime1.cpp

@@ -754,11 +754,11 @@ JRT_BLOCK_ENTRY(void, Runtime1::monitorenter(JavaThread* current, oopDesc* obj,
     _monitorenter_slowcase_cnt++;
   }
 #endif
-  if (UseHeavyMonitors) {
+  if (LockingMode == LM_MONITOR) {
     lock->set_obj(obj);
   }
-  assert(obj == lock->obj(), "must match");
+  assert(LockingMode == LM_LIGHTWEIGHT || obj == lock->obj(), "must match");
-  SharedRuntime::monitor_enter_helper(obj, lock->lock(), current);
+  SharedRuntime::monitor_enter_helper(obj, LockingMode == LM_LIGHTWEIGHT ? nullptr : lock->lock(), current);
 JRT_END
 
 

src/hotspot/share/interpreter/interpreterRuntime.cpp

@@ -735,6 +735,7 @@ void InterpreterRuntime::resolve_get_put(JavaThread* current, Bytecodes::Code by
 
 //%note monitor_1
 JRT_ENTRY_NO_ASYNC(void, InterpreterRuntime::monitorenter(JavaThread* current, BasicObjectLock* elem))
+  assert(LockingMode != LM_LIGHTWEIGHT, "Should call monitorenter_obj() when using the new lightweight locking");
 #ifdef ASSERT
   current->last_frame().interpreter_frame_verify_monitor(elem);
 #endif
@@ -749,6 +750,22 @@ JRT_ENTRY_NO_ASYNC(void, InterpreterRuntime::monitorenter(JavaThread* current, B
 #endif
 JRT_END
 
+// NOTE: We provide a separate implementation for the new lightweight locking to workaround a limitation
+// of registers in x86_32. This entry point accepts an oop instead of a BasicObjectLock*.
+// The problem is that we would need to preserve the register that holds the BasicObjectLock,
+// but we are using that register to hold the thread. We don't have enough registers to
+// also keep the BasicObjectLock, but we don't really need it anyway, we only need
+// the object. See also InterpreterMacroAssembler::lock_object().
+// As soon as legacy stack-locking goes away we could remove the other monitorenter() entry
+// point, and only use oop-accepting entries (same for monitorexit() below).
+JRT_ENTRY_NO_ASYNC(void, InterpreterRuntime::monitorenter_obj(JavaThread* current, oopDesc* obj))
+  assert(LockingMode == LM_LIGHTWEIGHT, "Should call monitorenter() when not using the new lightweight locking");
+  Handle h_obj(current, cast_to_oop(obj));
+  assert(Universe::heap()->is_in_or_null(h_obj()),
+         "must be null or an object");
+  ObjectSynchronizer::enter(h_obj, nullptr, current);
+  return;
+JRT_END
 
 JRT_LEAF(void, InterpreterRuntime::monitorexit(BasicObjectLock* elem))
   oop obj = elem->obj();

src/hotspot/share/interpreter/interpreterRuntime.hpp

@@ -103,6 +103,7 @@ class InterpreterRuntime: AllStatic {
  public:
   // Synchronization
   static void    monitorenter(JavaThread* current, BasicObjectLock* elem);
+  static void    monitorenter_obj(JavaThread* current, oopDesc* obj);
   static void    monitorexit (BasicObjectLock* elem);
 
   static void    throw_illegal_monitor_state_exception(JavaThread* current);

src/hotspot/share/interpreter/zero/bytecodeInterpreter.cpp

@@ -624,7 +624,7 @@ void BytecodeInterpreter::run(interpreterState istate) {
         // Traditional lightweight locking.
         markWord displaced = rcvr->mark().set_unlocked();
         mon->lock()->set_displaced_header(displaced);
-        bool call_vm = UseHeavyMonitors;
+        bool call_vm = (LockingMode == LM_MONITOR);
         bool inc_monitor_count = true;
         if (call_vm || rcvr->cas_set_mark(markWord::from_pointer(mon), displaced) != displaced) {
           // Is it simple recursive case?
@@ -723,7 +723,7 @@ void BytecodeInterpreter::run(interpreterState istate) {
       // traditional lightweight locking
       markWord displaced = lockee->mark().set_unlocked();
       entry->lock()->set_displaced_header(displaced);
-      bool call_vm = UseHeavyMonitors;
+      bool call_vm = (LockingMode == LM_MONITOR);
       bool inc_monitor_count = true;
       if (call_vm || lockee->cas_set_mark(markWord::from_pointer(entry), displaced) != displaced) {
         // Is it simple recursive case?
@@ -1653,7 +1653,7 @@ run:
           // traditional lightweight locking
           markWord displaced = lockee->mark().set_unlocked();
           entry->lock()->set_displaced_header(displaced);
-          bool call_vm = UseHeavyMonitors;
+          bool call_vm = (LockingMode == LM_MONITOR);
           bool inc_monitor_count = true;
           if (call_vm || lockee->cas_set_mark(markWord::from_pointer(entry), displaced) != displaced) {
             // Is it simple recursive case?
@@ -1689,7 +1689,7 @@ run:
 
             // If it isn't recursive we either must swap old header or call the runtime
             bool dec_monitor_count = true;
-            bool call_vm = UseHeavyMonitors;
+            bool call_vm = (LockingMode == LM_MONITOR);
             if (header.to_pointer() != nullptr || call_vm) {
               markWord old_header = markWord::encode(lock);
               if (call_vm || lockee->cas_set_mark(header, old_header) != old_header) {
@@ -3189,7 +3189,7 @@ run:
               illegal_state_oop = Handle(THREAD, THREAD->pending_exception());
               THREAD->clear_pending_exception();
             }
-          } else if (UseHeavyMonitors) {
+          } else if (LockingMode == LM_MONITOR) {
             InterpreterRuntime::monitorexit(base);
             if (THREAD->has_pending_exception()) {
               if (!suppress_error) illegal_state_oop = Handle(THREAD, THREAD->pending_exception());

src/hotspot/share/logging/logTag.hpp

@@ -77,6 +77,7 @@ class outputStream;
   LOG_TAG(event) \
   LOG_TAG(exceptions) \
   LOG_TAG(exit) \
+  LOG_TAG(fastlock) \
   LOG_TAG(finalizer) \
   LOG_TAG(fingerprint) \
   NOT_PRODUCT(LOG_TAG(foreign)) \

src/hotspot/share/oops/markWord.hpp

@@ -50,11 +50,12 @@
 //
 //  - the two lock bits are used to describe three states: locked/unlocked and monitor.
 //
-//    [ptr             | 00]  locked             ptr points to real header on stack
+//    [ptr             | 00]  locked             ptr points to real header on stack (stack-locking in use)
+//    [header          | 00]  locked             locked regular object header (fast-locking in use)
 //    [header          | 01]  unlocked           regular object header
-//    [ptr             | 10]  monitor            inflated lock (header is wapped out)
+//    [ptr             | 10]  monitor            inflated lock (header is swapped out)
 //    [ptr             | 11]  marked             used to mark an object
-//    [0 ............ 0| 00]  inflating          inflation in progress
+//    [0 ............ 0| 00]  inflating          inflation in progress (stack-locking in use)
 //
 //    We assume that stack/thread pointers have the lowest two bits cleared.
 //
@@ -156,6 +157,7 @@ class markWord {
   // check for and avoid overwriting a 0 value installed by some
   // other thread.  (They should spin or block instead.  The 0 value
   // is transient and *should* be short-lived).
+  // Fast-locking does not use INFLATING.
   static markWord INFLATING() { return zero(); }    // inflate-in-progress
 
   // Should this header be preserved during GC?
@@ -170,12 +172,23 @@ class markWord {
     return markWord(value() | unlocked_value);
   }
   bool has_locker() const {
-    return ((value() & lock_mask_in_place) == locked_value);
+    assert(LockingMode == LM_LEGACY, "should only be called with legacy stack locking");
+    return (value() & lock_mask_in_place) == locked_value;
   }
   BasicLock* locker() const {
     assert(has_locker(), "check");
     return (BasicLock*) value();
   }
+
+  bool is_fast_locked() const {
+    assert(LockingMode == LM_LIGHTWEIGHT, "should only be called with new lightweight locking");
+    return (value() & lock_mask_in_place) == locked_value;
+  }
+  markWord set_fast_locked() const {
+    // Clear the lock_mask_in_place bits to set locked_value:
+    return markWord(value() & ~lock_mask_in_place);
+  }
+
   bool has_monitor() const {
     return ((value() & lock_mask_in_place) == monitor_value);
   }
@@ -185,7 +198,9 @@ class markWord {
     return (ObjectMonitor*) (value() ^ monitor_value);
   }
   bool has_displaced_mark_helper() const {
-    return ((value() & unlocked_value) == 0);
+    intptr_t lockbits = value() & lock_mask_in_place;
+    return LockingMode == LM_LIGHTWEIGHT  ? lockbits == monitor_value   // monitor?
+                                          : (lockbits & unlocked_value) == 0; // monitor | stack-locked?
   }
   markWord displaced_mark_helper() const;
   void set_displaced_mark_helper(markWord m) const;

src/hotspot/share/oops/oop.cpp

@@ -119,7 +119,7 @@ bool oopDesc::is_oop(oop obj, bool ignore_mark_word) {
   }
 
   // Header verification: the mark is typically non-zero. If we're
-  // at a safepoint, it must not be zero.
+  // at a safepoint, it must not be zero, except when using the new lightweight locking.
   // Outside of a safepoint, the header could be changing (for example,
   // another thread could be inflating a lock on this object).
   if (ignore_mark_word) {
@@ -128,7 +128,7 @@ bool oopDesc::is_oop(oop obj, bool ignore_mark_word) {
   if (obj->mark().value() != 0) {
     return true;
   }
-  return !SafepointSynchronize::is_at_safepoint();
+  return LockingMode == LM_LIGHTWEIGHT || !SafepointSynchronize::is_at_safepoint();
 }
 
 // used only for asserts and guarantees

src/hotspot/share/opto/c2_CodeStubs.hpp

@@ -86,4 +86,19 @@ public:
   void emit(C2_MacroAssembler& masm);
 };
 
+#ifdef _LP64
+class C2HandleAnonOMOwnerStub : public C2CodeStub {
+private:
+  Register _monitor;
+  Register _tmp;
+public:
+  C2HandleAnonOMOwnerStub(Register monitor, Register tmp = noreg) : C2CodeStub(),
+    _monitor(monitor), _tmp(tmp) {}
+  Register monitor() { return _monitor; }
+  Register tmp() { return _tmp; }
+  int max_size() const;
+  void emit(C2_MacroAssembler& masm);
+};
+#endif
+
 #endif // SHARE_OPTO_C2_CODESTUBS_HPP

src/hotspot/share/runtime/arguments.cpp

@@ -1939,27 +1939,43 @@ bool Arguments::check_vm_args_consistency() {
   }
 #endif
 
-#if !defined(X86) && !defined(AARCH64) && !defined(PPC64) && !defined(RISCV64)
+
+#if !defined(X86) && !defined(AARCH64) && !defined(RISCV64) && !defined(ARM)
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    FLAG_SET_CMDLINE(LockingMode, LM_LEGACY);
+    warning("New lightweight locking not supported on this platform");
+  }
+#endif
+
   if (UseHeavyMonitors) {
+    if (FLAG_IS_CMDLINE(LockingMode) && LockingMode != LM_MONITOR) {
       jio_fprintf(defaultStream::error_stream(),
-                "UseHeavyMonitors is not fully implemented on this architecture");
+                  "Conflicting -XX:+UseHeavyMonitors and -XX:LockingMode=%d flags", LockingMode);
+      return false;
+    }
+    FLAG_SET_CMDLINE(LockingMode, LM_MONITOR);
+  }
+
+#if !defined(X86) && !defined(AARCH64) && !defined(PPC64) && !defined(RISCV64)
+  if (LockingMode == LM_MONITOR) {
+    jio_fprintf(defaultStream::error_stream(),
+                "LockingMode == 0 (LM_MONITOR) is not fully implemented on this architecture");
     return false;
   }
 #endif
 #if (defined(X86) || defined(PPC64)) && !defined(ZERO)
-  if (UseHeavyMonitors && UseRTMForStackLocks) {
+  if (LockingMode == LM_MONITOR && UseRTMForStackLocks) {
     jio_fprintf(defaultStream::error_stream(),
-                "-XX:+UseHeavyMonitors and -XX:+UseRTMForStackLocks are mutually exclusive");
+                "LockingMode == 0 (LM_MONITOR) and -XX:+UseRTMForStackLocks are mutually exclusive");
 
     return false;
   }
 #endif
-  if (VerifyHeavyMonitors && !UseHeavyMonitors) {
+  if (VerifyHeavyMonitors && LockingMode != LM_MONITOR) {
     jio_fprintf(defaultStream::error_stream(),
-                "-XX:+VerifyHeavyMonitors requires -XX:+UseHeavyMonitors");
+                "-XX:+VerifyHeavyMonitors requires LockingMode == 0 (LM_MONITOR)");
     return false;
   }
-
   return status;
 }
 

src/hotspot/share/runtime/deoptimization.cpp

@@ -1630,7 +1630,7 @@ bool Deoptimization::relock_objects(JavaThread* thread, GrowableArray<MonitorInf
         Handle obj(thread, mon_info->owner());
         markWord mark = obj->mark();
         if (exec_mode == Unpack_none) {
-          if (mark.has_locker() && fr.sp() > (intptr_t*)mark.locker()) {
+          if (LockingMode == LM_LEGACY && mark.has_locker() && fr.sp() > (intptr_t*)mark.locker()) {
             // With exec_mode == Unpack_none obj may be thread local and locked in
             // a callee frame. Make the lock in the callee a recursive lock and restore the displaced header.
             markWord dmw = mark.displaced_mark_helper();

src/hotspot/share/runtime/globals.hpp

@@ -1979,6 +1979,13 @@ const int ObjectAlignmentInBytes = 8;
           false AARCH64_ONLY(DEBUG_ONLY(||true)),                           \
              "Mark all threads after a safepoint, and clear on a modify "   \
              "fence. Add cleanliness checks.")                              \
+                                                                            \
+  product(int, LockingMode, LM_LEGACY, EXPERIMENTAL,                        \
+          "Select locking mode: "                                           \
+          "0: monitors only (LM_MONITOR), "                                 \
+          "1: monitors & legacy stack-locking (LM_LEGACY, default), "       \
+          "2: monitors & new lightweight locking (LM_LIGHTWEIGHT)")         \
+          range(0, 2)                                                       \
 
 // end of RUNTIME_FLAGS
 

src/hotspot/share/runtime/javaThread.cpp

@@ -70,6 +70,7 @@
 #include "runtime/javaCalls.hpp"
 #include "runtime/javaThread.inline.hpp"
 #include "runtime/jniHandles.inline.hpp"
+#include "runtime/lockStack.inline.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/orderAccess.hpp"
 #include "runtime/osThread.hpp"
@@ -490,8 +491,9 @@ JavaThread::JavaThread() :
 
   _class_to_be_initialized(nullptr),
 
-  _SleepEvent(ParkEvent::Allocate(this))
+  _SleepEvent(ParkEvent::Allocate(this)),
-{
+
+  _lock_stack(this) {
   set_jni_functions(jni_functions());
 
 #if INCLUDE_JVMCI
@@ -994,6 +996,7 @@ JavaThread* JavaThread::active() {
 }
 
 bool JavaThread::is_lock_owned(address adr) const {
+  assert(LockingMode != LM_LIGHTWEIGHT, "should not be called with new lightweight locking");
   if (Thread::is_lock_owned(adr)) return true;
 
   for (MonitorChunk* chunk = monitor_chunks(); chunk != nullptr; chunk = chunk->next()) {
@@ -1387,6 +1390,10 @@ void JavaThread::oops_do_no_frames(OopClosure* f, CodeBlobClosure* cf) {
     f->do_oop((oop*)entry->chunk_addr());
     entry = entry->parent();
   }
+
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    lock_stack().oops_do(f);
+  }
 }
 
 void JavaThread::oops_do_frames(OopClosure* f, CodeBlobClosure* cf) {

src/hotspot/share/runtime/javaThread.hpp

@@ -34,6 +34,7 @@
 #include "runtime/globals.hpp"
 #include "runtime/handshake.hpp"
 #include "runtime/javaFrameAnchor.hpp"
+#include "runtime/lockStack.hpp"
 #include "runtime/park.hpp"
 #include "runtime/safepointMechanism.hpp"
 #include "runtime/stackWatermarkSet.hpp"
@@ -1146,6 +1147,19 @@ public:
   void interrupt();
   bool is_interrupted(bool clear_interrupted);
 
+private:
+  LockStack _lock_stack;
+
+public:
+  LockStack& lock_stack() { return _lock_stack; }
+
+  static ByteSize lock_stack_offset()      { return byte_offset_of(JavaThread, _lock_stack); }
+  // Those offsets are used in code generators to access the LockStack that is embedded in this
+  // JavaThread structure. Those accesses are relative to the current thread, which
+  // is typically in a dedicated register.
+  static ByteSize lock_stack_top_offset()  { return lock_stack_offset() + LockStack::top_offset(); }
+  static ByteSize lock_stack_base_offset() { return lock_stack_offset() + LockStack::base_offset(); }
+
   static OopStorage* thread_oop_storage();
 
   static void verify_cross_modify_fence_failure(JavaThread *thread) PRODUCT_RETURN;

src/hotspot/share/runtime/lockStack.cpp

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/lockStack.inline.hpp"
+#include "runtime/safepoint.hpp"
+#include "runtime/stackWatermark.hpp"
+#include "runtime/stackWatermarkSet.inline.hpp"
+#include "runtime/thread.hpp"
+#include "utilities/copy.hpp"
+#include "utilities/ostream.hpp"
+
+const int LockStack::lock_stack_offset =      in_bytes(JavaThread::lock_stack_offset());
+const int LockStack::lock_stack_top_offset =  in_bytes(JavaThread::lock_stack_top_offset());
+const int LockStack::lock_stack_base_offset = in_bytes(JavaThread::lock_stack_base_offset());
+
+LockStack::LockStack(JavaThread* jt) :
+  _top(lock_stack_base_offset), _base() {
+#ifdef ASSERT
+  for (int i = 0; i < CAPACITY; i++) {
+    _base[i] = nullptr;
+  }
+#endif
+}
+
+uint32_t LockStack::start_offset() {
+  int offset = lock_stack_base_offset;
+  assert(offset > 0, "must be positive offset");
+  return static_cast<uint32_t>(offset);
+}
+
+uint32_t LockStack::end_offset() {
+  int offset = lock_stack_base_offset + CAPACITY * oopSize;
+  assert(offset > 0, "must be positive offset");
+  return static_cast<uint32_t>(offset);
+}
+
+#ifndef PRODUCT
+void LockStack::verify(const char* msg) const {
+  assert(LockingMode == LM_LIGHTWEIGHT, "never use lock-stack when light weight locking is disabled");
+  assert((_top <= end_offset()), "lockstack overflow: _top %d end_offset %d", _top, end_offset());
+  assert((_top >= start_offset()), "lockstack underflow: _top %d end_offset %d", _top, start_offset());
+  if (SafepointSynchronize::is_at_safepoint() || (Thread::current()->is_Java_thread() && is_owning_thread())) {
+    int top = to_index(_top);
+    for (int i = 0; i < top; i++) {
+      assert(_base[i] != nullptr, "no zapped before top");
+      for (int j = i + 1; j < top; j++) {
+        assert(_base[i] != _base[j], "entries must be unique: %s", msg);
+      }
+    }
+    for (int i = top; i < CAPACITY; i++) {
+      assert(_base[i] == nullptr, "only zapped entries after top: i: %d, top: %d, entry: " PTR_FORMAT, i, top, p2i(_base[i]));
+    }
+  }
+}
+#endif

src/hotspot/share/runtime/lockStack.hpp

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_RUNTIME_LOCKSTACK_HPP
+#define SHARE_RUNTIME_LOCKSTACK_HPP
+
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/sizes.hpp"
+
+class Thread;
+class OopClosure;
+
+class LockStack {
+  friend class VMStructs;
+private:
+  static const int CAPACITY = 8;
+
+  // TODO: It would be very useful if JavaThread::lock_stack_offset() and friends were constexpr,
+  // but this is currently not the case because we're using offset_of() which is non-constexpr,
+  // GCC would warn about non-standard-layout types if we were using offsetof() (which *is* constexpr).
+  static const int lock_stack_offset;
+  static const int lock_stack_top_offset;
+  static const int lock_stack_base_offset;
+
+  // The offset of the next element, in bytes, relative to the JavaThread structure.
+  // We do this instead of a simple index into the array because this allows for
+  // efficient addressing in generated code.
+  uint32_t _top;
+  oop _base[CAPACITY];
+
+  // Get the owning thread of this lock-stack.
+  inline JavaThread* get_thread() const;
+
+  // Tests if the calling thread is the thread that owns this lock-stack.
+  bool is_owning_thread() const;
+
+  // Verifies consistency of the lock-stack.
+  void verify(const char* msg) const PRODUCT_RETURN;
+
+  // Given an offset (in bytes) calculate the index into the lock-stack.
+  static inline int to_index(uint32_t offset);
+
+public:
+  static ByteSize top_offset()  { return byte_offset_of(LockStack, _top); }
+  static ByteSize base_offset() { return byte_offset_of(LockStack, _base); }
+
+  LockStack(JavaThread* jt);
+
+  // The boundary indicies of the lock-stack.
+  static uint32_t start_offset();
+  static uint32_t end_offset();
+
+  // Return true if we have room to push onto this lock-stack, false otherwise.
+  inline bool can_push() const;
+
+  // Pushes an oop on this lock-stack.
+  inline void push(oop o);
+
+  // Pops an oop from this lock-stack.
+  inline oop pop();
+
+  // Removes an oop from an arbitrary location of this lock-stack.
+  inline void remove(oop o);
+
+  // Tests whether the oop is on this lock-stack.
+  inline bool contains(oop o) const;
+
+  // GC support
+  inline void oops_do(OopClosure* cl);
+
+};
+
+#endif // SHARE_RUNTIME_LOCKSTACK_HPP

src/hotspot/share/runtime/lockStack.inline.hpp

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_RUNTIME_LOCKSTACK_INLINE_HPP
+#define SHARE_RUNTIME_LOCKSTACK_INLINE_HPP
+
+#include "memory/iterator.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/lockStack.hpp"
+#include "runtime/safepoint.hpp"
+#include "runtime/stackWatermark.hpp"
+#include "runtime/stackWatermarkSet.inline.hpp"
+
+inline int LockStack::to_index(uint32_t offset) {
+  return (offset - lock_stack_base_offset) / oopSize;
+}
+
+JavaThread* LockStack::get_thread() const {
+  char* addr = reinterpret_cast<char*>(const_cast<LockStack*>(this));
+  return reinterpret_cast<JavaThread*>(addr - lock_stack_offset);
+}
+
+inline bool LockStack::can_push() const {
+  return to_index(_top) < CAPACITY;
+}
+
+inline bool LockStack::is_owning_thread() const {
+  JavaThread* thread = JavaThread::current();
+  bool is_owning = &thread->lock_stack() == this;
+  assert(is_owning == (get_thread() == thread), "is_owning sanity");
+  return is_owning;
+}
+
+inline void LockStack::push(oop o) {
+  verify("pre-push");
+  assert(oopDesc::is_oop(o), "must be");
+  assert(!contains(o), "entries must be unique");
+  assert(can_push(), "must have room");
+  assert(_base[to_index(_top)] == nullptr, "expect zapped entry");
+  _base[to_index(_top)] = o;
+  _top += oopSize;
+  verify("post-push");
+}
+
+inline oop LockStack::pop() {
+  verify("pre-pop");
+  assert(to_index(_top) > 0, "underflow, probably unbalanced push/pop");
+  _top -= oopSize;
+  oop o = _base[to_index(_top)];
+#ifdef ASSERT
+  _base[to_index(_top)] = nullptr;
+#endif
+  assert(!contains(o), "entries must be unique: " PTR_FORMAT, p2i(o));
+  verify("post-pop");
+  return o;
+}
+
+inline void LockStack::remove(oop o) {
+  verify("pre-remove");
+  assert(contains(o), "entry must be present: " PTR_FORMAT, p2i(o));
+  int end = to_index(_top);
+  for (int i = 0; i < end; i++) {
+    if (_base[i] == o) {
+      int last = end - 1;
+      for (; i < last; i++) {
+        _base[i] = _base[i + 1];
+      }
+      _top -= oopSize;
+#ifdef ASSERT
+      _base[to_index(_top)] = nullptr;
+#endif
+      break;
+    }
+  }
+  assert(!contains(o), "entries must be unique: " PTR_FORMAT, p2i(o));
+  verify("post-remove");
+}
+
+inline bool LockStack::contains(oop o) const {
+  verify("pre-contains");
+  if (!SafepointSynchronize::is_at_safepoint() && !is_owning_thread()) {
+    // When a foreign thread inspects this thread's lock-stack, it may see
+    // bad references here when a concurrent collector has not gotten
+    // to processing the lock-stack, yet. Call StackWaterMark::start_processing()
+    // to ensure that all references are valid.
+    StackWatermark* watermark = StackWatermarkSet::get(get_thread(), StackWatermarkKind::gc);
+    if (watermark != nullptr) {
+      watermark->start_processing();
+    }
+  }
+  int end = to_index(_top);
+  for (int i = end - 1; i >= 0; i--) {
+    if (_base[i] == o) {
+      verify("post-contains");
+      return true;
+    }
+  }
+  verify("post-contains");
+  return false;
+}
+
+inline void LockStack::oops_do(OopClosure* cl) {
+  verify("pre-oops-do");
+  int end = to_index(_top);
+  for (int i = 0; i < end; i++) {
+    cl->do_oop(&_base[i]);
+  }
+  verify("post-oops-do");
+}
+
+#endif // SHARE_RUNTIME_LOCKSTACK_INLINE_HPP

src/hotspot/share/runtime/objectMonitor.cpp

@@ -334,7 +334,7 @@ bool ObjectMonitor::enter(JavaThread* current) {
     return true;
   }
 
-  if (current->is_lock_owned((address)cur)) {
+  if (LockingMode != LM_LIGHTWEIGHT && current->is_lock_owned((address)cur)) {
     assert(_recursions == 0, "internal state error");
     _recursions = 1;
     set_owner_from_BasicLock(cur, current);  // Convert from BasicLock* to Thread*.
@@ -1135,7 +1135,7 @@ void ObjectMonitor::UnlinkAfterAcquire(JavaThread* current, ObjectWaiter* curren
 void ObjectMonitor::exit(JavaThread* current, bool not_suspended) {
   void* cur = owner_raw();
   if (current != cur) {
-    if (current->is_lock_owned((address)cur)) {
+    if (LockingMode != LM_LIGHTWEIGHT && current->is_lock_owned((address)cur)) {
       assert(_recursions == 0, "invariant");
       set_owner_from_BasicLock(cur, current);  // Convert from BasicLock* to Thread*.
       _recursions = 0;
@@ -1350,7 +1350,7 @@ intx ObjectMonitor::complete_exit(JavaThread* current) {
 
   void* cur = owner_raw();
   if (current != cur) {
-    if (current->is_lock_owned((address)cur)) {
+    if (LockingMode != LM_LIGHTWEIGHT && current->is_lock_owned((address)cur)) {
       assert(_recursions == 0, "internal state error");
       set_owner_from_BasicLock(cur, current);  // Convert from BasicLock* to Thread*.
       _recursions = 0;
@@ -1385,10 +1385,11 @@ intx ObjectMonitor::complete_exit(JavaThread* current) {
 bool ObjectMonitor::check_owner(TRAPS) {
   JavaThread* current = THREAD;
   void* cur = owner_raw();
+  assert(cur != anon_owner_ptr(), "no anon owner here");
   if (cur == current) {
     return true;
   }
-  if (current->is_lock_owned((address)cur)) {
+  if (LockingMode != LM_LIGHTWEIGHT && current->is_lock_owned((address)cur)) {
     set_owner_from_BasicLock(cur, current);  // Convert from BasicLock* to Thread*.
     _recursions = 0;
     return true;

src/hotspot/share/runtime/objectMonitor.hpp

@@ -144,8 +144,22 @@ class ObjectMonitor : public CHeapObj<mtObjectMonitor> {
   // its cache line with _header.
   DEFINE_PAD_MINUS_SIZE(0, OM_CACHE_LINE_SIZE, sizeof(volatile markWord) +
                         sizeof(WeakHandle));
-  // Used by async deflation as a marker in the _owner field:
+  // Used by async deflation as a marker in the _owner field.
-  #define DEFLATER_MARKER reinterpret_cast<void*>(-1)
+  // Note that the choice of the two markers is peculiar:
+  // - They need to represent values that cannot be pointers. In particular,
+  //   we achieve this by using the lowest two bits.
+  // - ANONYMOUS_OWNER should be a small value, it is used in generated code
+  //   and small values encode much better.
+  // - We test for anonymous owner by testing for the lowest bit, therefore
+  //   DEFLATER_MARKER must *not* have that bit set.
+  #define DEFLATER_MARKER reinterpret_cast<void*>(2)
+public:
+  // NOTE: Typed as uintptr_t so that we can pick it up in SA, via vmStructs.
+  static const uintptr_t ANONYMOUS_OWNER = 1;
+
+private:
+  static void* anon_owner_ptr() { return reinterpret_cast<void*>(ANONYMOUS_OWNER); }
+
   void* volatile _owner;            // pointer to owning thread OR BasicLock
   volatile uint64_t _previous_owner_tid;  // thread id of the previous owner of the monitor
   // Separate _owner and _next_om on different cache lines since
@@ -178,6 +192,7 @@ class ObjectMonitor : public CHeapObj<mtObjectMonitor> {
   volatile int _WaitSetLock;        // protects Wait Queue - simple spinlock
 
  public:
+
   static void Initialize();
 
   // Only perform a PerfData operation if the PerfData object has been
@@ -242,7 +257,7 @@ class ObjectMonitor : public CHeapObj<mtObjectMonitor> {
   }
   const char* is_busy_to_string(stringStream* ss);
 
-  intptr_t  is_entered(JavaThread* current) const;
+  bool is_entered(JavaThread* current) const;
 
   // Returns true if this OM has an owner, false otherwise.
   bool      has_owner() const;
@@ -263,6 +278,18 @@ class ObjectMonitor : public CHeapObj<mtObjectMonitor> {
   // _owner field. Returns the prior value of the _owner field.
   void*     try_set_owner_from(void* old_value, void* new_value);
 
+  void set_owner_anonymous() {
+    set_owner_from(nullptr, anon_owner_ptr());
+  }
+
+  bool is_owner_anonymous() const {
+    return owner_raw() == anon_owner_ptr();
+  }
+
+  void set_owner_from_anonymous(Thread* owner) {
+    set_owner_from(anon_owner_ptr(), owner);
+  }
+
   // Simply get _next_om field.
   ObjectMonitor* next_om() const;
   // Simply set _next_om field to new_value.

src/hotspot/share/runtime/objectMonitor.inline.hpp

@@ -30,14 +30,23 @@
 #include "logging/log.hpp"
 #include "oops/access.inline.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/lockStack.inline.hpp"
 #include "runtime/synchronizer.hpp"
 
-inline intptr_t ObjectMonitor::is_entered(JavaThread* current) const {
+inline bool ObjectMonitor::is_entered(JavaThread* current) const {
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    if (is_owner_anonymous()) {
+      return current->lock_stack().contains(object());
+    } else {
+      return current == owner_raw();
+    }
+  } else {
     void* owner = owner_raw();
     if (current == owner || current->is_lock_owned((address)owner)) {
-    return 1;
+      return true;
     }
-  return 0;
+  }
+  return false;
 }
 
 inline markWord ObjectMonitor::header() const {

src/hotspot/share/runtime/synchronizer.cpp

@@ -40,6 +40,7 @@
 #include "runtime/handshake.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/javaThread.hpp"
+#include "runtime/lockStack.inline.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/objectMonitor.inline.hpp"
@@ -324,11 +325,19 @@ bool ObjectSynchronizer::quick_notify(oopDesc* obj, JavaThread* current, bool al
   if (obj == nullptr) return false;  // slow-path for invalid obj
   const markWord mark = obj->mark();
 
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    if (mark.is_fast_locked() && current->lock_stack().contains(cast_to_oop(obj))) {
+      // Degenerate notify
+      // fast-locked by caller so by definition the implied waitset is empty.
+      return true;
+    }
+  } else if (LockingMode == LM_LEGACY) {
     if (mark.has_locker() && current->is_lock_owned((address)mark.locker())) {
       // Degenerate notify
       // stack-locked by caller so by definition the implied waitset is empty.
       return true;
     }
+  }
 
   if (mark.has_monitor()) {
     ObjectMonitor* const mon = mark.monitor();
@@ -398,6 +407,7 @@ bool ObjectSynchronizer::quick_enter(oop obj, JavaThread* current,
       return true;
     }
 
+    if (LockingMode != LM_LIGHTWEIGHT) {
       // This Java Monitor is inflated so obj's header will never be
       // displaced to this thread's BasicLock. Make the displaced header
       // non-null so this BasicLock is not seen as recursive nor as
@@ -408,6 +418,7 @@ bool ObjectSynchronizer::quick_enter(oop obj, JavaThread* current,
       // recursive stack-locking in the displaced header in the BasicLock,
       // and last are the inflated Java Monitor (ObjectMonitor) checks.
       lock->set_displaced_header(markWord::unused_mark());
+    }
 
     if (owner == nullptr && m->try_set_owner_from(nullptr, current) == nullptr) {
       assert(m->_recursions == 0, "invariant");
@@ -476,7 +487,7 @@ void ObjectSynchronizer::handle_sync_on_value_based_class(Handle obj, JavaThread
 
 static bool useHeavyMonitors() {
 #if defined(X86) || defined(AARCH64) || defined(PPC64) || defined(RISCV64)
-  return UseHeavyMonitors;
+  return LockingMode == LM_MONITOR;
 #else
   return false;
 #endif
@@ -496,6 +507,25 @@ void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, JavaThread* current)
   current->inc_held_monitor_count();
 
   if (!useHeavyMonitors()) {
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      // Fast-locking does not use the 'lock' argument.
+      LockStack& lock_stack = current->lock_stack();
+      if (lock_stack.can_push()) {
+        markWord mark = obj()->mark_acquire();
+        if (mark.is_neutral()) {
+          assert(!lock_stack.contains(obj()), "thread must not already hold the lock");
+          // Try to swing into 'fast-locked' state.
+          markWord locked_mark = mark.set_fast_locked();
+          markWord old_mark = obj()->cas_set_mark(locked_mark, mark);
+          if (old_mark == mark) {
+            // Successfully fast-locked, push object to lock-stack and return.
+            lock_stack.push(obj());
+            return;
+          }
+        }
+      }
+      // All other paths fall-through to inflate-enter.
+    } else if (LockingMode == LM_LEGACY) {
       markWord mark = obj->mark();
       if (mark.is_neutral()) {
         // Anticipate successful CAS -- the ST of the displaced mark must
@@ -506,9 +536,9 @@ void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, JavaThread* current)
         }
         // Fall through to inflate() ...
       } else if (mark.has_locker() &&
-               current->is_lock_owned((address)mark.locker())) {
+                 current->is_lock_owned((address) mark.locker())) {
         assert(lock != mark.locker(), "must not re-lock the same lock");
-      assert(lock != (BasicLock*)obj->mark().value(), "don't relock with same BasicLock");
+        assert(lock != (BasicLock*) obj->mark().value(), "don't relock with same BasicLock");
         lock->set_displaced_header(markWord::from_pointer(nullptr));
         return;
       }
@@ -518,8 +548,9 @@ void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, JavaThread* current)
       // must be non-zero to avoid looking like a re-entrant lock,
       // and must not look locked either.
       lock->set_displaced_header(markWord::unused_mark());
+    }
   } else if (VerifyHeavyMonitors) {
-    guarantee(!obj->mark().has_locker(), "must not be stack-locked");
+    guarantee((obj->mark().value() & markWord::lock_mask_in_place) != markWord::locked_value, "must not be lightweight/stack-locked");
   }
 
   // An async deflation can race after the inflate() call and before
@@ -538,7 +569,27 @@ void ObjectSynchronizer::exit(oop object, BasicLock* lock, JavaThread* current)
 
   if (!useHeavyMonitors()) {
     markWord mark = object->mark();
-
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      // Fast-locking does not use the 'lock' argument.
+      if (mark.is_fast_locked()) {
+        markWord unlocked_mark = mark.set_unlocked();
+        markWord old_mark = object->cas_set_mark(unlocked_mark, mark);
+        if (old_mark != mark) {
+          // Another thread won the CAS, it must have inflated the monitor.
+          // It can only have installed an anonymously locked monitor at this point.
+          // Fetch that monitor, set owner correctly to this thread, and
+          // exit it (allowing waiting threads to enter).
+          assert(old_mark.has_monitor(), "must have monitor");
+          ObjectMonitor* monitor = old_mark.monitor();
+          assert(monitor->is_owner_anonymous(), "must be anonymous owner");
+          monitor->set_owner_from_anonymous(current);
+          monitor->exit(current);
+        }
+        LockStack& lock_stack = current->lock_stack();
+        lock_stack.remove(object);
+        return;
+      }
+    } else if (LockingMode == LM_LEGACY) {
       markWord dhw = lock->displaced_header();
       if (dhw.value() == 0) {
         // If the displaced header is null, then this exit matches up with
@@ -577,14 +628,22 @@ void ObjectSynchronizer::exit(oop object, BasicLock* lock, JavaThread* current)
           return;
         }
       }
+    }
   } else if (VerifyHeavyMonitors) {
-    guarantee(!object->mark().has_locker(), "must not be stack-locked");
+    guarantee((object->mark().value() & markWord::lock_mask_in_place) != markWord::locked_value, "must not be lightweight/stack-locked");
   }
 
   // We have to take the slow-path of possible inflation and then exit.
   // The ObjectMonitor* can't be async deflated until ownership is
   // dropped inside exit() and the ObjectMonitor* must be !is_busy().
   ObjectMonitor* monitor = inflate(current, object, inflate_cause_vm_internal);
+  if (LockingMode == LM_LIGHTWEIGHT && monitor->is_owner_anonymous()) {
+    // It must be owned by us. Pop lock object from lock stack.
+    LockStack& lock_stack = current->lock_stack();
+    oop popped = lock_stack.pop();
+    assert(popped == object, "must be owned by this thread");
+    monitor->set_owner_from_anonymous(current);
+  }
   monitor->exit(current);
 }
 
@@ -675,10 +734,17 @@ void ObjectSynchronizer::notify(Handle obj, TRAPS) {
   JavaThread* current = THREAD;
 
   markWord mark = obj->mark();
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    if ((mark.is_fast_locked() && current->lock_stack().contains(obj()))) {
+      // Not inflated so there can't be any waiters to notify.
+      return;
+    }
+  } else if (LockingMode == LM_LEGACY) {
     if (mark.has_locker() && current->is_lock_owned((address)mark.locker())) {
       // Not inflated so there can't be any waiters to notify.
       return;
     }
+  }
   // The ObjectMonitor* can't be async deflated until ownership is
   // dropped by the calling thread.
   ObjectMonitor* monitor = inflate(current, obj(), inflate_cause_notify);
@@ -690,10 +756,17 @@ void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
   JavaThread* current = THREAD;
 
   markWord mark = obj->mark();
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    if ((mark.is_fast_locked() && current->lock_stack().contains(obj()))) {
+      // Not inflated so there can't be any waiters to notify.
+      return;
+    }
+  } else if (LockingMode == LM_LEGACY) {
     if (mark.has_locker() && current->is_lock_owned((address)mark.locker())) {
       // Not inflated so there can't be any waiters to notify.
       return;
     }
+  }
   // The ObjectMonitor* can't be async deflated until ownership is
   // dropped by the calling thread.
   ObjectMonitor* monitor = inflate(current, obj(), inflate_cause_notify);
@@ -718,7 +791,8 @@ static SharedGlobals GVars;
 
 static markWord read_stable_mark(oop obj) {
   markWord mark = obj->mark_acquire();
-  if (!mark.is_being_inflated()) {
+  if (!mark.is_being_inflated() || LockingMode == LM_LIGHTWEIGHT) {
+    // New lightweight locking does not use the markWord::INFLATING() protocol.
     return mark;       // normal fast-path return
   }
 
@@ -833,6 +907,13 @@ static inline intptr_t get_next_hash(Thread* current, oop obj) {
   return value;
 }
 
+// Can be called from non JavaThreads (e.g., VMThread) for FastHashCode
+// calculations as part of JVM/TI tagging.
+static bool is_lock_owned(Thread* thread, oop obj) {
+  assert(LockingMode == LM_LIGHTWEIGHT, "only call this with new lightweight locking enabled");
+  return thread->is_Java_thread() ? JavaThread::cast(thread)->lock_stack().contains(obj) : false;
+}
+
 intptr_t ObjectSynchronizer::FastHashCode(Thread* current, oop obj) {
 
   while (true) {
@@ -841,8 +922,8 @@ intptr_t ObjectSynchronizer::FastHashCode(Thread* current, oop obj) {
     intptr_t hash;
     markWord mark = read_stable_mark(obj);
     if (VerifyHeavyMonitors) {
-      assert(UseHeavyMonitors, "+VerifyHeavyMonitors requires +UseHeavyMonitors");
+      assert(LockingMode == LM_MONITOR, "+VerifyHeavyMonitors requires LockingMode == 0 (LM_MONITOR)");
-      guarantee(!mark.has_locker(), "must not be stack locked");
+      guarantee((obj->mark().value() & markWord::lock_mask_in_place) != markWord::locked_value, "must not be lightweight/stack-locked");
     }
     if (mark.is_neutral()) {               // if this is a normal header
       hash = mark.hash();
@@ -887,8 +968,15 @@ intptr_t ObjectSynchronizer::FastHashCode(Thread* current, oop obj) {
       }
       // Fall thru so we only have one place that installs the hash in
       // the ObjectMonitor.
-    } else if (current->is_lock_owned((address)mark.locker())) {
+    } else if (LockingMode == LM_LIGHTWEIGHT && mark.is_fast_locked() && is_lock_owned(current, obj)) {
-      // This is a stack lock owned by the calling thread so fetch the
+      // This is a fast-lock owned by the calling thread so use the
+      // markWord from the object.
+      hash = mark.hash();
+      if (hash != 0) {                  // if it has a hash, just return it
+        return hash;
+      }
+    } else if (LockingMode == LM_LEGACY && mark.has_locker() && current->is_lock_owned((address)mark.locker())) {
+      // This is a stack-lock owned by the calling thread so fetch the
       // displaced markWord from the BasicLock on the stack.
       temp = mark.displaced_mark_helper();
       assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
@@ -899,7 +987,7 @@ intptr_t ObjectSynchronizer::FastHashCode(Thread* current, oop obj) {
       // WARNING:
       // The displaced header in the BasicLock on a thread's stack
       // is strictly immutable. It CANNOT be changed in ANY cases.
-      // So we have to inflate the stack lock into an ObjectMonitor
+      // So we have to inflate the stack-lock into an ObjectMonitor
       // even if the current thread owns the lock. The BasicLock on
       // a thread's stack can be asynchronously read by other threads
       // during an inflate() call so any change to that stack memory
@@ -952,12 +1040,18 @@ bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* current,
 
   markWord mark = read_stable_mark(obj);
 
-  // Uncontended case, header points to stack
+  if (LockingMode == LM_LEGACY && mark.has_locker()) {
-  if (mark.has_locker()) {
+    // stack-locked case, header points into owner's stack
     return current->is_lock_owned((address)mark.locker());
   }
-  // Contended case, header points to ObjectMonitor (tagged pointer)
+
+  if (LockingMode == LM_LIGHTWEIGHT && mark.is_fast_locked()) {
+    // fast-locking case, see if lock is in current's lock stack
+    return current->lock_stack().contains(h_obj());
+  }
+
   if (mark.has_monitor()) {
+    // Inflated monitor so header points to ObjectMonitor (tagged pointer).
     // The first stage of async deflation does not affect any field
     // used by this comparison so the ObjectMonitor* is usable here.
     ObjectMonitor* monitor = mark.monitor();
@@ -970,27 +1064,28 @@ bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* current,
 
 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
   oop obj = h_obj();
-  address owner = nullptr;
-
   markWord mark = read_stable_mark(obj);
 
-  // Uncontended case, header points to stack
+  if (LockingMode == LM_LEGACY && mark.has_locker()) {
-  if (mark.has_locker()) {
+    // stack-locked so header points into owner's stack.
-    owner = (address) mark.locker();
+    // owning_thread_from_monitor_owner() may also return null here:
+    return Threads::owning_thread_from_monitor_owner(t_list, (address) mark.locker());
   }
 
-  // Contended case, header points to ObjectMonitor (tagged pointer)
+  if (LockingMode == LM_LIGHTWEIGHT && mark.is_fast_locked()) {
-  else if (mark.has_monitor()) {
+    // fast-locked so get owner from the object.
+    // owning_thread_from_object() may also return null here:
+    return Threads::owning_thread_from_object(t_list, h_obj());
+  }
+
+  if (mark.has_monitor()) {
+    // Inflated monitor so header points to ObjectMonitor (tagged pointer).
     // The first stage of async deflation does not affect any field
     // used by this comparison so the ObjectMonitor* is usable here.
     ObjectMonitor* monitor = mark.monitor();
     assert(monitor != nullptr, "monitor should be non-null");
-    owner = (address) monitor->owner();
+    // owning_thread_from_monitor() may also return null here:
-  }
+    return Threads::owning_thread_from_monitor(t_list, monitor);
-
-  if (owner != nullptr) {
-    // owning_thread_from_monitor_owner() may also return null here
-    return Threads::owning_thread_from_monitor_owner(t_list, owner);
   }
 
   // Unlocked case, header in place
@@ -1004,7 +1099,7 @@ JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_ob
 // Visitors ...
 
 // Iterate ObjectMonitors where the owner == thread; this does NOT include
-// ObjectMonitors where owner is set to a stack lock address in thread.
+// ObjectMonitors where owner is set to a stack-lock address in thread.
 //
 // This version of monitors_iterate() works with the in-use monitor list.
 //
@@ -1014,7 +1109,7 @@ void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure, JavaThread* t
     ObjectMonitor* mid = iter.next();
     if (mid->owner() != thread) {
       // Not owned by the target thread and intentionally skips when owner
-      // is set to a stack lock address in the target thread.
+      // is set to a stack-lock address in the target thread.
       continue;
     }
     if (!mid->is_being_async_deflated() && mid->object_peek() != nullptr) {
@@ -1040,7 +1135,7 @@ void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure,
   ObjectMonitorIterator iter(list->head());
   while (!iter.is_empty()) {
     ObjectMonitor* mid = *iter.next();
-    // Owner set to a stack lock address in thread should never be seen here:
+    // Owner set to a stack-lock address in thread should never be seen here:
     assert(mid->owner() == thread, "must be");
     if (!mid->is_being_async_deflated() && mid->object_peek() != nullptr) {
       // Only process with closure if the object is set.
@@ -1224,19 +1319,32 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
     const markWord mark = object->mark_acquire();
 
     // The mark can be in one of the following states:
-    // *  Inflated     - just return
+    // *  inflated     - Just return if using stack-locking.
-    // *  Stack-locked - coerce it to inflated
+    //                   If using fast-locking and the ObjectMonitor owner
-    // *  INFLATING    - busy wait for conversion to complete
+    //                   is anonymous and the current thread owns the
-    // *  Neutral      - aggressively inflate the object.
+    //                   object lock, then we make the current thread the
+    //                   ObjectMonitor owner and remove the lock from the
+    //                   current thread's lock stack.
+    // *  fast-locked  - Coerce it to inflated from fast-locked.
+    // *  stack-locked - Coerce it to inflated from stack-locked.
+    // *  INFLATING    - Busy wait for conversion from stack-locked to
+    //                   inflated.
+    // *  neutral      - Aggressively inflate the object.
 
     // CASE: inflated
     if (mark.has_monitor()) {
       ObjectMonitor* inf = mark.monitor();
       markWord dmw = inf->header();
       assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
+      if (LockingMode == LM_LIGHTWEIGHT && inf->is_owner_anonymous() && is_lock_owned(current, object)) {
+        inf->set_owner_from_anonymous(current);
+        JavaThread::cast(current)->lock_stack().remove(object);
+      }
       return inf;
     }
 
+    if (LockingMode != LM_LIGHTWEIGHT) {
+      // New lightweight locking does not use INFLATING.
       // CASE: inflation in progress - inflating over a stack-lock.
       // Some other thread is converting from stack-locked to inflated.
       // Only that thread can complete inflation -- other threads must wait.
@@ -1247,20 +1355,73 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
         read_stable_mark(object);
         continue;
       }
+    }
+
+    // CASE: fast-locked
+    // Could be fast-locked either by current or by some other thread.
+    //
+    // Note that we allocate the ObjectMonitor speculatively, _before_
+    // attempting to set the object's mark to the new ObjectMonitor. If
+    // this thread owns the monitor, then we set the ObjectMonitor's
+    // owner to this thread. Otherwise, we set the ObjectMonitor's owner
+    // to anonymous. If we lose the race to set the object's mark to the
+    // new ObjectMonitor, then we just delete it and loop around again.
+    //
+    LogStreamHandle(Trace, monitorinflation) lsh;
+    if (LockingMode == LM_LIGHTWEIGHT && mark.is_fast_locked()) {
+      ObjectMonitor* monitor = new ObjectMonitor(object);
+      monitor->set_header(mark.set_unlocked());
+      bool own = is_lock_owned(current, object);
+      if (own) {
+        // Owned by us.
+        monitor->set_owner_from(nullptr, current);
+      } else {
+        // Owned by somebody else.
+        monitor->set_owner_anonymous();
+      }
+      markWord monitor_mark = markWord::encode(monitor);
+      markWord old_mark = object->cas_set_mark(monitor_mark, mark);
+      if (old_mark == mark) {
+        // Success! Return inflated monitor.
+        if (own) {
+          JavaThread::cast(current)->lock_stack().remove(object);
+        }
+        // Once the ObjectMonitor is configured and object is associated
+        // with the ObjectMonitor, it is safe to allow async deflation:
+        _in_use_list.add(monitor);
+
+        // Hopefully the performance counters are allocated on distinct
+        // cache lines to avoid false sharing on MP systems ...
+        OM_PERFDATA_OP(Inflations, inc());
+        if (log_is_enabled(Trace, monitorinflation)) {
+          ResourceMark rm(current);
+          lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
+                       INTPTR_FORMAT ", type='%s'", p2i(object),
+                       object->mark().value(), object->klass()->external_name());
+        }
+        if (event.should_commit()) {
+          post_monitor_inflate_event(&event, object, cause);
+        }
+        return monitor;
+      } else {
+        delete monitor;
+        continue;  // Interference -- just retry
+      }
+    }
 
     // CASE: stack-locked
-    // Could be stack-locked either by this thread or by some other thread.
+    // Could be stack-locked either by current or by some other thread.
     //
     // Note that we allocate the ObjectMonitor speculatively, _before_ attempting
     // to install INFLATING into the mark word.  We originally installed INFLATING,
     // allocated the ObjectMonitor, and then finally STed the address of the
     // ObjectMonitor into the mark.  This was correct, but artificially lengthened
     // the interval in which INFLATING appeared in the mark, thus increasing
-    // the odds of inflation contention.
+    // the odds of inflation contention. If we lose the race to set INFLATING,
-
+    // then we just delete the ObjectMonitor and loop around again.
-    LogStreamHandle(Trace, monitorinflation) lsh;
+    //
-
+    if (LockingMode == LM_LEGACY && mark.has_locker()) {
-    if (mark.has_locker()) {
+      assert(LockingMode != LM_LIGHTWEIGHT, "cannot happen with new lightweight locking");
       ObjectMonitor* m = new ObjectMonitor(object);
       // Optimistically prepare the ObjectMonitor - anticipate successful CAS
       // We do this before the CAS in order to minimize the length of time
@@ -1422,10 +1583,10 @@ void ObjectSynchronizer::chk_for_block_req(JavaThread* current, const char* op_n
 //
 // If table != nullptr, we gather owned ObjectMonitors indexed by the
 // owner in the table. Please note that ObjectMonitors where the owner
-// is set to a stack lock address are NOT associated with the JavaThread
+// is set to a stack-lock address are NOT associated with the JavaThread
-// that holds that stack lock. All of the current consumers of
+// that holds that stack-lock. All of the current consumers of
 // ObjectMonitorsHashtable info only care about JNI locked monitors and
-// those do not have the owner set to a stack lock address.
+// those do not have the owner set to a stack-lock address.
 //
 size_t ObjectSynchronizer::deflate_monitor_list(Thread* current, LogStream* ls,
                                                 elapsedTimer* timer_p,
@@ -1442,7 +1603,7 @@ size_t ObjectSynchronizer::deflate_monitor_list(Thread* current, LogStream* ls,
       deflated_count++;
     } else if (table != nullptr) {
       // The caller is interested in the owned ObjectMonitors. This does
-      // not include when owner is set to a stack lock address in thread.
+      // not include when owner is set to a stack-lock address in thread.
       // This also does not capture unowned ObjectMonitors that cannot be
       // deflated because of a waiter.
       void* key = mid->owner();

src/hotspot/share/runtime/thread.cpp

@@ -526,6 +526,7 @@ void Thread::print_owned_locks_on(outputStream* st) const {
 // should be revisited, and they should be removed if possible.
 
 bool Thread::is_lock_owned(address adr) const {
+  assert(LockingMode != LM_LIGHTWEIGHT, "should not be called with new lightweight locking");
   return is_in_full_stack(adr);
 }
 

src/hotspot/share/runtime/threads.cpp

@@ -69,6 +69,7 @@
 #include "runtime/javaThread.inline.hpp"
 #include "runtime/jniHandles.inline.hpp"
 #include "runtime/jniPeriodicChecker.hpp"
+#include "runtime/lockStack.inline.hpp"
 #include "runtime/monitorDeflationThread.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/nonJavaThread.hpp"
@@ -1177,6 +1178,7 @@ GrowableArray<JavaThread*>* Threads::get_pending_threads(ThreadsList * t_list,
 
 JavaThread *Threads::owning_thread_from_monitor_owner(ThreadsList * t_list,
                                                       address owner) {
+  assert(LockingMode != LM_LIGHTWEIGHT, "Not with new lightweight locking");
   // null owner means not locked so we can skip the search
   if (owner == nullptr) return nullptr;
 
@@ -1188,7 +1190,7 @@ JavaThread *Threads::owning_thread_from_monitor_owner(ThreadsList * t_list,
   // Cannot assert on lack of success here since this function may be
   // used by code that is trying to report useful problem information
   // like deadlock detection.
-  if (UseHeavyMonitors) return nullptr;
+  if (LockingMode == LM_MONITOR) return nullptr;
 
   // If we didn't find a matching Java thread and we didn't force use of
   // heavyweight monitors, then the owner is the stack address of the
@@ -1206,9 +1208,29 @@ JavaThread *Threads::owning_thread_from_monitor_owner(ThreadsList * t_list,
   return the_owner;
 }
 
+JavaThread* Threads::owning_thread_from_object(ThreadsList * t_list, oop obj) {
+  assert(LockingMode == LM_LIGHTWEIGHT, "Only with new lightweight locking");
+  for (JavaThread* q : *t_list) {
+    if (q->lock_stack().contains(obj)) {
+      return q;
+    }
+  }
+  return nullptr;
+}
+
 JavaThread* Threads::owning_thread_from_monitor(ThreadsList* t_list, ObjectMonitor* monitor) {
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    if (monitor->is_owner_anonymous()) {
+      return owning_thread_from_object(t_list, monitor->object());
+    } else {
+      Thread* owner = reinterpret_cast<Thread*>(monitor->owner());
+      assert(owner == nullptr || owner->is_Java_thread(), "only JavaThreads own monitors");
+      return reinterpret_cast<JavaThread*>(owner);
+    }
+  } else {
     address owner = (address)monitor->owner();
     return owning_thread_from_monitor_owner(t_list, owner);
+  }
 }
 
 class PrintOnClosure : public ThreadClosure {

src/hotspot/share/runtime/threads.hpp

@@ -139,6 +139,7 @@ public:
   static JavaThread *owning_thread_from_monitor_owner(ThreadsList * t_list,
                                                       address owner);
 
+  static JavaThread* owning_thread_from_object(ThreadsList* t_list, oop obj);
   static JavaThread* owning_thread_from_monitor(ThreadsList* t_list, ObjectMonitor* owner);
 
   // Number of threads on the active threads list

src/hotspot/share/runtime/vmStructs.cpp

@@ -702,6 +702,9 @@
   nonstatic_field(ThreadShadow,                _exception_line,                               int)                                   \
   nonstatic_field(Thread,                      _tlab,                                         ThreadLocalAllocBuffer)                \
   nonstatic_field(Thread,                      _allocated_bytes,                              jlong)                                 \
+  nonstatic_field(JavaThread,                  _lock_stack,                                   LockStack)                             \
+  nonstatic_field(LockStack,                   _top,                                          uint32_t)                              \
+  nonstatic_field(LockStack,                   _base[0],                                      oop)                                   \
   nonstatic_field(NamedThread,                 _name,                                         char*)                                 \
   nonstatic_field(NamedThread,                 _processed_thread,                             Thread*)                               \
   nonstatic_field(JavaThread,                  _threadObj,                                    OopHandle)                             \
@@ -1317,6 +1320,7 @@
                                                                           \
   declare_toplevel_type(ThreadsSMRSupport)                                \
   declare_toplevel_type(ThreadsList)                                      \
+  declare_toplevel_type(LockStack)                                        \
                                                                           \
   /***************/                                                       \
   /* Interpreter */                                                       \
@@ -2413,6 +2417,14 @@
   declare_constant(T_NARROWKLASS_size)                                    \
   declare_constant(T_VOID_size)                                           \
                                                                           \
+  /**********************************************/                        \
+  /* LockingMode enum (globalDefinitions.hpp) */                          \
+  /**********************************************/                        \
+                                                                          \
+  declare_constant(LM_MONITOR)                                            \
+  declare_constant(LM_LEGACY)                                             \
+  declare_constant(LM_LIGHTWEIGHT)                                        \
+                                                                          \
   /*********************/                                                 \
   /* Matcher (C2 only) */                                                 \
   /*********************/                                                 \
@@ -2597,8 +2609,10 @@
                                                                           \
   /* InvocationCounter constants */                                       \
   declare_constant(InvocationCounter::count_increment)                    \
-  declare_constant(InvocationCounter::count_shift)
+  declare_constant(InvocationCounter::count_shift)                        \
-
+                                                                          \
+  /* ObjectMonitor constants */                                           \
+  declare_constant(ObjectMonitor::ANONYMOUS_OWNER)                        \
 
 //--------------------------------------------------------------------------------
 //

src/hotspot/share/utilities/globalDefinitions.hpp

@@ -1042,6 +1042,15 @@ enum JavaThreadState {
   _thread_max_state         = 12  // maximum thread state+1 - used for statistics allocation
 };
 
+enum LockingMode {
+  // Use only heavy monitors for locking
+  LM_MONITOR     = 0,
+  // Legacy stack-locking, with monitors as 2nd tier
+  LM_LEGACY      = 1,
+  // New lightweight locking, with monitors as 2nd tier
+  LM_LIGHTWEIGHT = 2
+};
+
 //----------------------------------------------------------------------------------------------------
 // Special constants for debugging
 

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/JavaThread.java

@@ -44,6 +44,8 @@ public class JavaThread extends Thread {
   private static final boolean DEBUG = System.getProperty("sun.jvm.hotspot.runtime.JavaThread.DEBUG") != null;
 
   private static long          threadObjFieldOffset;
+  private static long          lockStackTopOffset;
+  private static long          lockStackBaseOffset;
   private static AddressField  anchorField;
   private static AddressField  lastJavaSPField;
   private static AddressField  lastJavaPCField;
@@ -53,6 +55,7 @@ public class JavaThread extends Thread {
   private static CIntegerField stackSizeField;
   private static CIntegerField terminatedField;
   private static AddressField activeHandlesField;
+  private static long oopPtrSize;
 
   private static JavaThreadPDAccess access;
 
@@ -85,6 +88,7 @@ public class JavaThread extends Thread {
   private static synchronized void initialize(TypeDataBase db) {
     Type type = db.lookupType("JavaThread");
     Type anchorType = db.lookupType("JavaFrameAnchor");
+    Type typeLockStack = db.lookupType("LockStack");
 
     threadObjFieldOffset = type.getField("_threadObj").getOffset();
 
@@ -98,6 +102,10 @@ public class JavaThread extends Thread {
     terminatedField   = type.getCIntegerField("_terminated");
     activeHandlesField = type.getAddressField("_active_handles");
 
+    lockStackTopOffset = type.getField("_lock_stack").getOffset() + typeLockStack.getField("_top").getOffset();
+    lockStackBaseOffset = type.getField("_lock_stack").getOffset() + typeLockStack.getField("_base[0]").getOffset();
+    oopPtrSize = VM.getVM().getAddressSize();
+
     UNINITIALIZED     = db.lookupIntConstant("_thread_uninitialized").intValue();
     NEW               = db.lookupIntConstant("_thread_new").intValue();
     NEW_TRANS         = db.lookupIntConstant("_thread_new_trans").intValue();
@@ -394,6 +402,23 @@ public class JavaThread extends Thread {
     return stackBase.greaterThan(a) && sp.lessThanOrEqual(a);
   }
 
+  public boolean isLockOwned(OopHandle obj) {
+    long current = lockStackBaseOffset;
+    long end = addr.getJIntAt(lockStackTopOffset);
+    if (Assert.ASSERTS_ENABLED) {
+      Assert.that(current <= end, "current stack offset must be above base offset");
+    }
+
+    while (current < end) {
+      Address oop = addr.getAddressAt(current);
+      if (oop.equals(obj)) {
+        return true;
+      }
+      current += oopPtrSize;
+    }
+    return false;
+  }
+
   public boolean isLockOwned(Address a) {
     Address stackBase = getStackBase();
     Address stackLimit = stackBase.addOffsetTo(-getStackSize());

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/JavaVFrame.java

@@ -82,6 +82,10 @@ public abstract class JavaVFrame extends VFrame {
     if (mark.hasMonitor() &&
         ( // we have marked ourself as pending on this monitor
           mark.monitor().equals(thread.getCurrentPendingMonitor()) ||
+          // Owned anonymously means that we are not the owner of
+          // the monitor and must be waiting for the owner to
+          // exit it.
+          mark.monitor().isOwnedAnonymous() ||
           // we are not the owner of this monitor
           !mark.monitor().isEntered(thread)
         )) {

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/LockingMode.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+package sun.jvm.hotspot.runtime;
+
+import sun.jvm.hotspot.types.TypeDataBase;
+
+
+/** Encapsulates the LockingMode enum in globalDefinitions.hpp in
+    the VM. */
+
+public class LockingMode {
+  private static int monitor;
+  private static int legacy;
+  private static int lightweight;
+
+  static {
+    VM.registerVMInitializedObserver(
+        (o, d) -> initialize(VM.getVM().getTypeDataBase()));
+  }
+
+  private static synchronized void initialize(TypeDataBase db) {
+    monitor     = db.lookupIntConstant("LM_MONITOR").intValue();
+    legacy      = db.lookupIntConstant("LM_LEGACY").intValue();
+    lightweight = db.lookupIntConstant("LM_LIGHTWEIGHT").intValue();
+  }
+
+  public static int getMonitor() {
+    return monitor;
+  }
+
+  public static int getLegacy() {
+    return legacy;
+  }
+
+  public static int getLightweight() {
+    return lightweight;
+  }
+}

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/ObjectMonitor.java

@@ -44,6 +44,7 @@ public class ObjectMonitor extends VMObject {
   private static synchronized void initialize(TypeDataBase db) throws WrongTypeException {
     heap = VM.getVM().getObjectHeap();
     Type type  = db.lookupType("ObjectMonitor");
+
     sun.jvm.hotspot.types.Field f = type.getField("_header");
     headerFieldOffset = f.getOffset();
     f = type.getField("_object");
@@ -55,6 +56,8 @@ public class ObjectMonitor extends VMObject {
     contentionsField  = new CIntField(type.getCIntegerField("_contentions"), 0);
     waitersField      = new CIntField(type.getCIntegerField("_waiters"), 0);
     recursionsField   = type.getCIntegerField("_recursions");
+
+    ANONYMOUS_OWNER = db.lookupLongConstant("ObjectMonitor::ANONYMOUS_OWNER").longValue();
   }
 
   public ObjectMonitor(Address addr) {
@@ -79,6 +82,10 @@ public class ObjectMonitor extends VMObject {
     return false;
   }
 
+  public boolean isOwnedAnonymous() {
+    return addr.getAddressAt(ownerFieldOffset).asLongValue() == ANONYMOUS_OWNER;
+  }
+
   public Address owner() { return addr.getAddressAt(ownerFieldOffset); }
   // FIXME
   //  void      set_owner(void* owner);
@@ -114,5 +121,7 @@ public class ObjectMonitor extends VMObject {
   private static CIntField     contentionsField;
   private static CIntField     waitersField;
   private static CIntegerField recursionsField;
+  private static long          ANONYMOUS_OWNER;
+
   // FIXME: expose platform-dependent stuff
 }

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/Threads.java

@@ -211,6 +211,7 @@ public class Threads {
 
     // refer to Threads::owning_thread_from_monitor_owner
     public JavaThread owningThreadFromMonitor(Address o) {
+        assert(VM.getVM().getCommandLineFlag("LockingMode").getInt() != LockingMode.getLightweight());
         if (o == null) return null;
         for (int i = 0; i < getNumberOfThreads(); i++) {
             JavaThread thread = getJavaThreadAt(i);
@@ -228,8 +229,25 @@ public class Threads {
     }
 
     public JavaThread owningThreadFromMonitor(ObjectMonitor monitor) {
+        if (VM.getVM().getCommandLineFlag("LockingMode").getInt() == LockingMode.getLightweight()) {
+            if (monitor.isOwnedAnonymous()) {
+                OopHandle object = monitor.object();
+                for (int i = 0; i < getNumberOfThreads(); i++) {
+                    JavaThread thread = getJavaThreadAt(i);
+                    if (thread.isLockOwned(object)) {
+                        return thread;
+                     }
+                }
+                throw new InternalError("We should have found a thread that owns the anonymous lock");
+            }
+            // Owner can only be threads at this point.
+            Address o = monitor.owner();
+            if (o == null) return null;
+            return new JavaThread(o);
+        } else {
             return owningThreadFromMonitor(monitor.owner());
         }
+    }
 
     // refer to Threads::get_pending_threads
     // Get list of Java threads that are waiting to enter the specified monitor.