8210832: Remove sneaky locking in class Monitor

Removed sneaky locking and simplified vm monitors implementation

Co-authored-by: David Holmes <david.holmes@oracle.com>
Reviewed-by: rehn, dcubed, pliden, dholmes, coleenp

src/hotspot/share/runtime/interfaceSupport.inline.hpp

@@ -286,6 +286,69 @@ class ThreadBlockInVM : public ThreadStateTransition {
   }
 };
 
+// Unlike ThreadBlockInVM, this class is designed to avoid certain deadlock scenarios while making
+// transitions inside class Monitor in cases where we need to block for a safepoint or handshake. It
+// receives an extra argument compared to ThreadBlockInVM, the address of a pointer to the monitor we
+// are trying to acquire. This will be used to access and release the monitor if needed to avoid
+// said deadlocks.
+// It works like ThreadBlockInVM but differs from it in two ways:
+// - When transitioning in (constructor), it checks for safepoints without blocking, i.e., calls
+//   back if needed to allow a pending safepoint to continue but does not block in it.
+// - When transitioning back (destructor), if there is a pending safepoint or handshake it releases
+//   the monitor that is only partially acquired.
+class ThreadBlockInVMWithDeadlockCheck : public ThreadStateTransition {
+ private:
+  Monitor** _in_flight_monitor_adr;
+
+  void release_monitor() {
+    assert(_in_flight_monitor_adr != NULL, "_in_flight_monitor_adr should have been set on constructor");
+    Monitor* in_flight_monitor = *_in_flight_monitor_adr;
+    if (in_flight_monitor != NULL) {
+      in_flight_monitor->release_for_safepoint();
+      *_in_flight_monitor_adr = NULL;
+    }
+  }
+ public:
+  ThreadBlockInVMWithDeadlockCheck(JavaThread* thread, Monitor** in_flight_monitor_adr)
+  : ThreadStateTransition(thread), _in_flight_monitor_adr(in_flight_monitor_adr) {
+    // Once we are blocked vm expects stack to be walkable
+    thread->frame_anchor()->make_walkable(thread);
+
+    thread->set_thread_state((JavaThreadState)(_thread_in_vm + 1));
+    InterfaceSupport::serialize_thread_state_with_handler(thread);
+
+    SafepointMechanism::callback_if_safepoint(thread);
+
+    thread->set_thread_state(_thread_blocked);
+
+    CHECK_UNHANDLED_OOPS_ONLY(_thread->clear_unhandled_oops();)
+  }
+  ~ThreadBlockInVMWithDeadlockCheck() {
+    // Change to transition state
+    _thread->set_thread_state((JavaThreadState)(_thread_blocked + 1));
+
+    InterfaceSupport::serialize_thread_state_with_handler(_thread);
+
+    if (SafepointMechanism::should_block(_thread)) {
+      release_monitor();
+      SafepointMechanism::callback_if_safepoint(_thread);
+      // The VMThread might have read that we were in a _thread_blocked state
+      // and proceeded to process a handshake for us. If that's the case then
+      // we need to block.
+      // By doing this we are also making the current thread process its own
+      // handshake if there is one pending and the VMThread didn't try to process
+      // it yet. This is more of a side-effect and not really necessary; the
+      // handshake could be processed later on.
+      if (_thread->has_handshake()) {
+        _thread->handshake_process_by_self();
+      }
+    }
+
+    _thread->set_thread_state(_thread_in_vm);
+    CHECK_UNHANDLED_OOPS_ONLY(_thread->clear_unhandled_oops();)
+  }
+};
+
 
 // This special transition class is only used to prevent asynchronous exceptions
 // from being installed on vm exit in situations where we can't tolerate them.