diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp index e60495d1f47..b470eb9b838 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.cpp +++ b/src/hotspot/share/classfile/vmIntrinsics.cpp @@ -242,6 +242,8 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_Reference_get: case vmIntrinsics::_Continuation_doYield: case vmIntrinsics::_Continuation_enterSpecial: + case vmIntrinsics::_Continuation_pin: + case vmIntrinsics::_Continuation_unpin: break; default: return true; diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 7a5c2ee47bb..4b772c171d5 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -600,6 +600,8 @@ class methodHandle; do_alias(continuationOnPinned_signature, int_void_signature) \ do_intrinsic(_Continuation_doYield, jdk_internal_vm_Continuation, doYield_name, continuationDoYield_signature, F_SN) \ do_alias( continuationDoYield_signature, void_int_signature) \ + do_intrinsic(_Continuation_pin, jdk_internal_vm_Continuation, pin_name, void_method_signature, F_SN) \ + do_intrinsic(_Continuation_unpin, jdk_internal_vm_Continuation, unpin_name, void_method_signature, F_SN) \ \ /* java/lang/VirtualThread */ \ do_intrinsic(_notifyJvmtiVThreadStart, java_lang_VirtualThread, notifyJvmtiStart_name, void_method_signature, F_RN) \ diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp index 8d1ae20eac0..c1ba050b454 100644 --- a/src/hotspot/share/classfile/vmSymbols.hpp +++ b/src/hotspot/share/classfile/vmSymbols.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -406,6 +406,8 @@ class SerializeClosure; template(onContinue_name, "onContinue0") \ template(scope_name, "scope") \ template(yieldInfo_name, "yieldInfo") \ + template(pin_name, "pin") \ + template(unpin_name, "unpin") \ template(tail_name, "tail") \ template(size_name, "size") \ template(bottom_name, "bottom") \ diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp index 688691fb976..1526f81295b 100644 --- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp +++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp @@ -35,6 +35,7 @@ #include "oops/methodCounters.hpp" #include "oops/objArrayKlass.hpp" #include "prims/jvmtiThreadState.hpp" +#include "runtime/continuationEntry.hpp" #include "runtime/deoptimization.hpp" #include "runtime/flags/jvmFlag.hpp" #include "runtime/osThread.hpp" @@ -244,10 +245,12 @@ nonstatic_field(JavaThread, _held_monitor_count, intx) \ nonstatic_field(JavaThread, _lock_stack, LockStack) \ nonstatic_field(JavaThread, _om_cache, OMCache) \ + nonstatic_field(JavaThread, _cont_entry, ContinuationEntry*) \ JVMTI_ONLY(nonstatic_field(JavaThread, _is_in_VTMS_transition, bool)) \ JVMTI_ONLY(nonstatic_field(JavaThread, _is_in_tmp_VTMS_transition, bool)) \ JVMTI_ONLY(nonstatic_field(JavaThread, _is_disable_suspend, bool)) \ \ + nonstatic_field(ContinuationEntry, _pin_count, uint32_t) \ nonstatic_field(LockStack, _top, uint32_t) \ \ JVMTI_ONLY(static_field(JvmtiVTMSTransitionDisabler, _VTMS_notify_jvmti_events, bool)) \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index c5e17478477..2f087858efd 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -731,6 +731,8 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { case vmIntrinsics::_setCurrentThread: case vmIntrinsics::_scopedValueCache: case vmIntrinsics::_setScopedValueCache: + case vmIntrinsics::_Continuation_pin: + case vmIntrinsics::_Continuation_unpin: #ifdef JFR_HAVE_INTRINSICS case vmIntrinsics::_counterTime: case vmIntrinsics::_getEventWriter: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index eced285f8cb..12cd005f93e 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -482,6 +482,9 @@ bool LibraryCallKit::try_to_inline(int predicate) { case vmIntrinsics::_scopedValueCache: return inline_native_scopedValueCache(); case vmIntrinsics::_setScopedValueCache: return inline_native_setScopedValueCache(); + case vmIntrinsics::_Continuation_pin: return inline_native_Continuation_pinning(false); + case vmIntrinsics::_Continuation_unpin: return inline_native_Continuation_pinning(true); + #if INCLUDE_JVMTI case vmIntrinsics::_notifyJvmtiVThreadStart: return inline_native_notify_jvmti_funcs(CAST_FROM_FN_PTR(address, OptoRuntime::notify_jvmti_vthread_start()), "notifyJvmtiStart", true, false); @@ -3715,6 +3718,93 @@ bool LibraryCallKit::inline_native_setScopedValueCache() { return true; } +//------------------------inline_native_Continuation_pin and unpin----------- + +// Shared implementation routine for both pin and unpin. +bool LibraryCallKit::inline_native_Continuation_pinning(bool unpin) { + enum { _true_path = 1, _false_path = 2, PATH_LIMIT }; + + // Save input memory. + Node* input_memory_state = reset_memory(); + set_all_memory(input_memory_state); + + // TLS + Node* tls_ptr = _gvn.transform(new ThreadLocalNode()); + Node* last_continuation_offset = basic_plus_adr(top(), tls_ptr, in_bytes(JavaThread::cont_entry_offset())); + Node* last_continuation = make_load(control(), last_continuation_offset, last_continuation_offset->get_ptr_type(), T_ADDRESS, MemNode::unordered); + + // Null check the last continuation object. + Node* continuation_cmp_null = _gvn.transform(new CmpPNode(last_continuation, null())); + Node* test_continuation_not_equal_null = _gvn.transform(new BoolNode(continuation_cmp_null, BoolTest::ne)); + IfNode* iff_continuation_not_equal_null = create_and_map_if(control(), test_continuation_not_equal_null, PROB_MAX, COUNT_UNKNOWN); + + // False path, last continuation is null. + Node* continuation_is_null = _gvn.transform(new IfFalseNode(iff_continuation_not_equal_null)); + + // True path, last continuation is not null. + Node* continuation_is_not_null = _gvn.transform(new IfTrueNode(iff_continuation_not_equal_null)); + + set_control(continuation_is_not_null); + + // Load the pin count from the last continuation. + Node* pin_count_offset = basic_plus_adr(top(), last_continuation, in_bytes(ContinuationEntry::pin_count_offset())); + Node* pin_count = make_load(control(), pin_count_offset, TypeInt::INT, T_INT, MemNode::unordered); + + // The loaded pin count is compared against a context specific rhs for over/underflow detection. + Node* pin_count_rhs; + if (unpin) { + pin_count_rhs = _gvn.intcon(0); + } else { + pin_count_rhs = _gvn.intcon(UINT32_MAX); + } + Node* pin_count_cmp = _gvn.transform(new CmpUNode(_gvn.transform(pin_count), pin_count_rhs)); + Node* test_pin_count_over_underflow = _gvn.transform(new BoolNode(pin_count_cmp, BoolTest::eq)); + IfNode* iff_pin_count_over_underflow = create_and_map_if(control(), test_pin_count_over_underflow, PROB_MIN, COUNT_UNKNOWN); + + // False branch, no pin count over/underflow. Increment or decrement pin count and store back. + Node* valid_pin_count = _gvn.transform(new IfFalseNode(iff_pin_count_over_underflow)); + set_control(valid_pin_count); + + Node* next_pin_count; + if (unpin) { + next_pin_count = _gvn.transform(new SubINode(pin_count, _gvn.intcon(1))); + } else { + next_pin_count = _gvn.transform(new AddINode(pin_count, _gvn.intcon(1))); + } + + Node* updated_pin_count_memory = store_to_memory(control(), pin_count_offset, next_pin_count, T_INT, Compile::AliasIdxRaw, MemNode::unordered); + + // True branch, pin count over/underflow. + Node* pin_count_over_underflow = _gvn.transform(new IfTrueNode(iff_pin_count_over_underflow)); + { + // Trap (but not deoptimize (Action_none)) and continue in the interpreter + // which will throw IllegalStateException for pin count over/underflow. + PreserveJVMState pjvms(this); + set_control(pin_count_over_underflow); + set_all_memory(input_memory_state); + uncommon_trap_exact(Deoptimization::Reason_intrinsic, + Deoptimization::Action_none); + assert(stopped(), "invariant"); + } + + // Result of top level CFG and Memory. + RegionNode* result_rgn = new RegionNode(PATH_LIMIT); + record_for_igvn(result_rgn); + PhiNode* result_mem = new PhiNode(result_rgn, Type::MEMORY, TypePtr::BOTTOM); + record_for_igvn(result_mem); + + result_rgn->init_req(_true_path, _gvn.transform(valid_pin_count)); + result_rgn->init_req(_false_path, _gvn.transform(continuation_is_null)); + result_mem->init_req(_true_path, _gvn.transform(updated_pin_count_memory)); + result_mem->init_req(_false_path, _gvn.transform(input_memory_state)); + + // Set output state. + set_control(_gvn.transform(result_rgn)); + set_all_memory(_gvn.transform(result_mem)); + + return true; +} + //---------------------------load_mirror_from_klass---------------------------- // Given a klass oop, load its java mirror (a java.lang.Class oop). Node* LibraryCallKit::load_mirror_from_klass(Node* klass) { diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 313e8c39544..4ae9d0216e9 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -241,6 +241,7 @@ class LibraryCallKit : public GraphKit { const Type* scopedValueCache_type(); Node* scopedValueCache_helper(); bool inline_native_setScopedValueCache(); + bool inline_native_Continuation_pinning(bool unpin); bool inline_native_time_funcs(address method, const char* funcName); #if INCLUDE_JVMTI diff --git a/src/hotspot/share/runtime/continuationEntry.hpp b/src/hotspot/share/runtime/continuationEntry.hpp index 5930b008d27..ac76cd6f088 100644 --- a/src/hotspot/share/runtime/continuationEntry.hpp +++ b/src/hotspot/share/runtime/continuationEntry.hpp @@ -39,6 +39,7 @@ class RegisterMap; // Metadata stored in the continuation entry frame class ContinuationEntry { + friend class JVMCIVMStructs; ContinuationEntryPD _pd; #ifdef ASSERT private: @@ -78,7 +79,7 @@ private: #else int32_t _parent_held_monitor_count; #endif - uint _pin_count; + uint32_t _pin_count; public: static ByteSize parent_offset() { return byte_offset_of(ContinuationEntry, _parent); } @@ -108,7 +109,7 @@ public: bool is_pinned() { return _pin_count > 0; } bool pin() { - if (_pin_count == UINT_MAX) return false; + if (_pin_count == UINT32_MAX) return false; _pin_count++; return true; } diff --git a/src/java.base/share/classes/jdk/internal/vm/Continuation.java b/src/java.base/share/classes/jdk/internal/vm/Continuation.java index 5c1d41c36d9..99d0c62aaec 100644 --- a/src/java.base/share/classes/jdk/internal/vm/Continuation.java +++ b/src/java.base/share/classes/jdk/internal/vm/Continuation.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -427,6 +427,7 @@ public class Continuation { * Pins the current continuation (enters a critical section). * This increments an internal semaphore that, when greater than 0, pins the continuation. */ + @IntrinsicCandidate public static native void pin(); /** @@ -434,6 +435,7 @@ public class Continuation { * This decrements an internal semaphore that, when equal 0, unpins the current continuation * if pinned with {@link #pin()}. */ + @IntrinsicCandidate public static native void unpin(); /**