8292769: [JVMCI] OutOfMemoryError thrown when attaching the libgraal isolate causes HotSpot to crash.

Reviewed-by: dnsimon, never

src/hotspot/share/compiler/compileBroker.cpp

@@ -2172,18 +2172,21 @@ void CompileBroker::invoke_compiler_on_method(CompileTask* task) {
       compilable = ciEnv::MethodCompilable_never;
     } else {
       JVMCIEnv env(thread, &compile_state, __FILE__, __LINE__);
-      methodHandle method(thread, target_handle);
-      runtime = env.runtime();
-      runtime->compile_method(&env, jvmci, method, osr_bci);
-
       failure_reason = compile_state.failure_reason();
-      failure_reason_on_C_heap = compile_state.failure_reason_on_C_heap();
+      if (failure_reason == nullptr) {
-      if (!compile_state.retryable()) {
+        methodHandle method(thread, target_handle);
-        retry_message = "not retryable";
+        runtime = env.runtime();
-        compilable = ciEnv::MethodCompilable_not_at_tier;
+        runtime->compile_method(&env, jvmci, method, osr_bci);
-      }
+
-      if (!task->is_success()) {
+        failure_reason = compile_state.failure_reason();
-        assert(failure_reason != NULL, "must specify failure_reason");
+        failure_reason_on_C_heap = compile_state.failure_reason_on_C_heap();
+        if (!compile_state.retryable()) {
+          retry_message = "not retryable";
+          compilable = ciEnv::MethodCompilable_not_at_tier;
+        }
+        if (!task->is_success()) {
+          assert(failure_reason != NULL, "must specify failure_reason");
+        }
       }
     }
     if (!task->is_success()) {

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -156,7 +156,7 @@ void JVMCIEnv::copy_saved_properties() {
   }
 }
 
-void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env) {
+void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool attach_OOME_is_fatal) {
   assert(thread != NULL, "npe");
   _env = NULL;
   _pop_frame_on_close = false;
@@ -208,10 +208,16 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env) {
       attach_args.version = JNI_VERSION_1_2;
       attach_args.name = const_cast<char*>(thread->name());
       attach_args.group = NULL;
-      if (_runtime->AttachCurrentThread(thread, (void**) &_env, &attach_args) != JNI_OK) {
+      jint attach_result = _runtime->AttachCurrentThread(thread, (void**) &_env, &attach_args);
+      if (attach_result == JNI_OK) {
+        _detach_on_close = true;
+      } else if (!attach_OOME_is_fatal && attach_result == JNI_ENOMEM) {
+        _env = NULL;
+        _attach_threw_OOME = true;
+        return;
+      } else {
         fatal("Error attaching current thread (%s) to JVMCI shared library JNI interface", attach_args.name);
       }
-      _detach_on_close = true;
     }
   }
 
@@ -229,17 +235,22 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env) {
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, JVMCICompileState* compile_state, const char* file, int line):
-    _throw_to_caller(false), _file(file), _line(line), _compile_state(compile_state) {
+    _throw_to_caller(false), _file(file), _line(line), _attach_threw_OOME(false), _compile_state(compile_state) {
-  init_env_mode_runtime(thread, NULL);
+  // In case of OOME, there's a good chance a subsequent attempt to attach might succeed.
+  // Other errors most likely indicate a non-recoverable error in the JVMCI runtime.
+  init_env_mode_runtime(thread, NULL, false);
+  if (_attach_threw_OOME) {
+    compile_state->set_failure(true, "Out of memory while attaching JVMCI compiler to current thread");
+  }
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, const char* file, int line):
-    _throw_to_caller(false), _file(file), _line(line), _compile_state(NULL) {
+    _throw_to_caller(false), _file(file), _line(line), _attach_threw_OOME(false), _compile_state(NULL) {
   init_env_mode_runtime(thread, NULL);
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, JNIEnv* parent_env, const char* file, int line):
-    _throw_to_caller(true), _file(file), _line(line), _compile_state(NULL) {
+    _throw_to_caller(true), _file(file), _line(line), _attach_threw_OOME(false), _compile_state(NULL) {
   init_env_mode_runtime(thread, parent_env);
   assert(_env == NULL || parent_env == _env, "mismatched JNIEnvironment");
 }
@@ -249,6 +260,7 @@ void JVMCIEnv::init(JavaThread* thread, bool is_hotspot, const char* file, int l
   _throw_to_caller = false;
   _file = file;
   _line = line;
+  _attach_threw_OOME = false;
   if (is_hotspot) {
     _env = NULL;
     _pop_frame_on_close = false;
@@ -415,6 +427,9 @@ jboolean JVMCIEnv::transfer_pending_exception(JavaThread* THREAD, JVMCIEnv* peer
 
 
 JVMCIEnv::~JVMCIEnv() {
+  if (_attach_threw_OOME) {
+    return;
+  }
   if (_throw_to_caller) {
     if (is_hotspot()) {
       // Nothing to do

src/hotspot/share/jvmci/jvmciEnv.hpp

@@ -156,7 +156,7 @@ class JVMCIEnv : public ResourceObj {
   friend class JNIAccessMark;
 
   // Initializes the _env, _mode and _runtime fields.
-  void init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env);
+  void init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool attach_OOME_is_fatal = true);
 
   void init(JavaThread* thread, bool is_hotspot, const char* file, int line);
 
@@ -168,6 +168,7 @@ class JVMCIEnv : public ResourceObj {
   bool        _throw_to_caller;  // Propagate an exception raised in this env to the caller?
   const char*            _file;  // The file and ...
   int                    _line;  // ... line where this JNIEnv was created
+  bool    _attach_threw_OOME;    // Failed to attach thread due to OutOfMemoryError, the JVMCIEnv is invalid
 
   // Translates an exception on the HotSpot heap (i.e., hotspot_env) to an exception on
   // the shared library heap (i.e., jni_env). The translation includes the stack and cause(s) of `throwable`.