8335553: [Graal] Compiler thread calls into jdk.internal.vm.VMSupport.decodeAndThrowThrowable and crashes in OOM situation

Reviewed-by: yzheng, never, dholmes

src/java.base/share/classes/jdk/internal/vm/TranslatedException.java

@@ -42,9 +42,13 @@ import java.util.zip.GZIPOutputStream;
 
 /**
  * Support for translating exceptions between the HotSpot heap and libjvmci heap.
+ *
+ * Successfully translated exceptions are wrapped in a TranslatedException instance.
+ * This allows callers to distiguish between a translated exception and an error
+ * that arose during translation.
  */
 @SuppressWarnings("serial")
-final class TranslatedException extends Exception {
+public final class TranslatedException extends Exception {
 
     /**
      * The value returned by {@link #encodeThrowable(Throwable)} when encoding
@@ -61,15 +65,18 @@ final class TranslatedException extends Exception {
         maybeFailClinit();
         try {
             FALLBACK_ENCODED_THROWABLE_BYTES =
-                encodeThrowable(new TranslatedException("error during encoding",
-                                                        "<unknown>"), false);
+                encodeThrowable(translationFailure("error during encoding"), false);
             FALLBACK_ENCODED_OUTOFMEMORYERROR_BYTES =
-                encodeThrowable(new OutOfMemoryError(), false);
+                encodeThrowable(translationFailure("OutOfMemoryError during encoding"), false);
         } catch (IOException e) {
             throw new InternalError(e);
         }
     }
 
+    private static InternalError translationFailure(String messageFormat, Object... messageArgs) {
+        return new InternalError(messageFormat.formatted(messageArgs));
+    }
+
     /**
      * Helper to test exception translation.
      */
@@ -86,14 +93,8 @@ final class TranslatedException extends Exception {
         }
     }
 
-    /**
-     * Class name of exception that could not be instantiated.
-     */
-    private String originalExceptionClassName;
-
-    private TranslatedException(String message, String originalExceptionClassName) {
-        super(message);
-        this.originalExceptionClassName = originalExceptionClassName;
+    TranslatedException(Throwable translated) {
+        super(translated);
     }
 
     /**
@@ -106,18 +107,6 @@ final class TranslatedException extends Exception {
         return this;
     }
 
-    @Override
-    public String toString() {
-        String s;
-        if (originalExceptionClassName.equals(TranslatedException.class.getName())) {
-            s = getClass().getName();
-        } else {
-            s = getClass().getName() + "[" + originalExceptionClassName + "]";
-        }
-        String message = getMessage();
-        return (message != null) ? (s + ": " + message) : s;
-    }
-
     /**
      * Prints a stack trace for {@code throwable} if the system property
      * {@code "jdk.internal.vm.TranslatedException.debug"} is true.
@@ -163,7 +152,7 @@ final class TranslatedException extends Exception {
             return initCause((Throwable) cons.newInstance(message), cause, debug);
         } catch (Throwable translationFailure) {
             debugPrintStackTrace(translationFailure, debug);
-            return initCause(new TranslatedException(message, className), cause, debug);
+            return initCause(translationFailure("%s [%s]", message, className), cause, debug);
         }
     }
 
@@ -308,11 +297,10 @@ final class TranslatedException extends Exception {
                 throwable.setStackTrace(stackTrace);
                 cause = throwable;
             }
-            return throwable;
+            return new TranslatedException(throwable);
         } catch (Throwable translationFailure) {
             debugPrintStackTrace(translationFailure, debug);
-            return new TranslatedException("Error decoding exception: " + encodedThrowable,
-                                           translationFailure.getClass().getName());
+            return translationFailure("error decoding exception: %s", encodedThrowable);
         }
     }
 }

src/java.base/share/classes/jdk/internal/vm/VMSupport.java

@@ -122,6 +122,8 @@ public class VMSupport {
      *             2: an OutOfMemoryError was thrown while encoding the exception
      *             3: some other problem occured while encoding the exception. If {@code buffer != 0},
      *                it contains a {@code struct { u4 len; char[len] desc}} where {@code desc} describes the problem
+     *             4: an OutOfMemoryError thrown from within VM code on a
+     *                thread that cannot call Java (OOME has no stack trace)
      *            </pre>
      * @param buffer encoded info about the exception to throw (depends on {@code format})
      * @param inJVMHeap [@code true} if executing in the JVM heap, {@code false} otherwise
@@ -129,13 +131,16 @@ public class VMSupport {
      */
     public static void decodeAndThrowThrowable(int format, long buffer, boolean inJVMHeap, boolean debug) throws Throwable {
         if (format != 0) {
+            if (format == 4) {
+                throw new TranslatedException(new OutOfMemoryError("in VM code and current thread cannot call Java"));
+            }
             String context = String.format("while encoding an exception to translate it %s the JVM heap",
                     inJVMHeap ? "to" : "from");
             if (format == 1) {
                 throw new InternalError("native buffer could not be allocated " + context);
             }
             if (format == 2) {
-                throw new OutOfMemoryError("OutOfMemoryError occurred " + context);
+                throw new OutOfMemoryError(context);
             }
             if (format == 3 && buffer != 0L) {
                 byte[] bytes = bufferToBytes(buffer);