8187222: ClassLoader.getSystemClassLoader not clear if recursive initialization leads to ISE or unspecified error

Reviewed-by: alanb, mchung

src/java.base/share/classes/java/lang/ClassLoader.java

@@ -30,6 +30,7 @@ import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.io.File;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.net.URL;
 import java.security.AccessController;
 import java.security.AccessControlContext;
@@ -1867,7 +1868,7 @@ public abstract class ClassLoader {
      * to be the system class loader. During construction, the class loader
      * should take great care to avoid calling {@code getSystemClassLoader()}.
      * If circular initialization of the system class loader is detected then
-     * an unspecified error or exception is thrown.
+     * an {@code IllegalStateException} is thrown.
      *
      * @implNote The system property to override the system class loader is not
      * examined until the VM is almost fully initialized. Code that executes
@@ -1918,8 +1919,8 @@ public abstract class ClassLoader {
                 // the system class loader is the built-in app class loader during startup
                 return getBuiltinAppClassLoader();
             case 3:
-                String msg = "getSystemClassLoader should only be called after VM booted";
-                throw new InternalError(msg);
+                String msg = "getSystemClassLoader cannot be called during the system class loader instantiation";
+                throw new IllegalStateException(msg);
             case 4:
                 // system fully initialized
                 assert VM.isBooted() && scl != null;
@@ -1969,7 +1970,17 @@ public abstract class ClassLoader {
                                            .getDeclaredConstructor(ClassLoader.class);
                 scl = (ClassLoader) ctor.newInstance(builtinLoader);
             } catch (Exception e) {
-                throw new Error(e);
+                Throwable cause = e;
+                if (e instanceof InvocationTargetException) {
+                    cause = e.getCause();
+                    if (cause instanceof Error) {
+                        throw (Error) cause;
+                    }
+                }
+                if (cause instanceof RuntimeException) {
+                    throw (RuntimeException) cause;
+                }
+                throw new Error(cause.getMessage(), cause);
             }
         } else {
             scl = builtinLoader;

test/jdk/java/lang/ClassLoader/RecursiveSystemLoader.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8187222
+ * @run main/othervm -Djava.system.class.loader=RecursiveSystemLoader RecursiveSystemLoader
+ * @summary Test for IllegalStateException if a custom system loader recursively calls getSystemClassLoader()
+ */
+public class RecursiveSystemLoader extends ClassLoader {
+    public static void main(String[] args) {
+        ClassLoader sys = ClassLoader.getSystemClassLoader();
+        if (!(sys instanceof RecursiveSystemLoader)) {
+            throw new RuntimeException("Unexpected system classloader: " + sys);
+        }
+    }
+    public RecursiveSystemLoader(ClassLoader classLoader) {
+        super("RecursiveSystemLoader", classLoader);
+
+        // Calling ClassLoader.getSystemClassLoader() before the VM is booted
+        // should throw an IllegalStateException.
+        try {
+            ClassLoader.getSystemClassLoader();
+        } catch(IllegalStateException ise) {
+            System.err.println("Caught expected exception:");
+            ise.printStackTrace();
+            return;
+        }
+        throw new RuntimeException("Expected IllegalStateException was not thrown.");
+    }
+
+    @Override
+    public Class<?> loadClass(String name) throws ClassNotFoundException {
+        return super.loadClass(name);
+    }
+}