8267347: CDS record_linking_constraint asserts with unregistered class

Reviewed-by: minqi, ccheung

src/hotspot/share/classfile/systemDictionaryShared.cpp

@@ -1855,8 +1855,6 @@ void SystemDictionaryShared::record_linking_constraint(Symbol* name, InstanceKla
   // either of these two loaders. The check itself does not
   // try to resolve T.
   oop klass_loader = klass->class_loader();
-  assert(klass_loader != NULL, "should not be called for boot loader");
-  assert(loader1 != loader2, "must be");
 
   if (!is_system_class_loader(klass_loader) &&
       !is_platform_class_loader(klass_loader)) {
@@ -1872,6 +1870,17 @@ void SystemDictionaryShared::record_linking_constraint(Symbol* name, InstanceKla
     return;
   }
 
+  if (DumpSharedSpaces && !is_builtin(klass)) {
+    // During static dump, unregistered classes (those intended for
+    // custom loaders) are loaded by the boot loader. Need to
+    // exclude these for the same reason as above.
+    // This should be fixed by JDK-8261941.
+    return;
+  }
+
+  assert(klass_loader != NULL, "should not be called for boot loader");
+  assert(loader1 != loader2, "must be");
+
   if (DynamicDumpSharedSpaces && Thread::current()->is_VM_thread()) {
     // We are re-laying out the vtable/itables of the *copy* of
     // a class during the final stage of dynamic dumping. The

test/hotspot/jtreg/runtime/cds/appcds/loaderConstraints/DynamicLoaderConstraintsTest.java

@@ -25,6 +25,7 @@
  * @test
  * @requires vm.cds
  * @summary Test class loader constraint checks for archived classes (dynamic archive)
+ * @bug 8267347
  * @library /test/lib
  *          /test/hotspot/jtreg/runtime/cds/appcds
  *          /test/hotspot/jtreg/runtime/cds/appcds/test-classes
@@ -40,6 +41,7 @@ import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 import jdk.test.lib.Asserts;
 import jdk.test.lib.helpers.ClassFileInstaller;
+import jdk.test.lib.Platform;
 
 public class DynamicLoaderConstraintsTest extends DynamicArchiveTestBase {
     static String mainClass = LoaderConstraintsApp.class.getName();
@@ -55,14 +57,27 @@ public class DynamicLoaderConstraintsTest extends DynamicArchiveTestBase {
         MyClassLoader.class.getName()
     };
 
+    static String loaderMainClass = CustomAppLoader.class.getName();
+    static String loaderJar = null;
+    static String loaderClasses[] = {
+        loaderMainClass
+    };
+
     public static void main(String[] args) throws Exception {
         runTest(DynamicLoaderConstraintsTest::doTest);
     }
 
     static void doTest() throws Exception  {
         appJar = ClassFileInstaller.writeJar("loader_constraints.jar", appClasses);
-        doTest(false);
+        doTest(false, false);
-        doTest(true);
+        doTest(true,  false);
+
+        if (!Platform.isWindows()) {
+            // custom loaders are not supported on Windows yet.
+            loaderJar = ClassFileInstaller.writeJar("custom_app_loader.jar", loaderClasses);
+            doTest(false, true);
+            doTest(true,  true);
+        }
     }
 
     /*
@@ -74,21 +89,34 @@ public class DynamicLoaderConstraintsTest extends DynamicArchiveTestBase {
      *        causing LinkageError. This ensures the test classes will be
      *        archived so we can test CDS's handling of loader constraints during
      *        run time.
+     *
+     * useCustomLoader: if true, load the LoaderConstraintsApp in a custom loader before executing it.
+     *                  if false, LoaderConstraintsApp will be loaded by the built-in AppClassLoader.
      */
-    static void doTest(boolean errorInDump) throws Exception  {
+  static void doTest(boolean errorInDump, boolean useCustomLoader) throws Exception  {
         for (int i = 1; i <= 3; i++) {
+            System.out.println("========================================");
+            System.out.println("errorInDump: " + errorInDump + ", useCustomLoader: " + useCustomLoader + ", case: " + i);
+            System.out.println("========================================");
             String topArchiveName = getNewArchiveName();
             String testCase = Integer.toString(i);
             String cmdLine[] = new String[] {
-                "-cp", appJar,
                 "--add-modules",
                 "java.base,jdk.httpserver",
                 "--add-exports",
                 "java.base/jdk.internal.misc=ALL-UNNAMED",
                 "-Xlog:class+load,class+loader+constraints",
-                mainClass, testCase
             };
 
+            if (useCustomLoader) {
+                cmdLine = TestCommon.concat(cmdLine, "-cp", loaderJar,
+                                          loaderMainClass, appJar);
+            } else {
+                cmdLine = TestCommon.concat(cmdLine, "-cp", appJar);
+            }
+
+            cmdLine = TestCommon.concat(cmdLine, mainClass, testCase);
+
             String[] dumpCmdLine = cmdLine;
             if (!errorInDump) {
                 dumpCmdLine = TestCommon.concat(dumpCmdLine, "loadClassOnly");

test/hotspot/jtreg/runtime/cds/appcds/loaderConstraints/LoaderConstraintsApp.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -36,7 +36,7 @@ import java.net.URL;
 //        then try to define HttpExchange in child (app) loader => fail.
 //     3. no LinkageError should be throw when linking a class that does not override/implement any
 //        methods.
-class LoaderConstraintsApp {
+public class LoaderConstraintsApp {
     static void defineHttpExchangeWithAppLoader() throws Exception {
         Unsafe unsafe = Unsafe.getUnsafe();
         URL url = new URL("jrt://modules/jdk.httpserver/com/sun/net/httpserver/HttpExchange.class");

test/hotspot/jtreg/runtime/cds/appcds/loaderConstraints/LoaderConstraintsTest.java

@@ -25,6 +25,7 @@
  * @test
  * @requires vm.cds
  * @summary Test class loader constraint checks for archived classes
+ * @bug 8267347
  * @library /test/lib
  *          /test/hotspot/jtreg/runtime/cds/appcds
  *          /test/hotspot/jtreg/runtime/cds/appcds/test-classes
@@ -37,14 +38,17 @@ import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 import jdk.test.lib.Asserts;
 import jdk.test.lib.helpers.ClassFileInstaller;
+import jdk.test.lib.Platform;
 
 public class LoaderConstraintsTest  {
     static String mainClass = LoaderConstraintsApp.class.getName();
+    static String httpHandlerClass = HttpHandler.class.getName().replace(".", "/");
+    static String httpExchangeClass = HttpExchange.class.getName().replace(".", "/");
     static String appJar = null;
     static String appClasses[] = {
         mainClass,
-        HttpHandler.class.getName(),
+        httpHandlerClass,
-        HttpExchange.class.getName(),
+        httpExchangeClass,
         Asserts.class.getName(),
         MyHttpHandler.class.getName(),
         MyHttpHandlerB.class.getName(),
@@ -52,18 +56,50 @@ public class LoaderConstraintsTest  {
         MyClassLoader.class.getName()
     };
 
+    static String loaderMainClass = CustomAppLoader.class.getName();
+    static String loaderJar = null;
+    static String loaderClasses[] = {
+        loaderMainClass
+    };
+
     static void doTest() throws Exception  {
-        appJar = ClassFileInstaller.writeJar("loader_constraints.jar", appClasses);
+        TestCommon.dump(appJar, appClasses, "-Xlog:cds");
-        TestCommon.dump(appJar, appClasses, "-Xlog:cds+load");
+        String cmdLine[] =
-        String joptsMain[] = TestCommon.concat("-cp", appJar,
+            TestCommon.concat("-cp", appJar,
-                                          "-Xlog:cds",
+                              "-Xlog:cds",
-                                          "-Xlog:class+loader+constraints=debug",
+                              "-Xlog:class+loader+constraints=debug",
-                                          "--add-exports",
+                              "--add-exports",
-                                          "java.base/jdk.internal.misc=ALL-UNNAMED",
+                              "java.base/jdk.internal.misc=ALL-UNNAMED",
-                                          mainClass);
+                              mainClass);
-        runWithArchive(joptsMain, "1");
+        runWithArchive(cmdLine, "1");
-        runWithArchive(joptsMain, "2");
+        runWithArchive(cmdLine, "2");
-        runWithArchive(joptsMain, "3");
+        runWithArchive(cmdLine, "3");
+    }
+
+    // Same as doTest, except that LoaderConstraintsApp and MyHttpHandler* are loaded
+    // by a custom loader. This is test case for JDK-8267347.
+    static void doTestCustomLoader() throws Exception  {
+        String src = " source: " + appJar;
+        String classList[] =
+            TestCommon.concat(loaderClasses,
+                              "java/lang/Object id: 1",
+                              mainClass + " id: 2 super: 1" + src,
+                              httpHandlerClass + " id: 3",
+                              "MyHttpHandler id: 5 super: 1 interfaces: 3" + src,
+                              "MyHttpHandlerB id: 6 super: 1 interfaces: 3" + src,
+                              "MyHttpHandlerC id: 7 super: 1 interfaces: 3" + src);
+        TestCommon.dump(loaderJar, classList, "-Xlog:cds");
+
+        String cmdLine[] =
+            TestCommon.concat("-cp", loaderJar,
+                              "-Xlog:cds",
+                              "-Xlog:class+loader+constraints=debug",
+                              "--add-exports",
+                              "java.base/jdk.internal.misc=ALL-UNNAMED",
+                              loaderMainClass, appJar, mainClass);
+        runWithArchive(cmdLine, "1");
+        runWithArchive(cmdLine, "2");
+        runWithArchive(cmdLine, "3");
     }
 
     static void runWithArchive(String[] optsMain, String arg) throws Exception {
@@ -72,7 +108,13 @@ public class LoaderConstraintsTest  {
     }
 
     public static void main(String... args) throws Exception {
+        appJar = ClassFileInstaller.writeJar("loader_constraints.jar", appClasses);
         doTest();
+        if (!Platform.isWindows()) {
+            // custom loaders are not supported on Windows yet.
+            loaderJar = ClassFileInstaller.writeJar("custom_app_loader.jar", loaderClasses);
+            doTestCustomLoader();
+        }
     }
 }
 

test/hotspot/jtreg/runtime/cds/appcds/test-classes/CustomAppLoader.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+
+// This is a handy class for running an application inside a custom class loader. This
+// is used for testing CDS handling of unregistered classes (i.e., archived classes loaded
+// by custom class loaders).
+//
+// See test/hotspot/jtreg/runtime/cds/appcds/loaderConstraints/LoaderConstraintsTest.java
+// for an example.
+public class CustomAppLoader {
+    // args[0] = App JAR file
+    // args[1] = App main class
+    // args[2...] = arguments for the main class
+    public static void main(String args[]) throws Throwable {
+        File f = new File(args[0]);
+        URL[] classLoaderUrls = new URL[] {new URL("file://" + f.getCanonicalPath())};
+        URLClassLoader loader = new URLClassLoader(classLoaderUrls, CustomAppLoader.class.getClassLoader());
+        Class k = Class.forName(args[1], true, loader);
+        Class parameterTypes[] = new Class[] {String[].class};
+        Method mainMethod = k.getDeclaredMethod("main", parameterTypes);
+        String appArgs[] = new String[args.length - 2];
+        Object invokeArgs[] = new Object[] {appArgs};
+        for (int i = 0; i < appArgs.length; i++) {
+            appArgs[i] = args[i + 2];
+        }
+        mainMethod.invoke(null, invokeArgs);
+    }
+}