8231584: Deadlock with ClassLoader.findLibrary and System.loadLibrary call

Reviewed-by: mchung

src/java.base/share/classes/java/lang/ClassLoader.java

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2004,6 +2005,17 @@ public abstract class ClassLoader {
         return scl;
     }
 
+    /*
+     * Initialize default paths for native libraries search.
+     * Must be done early as JDK may load libraries during bootstrap.
+     *
+     * @see java.lang.System#initPhase1
+     */
+    static void initLibraryPaths() {
+        usr_paths = initializePath("java.library.path");
+        sys_paths = initializePath("sun.boot.library.path");
+    }
+
     // Returns true if the specified class loader can be found in this class
     // loader's delegation chain.
     boolean isAncestor(ClassLoader cl) {
@@ -2473,8 +2485,7 @@ public abstract class ClassLoader {
                  *
                  * We use a static stack to hold the list of libraries we are
                  * loading because this can happen only when called by the
-                 * same thread because Runtime.load and Runtime.loadLibrary
-                 * are synchronous.
+                 * same thread because this block is synchronous.
                  *
                  * If there is a pending load operation for the library, we
                  * immediately return success; otherwise, we raise
@@ -2619,10 +2630,9 @@ public abstract class ClassLoader {
                             boolean isAbsolute) {
         ClassLoader loader =
             (fromClass == null) ? null : fromClass.getClassLoader();
-        if (sys_paths == null) {
-            usr_paths = initializePath("java.library.path");
-            sys_paths = initializePath("sun.boot.library.path");
-        }
+        assert sys_paths != null : "should be initialized at this point";
+        assert usr_paths != null : "should be initialized at this point";
+
         if (isAbsolute) {
             if (loadLibrary0(fromClass, new File(name))) {
                 return;

src/java.base/share/classes/java/lang/Runtime.java

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 1995, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -732,7 +733,7 @@ public class Runtime {
         load0(Reflection.getCallerClass(), filename);
     }
 
-    synchronized void load0(Class<?> fromClass, String filename) {
+    void load0(Class<?> fromClass, String filename) {
         SecurityManager security = System.getSecurityManager();
         if (security != null) {
             security.checkLink(filename);
@@ -794,7 +795,7 @@ public class Runtime {
         loadLibrary0(Reflection.getCallerClass(), libname);
     }
 
-    synchronized void loadLibrary0(Class<?> fromClass, String libname) {
+    void loadLibrary0(Class<?> fromClass, String libname) {
         SecurityManager security = System.getSecurityManager();
         if (security != null) {
             security.checkLink(libname);

src/java.base/share/classes/java/lang/System.java

@@ -2045,6 +2045,8 @@ public final class System {
         // register shared secrets
         setJavaLangAccess();
 
+        ClassLoader.initLibraryPaths();
+
         // Subsystems that are invoked during initialization can invoke
         // VM.isBooted() in order to avoid doing things that should
         // wait until the VM is fully initialized. The initialization level

test/jdk/java/lang/Runtime/loadLibrary/LoadLibraryTest.java

@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2018, Amazon and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8231584
+ * @library /test/lib
+ * @run main/othervm LoadLibraryTest
+ */
+
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.Path;
+import java.net.MalformedURLException;
+import java.net.URLClassLoader;
+import java.net.URL;
+
+import jdk.test.lib.compiler.CompilerUtils;
+
+public class LoadLibraryTest {
+    static Thread thread1 = null;
+    static Thread thread2 = null;
+
+    static volatile boolean thread1Ready = false;
+
+    private static final String TEST_SRC = System.getProperty("test.src");
+    private static final Path SRC_DIR = Paths.get(TEST_SRC, "src");
+    private static final Path CLS_DIR = Paths.get("classes");
+
+    static TestClassLoader loader;
+    static void someLibLoad() {
+        try {
+/*
+            FileSystems.getDefault();
+
+            // jdk/jdk: loads directly from Bootstrap Classloader (doesn't take lock on Runtime)
+            java.net.NetworkInterface.getNetworkInterfaces();
+
+            System.out.println(jdk.net.ExtendedSocketOptions.SO_FLOW_SLA);
+*/
+            Class c = Class.forName("Target2", true, loader);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static class TestClassLoader extends URLClassLoader {
+        boolean passed = false;
+
+        public boolean passed() {
+            return passed;
+        }
+
+        TestClassLoader() throws MalformedURLException {
+            super(new URL[] { new URL("file://" + CLS_DIR.toAbsolutePath().toString() + '/') });
+        }
+
+        public String findLibrary(String name) {
+            System.out.println("findLibrary " + name);
+
+            if ("someLibrary".equals(name)) {
+                try {
+                    synchronized(thread1) {
+                        while(!thread1Ready) {
+                            thread1.wait();
+                        }
+                        thread1.notifyAll();
+                    }
+
+                    Thread.sleep(10000);
+
+                    System.out.println("Thread2 load");
+                    someLibLoad();
+
+                    // no deadlock happened
+                    passed = true;
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+                return null;
+            }
+
+            return super.findLibrary(name);
+        }
+    }
+
+
+    public static void main(String[] args) throws Exception {
+        loader = new TestClassLoader();
+
+        if (!CompilerUtils.compile(SRC_DIR, CLS_DIR)) {
+            throw new Exception("Can't compile");
+        }
+
+        thread1 = new Thread() {
+            public void run() {
+                try {
+                    synchronized(this) {
+                        thread1Ready = true;
+                        thread1.notifyAll();
+                        thread1.wait();
+                    }
+                } catch(InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+
+                System.out.println("Thread1 load");
+                someLibLoad();
+            };
+        };
+
+        thread2 = new Thread() {
+            public void run() {
+                try {
+                    Class c = Class.forName("Target", true, loader);
+                    System.out.println(c);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            };
+        };
+
+        thread1.setDaemon(true);
+        thread2.setDaemon(true);
+
+        thread1.start();
+        thread2.start();
+
+        thread1.join();
+        thread2.join();
+
+        if (!loader.passed()) {
+            throw new RuntimeException("FAIL");
+        }
+    }
+}

test/jdk/java/lang/Runtime/loadLibrary/src/Target.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+class Target {
+    static {
+        try {
+            System.loadLibrary("someLibrary");
+            throw new RuntimeException("someLibrary was loaded");
+        } catch (UnsatisfiedLinkError e) {
+            // expected: we do not have a someLibrary
+        }
+    }
+}
+

test/jdk/java/lang/Runtime/loadLibrary/src/Target2.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+class Target2 {
+    static {
+        System.loadLibrary("awt");
+    }
+}
+