8268974: GetJREPath() JLI function fails to locate libjava.so if not standard Java launcher is used

Reviewed-by: almatvee, herrick, alanb

src/java.base/unix/native/libjli/java_md_common.c

@@ -45,20 +45,27 @@ static char* findLastPathComponent(char *buffer, const char *comp) {
 /*
  * Removes the trailing file name and any intermediate platform
  * directories, if any, and its enclosing directory.
+ * Second parameter is a hint about the type of a file. JNI_TRUE is for
+ * shared libraries and JNI_FALSE is for executables.
  * Ex: if a buffer contains "/foo/bin/javac" or "/foo/bin/x64/javac", the
  * truncated resulting buffer will contain "/foo".
  */
 static jboolean
-TruncatePath(char *buf)
+TruncatePath(char *buf, jboolean pathisdll)
 {
-    // try bin directory, maybe an executable
+    /*
-    char *p = findLastPathComponent(buf, "/bin/");
+     * If the file is a library, try lib directory first and then bin
+     * directory.
+     * If the file is an executable, try bin directory first and then lib
+     * directory.
+     */
+
+    char *p = findLastPathComponent(buf, pathisdll ? "/lib/" : "/bin/");
     if (p != NULL) {
         *p = '\0';
         return JNI_TRUE;
     }
-    // try lib directory, maybe a library
+    p = findLastPathComponent(buf, pathisdll ? "/bin/" : "/lib/");
-    p = findLastPathComponent(buf, "/lib/");
     if (p != NULL) {
         *p = '\0';
         return JNI_TRUE;
@@ -80,7 +87,7 @@ GetApplicationHome(char *buf, jint bufsize)
     } else {
         return JNI_FALSE;
     }
-    return TruncatePath(buf);
+    return TruncatePath(buf, JNI_FALSE);
 }
 
 /*
@@ -95,7 +102,7 @@ GetApplicationHomeFromDll(char *buf, jint bufsize)
     if (dladdr((void*)&GetApplicationHomeFromDll, &info) != 0) {
         char *path = realpath(info.dli_fname, buf);
         if (path == buf) {
-            return TruncatePath(buf);
+            return TruncatePath(buf, JNI_TRUE);
         }
     }
     return JNI_FALSE;

test/jdk/tools/jpackage/linux/LinuxWeirdOutputDirTest.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import jdk.jpackage.test.Annotations.Test;
+import jdk.jpackage.test.Annotations.Parameter;
+import jdk.jpackage.test.JPackageCommand;
+
+/*
+ * @test
+ * @summary jpackage with values of --dest parameter breaking jpackage launcher
+ * @requires (os.family == "linux")
+ * @bug 8268974
+ * @library ../helpers
+ * @build jdk.jpackage.test.*
+ * @modules jdk.jpackage/jdk.jpackage.internal
+ * @compile LinuxWeirdOutputDirTest.java
+ * @run main/othervm/timeout=540 -Xmx512m jdk.jpackage.test.Main
+ *  --jpt-run=LinuxWeirdOutputDirTest
+ */
+public class LinuxWeirdOutputDirTest {
+
+    @Test
+    @Parameter("bin")
+    @Parameter("lib")
+    public void test(String outputPath) {
+        JPackageCommand cmd = JPackageCommand.helloAppImage();
+        cmd.setArgumentValue("--dest", cmd.outputDir().resolve(outputPath));
+        cmd.executeAndAssertHelloAppImageCreated();
+    }
+}