8059586: hs_err report should treat redirected core pattern

Reviewed-by: dholmes, sla, stuefe

hotspot/src/os/aix/vm/os_aix.cpp

@@ -5090,6 +5090,9 @@ int os::get_core_path(char* buffer, size_t bufferSize) {
     return 0;
   }
 
+  jio_snprintf(buffer, bufferSize, "%s/core or core.%d",
+                                               p, current_process_id());
+
   return strlen(buffer);
 }
 

hotspot/src/os/bsd/vm/os_bsd.cpp

@@ -4673,7 +4673,7 @@ bool os::is_headless_jre() {
 // Get the default path to the core file
 // Returns the length of the string
 int os::get_core_path(char* buffer, size_t bufferSize) {
-  int n = jio_snprintf(buffer, bufferSize, "/cores");
+  int n = jio_snprintf(buffer, bufferSize, "/cores/core.%d", current_process_id());
 
   // Truncate if theoretical string was longer than bufferSize
   n = MIN2(n, (int)bufferSize);

hotspot/src/os/linux/vm/os_linux.cpp

@@ -5988,13 +5988,70 @@ bool os::is_headless_jre() {
 // Get the default path to the core file
 // Returns the length of the string
 int os::get_core_path(char* buffer, size_t bufferSize) {
-  const char* p = get_current_directory(buffer, bufferSize);
+  /*
+   * Max length of /proc/sys/kernel/core_pattern is 128 characters.
+   * See https://www.kernel.org/doc/Documentation/sysctl/kernel.txt
+   */
+  const int core_pattern_len = 129;
+  char core_pattern[core_pattern_len] = {0};
 
+  int core_pattern_file = ::open("/proc/sys/kernel/core_pattern", O_RDONLY);
+  if (core_pattern_file != -1) {
+    ssize_t ret = ::read(core_pattern_file, core_pattern, core_pattern_len);
+    ::close(core_pattern_file);
+
+    if (ret > 0) {
+      char *last_char = core_pattern + strlen(core_pattern) - 1;
+
+      if (*last_char == '\n') {
+        *last_char = '\0';
+      }
+    }
+  }
+
+  if (strlen(core_pattern) == 0) {
+    return 0;
+  }
+
+  char *pid_pos = strstr(core_pattern, "%p");
+  size_t written;
+
+  if (core_pattern[0] == '/') {
+    written = jio_snprintf(buffer, bufferSize, core_pattern);
+  } else {
+    char cwd[PATH_MAX];
+
+    const char* p = get_current_directory(cwd, PATH_MAX);
     if (p == NULL) {
       assert(p != NULL, "failed to get current directory");
       return 0;
     }
 
+    if (core_pattern[0] == '|') {
+      written = jio_snprintf(buffer, bufferSize,
+                        "\"%s\" (or dumping to %s/core.%d)",
+                                     &core_pattern[1], p, current_process_id());
+    } else {
+      written = jio_snprintf(buffer, bufferSize, "%s/%s", p, core_pattern);
+    }
+  }
+
+  if ((written >= 0) && (written < bufferSize)
+            && (pid_pos == NULL) && (core_pattern[0] != '|')) {
+    int core_uses_pid_file = ::open("/proc/sys/kernel/core_uses_pid", O_RDONLY);
+
+    if (core_uses_pid_file != -1) {
+      char core_uses_pid = 0;
+      ssize_t ret = ::read(core_uses_pid_file, &core_uses_pid, 1);
+      ::close(core_uses_pid_file);
+
+      if (core_uses_pid == '1'){
+        jio_snprintf(buffer + written, bufferSize - written,
+                                          ".%d", current_process_id());
+      }
+    }
+  }
+
   return strlen(buffer);
 }
 

hotspot/src/os/posix/vm/os_posix.cpp

@@ -51,15 +51,24 @@ void os::check_or_create_dump(void* exceptionRecord, void* contextRecord, char*
   struct rlimit rlim;
   bool success;
 
-  n = get_core_path(buffer, bufferSize);
+  char core_path[PATH_MAX];
+  n = get_core_path(core_path, PATH_MAX);
 
-  if (getrlimit(RLIMIT_CORE, &rlim) != 0) {
-    jio_snprintf(buffer + n, bufferSize - n, "/core or core.%d (may not exist)", current_process_id());
+  if (n <= 0) {
+    jio_snprintf(buffer, bufferSize, "core.%d (may not exist)", current_process_id());
+    success = true;
+#ifdef LINUX
+  } else if (core_path[0] == '"') { // redirect to user process
+    jio_snprintf(buffer, bufferSize, "Core dumps may be processed with %s", core_path);
+    success = true;
+#endif
+  } else if (getrlimit(RLIMIT_CORE, &rlim) != 0) {
+    jio_snprintf(buffer, bufferSize, "%s (may not exist)", core_path);
     success = true;
   } else {
     switch(rlim.rlim_cur) {
       case RLIM_INFINITY:
-        jio_snprintf(buffer + n, bufferSize - n, "/core or core.%d", current_process_id());
+        jio_snprintf(buffer, bufferSize, "%s", core_path);
         success = true;
         break;
       case 0:
@@ -67,11 +76,12 @@ void os::check_or_create_dump(void* exceptionRecord, void* contextRecord, char*
         success = false;
         break;
       default:
-        jio_snprintf(buffer + n, bufferSize - n, "/core or core.%d (max size %lu kB). To ensure a full core dump, try \"ulimit -c unlimited\" before starting Java again", current_process_id(), (unsigned long)(rlim.rlim_cur >> 10));
+        jio_snprintf(buffer, bufferSize, "%s (max size %lu kB). To ensure a full core dump, try \"ulimit -c unlimited\" before starting Java again", core_path, (unsigned long)(rlim.rlim_cur >> 10));
         success = true;
         break;
     }
   }
+
   VMError::report_coredump_status(buffer, success);
 }
 

hotspot/src/os/solaris/vm/os_solaris.cpp

@@ -5979,6 +5979,9 @@ int os::get_core_path(char* buffer, size_t bufferSize) {
     return 0;
   }
 
+  jio_snprintf(buffer, bufferSize, "%s/core or core.%d",
+                                              p, current_process_id());
+
   return strlen(buffer);
 }