8158033: Notify_tracing() misplaced for intended purpose

Reviewed-by: egahlin, dholmes

hotspot/src/share/vm/runtime/java.cpp

@@ -446,6 +446,14 @@ void before_exit(JavaThread* thread) {
     os::infinite_sleep();
   }
 
+  EventThreadEnd event;
+  if (event.should_commit()) {
+    event.set_thread(THREAD_TRACE_ID(thread));
+    event.commit();
+  }
+
+  TRACE_VM_EXIT();
+
   // Stop the WatcherThread. We do this before disenrolling various
   // PeriodicTasks to reduce the likelihood of races.
   if (PeriodicTask::num_tasks() > 0) {
@@ -484,13 +492,6 @@ void before_exit(JavaThread* thread) {
     JvmtiExport::post_thread_end(thread);
   }
 
-
-  EventThreadEnd event;
-  if (event.should_commit()) {
-      event.set_thread(THREAD_TRACE_ID(thread));
-      event.commit();
-  }
-
   // Always call even when there are not JVMTI environments yet, since environments
   // may be attached late and JVMTI must track phases of VM execution
   JvmtiExport::post_vm_death();

hotspot/src/share/vm/trace/traceBackend.hpp

@@ -48,9 +48,6 @@ public:
   static void on_unloading_classes(void) {
   }
 
-  static void on_vm_error(bool) {
-  }
-
 };
 
 class TraceThreadData {

hotspot/src/share/vm/trace/traceMacros.hpp

@@ -43,6 +43,8 @@ extern "C" void JNICALL trace_register_natives(JNIEnv*, jclass);
 #define TRACE_REGISTER_NATIVES ((void*)((address_word)(&trace_register_natives)))
 #define TRACE_START() JNI_OK
 #define TRACE_INITIALIZE() JNI_OK
+#define TRACE_VM_EXIT()
+#define TRACE_VM_ERROR()
 
 #define TRACE_DEFINE_TRACE_ID_METHODS typedef int ___IGNORED_hs_trace_type1
 #define TRACE_DEFINE_TRACE_ID_FIELD typedef int ___IGNORED_hs_trace_type2

hotspot/src/share/vm/utilities/debug.cpp

@@ -54,10 +54,6 @@
 #include "utilities/macros.hpp"
 #include "utilities/vmError.hpp"
 
-#if INCLUDE_TRACE
-#include "trace/tracing.hpp"
-#endif
-
 #include <stdio.h>
 
 #ifndef ASSERT
@@ -306,11 +302,6 @@ void report_out_of_shared_space(SharedSpaceType shared_space) {
    exit(2);
 }
 
-static void notify_tracing() {
-#if INCLUDE_TRACE
-  Tracing::on_vm_error(true);
-#endif
-}
 
 void report_insufficient_metaspace(size_t required_size) {
   warning("\nThe MaxMetaspaceSize of " SIZE_FORMAT " bytes is not large enough.\n"
@@ -334,8 +325,6 @@ void report_java_out_of_memory(const char* message) {
       HeapDumper::dump_heap_from_oome();
     }
 
-    notify_tracing();
-
     if (OnOutOfMemoryError && OnOutOfMemoryError[0]) {
       VMError::report_java_out_of_memory(message);
     }

hotspot/src/share/vm/utilities/vmError.cpp

@@ -39,6 +39,7 @@
 #include "runtime/vmThread.hpp"
 #include "runtime/vm_operations.hpp"
 #include "services/memTracker.hpp"
+#include "trace/traceMacros.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/decoder.hpp"
 #include "utilities/defaultStream.hpp"
@@ -1165,6 +1166,8 @@ void VMError::report_and_die(int id, const char* message, const char* detail_fmt
     // are handled properly.
     reset_signal_handlers();
 
+    TRACE_VM_ERROR();
+
   } else {
     // If UseOsErrorReporting we call this for each level of the call stack
     // while searching for the exception handler.  Only the first level needs