8307517: Add VMErrorCallback infrastructure to extend hs_err dumping

Reviewed-by: eosterlund, aboldtch, dholmes, stuefe

src/hotspot/share/runtime/thread.cpp

@@ -98,6 +98,7 @@ Thread::Thread() {
   _jvmti_env_iteration_count = 0;
   set_allocated_bytes(0);
   _current_pending_raw_monitor = nullptr;
+  _vm_error_callbacks = nullptr;
 
   // thread-specific hashCode stream generator state - Marsaglia shift-xor form
   _hashStateX = os::random();

src/hotspot/share/runtime/thread.hpp

@@ -55,6 +55,7 @@ class SafeThreadsListPtr;
 class ThreadClosure;
 class ThreadsList;
 class ThreadsSMRSupport;
+class VMErrorCallback;
 
 class OopClosure;
 class CodeBlobClosure;
@@ -104,6 +105,8 @@ class JavaThread;
 //       - this->entry_point()  // set differently for each kind of JavaThread
 
 class Thread: public ThreadShadow {
+  friend class VMError;
+  friend class VMErrorCallbackMark;
   friend class VMStructs;
   friend class JVMCIVMStructs;
  private:
@@ -634,6 +637,9 @@ protected:
     Thread *cur = Thread::current_or_null_safe();
     return cur != nullptr && cur->in_asgct();
   }
+
+ private:
+  VMErrorCallback* _vm_error_callbacks;
 };
 
 class ThreadInAsgct {

src/hotspot/share/utilities/vmError.cpp

@@ -948,6 +948,14 @@ void VMError::report(outputStream* st, bool _verbose) {
       st->cr();
     }
 
+  STEP_IF("printing registered callbacks", _verbose && _thread != nullptr);
+    for (VMErrorCallback* callback = _thread->_vm_error_callbacks;
+        callback != nullptr;
+        callback = callback->_next) {
+      callback->call(st);
+      st->cr();
+    }
+
   STEP("printing process")
 
   STEP_IF("printing process", _verbose)
@@ -1896,3 +1904,14 @@ void VMError::controlled_crash(int how) {
   ShouldNotReachHere();
 }
 #endif // !ASSERT
+
+VMErrorCallbackMark::VMErrorCallbackMark(VMErrorCallback* callback)
+  : _thread(Thread::current()) {
+  callback->_next = _thread->_vm_error_callbacks;
+  _thread->_vm_error_callbacks = callback;
+}
+
+VMErrorCallbackMark::~VMErrorCallbackMark() {
+  assert(_thread->_vm_error_callbacks != nullptr, "Popped too far");
+  _thread->_vm_error_callbacks = _thread->_vm_error_callbacks->_next;
+}

src/hotspot/share/utilities/vmError.hpp

@@ -213,4 +213,27 @@ public:
   static int prepare_log_file(const char* pattern, const char* default_pattern, bool overwrite_existing, char* buf, size_t buflen);
 
 };
+
+class VMErrorCallback {
+  friend class VMError;
+  friend class VMErrorCallbackMark;
+
+  // Link through all callbacks active on a thread
+  VMErrorCallback* _next;
+
+  // Called by VMError reporting
+  virtual void call(outputStream* st) = 0;
+
+public:
+  VMErrorCallback() : _next(nullptr) {}
+};
+
+class VMErrorCallbackMark : public StackObj {
+  Thread* _thread;
+
+public:
+  VMErrorCallbackMark(VMErrorCallback* callback);
+  ~VMErrorCallbackMark();
+};
+
 #endif // SHARE_UTILITIES_VMERROR_HPP