8031968: Mac OS X: VM starts the agent by calling both Agent_OnAttach and Agent_OnAttach_L functions if its agent library is dynamically linked

Make sure we only look for statically linked agents in the main process image

Reviewed-by: dsamersoff, bpittore, dcubed

hotspot/src/os/bsd/vm/os_bsd.cpp

@@ -1557,6 +1557,17 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen)
 }
 #endif /* !__APPLE__ */
 
+void* os::get_default_process_handle() {
+#ifdef __APPLE__
+  // MacOS X needs to use RTLD_FIRST instead of RTLD_LAZY
+  // to avoid finding unexpected symbols on second (or later)
+  // loads of a library.
+  return (void*)::dlopen(NULL, RTLD_FIRST);
+#else
+  return (void*)::dlopen(NULL, RTLD_LAZY);
+#endif
+}
+
 // XXX: Do we need a lock around this as per Linux?
 void* os::dll_lookup(void* handle, const char* name) {
   return dlsym(handle, name);

hotspot/src/os/linux/vm/os_linux.cpp

@@ -2104,6 +2104,9 @@ void* os::dll_lookup(void* handle, const char* name) {
   return res;
 }
 
+void* os::get_default_process_handle() {
+  return (void*)::dlopen(NULL, RTLD_LAZY);
+}
 
 static bool _print_ascii_file(const char* filename, outputStream* st) {
   int fd = ::open(filename, O_RDONLY);

hotspot/src/os/posix/vm/os_posix.cpp

@@ -262,10 +262,6 @@ FILE* os::open(int fd, const char* mode) {
   return ::fdopen(fd, mode);
 }
 
-void* os::get_default_process_handle() {
-  return (void*)::dlopen(NULL, RTLD_LAZY);
-}
-
 // Builds a platform dependent Agent_OnLoad_<lib_name> function name
 // which is used to find statically linked in agents.
 // Parameters:

hotspot/src/os/solaris/vm/os_solaris.cpp

@@ -2146,6 +2146,10 @@ void* os::dll_lookup(void* handle, const char* name) {
   return dlsym(handle, name);
 }
 
+void* os::get_default_process_handle() {
+  return (void*)::dlopen(NULL, RTLD_LAZY);
+}
+
 int os::stat(const char *path, struct stat *sbuf) {
   char pathbuf[MAX_PATH];
   if (strlen(path) > MAX_PATH - 1) {