Merge

hotspot/make/windows/makefiles/compile.make

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -110,6 +110,7 @@ CXX_FLAGS=$(CXX_FLAGS) /D TARGET_COMPILER_visCPP
 #      1400 is for VS2005
 #      1500 is for VS2008
 #      1600 is for VS2010
+#      1700 is for VS2012
 #    Do not confuse this MSC_VER with the predefined macro _MSC_VER that the
 #    compiler provides, when MSC_VER==1399, _MSC_VER will be 1400.
 #    Normally they are the same, but a pre-release of the VS2005 compilers
@@ -142,6 +143,9 @@ COMPILER_NAME=VS2008
 !if "$(MSC_VER)" == "1600"
 COMPILER_NAME=VS2010
 !endif
+!if "$(MSC_VER)" == "1700"
+COMPILER_NAME=VS2012
+!endif
 !endif
 
 # By default, we do not want to use the debug version of the msvcrt.dll file
@@ -151,9 +155,13 @@ MS_RUNTIME_OPTION = /MD
 MS_RUNTIME_OPTION = /MTd /D "_DEBUG"
 !endif
 
+# VS2012 and later won't work with:
+#     /D _STATIC_CPPLIB /D _DISABLE_DEPRECATE_STATIC_CPPLIB
+!if "$(MSC_VER)" < "1700"
 # Always add the _STATIC_CPPLIB flag
 STATIC_CPPLIB_OPTION = /D _STATIC_CPPLIB /D _DISABLE_DEPRECATE_STATIC_CPPLIB
 MS_RUNTIME_OPTION = $(MS_RUNTIME_OPTION) $(STATIC_CPPLIB_OPTION)
+!endif
 CXX_FLAGS=$(CXX_FLAGS) $(MS_RUNTIME_OPTION)
 
 # How /GX option is spelled
@@ -221,6 +229,22 @@ LD_FLAGS = /SAFESEH $(LD_FLAGS)
 !endif
 !endif
 
+!if "$(COMPILER_NAME)" == "VS2012"
+PRODUCT_OPT_OPTION   = /O2 /Oy-
+FASTDEBUG_OPT_OPTION = /O2 /Oy-
+DEBUG_OPT_OPTION     = /Od
+GX_OPTION = /EHsc
+LD_FLAGS = /manifest $(LD_FLAGS)
+# Manifest Tool - used in VS2005 and later to adjust manifests stored
+# as resources inside build artifacts.
+!if "x$(MT)" == "x"
+MT=mt.exe
+!endif
+!if "$(BUILDARCH)" == "i486"
+LD_FLAGS = /SAFESEH $(LD_FLAGS)
+!endif
+!endif
+
 # If NO_OPTIMIZATIONS is defined in the environment, turn everything off
 !ifdef NO_OPTIMIZATIONS
 PRODUCT_OPT_OPTION   = $(DEBUG_OPT_OPTION)

hotspot/make/windows/makefiles/sanity.make

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2006, 2012, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -27,9 +27,9 @@
 all: checkCL checkLink
 
 checkCL:
-	@ if "$(MSC_VER)" NEQ "1310" if "$(MSC_VER)" NEQ "1399" if "$(MSC_VER)" NEQ "1400" if "$(MSC_VER)" NEQ "1500" if "$(MSC_VER)" NEQ "1600" \
+	@ if "$(MSC_VER)" NEQ "1310" if "$(MSC_VER)" NEQ "1399" if "$(MSC_VER)" NEQ "1400" if "$(MSC_VER)" NEQ "1500" if "$(MSC_VER)" NEQ "1600" if "$(MSC_VER)" NEQ "1700" \
 	echo *** WARNING *** unrecognized cl.exe version $(MSC_VER) ($(RAW_MSC_VER)).  Use FORCE_MSC_VER to override automatic detection.
 
 checkLink:
-	@ if "$(LD_VER)" NEQ "710" if "$(LD_VER)" NEQ "800" if "$(LD_VER)" NEQ "900" if "$(LD_VER)" NEQ "1000" \
+	@ if "$(LD_VER)" NEQ "710" if "$(LD_VER)" NEQ "800" if "$(LD_VER)" NEQ "900" if "$(LD_VER)" NEQ "1000" if "$(LD_VER)" NEQ "1100" \
 	echo *** WARNING *** unrecognized link.exe version $(LD_VER) ($(RAW_LD_VER)).  Use FORCE_LD_VER to override automatic detection.

hotspot/make/windows/makefiles/vm.make

@@ -132,6 +132,10 @@ CXX_DONT_USE_PCH=/D DONT_USE_PRECOMPILED_HEADER
 
 !if "$(USE_PRECOMPILED_HEADER)" != "0"
 CXX_USE_PCH=/Fp"vm.pch" /Yu"precompiled.hpp"
+!if "$(COMPILER_NAME)" == "VS2012"
+# VS2012 requires this object file to be listed:
+LD_FLAGS=$(LD_FLAGS) _build_pch_file.obj
+!endif
 !else
 CXX_USE_PCH=$(CXX_DONT_USE_PCH)
 !endif

hotspot/src/cpu/sparc/vm/frame_sparc.inline.hpp

@@ -240,10 +240,10 @@ inline ConstantPoolCache** frame::interpreter_frame_cache_addr() const {
 #endif // CC_INTERP
 
 
-inline JavaCallWrapper* frame::entry_frame_call_wrapper() const {
+inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const {
   // note: adjust this code if the link argument in StubGenerator::call_stub() changes!
   const Argument link = Argument(0, false);
-  return (JavaCallWrapper*)sp()[link.as_in().as_register()->sp_offset_in_saved_window()];
+  return (JavaCallWrapper**)&sp()[link.as_in().as_register()->sp_offset_in_saved_window()];
 }
 
 

hotspot/src/cpu/x86/vm/frame_x86.inline.hpp

@@ -272,11 +272,10 @@ inline jint frame::interpreter_frame_expression_stack_direction() { return -1; }
 
 // Entry frames
 
-inline JavaCallWrapper* frame::entry_frame_call_wrapper() const {
+inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const {
- return (JavaCallWrapper*)at(entry_frame_call_wrapper_offset);
+ return (JavaCallWrapper**)addr_at(entry_frame_call_wrapper_offset);
 }
 
-
 // Compiled frames
 
 inline int frame::local_offset_for_compiler(int local_index, int nof_args, int max_nof_locals, int max_nof_monitors) {

hotspot/src/os/bsd/vm/attachListener_bsd.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -437,6 +437,30 @@ AttachOperation* AttachListener::dequeue() {
   return op;
 }
 
+
+// Performs initialization at vm startup
+// For BSD we remove any stale .java_pid file which could cause
+// an attaching process to think we are ready to receive on the
+// domain socket before we are properly initialized
+
+void AttachListener::vm_start() {
+  char fn[UNIX_PATH_MAX];
+  struct stat64 st;
+  int ret;
+
+  int n = snprintf(fn, UNIX_PATH_MAX, "%s/.java_pid%d",
+           os::get_temp_directory(), os::current_process_id());
+  assert(n < (int)UNIX_PATH_MAX, "java_pid file name buffer overflow");
+
+  RESTARTABLE(::stat64(fn, &st), ret);
+  if (ret == 0) {
+    ret = ::unlink(fn);
+    if (ret == -1) {
+      debug_only(warning("failed to remove stale attach pid file at %s", fn));
+    }
+  }
+}
+
 int AttachListener::pd_init() {
   JavaThread* thread = JavaThread::current();
   ThreadBlockInVM tbivm(thread);

hotspot/src/os/linux/vm/attachListener_linux.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -432,6 +432,30 @@ AttachOperation* AttachListener::dequeue() {
   return op;
 }
 
+
+// Performs initialization at vm startup
+// For Linux we remove any stale .java_pid file which could cause
+// an attaching process to think we are ready to receive on the
+// domain socket before we are properly initialized
+
+void AttachListener::vm_start() {
+  char fn[UNIX_PATH_MAX];
+  struct stat64 st;
+  int ret;
+
+  int n = snprintf(fn, UNIX_PATH_MAX, "%s/.java_pid%d",
+           os::get_temp_directory(), os::current_process_id());
+  assert(n < (int)UNIX_PATH_MAX, "java_pid file name buffer overflow");
+
+  RESTARTABLE(::stat64(fn, &st), ret);
+  if (ret == 0) {
+    ret = ::unlink(fn);
+    if (ret == -1) {
+      debug_only(warning("failed to remove stale attach pid file at %s", fn));
+    }
+  }
+}
+
 int AttachListener::pd_init() {
   JavaThread* thread = JavaThread::current();
   ThreadBlockInVM tbivm(thread);

hotspot/src/os/linux/vm/jsig.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -107,7 +107,7 @@ static sa_handler_t set_signal(int sig, sa_handler_t disp, bool is_sigset) {
 
   signal_lock();
 
-  sigused = (MASK(sig) & jvmsigs) != 0;
+  sigused = (sig < MAXSIGNUM) && ((MASK(sig) & jvmsigs) != 0);
   if (jvm_signal_installed && sigused) {
     /* jvm has installed its signal handler for this signal. */
     /* Save the handler. Don't really install it. */
@@ -116,7 +116,7 @@ static sa_handler_t set_signal(int sig, sa_handler_t disp, bool is_sigset) {
 
     signal_unlock();
     return oldhandler;
-  } else if (jvm_signal_installing) {
+  } else if (sig < MAXSIGNUM && jvm_signal_installing) {
     /* jvm is installing its signal handlers. Install the new
      * handlers and save the old ones. jvm uses sigaction().
      * Leave the piece here just in case. */
@@ -165,7 +165,7 @@ int sigaction(int sig, const struct sigaction *act, struct sigaction *oact) {
 
   signal_lock();
 
-  sigused = (MASK(sig) & jvmsigs) != 0;
+  sigused = (sig < MAXSIGNUM) && ((MASK(sig) & jvmsigs) != 0);
   if (jvm_signal_installed && sigused) {
     /* jvm has installed its signal handler for this signal. */
     /* Save the handler. Don't really install it. */
@@ -178,7 +178,7 @@ int sigaction(int sig, const struct sigaction *act, struct sigaction *oact) {
 
     signal_unlock();
     return 0;
-  } else if (jvm_signal_installing) {
+  } else if (sig < MAXSIGNUM && jvm_signal_installing) {
     /* jvm is installing its signal handlers. Install the new
      * handlers and save the old ones. */
     res = call_os_sigaction(sig, act, &oldAct);

hotspot/src/os/solaris/vm/attachListener_solaris.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -576,6 +576,30 @@ AttachOperation* AttachListener::dequeue() {
   return op;
 }
 
+
+// Performs initialization at vm startup
+// For Solaris we remove any stale .java_pid file which could cause
+// an attaching process to think we are ready to receive a door_call
+// before we are properly initialized
+
+void AttachListener::vm_start() {
+  char fn[PATH_MAX+1];
+  struct stat64 st;
+  int ret;
+
+  int n = snprintf(fn, sizeof(fn), "%s/.java_pid%d",
+           os::get_temp_directory(), os::current_process_id());
+  assert(n < sizeof(fn), "java_pid file name buffer overflow");
+
+  RESTARTABLE(::stat64(fn, &st), ret);
+  if (ret == 0) {
+    ret = ::unlink(fn);
+    if (ret == -1) {
+      debug_only(warning("failed to remove stale attach pid file at %s", fn));
+    }
+  }
+}
+
 int AttachListener::pd_init() {
   JavaThread* thread = JavaThread::current();
   ThreadBlockInVM tbivm(thread);

hotspot/src/os/windows/vm/attachListener_windows.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -358,6 +358,10 @@ AttachOperation* AttachListener::dequeue() {
   return op;
 }
 
+void AttachListener::vm_start() {
+  // nothing to do
+}
+
 int AttachListener::pd_init() {
   return Win32AttachListener::init();
 }

hotspot/src/os_cpu/windows_x86/vm/unwind_windows_x86.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,10 +29,15 @@
 #ifdef AMD64
 typedef unsigned char UBYTE;
 
+#if _MSC_VER < 1700
+
+/* Not needed for VS2012 compiler, comes from winnt.h. */
 #define UNW_FLAG_EHANDLER  0x01
 #define UNW_FLAG_UHANDLER  0x02
 #define UNW_FLAG_CHAININFO 0x04
 
+#endif
+
 // This structure is used to define an UNWIND_INFO that
 // only has an ExceptionHandler.  There are no UnwindCodes
 // declared.
@@ -59,6 +64,9 @@ typedef struct _RUNTIME_FUNCTION {
 } RUNTIME_FUNCTION, *PRUNTIME_FUNCTION;
 */
 
+#if _MSC_VER < 1700
+
+/* Not needed for VS2012 compiler, comes from winnt.h. */
 typedef struct _DISPATCHER_CONTEXT {
     ULONG64 ControlPc;
     ULONG64 ImageBase;
@@ -71,6 +79,8 @@ typedef struct _DISPATCHER_CONTEXT {
     PVOID HandlerData;
 } DISPATCHER_CONTEXT, *PDISPATCHER_CONTEXT;
 
+#endif
+
 #if _MSC_VER < 1500
 
 /* Not needed for VS2008 compiler, comes from winnt.h. */

hotspot/src/share/vm/classfile/classFileParser.cpp

@@ -3647,8 +3647,7 @@ instanceKlassHandle ClassFileParser::parseClassFile(Symbol* name,
   // If RedefineClasses() was used before the retransformable
   // agent attached, then the cached class bytes may not be the
   // original class bytes.
-  unsigned char *cached_class_file_bytes = NULL;
+  JvmtiCachedClassFileData *cached_class_file = NULL;
-  jint cached_class_file_length;
   Handle class_loader(THREAD, loader_data->class_loader());
   bool has_default_methods = false;
   ResourceMark rm(THREAD);
@@ -3680,10 +3679,7 @@ instanceKlassHandle ClassFileParser::parseClassFile(Symbol* name,
       if (h_class_being_redefined != NULL) {
         instanceKlassHandle ikh_class_being_redefined =
           instanceKlassHandle(THREAD, (*h_class_being_redefined)());
-        cached_class_file_bytes =
+        cached_class_file = ikh_class_being_redefined->get_cached_class_file();
-          ikh_class_being_redefined->get_cached_class_file_bytes();
-        cached_class_file_length =
-          ikh_class_being_redefined->get_cached_class_file_len();
       }
     }
 
@@ -3691,9 +3687,7 @@ instanceKlassHandle ClassFileParser::parseClassFile(Symbol* name,
     unsigned char* end_ptr = cfs->buffer() + cfs->length();
 
     JvmtiExport::post_class_file_load_hook(name, class_loader(), protection_domain,
-                                           &ptr, &end_ptr,
+                                           &ptr, &end_ptr, &cached_class_file);
-                                           &cached_class_file_bytes,
-                                           &cached_class_file_length);
 
     if (ptr != cfs->buffer()) {
       // JVMTI agent has modified class file data.
@@ -4011,10 +4005,9 @@ instanceKlassHandle ClassFileParser::parseClassFile(Symbol* name,
       }
     }
 
-    if (cached_class_file_bytes != NULL) {
+    if (cached_class_file != NULL) {
       // JVMTI: we have an InstanceKlass now, tell it about the cached bytes
-      this_klass->set_cached_class_file(cached_class_file_bytes,
+      this_klass->set_cached_class_file(cached_class_file);
-                                        cached_class_file_length);
     }
 
     // Fill in field values obtained by parse_classfile_attributes

hotspot/src/share/vm/memory/resourceArea.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -83,6 +83,10 @@ protected:
   Chunk *_chunk;                // saved arena chunk
   char *_hwm, *_max;
   size_t _size_in_bytes;
+#ifdef ASSERT
+  Thread* _thread;
+  ResourceMark* _previous_resource_mark;
+#endif //ASSERT
 
   void initialize(Thread *thread) {
     _area = thread->resource_area();
@@ -92,6 +96,11 @@ protected:
     _size_in_bytes = _area->size_in_bytes();
     debug_only(_area->_nesting++;)
     assert( _area->_nesting > 0, "must stack allocate RMs" );
+#ifdef ASSERT
+    _thread = thread;
+    _previous_resource_mark = thread->current_resource_mark();
+    thread->set_current_resource_mark(this);
+#endif // ASSERT
   }
  public:
 
@@ -111,6 +120,17 @@ protected:
     _size_in_bytes = r->_size_in_bytes;
     debug_only(_area->_nesting++;)
     assert( _area->_nesting > 0, "must stack allocate RMs" );
+#ifdef ASSERT
+    Thread* thread = ThreadLocalStorage::thread();
+    if (thread != NULL) {
+      _thread = thread;
+      _previous_resource_mark = thread->current_resource_mark();
+      thread->set_current_resource_mark(this);
+    } else {
+      _thread = NULL;
+      _previous_resource_mark = NULL;
+    }
+#endif // ASSERT
   }
 
   void reset_to_mark() {
@@ -137,6 +157,11 @@ protected:
     assert( _area->_nesting > 0, "must stack allocate RMs" );
     debug_only(_area->_nesting--;)
     reset_to_mark();
+#ifdef ASSERT
+    if (_thread != NULL) {
+      _thread->set_current_resource_mark(_previous_resource_mark);
+    }
+#endif // ASSERT
   }
 
 

hotspot/src/share/vm/oops/cpCache.hpp

@@ -140,8 +140,15 @@ class ConstantPoolCacheEntry VALUE_OBJ_CLASS_SPEC {
     _f1 = f1;
   }
   void release_set_f1(Metadata* f1);
-  void set_f2(intx f2)                           { assert(_f2 == 0 || _f2 == f2,            "illegal field change"); _f2 = f2; }
+  void set_f2(intx f2) {
-  void set_f2_as_vfinal_method(Method* f2)     { assert(_f2 == 0 || _f2 == (intptr_t) f2, "illegal field change"); assert(is_vfinal(), "flags must be set"); _f2 = (intptr_t) f2; }
+    intx existing_f2 = _f2; // read once
+    assert(existing_f2 == 0 || existing_f2 == f2, "illegal field change");
+    _f2 = f2;
+  }
+  void set_f2_as_vfinal_method(Method* f2) {
+    assert(is_vfinal(), "flags must be set");
+    set_f2((intx)f2);
+  }
   int make_flags(TosState state, int option_bits, int field_index_or_method_params);
   void set_flags(intx flags)                     { _flags = flags; }
   bool init_flags_atomic(intx flags);

hotspot/src/share/vm/oops/instanceKlass.cpp

@@ -48,6 +48,7 @@
 #include "oops/symbol.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "prims/jvmtiRedefineClassesTrace.hpp"
+#include "prims/jvmtiRedefineClasses.hpp"
 #include "prims/methodComparator.hpp"
 #include "runtime/fieldDescriptor.hpp"
 #include "runtime/handles.inline.hpp"
@@ -291,7 +292,7 @@ InstanceKlass::InstanceKlass(int vtable_len,
   set_initial_method_idnum(0);
   _dependencies = NULL;
   set_jvmti_cached_class_field_map(NULL);
-  set_cached_class_file(NULL, 0);
+  set_cached_class_file(NULL);
   set_initial_method_idnum(0);
   set_minor_version(0);
   set_major_version(0);
@@ -2357,10 +2358,9 @@ void InstanceKlass::release_C_heap_structures() {
   }
 
   // deallocate the cached class file
-  if (_cached_class_file_bytes != NULL) {
+  if (_cached_class_file != NULL) {
-    os::free(_cached_class_file_bytes, mtClass);
+    os::free(_cached_class_file, mtClass);
-    _cached_class_file_bytes = NULL;
+    _cached_class_file = NULL;
-    _cached_class_file_len = 0;
   }
 
   // Decrement symbol reference counts associated with the unloaded class.
@@ -3530,6 +3530,14 @@ Method* InstanceKlass::method_with_idnum(int idnum) {
   return m;
 }
 
+jint InstanceKlass::get_cached_class_file_len() {
+  return VM_RedefineClasses::get_cached_class_file_len(_cached_class_file);
+}
+
+unsigned char * InstanceKlass::get_cached_class_file_bytes() {
+  return VM_RedefineClasses::get_cached_class_file_bytes(_cached_class_file);
+}
+
 
 // Construct a PreviousVersionNode entry for the array hung off
 // the InstanceKlass.

hotspot/src/share/vm/oops/instanceKlass.hpp

@@ -133,6 +133,8 @@ class OopMapBlock VALUE_OBJ_CLASS_SPEC {
   uint _count;
 };
 
+struct JvmtiCachedClassFileData;
+
 class InstanceKlass: public Klass {
   friend class VMStructs;
   friend class ClassFileParser;
@@ -249,8 +251,8 @@ class InstanceKlass: public Klass {
   // InstanceKlass. See PreviousVersionWalker below.
   GrowableArray<PreviousVersionNode *>* _previous_versions;
   // JVMTI fields can be moved to their own structure - see 6315920
-  unsigned char * _cached_class_file_bytes;       // JVMTI: cached class file, before retransformable agent modified it in CFLH
+  // JVMTI: cached class file, before retransformable agent modified it in CFLH
-  jint            _cached_class_file_len;         // JVMTI: length of above
+  JvmtiCachedClassFileData* _cached_class_file;
 
   volatile u2     _idnum_allocated_count;         // JNI/JVMTI: increments with the addition of methods, old ids don't change
 
@@ -615,11 +617,12 @@ class InstanceKlass: public Klass {
   static void purge_previous_versions(InstanceKlass* ik);
 
   // JVMTI: Support for caching a class file before it is modified by an agent that can do retransformation
-  void set_cached_class_file(unsigned char *class_file_bytes,
+  void set_cached_class_file(JvmtiCachedClassFileData *data) {
-                             jint class_file_len)     { _cached_class_file_len = class_file_len;
+    _cached_class_file = data;
-                                                        _cached_class_file_bytes = class_file_bytes; }
+  }
-  jint get_cached_class_file_len()                    { return _cached_class_file_len; }
+  JvmtiCachedClassFileData * get_cached_class_file() { return _cached_class_file; }
-  unsigned char * get_cached_class_file_bytes()       { return _cached_class_file_bytes; }
+  jint get_cached_class_file_len();
+  unsigned char * get_cached_class_file_bytes();
 
   // JVMTI: Support for caching of field indices, types, and offsets
   void set_jvmti_cached_class_field_map(JvmtiCachedClassFieldMap* descriptor) {

hotspot/src/share/vm/prims/forte.cpp

@@ -31,6 +31,7 @@
 #include "oops/oop.inline.hpp"
 #include "oops/oop.inline2.hpp"
 #include "prims/forte.hpp"
+#include "runtime/javaCalls.hpp"
 #include "runtime/thread.hpp"
 #include "runtime/vframe.hpp"
 #include "runtime/vframeArray.hpp"
@@ -308,11 +309,15 @@ static bool find_initial_Java_frame(JavaThread* thread,
 
   for (loop_count = 0; loop_count < loop_max; loop_count++) {
 
-    if (candidate.is_first_frame()) {
+    if (candidate.is_entry_frame()) {
+      // jcw is NULL if the java call wrapper couldn't be found
+      JavaCallWrapper *jcw = candidate.entry_frame_call_wrapper_if_safe(thread);
       // If initial frame is frame from StubGenerator and there is no
       // previous anchor, there are no java frames associated with a method
+      if (jcw == NULL || jcw->is_first_frame()) {
         return false;
       }
+    }
 
     if (candidate.is_interpreted_frame()) {
       if (is_decipherable_interpreted_frame(thread, &candidate, method_p, bci_p)) {

hotspot/src/share/vm/prims/jniCheck.cpp

@@ -126,6 +126,7 @@ static const char * fatal_wrong_class_or_method = "Wrong object class or methodI
 static const char * fatal_non_weak_method = "non-weak methodID passed to JNI call";
 static const char * fatal_unknown_array_object = "Unknown array object passed to JNI array operations";
 static const char * fatal_object_array_expected = "Object array expected but not received for JNI array operation";
+static const char * fatal_prim_type_array_expected = "Primitive type array expected but not received for JNI array operation";
 static const char * fatal_non_array  = "Non-array passed to JNI array operations";
 static const char * fatal_element_type_mismatch = "Array element type mismatch in JNI";
 static const char * fatal_should_be_static = "Non-static field ID passed to JNI";
@@ -278,30 +279,49 @@ checkString(JavaThread* thr, jstring js)
     ReportJNIFatalError(thr, fatal_non_string);
 }
 
-static inline void
+static inline arrayOop
-checkArray(JavaThread* thr, jarray jArray, int elementType)
+check_is_array(JavaThread* thr, jarray jArray)
 {
   ASSERT_OOPS_ALLOWED;
   arrayOop aOop;
 
   aOop = (arrayOop)jniCheck::validate_object(thr, jArray);
-  if (aOop == NULL || !aOop->is_array())
+  if (aOop == NULL || !aOop->is_array()) {
     ReportJNIFatalError(thr, fatal_non_array);
+  }
+  return aOop;
+}
 
-  if (elementType != -1) {
+static inline arrayOop
-    if (aOop->is_typeArray()) {
+check_is_primitive_array(JavaThread* thr, jarray jArray) {
-      BasicType array_type = TypeArrayKlass::cast(aOop->klass())->element_type();
+  arrayOop aOop = check_is_array(thr, jArray);
-      if (array_type != elementType)
+
+  if (!aOop->is_typeArray()) {
+     ReportJNIFatalError(thr, fatal_prim_type_array_expected);
+  }
+  return aOop;
+}
+
+static inline void
+check_primitive_array_type(JavaThread* thr, jarray jArray, BasicType elementType)
+{
+  BasicType array_type;
+  arrayOop aOop;
+
+  aOop = check_is_primitive_array(thr, jArray);
+  array_type = TypeArrayKlass::cast(aOop->klass())->element_type();
+  if (array_type != elementType) {
     ReportJNIFatalError(thr, fatal_element_type_mismatch);
-      } else if (aOop->is_objArray()) {
-        if ( T_OBJECT != elementType)
-          ReportJNIFatalError(thr, fatal_object_array_expected);
-      } else {
-        ReportJNIFatalError(thr, fatal_unknown_array_object);
-    }
   }
 }
 
+static inline void
+check_is_obj_array(JavaThread* thr, jarray jArray) {
+  arrayOop aOop = check_is_array(thr, jArray);
+  if (!aOop->is_objArray()) {
+    ReportJNIFatalError(thr, fatal_object_array_expected);
+  }
+}
 
 oop jniCheck::validate_handle(JavaThread* thr, jobject obj) {
   if (JNIHandles::is_frame_handle(thr, obj) ||
@@ -1417,7 +1437,7 @@ JNI_ENTRY_CHECKED(jsize,
                              jarray array))
     functionEnter(thr);
     IN_VM(
-      checkArray(thr, array, -1);
+      check_is_array(thr, array);
     )
     jsize result = UNCHECKED()->GetArrayLength(env,array);
     functionExit(env);
@@ -1441,7 +1461,7 @@ JNI_ENTRY_CHECKED(jobject,
                                     jsize index))
     functionEnter(thr);
     IN_VM(
-      checkArray(thr, array, T_OBJECT);
+      check_is_obj_array(thr, array);
     )
     jobject result = UNCHECKED()->GetObjectArrayElement(env,array,index);
     functionExit(env);
@@ -1455,7 +1475,7 @@ JNI_ENTRY_CHECKED(void,
                                     jobject val))
     functionEnter(thr);
     IN_VM(
-      checkArray(thr, array, T_OBJECT);
+      check_is_obj_array(thr, array);
     )
     UNCHECKED()->SetObjectArrayElement(env,array,index,val);
     functionExit(env);
@@ -1487,7 +1507,7 @@ JNI_ENTRY_CHECKED(ElementType *,  \
                                          jboolean *isCopy)) \
     functionEnter(thr); \
     IN_VM( \
-      checkArray(thr, array, ElementTag); \
+      check_primitive_array_type(thr, array, ElementTag); \
     ) \
     ElementType *result = UNCHECKED()->Get##Result##ArrayElements(env, \
                                                                   array, \
@@ -1513,7 +1533,7 @@ JNI_ENTRY_CHECKED(void,  \
                                              jint mode)) \
     functionEnterExceptionAllowed(thr); \
     IN_VM( \
-      checkArray(thr, array, ElementTag); \
+      check_primitive_array_type(thr, array, ElementTag); \
       ASSERT_OOPS_ALLOWED; \
       typeArrayOop a = typeArrayOop(JNIHandles::resolve_non_null(array)); \
       /* cannot check validity of copy, unless every request is logged by
@@ -1543,7 +1563,7 @@ JNI_ENTRY_CHECKED(void,  \
                                        ElementType *buf)) \
     functionEnter(thr); \
     IN_VM( \
-      checkArray(thr, array, ElementTag); \
+      check_primitive_array_type(thr, array, ElementTag); \
     ) \
     UNCHECKED()->Get##Result##ArrayRegion(env,array,start,len,buf); \
     functionExit(env); \
@@ -1567,7 +1587,7 @@ JNI_ENTRY_CHECKED(void,  \
                                        const ElementType *buf)) \
     functionEnter(thr); \
     IN_VM( \
-      checkArray(thr, array, ElementTag); \
+      check_primitive_array_type(thr, array, ElementTag); \
     ) \
     UNCHECKED()->Set##Result##ArrayRegion(env,array,start,len,buf); \
     functionExit(env); \
@@ -1669,7 +1689,7 @@ JNI_ENTRY_CHECKED(void *,
                                         jboolean *isCopy))
     functionEnterCritical(thr);
     IN_VM(
-      checkArray(thr, array, -1);
+      check_is_primitive_array(thr, array);
     )
     void *result = UNCHECKED()->GetPrimitiveArrayCritical(env, array, isCopy);
     functionExit(env);
@@ -1683,7 +1703,7 @@ JNI_ENTRY_CHECKED(void,
                                             jint mode))
     functionEnterCriticalExceptionAllowed(thr);
     IN_VM(
-      checkArray(thr, array, -1);
+      check_is_primitive_array(thr, array);
     )
     /* The Hotspot JNI code does not use the parameters, so just check the
      * array parameter as a minor sanity check

hotspot/src/share/vm/prims/jvmtiExport.cpp

@@ -41,6 +41,7 @@
 #include "prims/jvmtiRawMonitor.hpp"
 #include "prims/jvmtiTagMap.hpp"
 #include "prims/jvmtiThreadState.inline.hpp"
+#include "prims/jvmtiRedefineClasses.hpp"
 #include "runtime/arguments.hpp"
 #include "runtime/handles.hpp"
 #include "runtime/interfaceSupport.hpp"
@@ -516,8 +517,7 @@ class JvmtiClassFileLoadHookPoster : public StackObj {
   jint                 _curr_len;
   unsigned char *      _curr_data;
   JvmtiEnv *           _curr_env;
-  jint *               _cached_length_ptr;
+  JvmtiCachedClassFileData ** _cached_class_file_ptr;
-  unsigned char **     _cached_data_ptr;
   JvmtiThreadState *   _state;
   KlassHandle *        _h_class_being_redefined;
   JvmtiClassLoadKind   _load_kind;
@@ -526,8 +526,7 @@ class JvmtiClassFileLoadHookPoster : public StackObj {
   inline JvmtiClassFileLoadHookPoster(Symbol* h_name, Handle class_loader,
                                       Handle h_protection_domain,
                                       unsigned char **data_ptr, unsigned char **end_ptr,
-                                      unsigned char **cached_data_ptr,
+                                      JvmtiCachedClassFileData **cache_ptr) {
-                                      jint *cached_length_ptr) {
     _h_name = h_name;
     _class_loader = class_loader;
     _h_protection_domain = h_protection_domain;
@@ -537,8 +536,7 @@ class JvmtiClassFileLoadHookPoster : public StackObj {
     _curr_len = *end_ptr - *data_ptr;
     _curr_data = *data_ptr;
     _curr_env = NULL;
-    _cached_length_ptr = cached_length_ptr;
+    _cached_class_file_ptr = cache_ptr;
-    _cached_data_ptr = cached_data_ptr;
 
     _state = _thread->jvmti_thread_state();
     if (_state != NULL) {
@@ -615,15 +613,20 @@ class JvmtiClassFileLoadHookPoster : public StackObj {
     }
     if (new_data != NULL) {
       // this agent has modified class data.
-      if (caching_needed && *_cached_data_ptr == NULL) {
+      if (caching_needed && *_cached_class_file_ptr == NULL) {
         // data has been changed by the new retransformable agent
         // and it hasn't already been cached, cache it
-        *_cached_data_ptr = (unsigned char *)os::malloc(_curr_len, mtInternal);
+        JvmtiCachedClassFileData *p;
-        if (*_cached_data_ptr == NULL) {
+        p = (JvmtiCachedClassFileData *)os::malloc(
-          vm_exit_out_of_memory(_curr_len, OOM_MALLOC_ERROR, "unable to allocate cached copy of original class bytes");
+          offset_of(JvmtiCachedClassFileData, data) + _curr_len, mtInternal);
+        if (p == NULL) {
+          vm_exit_out_of_memory(offset_of(JvmtiCachedClassFileData, data) + _curr_len,
+            OOM_MALLOC_ERROR,
+            "unable to allocate cached copy of original class bytes");
         }
-        memcpy(*_cached_data_ptr, _curr_data, _curr_len);
+        p->length = _curr_len;
-        *_cached_length_ptr = _curr_len;
+        memcpy(p->data, _curr_data, _curr_len);
+        *_cached_class_file_ptr = p;
       }
 
       if (_curr_data != *_data_ptr) {
@@ -662,13 +665,11 @@ void JvmtiExport::post_class_file_load_hook(Symbol* h_name,
                                             Handle h_protection_domain,
                                             unsigned char **data_ptr,
                                             unsigned char **end_ptr,
-                                            unsigned char **cached_data_ptr,
+                                            JvmtiCachedClassFileData **cache_ptr) {
-                                            jint *cached_length_ptr) {
   JvmtiClassFileLoadHookPoster poster(h_name, class_loader,
                                       h_protection_domain,
                                       data_ptr, end_ptr,
-                                      cached_data_ptr,
+                                      cache_ptr);
-                                      cached_length_ptr);
   poster.post();
 }
 

hotspot/src/share/vm/prims/jvmtiExport.hpp

@@ -323,8 +323,7 @@ class JvmtiExport : public AllStatic {
   static void post_class_file_load_hook(Symbol* h_name, Handle class_loader,
                                         Handle h_protection_domain,
                                         unsigned char **data_ptr, unsigned char **end_ptr,
-                                        unsigned char **cached_data_ptr,
+                                        JvmtiCachedClassFileData **cache_ptr) NOT_JVMTI_RETURN;
-                                        jint *cached_length_ptr) NOT_JVMTI_RETURN;
   static void post_native_method_bind(Method* method, address* function_ptr) NOT_JVMTI_RETURN;
   static void post_compiled_method_load(nmethod *nm) NOT_JVMTI_RETURN;
   static void post_dynamic_code_generated(const char *name, const void *code_begin, const void *code_end) NOT_JVMTI_RETURN;

hotspot/src/share/vm/prims/jvmtiRedefineClasses.cpp

@@ -3342,9 +3342,7 @@ void VM_RedefineClasses::redefine_single_class(jclass the_jclass,
   // should get cleared in the_class too.
   if (the_class->get_cached_class_file_bytes() == 0) {
     // the_class doesn't have a cache yet so copy it
-    the_class->set_cached_class_file(
+    the_class->set_cached_class_file(scratch_class->get_cached_class_file());
-      scratch_class->get_cached_class_file_bytes(),
-      scratch_class->get_cached_class_file_len());
   }
 #ifndef PRODUCT
   else {
@@ -3357,7 +3355,7 @@ void VM_RedefineClasses::redefine_single_class(jclass the_jclass,
 
   // NULL out in scratch class to not delete twice.  The class to be redefined
   // always owns these bytes.
-  scratch_class->set_cached_class_file(NULL, 0);
+  scratch_class->set_cached_class_file(NULL);
 
   // Replace inner_classes
   Array<u2>* old_inner_classes = the_class->inner_classes();

hotspot/src/share/vm/prims/jvmtiRedefineClasses.hpp

@@ -331,6 +331,11 @@
 //   coordinate a cleanup of these constants with Runtime.
 //
 
+struct JvmtiCachedClassFileData {
+  jint length;
+  unsigned char data[1];
+};
+
 class VM_RedefineClasses: public VM_Operation {
  private:
   // These static fields are needed by ClassLoaderDataGraph::classes_do()
@@ -509,5 +514,12 @@ class VM_RedefineClasses: public VM_Operation {
   // Modifiable test must be shared between IsModifiableClass query
   // and redefine implementation
   static bool is_modifiable_class(oop klass_mirror);
+
+  static jint get_cached_class_file_len(JvmtiCachedClassFileData *cache) {
+    return cache == NULL ? 0 : cache->length;
+  }
+  static unsigned char * get_cached_class_file_bytes(JvmtiCachedClassFileData *cache) {
+    return cache == NULL ? NULL : cache->data;
+  }
 };
 #endif // SHARE_VM_PRIMS_JVMTIREDEFINECLASSES_HPP

hotspot/src/share/vm/runtime/frame.cpp

@@ -221,9 +221,20 @@ bool frame::is_first_java_frame() const {
 
 
 bool frame::entry_frame_is_first() const {
-  return entry_frame_call_wrapper()->anchor()->last_Java_sp() == NULL;
+  return entry_frame_call_wrapper()->is_first_frame();
 }
 
+JavaCallWrapper* frame::entry_frame_call_wrapper_if_safe(JavaThread* thread) const {
+  JavaCallWrapper** jcw = entry_frame_call_wrapper_addr();
+  address addr = (address) jcw;
+
+  // addr must be within the usable part of the stack
+  if (thread->is_in_usable_stack(addr)) {
+    return *jcw;
+  }
+
+  return NULL;
+}
 
 bool frame::should_be_deoptimized() const {
   if (_deopt_state == is_deoptimized ||

hotspot/src/share/vm/runtime/frame.hpp

@@ -353,7 +353,9 @@ class frame VALUE_OBJ_CLASS_SPEC {
 
  public:
   // Entry frames
-  JavaCallWrapper* entry_frame_call_wrapper() const;
+  JavaCallWrapper* entry_frame_call_wrapper() const { return *entry_frame_call_wrapper_addr(); }
+  JavaCallWrapper* entry_frame_call_wrapper_if_safe(JavaThread* thread) const;
+  JavaCallWrapper** entry_frame_call_wrapper_addr() const;
   intptr_t* entry_frame_argument_at(int offset) const;
 
   // tells whether there is another chunk of Delta stack above

hotspot/src/share/vm/runtime/javaCalls.hpp

@@ -80,6 +80,8 @@ class JavaCallWrapper: StackObj {
   oop              receiver()               { return _receiver; }
   void             oops_do(OopClosure* f);
 
+  bool             is_first_frame() const   { return _anchor.last_Java_sp() == NULL; }
+
 };
 
 

hotspot/src/share/vm/runtime/thread.cpp

@@ -218,6 +218,7 @@ Thread::Thread() {
   // allocated data structures
   set_osthread(NULL);
   set_resource_area(new (mtThread)ResourceArea());
+  DEBUG_ONLY(_current_resource_mark = NULL;)
   set_handle_area(new (mtThread) HandleArea(NULL));
   set_metadata_handles(new (ResourceObj::C_HEAP, mtClass) GrowableArray<Metadata*>(30, true));
   set_active_handles(NULL);
@@ -953,6 +954,14 @@ bool Thread::is_in_stack(address adr) const {
 }
 
 
+bool Thread::is_in_usable_stack(address adr) const {
+  size_t stack_guard_size = os::uses_stack_guard_pages() ? (StackYellowPages + StackRedPages) * os::vm_page_size() : 0;
+  size_t usable_stack_size = _stack_size - stack_guard_size;
+
+  return ((adr < stack_base()) && (adr >= stack_base() - usable_stack_size));
+}
+
+
 // We had to move these methods here, because vm threads get into ObjectSynchronizer::enter
 // However, there is a note in JavaThread::is_lock_owned() about the VM threads not being
 // used for compilation in the future. If that change is made, the need for these methods
@@ -3636,6 +3645,7 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
 
   // Start Attach Listener if +StartAttachListener or it can't be started lazily
   if (!DisableAttachMechanism) {
+    AttachListener::vm_start();
     if (StartAttachListener || AttachListener::init_at_startup()) {
       AttachListener::init();
     }

hotspot/src/share/vm/runtime/thread.hpp

@@ -86,6 +86,8 @@ class GCTaskQueue;
 class ThreadClosure;
 class IdealGraphPrinter;
 
+DEBUG_ONLY(class ResourceMark;)
+
 class WorkerThread;
 
 // Class hierarchy
@@ -519,6 +521,9 @@ public:
   // Check if address is in the stack of the thread (not just for locks).
   // Warning: the method can only be used on the running thread
   bool is_in_stack(address adr) const;
+  // Check if address is in the usable part of the stack (excludes protected
+  // guard pages)
+  bool is_in_usable_stack(address adr) const;
 
   // Sets this thread as starting thread. Returns failure if thread
   // creation fails due to lack of memory, too many threads etc.
@@ -531,6 +536,8 @@ public:
   // Thread local resource area for temporary allocation within the VM
   ResourceArea* _resource_area;
 
+  DEBUG_ONLY(ResourceMark* _current_resource_mark;)
+
   // Thread local handle area for allocation of handles within the VM
   HandleArea* _handle_area;
   GrowableArray<Metadata*>* _metadata_handles;
@@ -585,6 +592,8 @@ public:
 
   // Deadlock detection
   bool allow_allocation()                        { return _allow_allocation_count == 0; }
+  ResourceMark* current_resource_mark()          { return _current_resource_mark; }
+  void set_current_resource_mark(ResourceMark* rm) { _current_resource_mark = rm; }
 #endif
 
   void check_for_valid_safepoint_state(bool potential_vm_operation) PRODUCT_RETURN;

hotspot/src/share/vm/services/attachListener.hpp

@@ -50,6 +50,7 @@ struct AttachOperationFunctionInfo {
 
 class AttachListener: AllStatic {
  public:
+  static void vm_start() NOT_SERVICES_RETURN;
   static void init()  NOT_SERVICES_RETURN;
   static void abort() NOT_SERVICES_RETURN;
 

hotspot/src/share/vm/services/memTracker.cpp

@@ -385,6 +385,7 @@ void MemTracker::enqueue_pending_recorder(MemRecorder* rec) {
 #define SAFE_SEQUENCE_THRESHOLD    30
 #define HIGH_GENERATION_THRESHOLD  60
 #define MAX_RECORDER_THREAD_RATIO  30
+#define MAX_RECORDER_PER_THREAD    100
 
 void MemTracker::sync() {
   assert(_tracking_level > NMT_off, "NMT is not enabled");
@@ -437,6 +438,11 @@ void MemTracker::sync() {
         // means that worker thread is lagging behind in processing them.
         if (!AutoShutdownNMT) {
           _slowdown_calling_thread = (MemRecorder::_instance_count > MAX_RECORDER_THREAD_RATIO * _thread_count);
+        } else {
+          // If auto shutdown is on, enforce MAX_RECORDER_PER_THREAD threshold to prevent OOM
+          if (MemRecorder::_instance_count >= _thread_count * MAX_RECORDER_PER_THREAD) {
+            shutdown(NMT_out_of_memory);
+          }
         }
 
         // check _worker_thread with lock to avoid racing condition

hotspot/src/share/vm/utilities/ostream.cpp

@@ -296,6 +296,7 @@ stringStream::stringStream(size_t initial_size) : outputStream() {
   buffer        = NEW_RESOURCE_ARRAY(char, buffer_length);
   buffer_pos    = 0;
   buffer_fixed  = false;
+  DEBUG_ONLY(rm = Thread::current()->current_resource_mark();)
 }
 
 // useful for output to fixed chunks of memory, such as performance counters
@@ -321,6 +322,8 @@ void stringStream::write(const char* s, size_t len) {
         end = buffer_length * 2;
       }
       char* oldbuf = buffer;
+      assert(rm == NULL || Thread::current()->current_resource_mark() == rm,
+             "stringStream is re-allocated with a different ResourceMark");
       buffer = NEW_RESOURCE_ARRAY(char, end);
       strncpy(buffer, oldbuf, buffer_pos);
       buffer_length = end;

hotspot/src/share/vm/utilities/ostream.hpp

@@ -28,6 +28,8 @@
 #include "memory/allocation.hpp"
 #include "runtime/timer.hpp"
 
+DEBUG_ONLY(class ResourceMark;)
+
 // Output streams for printing
 //
 // Printing guidelines:
@@ -177,6 +179,7 @@ class stringStream : public outputStream {
   size_t buffer_pos;
   size_t buffer_length;
   bool   buffer_fixed;
+  DEBUG_ONLY(ResourceMark* rm;)
  public:
   stringStream(size_t initial_bufsize = 256);
   stringStream(char* fixed_buffer, size_t fixed_buffer_size);

hotspot/test/runtime/jsig/Test8017498.sh

@@ -0,0 +1,95 @@
+#!/bin/sh
+
+#
+#  Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+#  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+#  This code is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License version 2 only, as
+#  published by the Free Software Foundation.
+#
+#  This code is distributed in the hope that it will be useful, but WITHOUT
+#  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+#  version 2 for more details (a copy is included in the LICENSE file that
+#  accompanied this code).
+#
+#  You should have received a copy of the GNU General Public License version
+#  2 along with this work; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+#  Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+#  or visit www.oracle.com if you need additional information or have any
+#  questions.
+#
+
+##
+## @test Test8017498.sh
+## @bug 8017498
+## @summary sigaction(sig) results in process hang/timed-out if sig is much greater than SIGRTMAX
+## @run shell Test8017498.sh
+##
+
+if [ "${TESTSRC}" = "" ]
+then
+  TESTSRC=${PWD}
+  echo "TESTSRC not set.  Using "${TESTSRC}" as default"
+fi
+echo "TESTSRC=${TESTSRC}"
+## Adding common setup Variables for running shell tests.
+. ${TESTSRC}/../../test_env.sh
+
+# set platform-dependent variables
+OS=`uname -s`
+case "$OS" in
+  Linux)
+    echo "Testing on Linux"
+    if [ "$VM_BITS" = "64" ]
+    then
+        LD_PRELOAD=${TESTJAVA}${FS}jre${FS}lib${FS}amd64${FS}libjsig.so
+    else
+        LD_PRELOAD=${TESTJAVA}${FS}jre${FS}lib${FS}i386${FS}libjsig.so
+    fi
+    echo LD_PRELOAD = ${LD_PRELOAD}
+    export LD_PRELOAD=${LD_PRELOAD}
+    ;;
+  *)
+    NULL=NUL
+    PS=";"
+    FS="\\"
+    echo "Test passed; only valid for Linux"
+    exit 0;
+    ;;
+esac
+
+THIS_DIR=.
+
+cp ${TESTSRC}${FS}*.java ${THIS_DIR}
+${TESTJAVA}${FS}bin${FS}javac *.java
+
+gcc -fPIC -shared -o ${TESTSRC}${FS}libTestJNI.so -I${TESTJAVA}${FS}include -I${TESTJAVA}${FS}include${FS}linux ${TESTSRC}${FS}TestJNI.c
+
+# run the java test in the background
+echo ${TESTJAVA}${FS}bin${FS}java -Djava.library.path=${TESTSRC}${FS} -server TestJNI 100 > test.out 2>&1 &
+${TESTJAVA}${FS}bin${FS}java -Djava.library.path=${TESTSRC}${FS} -server TestJNI 100 > test.out 2>&1 &
+
+# obtain the process id
+C_PID=$!
+
+# sleep for 1s
+sleep 1
+
+# reset LD_PRELOAD
+unset LD_PRELOAD
+
+# check the output file (test.out)
+grep "old handler" test.out > ${NULL}
+if [ $? = 0 ]
+then
+    echo "Test Passed"
+    exit 0
+else
+    kill -9 ${C_PID}
+    echo "Test Failed"
+    exit 1
+fi

hotspot/test/runtime/jsig/TestJNI.c

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <stdio.h>
+#include <jni.h>
+#define __USE_GNU
+#include <signal.h>
+#include <sys/ucontext.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void sig_handler(int sig, siginfo_t *info, ucontext_t *context) {
+    int thrNum;
+
+    printf( " HANDLER (1) " );
+    // Move forward RIP to skip failing instruction
+    context->uc_mcontext.gregs[REG_RIP] += 6;
+}
+
+JNIEXPORT void JNICALL Java_TestJNI_doSomething(JNIEnv *env, jclass klass, jint val) {
+    struct sigaction act;
+    struct sigaction oact;
+    pthread_attr_t attr;
+    stack_t stack;
+
+    act.sa_flags = SA_ONSTACK|SA_RESTART|SA_SIGINFO;
+    sigfillset(&act.sa_mask);
+    act.sa_handler = SIG_DFL;
+    act.sa_sigaction = (void (*)())sig_handler;
+    sigaction(0x20+val, &act, &oact);
+
+    printf( " doSomething(%d) " , val);
+    printf( " old handler = %p " , oact.sa_handler);
+}
+
+#ifdef __cplusplus
+}
+#endif
+

hotspot/test/runtime/jsig/TestJNI.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+public class TestJNI {
+    static {
+        System.loadLibrary("TestJNI");
+    }
+    public static native void doSomething(int val);
+    public static void main(String[] args) {
+        int intArg = 43;
+        if (args.length > 0) {
+            try {
+                intArg = Integer.parseInt(args[0]);
+            } catch (NumberFormatException e) {
+                System.err.println("arg " + args[0] + " must be an integer");
+                System.exit(1);
+            }
+        }
+        TestJNI.doSomething(intArg);
+    }
+}
+

hotspot/test/serviceability/attach/AttachWithStalePidFile.java

@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 7162400
+ * @key regression
+ * @summary Regression test for attach issue where stale pid files in /tmp lead to connection issues
+ * @library /testlibrary
+ * @compile AttachWithStalePidFileTarget.java
+ * @run main AttachWithStalePidFile
+ */
+
+import com.oracle.java.testlibrary.*;
+import com.sun.tools.attach.VirtualMachine;
+import sun.tools.attach.HotSpotVirtualMachine;
+import java.lang.reflect.Field;
+import java.nio.file.*;
+import java.nio.file.attribute.*;
+import java.io.*;
+
+public class AttachWithStalePidFile {
+  public static void main(String... args) throws Exception {
+
+    // this test is only valid on non-Windows platforms
+    if(Platform.isWindows()) {
+      System.out.println("This test is only valid on non-Windows platforms.");
+      return;
+    }
+
+    // Since there might be stale pid-files owned by different
+    // users on the system we may need to retry the test in case we
+    // are unable to remove the existing file.
+    int retries = 5;
+    while(!runTest() && --retries > 0);
+
+    if(retries == 0) {
+      throw new RuntimeException("Test failed after 5 retries. " +
+        "Remove any /tmp/.java_pid* files and retry.");
+    }
+  }
+
+  public static boolean runTest() throws Exception {
+    ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+      "-XX:+UnlockDiagnosticVMOptions", "-XX:+PauseAtStartup", "AttachWithStalePidFileTarget");
+    Process target = pb.start();
+    Path pidFile = null;
+
+    try {
+      int pid = getUnixProcessId(target);
+
+      // create the stale .java_pid file. use hard-coded /tmp path as in th VM
+      pidFile = createJavaPidFile(pid);
+      if(pidFile == null) {
+        return false;
+      }
+
+      // wait for vm.paused file to be created and delete it once we find it.
+      waitForAndResumeVM(pid);
+
+      // unfortunately there's no reliable way to know the VM is ready to receive the
+      // attach request so we have to do an arbitrary sleep.
+      Thread.sleep(5000);
+
+      HotSpotVirtualMachine vm = (HotSpotVirtualMachine)VirtualMachine.attach(((Integer)pid).toString());
+      BufferedReader remoteDataReader = new BufferedReader(new InputStreamReader(vm.remoteDataDump()));
+      String line = null;
+      while((line = remoteDataReader.readLine()) != null);
+
+      vm.detach();
+      return true;
+    }
+    finally {
+      target.destroy();
+      target.waitFor();
+
+      if(pidFile != null && Files.exists(pidFile)) {
+        Files.delete(pidFile);
+      }
+    }
+  }
+
+  private static Path createJavaPidFile(int pid) throws Exception {
+    Path pidFile = Paths.get("/tmp/.java_pid" + pid);
+    if(Files.exists(pidFile)) {
+      try {
+        Files.delete(pidFile);
+      }
+      catch(FileSystemException e) {
+        if(e.getReason().equals("Operation not permitted")) {
+          System.out.println("Unable to remove exisiting stale PID file" + pidFile);
+          return null;
+        }
+        throw e;
+      }
+    }
+    return Files.createFile(pidFile,
+      PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------")));
+  }
+
+  private static void waitForAndResumeVM(int pid) throws Exception {
+    Path pauseFile = Paths.get("vm.paused." + pid);
+    int retries = 60;
+    while(!Files.exists(pauseFile) && --retries > 0) {
+      Thread.sleep(1000);
+    }
+    if(retries == 0) {
+      throw new RuntimeException("Timeout waiting for VM to start. " +
+        "vm.paused file not created within 60 seconds.");
+    }
+    Files.delete(pauseFile);
+  }
+
+  private static int getUnixProcessId(Process unixProcess) throws Exception {
+    Field pidField = unixProcess.getClass().getDeclaredField("pid");
+    pidField.setAccessible(true);
+    return (Integer)pidField.get(unixProcess);
+  }
+}

hotspot/test/serviceability/attach/AttachWithStalePidFileTarget.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+public class AttachWithStalePidFileTarget {
+  public static void main(String... args) throws Exception {
+    Thread.sleep(2*60*1000);
+  }
+}