8203356: VM Object Allocation Collector can infinite recurse

VM Event callback do not provoke a VM alloc event

Reviewed-by: sspitsyn, phh, amenkov, cjplummer

make/nb_native/nbproject/configurations.xml

@@ -6155,6 +6155,9 @@
                 <df name="HeapMonitorModule">
                   <in>libHeapMonitorTest.c</in>
                 </df>
+                <df name="VMEvent">
+                  <in>libVMEventTest.c</in>
+                </df>
                 <df name="ModuleAwareAgents">
                   <df name="ClassFileLoadHook">
                     <in>libMAAClassFileLoadHook.c</in>
@@ -40146,6 +40149,11 @@
             tool="0"
             flavor2="0">
       </item>
+      <item path="../../test/hotspot/jtreg/serviceability/jvmti/VMEvent/libVMEventTest.c"
+            ex="false"
+            tool="0"
+            flavor2="0">
+      </item>
       <item path="../../test/hotspot/jtreg/serviceability/jvmti/ModuleAwareAgents/ClassFileLoadHook/libMAAClassFileLoadHook.c"
             ex="false"
             tool="0"

src/hotspot/share/prims/jvmtiExport.cpp

@@ -2722,7 +2722,14 @@ void JvmtiEventCollector::setup_jvmti_thread_state() {
   // should not happen since we're trying to configure for event collection
   guarantee(state != NULL, "exiting thread called setup_jvmti_thread_state");
   if (is_vm_object_alloc_event()) {
-    _prev = state->get_vm_object_alloc_event_collector();
+    JvmtiVMObjectAllocEventCollector *prev = state->get_vm_object_alloc_event_collector();
+
+    // If we have a previous collector and it is disabled, it means this allocation came from a
+    // callback induced VM Object allocation, do not register this collector then.
+    if (prev && !prev->is_enabled()) {
+      return;
+    }
+    _prev = prev;
     state->set_vm_object_alloc_event_collector((JvmtiVMObjectAllocEventCollector *)this);
   } else if (is_dynamic_code_event()) {
     _prev = state->get_dynamic_code_event_collector();

test/hotspot/jtreg/serviceability/jvmti/VMEvent/MyPackage/VMEventRecursionTest.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, Google and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package MyPackage;
+
+/**
+ * @test
+ * @summary Verifies that a VM event callback does not recurse if a VM object is allocated during callback.
+ * @compile VMEventRecursionTest.java
+ * @run main/othervm/native -agentlib:VMEventTest MyPackage.VMEventRecursionTest
+ */
+public class VMEventRecursionTest implements Cloneable {
+    // Implement a simple clone. A call will provoke a JVMTI event for VM allocations, which tries to
+    // call this again.
+    public Object clone() throws CloneNotSupportedException {
+        return super.clone();
+    }
+
+    public static void main(String[] args) {
+        VMEventRecursionTest obj = new VMEventRecursionTest();
+        try {
+            obj.clone();
+        } catch(CloneNotSupportedException e) {
+            // NOP.
+        }
+    }
+}

test/hotspot/jtreg/serviceability/jvmti/VMEvent/libVMEventTest.c

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, Google and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <string.h>
+#include "jvmti.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef JNI_ENV_ARG
+
+#ifdef __cplusplus
+#define JNI_ENV_ARG(x)
+#define JNI_ENV_ARGS2(x, y) y
+#define JNI_ENV_ARGS3(x, y, z) y, z
+#define JNI_ENV_ARGS4(x, y, z, w) y, z, w
+#define JNI_ENV_PTR(x) x
+#else
+#define JNI_ENV_ARG(x) x
+#define JNI_ENV_ARGS2(x,y) x, y
+#define JNI_ENV_ARGS3(x, y, z) x, y, z
+#define JNI_ENV_ARGS4(x, y, z, w) x, y, z, w
+#define JNI_ENV_PTR(x) (*x)
+#endif
+
+#endif
+
+extern JNIEXPORT void JNICALL VMObjectAlloc(jvmtiEnv *jvmti,
+                                            JNIEnv* jni,
+                                            jthread thread,
+                                            jobject object,
+                                            jclass klass,
+                                            jlong size) {
+  char *signature = NULL;
+  jvmtiError error = (*jvmti)->GetClassSignature(jvmti, klass, &signature, NULL);
+
+  if (error != JVMTI_ERROR_NONE || signature == NULL) {
+    JNI_ENV_PTR(jni)->FatalError(
+        JNI_ENV_ARGS2(jni, "Failed during the GetClassSignature call"));
+  }
+
+  // If it is our test class, call clone now.
+  if (!strcmp(signature, "LMyPackage/VMEventRecursionTest;")) {
+    jmethodID clone_method =
+        JNI_ENV_PTR(jni)->GetMethodID(JNI_ENV_ARGS4(jni, klass, "clone", "()Ljava/lang/Object;"));
+
+    if (JNI_ENV_PTR(jni)->ExceptionOccurred(JNI_ENV_ARG(jni))) {
+      JNI_ENV_PTR(jni)->FatalError(
+          JNI_ENV_ARGS2(jni, "Failed during the GetMethodID call"));
+    }
+
+    JNI_ENV_PTR(jni)->CallObjectMethod(JNI_ENV_ARGS3(jni, object, clone_method));
+
+    if (JNI_ENV_PTR(jni)->ExceptionOccurred(JNI_ENV_ARG(jni))) {
+      JNI_ENV_PTR(jni)->FatalError(
+          JNI_ENV_ARGS2(jni, "Failed during the CallObjectMethod call"));
+    }
+  }
+}
+
+extern JNIEXPORT void JNICALL OnVMInit(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) {
+  (*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_VM_OBJECT_ALLOC, NULL);
+}
+
+extern JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM *jvm, char *options,
+                                           void *reserved) {
+  jvmtiEnv *jvmti;
+  jvmtiEventCallbacks callbacks;
+  jvmtiCapabilities caps;
+
+  if ((*jvm)->GetEnv(jvm, (void **) (&jvmti), JVMTI_VERSION) != JNI_OK) {
+    return JNI_ERR;
+  }
+
+  memset(&callbacks, 0, sizeof(callbacks));
+  callbacks.VMObjectAlloc = &VMObjectAlloc;
+  callbacks.VMInit = &OnVMInit;
+
+  memset(&caps, 0, sizeof(caps));
+  caps.can_generate_vm_object_alloc_events = 1;
+  (*jvmti)->AddCapabilities(jvmti, &caps);
+
+  (*jvmti)->SetEventCallbacks(jvmti, &callbacks, sizeof(jvmtiEventCallbacks));
+  (*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_VM_INIT, NULL);
+  return 0;
+}
+
+#ifdef __cplusplus
+}
+#endif