8006683: Add WhiteBox API to testing of compiler

Reviewed-by: kvn, vlivanov

hotspot/src/share/tools/whitebox/sun/hotspot/WhiteBox.java

@@ -23,6 +23,8 @@
  */
 
 package sun.hotspot;
+
+import java.lang.reflect.Method;
 import java.security.BasicPermission;
 import sun.hotspot.parser.DiagnosticCommand;
 
@@ -81,4 +83,15 @@ public class WhiteBox {
   public native boolean NMTAllocTest();
   public native boolean NMTFreeTestMemory();
   public native boolean NMTWaitForDataMerge();
+
+  // Compiler
+  public native void    deoptimizeAll();
+  public native boolean isMethodCompiled(Method method);
+  public native boolean isMethodCompilable(Method method);
+  public native boolean isMethodQueuedForCompilation(Method method);
+  public native int     deoptimizeMethod(Method method);
+  public native void    makeMethodNotCompilable(Method method);
+  public native int     getMethodCompilationLevel(Method method);
+  public native boolean setDontInlineMethod(Method method, boolean value);
+  public native int     getCompileQueuesSize();
 }

hotspot/src/share/vm/prims/wbtestmethods/parserTests.hpp

@@ -27,6 +27,6 @@
 #include "prims/jni.h"
 #include "prims/whitebox.hpp"
 
-WB_METHOD_DECLARE WB_ParseCommandLine(JNIEnv* env, jobject o, jstring args, jobjectArray arguments);
+WB_METHOD_DECLARE(jobjectArray) WB_ParseCommandLine(JNIEnv* env, jobject o, jstring args, jobjectArray arguments);
 
 #endif //SHARE_VM_PRIMS_WBTESTMETHODS_PARSERTESTS_H

hotspot/src/share/vm/prims/whitebox.cpp

@@ -48,6 +48,8 @@
 #include "services/memTracker.hpp"
 #endif // INCLUDE_NMT
 
+#include "compiler/compileBroker.hpp"
+
 bool WhiteBox::_used = false;
 
 WB_ENTRY(jlong, WB_GetObjectAddress(JNIEnv* env, jobject o, jobject obj))
@@ -169,6 +171,89 @@ WB_END
 
 #endif // INCLUDE_NMT
 
+static jmethodID reflected_method_to_jmid(JavaThread* thread, JNIEnv* env, jobject method) {
+  assert(method != NULL, "method should not be null");
+  ThreadToNativeFromVM ttn(thread);
+  return env->FromReflectedMethod(method);
+}
+
+WB_ENTRY(void, WB_DeoptimizeAll(JNIEnv* env, jobject o))
+  MutexLockerEx mu(Compile_lock);
+  CodeCache::mark_all_nmethods_for_deoptimization();
+  VM_Deoptimize op;
+  VMThread::execute(&op);
+WB_END
+
+WB_ENTRY(jint, WB_DeoptimizeMethod(JNIEnv* env, jobject o, jobject method))
+  jmethodID jmid = reflected_method_to_jmid(thread, env, method);
+  MutexLockerEx mu(Compile_lock);
+  methodHandle mh(THREAD, Method::checked_resolve_jmethod_id(jmid));
+  int result = 0;
+  nmethod* code = mh->code();
+  if (code != NULL) {
+    code->mark_for_deoptimization();
+    ++result;
+  }
+  result += CodeCache::mark_for_deoptimization(mh());
+  if (result > 0) {
+    VM_Deoptimize op;
+    VMThread::execute(&op);
+  }
+  return result;
+WB_END
+
+WB_ENTRY(jboolean, WB_IsMethodCompiled(JNIEnv* env, jobject o, jobject method))
+  jmethodID jmid = reflected_method_to_jmid(thread, env, method);
+  MutexLockerEx mu(Compile_lock);
+  methodHandle mh(THREAD, Method::checked_resolve_jmethod_id(jmid));
+  nmethod* code = mh->code();
+  if (code == NULL) {
+    return JNI_FALSE;
+  }
+  return (code->is_alive() && !code->is_marked_for_deoptimization());
+WB_END
+
+WB_ENTRY(jboolean, WB_IsMethodCompilable(JNIEnv* env, jobject o, jobject method))
+  jmethodID jmid = reflected_method_to_jmid(thread, env, method);
+  MutexLockerEx mu(Compile_lock);
+  methodHandle mh(THREAD, Method::checked_resolve_jmethod_id(jmid));
+  return !mh->is_not_compilable();
+WB_END
+
+WB_ENTRY(jboolean, WB_IsMethodQueuedForCompilation(JNIEnv* env, jobject o, jobject method))
+  jmethodID jmid = reflected_method_to_jmid(thread, env, method);
+  MutexLockerEx mu(Compile_lock);
+  methodHandle mh(THREAD, Method::checked_resolve_jmethod_id(jmid));
+  return mh->queued_for_compilation();
+WB_END
+
+WB_ENTRY(jint, WB_GetMethodCompilationLevel(JNIEnv* env, jobject o, jobject method))
+  jmethodID jmid = reflected_method_to_jmid(thread, env, method);
+  methodHandle mh(THREAD, Method::checked_resolve_jmethod_id(jmid));
+  nmethod* code = mh->code();
+  return (code != NULL ? code->comp_level() : CompLevel_none);
+WB_END
+
+
+WB_ENTRY(void, WB_MakeMethodNotCompilable(JNIEnv* env, jobject o, jobject method))
+  jmethodID jmid = reflected_method_to_jmid(thread, env, method);
+  methodHandle mh(THREAD, Method::checked_resolve_jmethod_id(jmid));
+  mh->set_not_compilable();
+WB_END
+
+WB_ENTRY(jboolean, WB_SetDontInlineMethod(JNIEnv* env, jobject o, jobject method, jboolean value))
+  jmethodID jmid = reflected_method_to_jmid(thread, env, method);
+  methodHandle mh(THREAD, Method::checked_resolve_jmethod_id(jmid));
+  bool result = mh->dont_inline();
+  mh->set_dont_inline(value == JNI_TRUE);
+  return result;
+WB_END
+
+WB_ENTRY(jint, WB_GetCompileQueuesSize(JNIEnv* env, jobject o))
+  return CompileBroker::queue_size(CompLevel_full_optimization) /* C2 */ +
+         CompileBroker::queue_size(CompLevel_full_profile) /* C1 */;
+WB_END
+
 //Some convenience methods to deal with objects from java
 int WhiteBox::offset_for_field(const char* field_name, oop object,
     Symbol* signature_symbol) {
@@ -225,9 +310,9 @@ bool WhiteBox::lookup_bool(const char* field_name, oop object) {
 static JNINativeMethod methods[] = {
   {CC"getObjectAddress",   CC"(Ljava/lang/Object;)J", (void*)&WB_GetObjectAddress  },
   {CC"getHeapOopSize",     CC"()I",                   (void*)&WB_GetHeapOopSize    },
-  {CC"isClassAlive0",       CC"(Ljava/lang/String;)Z", (void*)&WB_IsClassAlive      },
-  {CC "parseCommandLine",
-      CC "(Ljava/lang/String;[Lsun/hotspot/parser/DiagnosticCommand;)[Ljava/lang/Object;",
+  {CC"isClassAlive0",      CC"(Ljava/lang/String;)Z", (void*)&WB_IsClassAlive      },
+  {CC"parseCommandLine",
+      CC"(Ljava/lang/String;[Lsun/hotspot/parser/DiagnosticCommand;)[Ljava/lang/Object;",
       (void*) &WB_ParseCommandLine
   },
 #if INCLUDE_ALL_GCS
@@ -241,6 +326,23 @@ static JNINativeMethod methods[] = {
   {CC"NMTFreeTestMemory",  CC"()Z",                   (void*)&WB_NMTFreeTestMemory },
   {CC"NMTWaitForDataMerge",CC"()Z",                   (void*)&WB_NMTWaitForDataMerge},
 #endif // INCLUDE_NMT
+  {CC"deoptimizeAll",      CC"()V",                   (void*)&WB_DeoptimizeAll     },
+  {CC"deoptimizeMethod",   CC"(Ljava/lang/reflect/Method;)I",
+                                                      (void*)&WB_DeoptimizeMethod  },
+  {CC"isMethodCompiled",   CC"(Ljava/lang/reflect/Method;)Z",
+                                                      (void*)&WB_IsMethodCompiled  },
+  {CC"isMethodCompilable", CC"(Ljava/lang/reflect/Method;)Z",
+                                                      (void*)&WB_IsMethodCompilable},
+  {CC"isMethodQueuedForCompilation",
+      CC"(Ljava/lang/reflect/Method;)Z",              (void*)&WB_IsMethodQueuedForCompilation},
+  {CC"makeMethodNotCompilable",
+      CC"(Ljava/lang/reflect/Method;)V",              (void*)&WB_MakeMethodNotCompilable},
+  {CC"setDontInlineMethod",
+      CC"(Ljava/lang/reflect/Method;Z)Z",             (void*)&WB_SetDontInlineMethod},
+  {CC"getMethodCompilationLevel",
+      CC"(Ljava/lang/reflect/Method;)I",              (void*)&WB_GetMethodCompilationLevel},
+  {CC"getCompileQueuesSize",
+      CC"()I",                                        (void*)&WB_GetCompileQueuesSize},
 };
 
 #undef CC

hotspot/src/share/vm/prims/whitebox.hpp

@@ -34,7 +34,7 @@
 
 #define WB_ENTRY(result_type, header) JNI_ENTRY(result_type, header)
 #define WB_END JNI_END
-#define WB_METHOD_DECLARE extern "C" jobjectArray JNICALL
+#define WB_METHOD_DECLARE(result_type) extern "C" result_type JNICALL
 
 class WhiteBox : public AllStatic {
  private:

hotspot/test/compiler/whitebox/CompilerWhiteBoxTest.java

@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import sun.hotspot.WhiteBox;
+import sun.management.ManagementFactoryHelper;
+import com.sun.management.HotSpotDiagnosticMXBean;
+
+import java.lang.reflect.Method;
+
+/*
+ * @author igor.ignatyev@oracle.com
+ */
+public abstract class CompilerWhiteBoxTest {
+    protected static final WhiteBox WHITE_BOX = WhiteBox.getWhiteBox();
+    protected static final Method METHOD = getMethod("method");
+    protected static final int COMPILE_THRESHOLD
+            = Integer.parseInt(getVMOption("CompileThreshold", "10000"));
+
+    protected static Method getMethod(String name) {
+        try {
+            return CompilerWhiteBoxTest.class.getDeclaredMethod(name);
+        } catch (NoSuchMethodException | SecurityException e) {
+            throw new RuntimeException(
+                    "exception on getting method " + name, e);
+        }
+    }
+
+    protected static String getVMOption(String name, String defaultValue) {
+        String result;
+        HotSpotDiagnosticMXBean diagnostic
+                = ManagementFactoryHelper.getDiagnosticMXBean();
+        result = diagnostic.getVMOption(name).getValue();
+        return result == null ? defaultValue : result;
+    }
+
+    protected final void runTest() throws RuntimeException {
+        if (ManagementFactoryHelper.getCompilationMXBean() == null) {
+            System.err.println(
+                    "Warning: test is not applicable in interpreted mode");
+            return;
+        }
+        System.out.println("at test's start:");
+        printInfo(METHOD);
+        try {
+            test();
+        } catch (Exception e) {
+            System.out.printf("on exception '%s':", e.getMessage());
+            printInfo(METHOD);
+            throw new RuntimeException(e);
+        }
+        System.out.println("at test's end:");
+        printInfo(METHOD);
+    }
+
+    protected static void checkNotCompiled(Method method) {
+        if (WHITE_BOX.isMethodCompiled(method)) {
+            throw new RuntimeException(method + " must be not compiled");
+        }
+        if (WHITE_BOX.getMethodCompilationLevel(method) != 0) {
+            throw new RuntimeException(method + " comp_level must be == 0");
+        }
+    }
+
+    protected static void checkCompiled(Method method)
+            throws InterruptedException {
+        final long start = System.currentTimeMillis();
+        waitBackgroundCompilation(method);
+        if (WHITE_BOX.isMethodQueuedForCompilation(method)) {
+            System.err.printf("Warning: %s is still in queue after %dms%n",
+                    method, System.currentTimeMillis() - start);
+            return;
+        }
+        if (!WHITE_BOX.isMethodCompiled(method)) {
+            throw new RuntimeException(method + " must be compiled");
+        }
+        if (WHITE_BOX.getMethodCompilationLevel(method) == 0) {
+            throw new RuntimeException(method + " comp_level must be != 0");
+        }
+    }
+
+    protected static void waitBackgroundCompilation(Method method)
+            throws InterruptedException {
+        final Object obj = new Object();
+        synchronized (obj) {
+            for (int i = 0; i < 10; ++i) {
+                if (!WHITE_BOX.isMethodQueuedForCompilation(method)) {
+                    break;
+                }
+                obj.wait(1000);
+            }
+        }
+    }
+
+    protected static void printInfo(Method method) {
+        System.out.printf("%n%s:%n", method);
+        System.out.printf("\tcompilable:\t%b%n",
+                WHITE_BOX.isMethodCompilable(method));
+        System.out.printf("\tcompiled:\t%b%n",
+                WHITE_BOX.isMethodCompiled(method));
+        System.out.printf("\tcomp_level:\t%d%n",
+                WHITE_BOX.getMethodCompilationLevel(method));
+        System.out.printf("\tin_queue:\t%b%n",
+                WHITE_BOX.isMethodQueuedForCompilation(method));
+        System.out.printf("compile_queues_size:\t%d%n%n",
+                WHITE_BOX.getCompileQueuesSize());
+    }
+
+    protected abstract void test() throws Exception;
+
+    protected final int compile() {
+        int result = 0;
+        for (int i = 0; i < COMPILE_THRESHOLD; ++i) {
+            result += method();
+        }
+        return result;
+    }
+
+
+    protected int method() {
+        return 42;
+    }
+}

hotspot/test/compiler/whitebox/DeoptimizeAllTest.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test DeoptimizeAllTest
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI CompilerWhiteBoxTest.java
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI DeoptimizeAllTest.java
+ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI DeoptimizeAllTest
+ * @author igor.ignatyev@oracle.com
+ */
+public class DeoptimizeAllTest extends CompilerWhiteBoxTest {
+
+    public static void main(String[] args) throws Exception {
+        new DeoptimizeAllTest().runTest();
+    }
+
+    protected void test() throws Exception {
+        // to prevent inlining #method into #compile()
+        WHITE_BOX.setDontInlineMethod(METHOD, true);
+        compile();
+        checkCompiled(METHOD);
+        WHITE_BOX.deoptimizeAll();
+        checkNotCompiled(METHOD);
+    }
+}

hotspot/test/compiler/whitebox/DeoptimizeMethodTest.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test DeoptimizeMethodTest
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI CompilerWhiteBoxTest.java
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI DeoptimizeMethodTest.java
+ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI DeoptimizeMethodTest
+ * @author igor.ignatyev@oracle.com
+ */
+public class DeoptimizeMethodTest extends CompilerWhiteBoxTest {
+
+    public static void main(String[] args) throws Exception {
+        new DeoptimizeMethodTest().runTest();
+    }
+
+    protected void test() throws Exception {
+        // to prevent inlining #method into #compile()
+        WHITE_BOX.setDontInlineMethod(METHOD, true);
+        compile();
+        checkCompiled(METHOD);
+        WHITE_BOX.deoptimizeMethod(METHOD);
+        checkNotCompiled(METHOD);
+    }
+}

hotspot/test/compiler/whitebox/IsMethodCompilableTest.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test IsMethodCompilableTest
+ * @bug 8007270
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI CompilerWhiteBoxTest.java
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI IsMethodCompilableTest.java
+ * @run main/othervm/timeout=600 -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI IsMethodCompilableTest
+ * @author igor.ignatyev@oracle.com
+ */
+public class IsMethodCompilableTest extends CompilerWhiteBoxTest {
+    protected static final long PER_METHOD_RECOMPILATION_CUTOFF;
+
+    static {
+        long tmp = Long.parseLong(
+                getVMOption("PerMethodRecompilationCutoff", "400"));
+        if (tmp == -1) {
+            PER_METHOD_RECOMPILATION_CUTOFF = -1 /* Inf */;
+        } else {
+            PER_METHOD_RECOMPILATION_CUTOFF = 1 + (0xFFFFFFFFL & tmp);
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        new IsMethodCompilableTest().runTest();
+    }
+
+    protected void test() throws Exception {
+        if (!WHITE_BOX.isMethodCompilable(METHOD)) {
+            throw new RuntimeException(METHOD + " must be compilable");
+        }
+        System.out.println("PerMethodRecompilationCutoff = "
+                + PER_METHOD_RECOMPILATION_CUTOFF);
+        if (PER_METHOD_RECOMPILATION_CUTOFF == -1) {
+            System.err.println(
+                    "Warning: test is not applicable if PerMethodRecompilationCutoff == Inf");
+            return;
+        }
+        // to prevent inlining #method into #compile()
+        WHITE_BOX.setDontInlineMethod(METHOD, true);
+        boolean madeNotCompilable = false;
+
+        for (long i = 0; i < PER_METHOD_RECOMPILATION_CUTOFF; ++i) {
+            compile();
+            waitBackgroundCompilation(METHOD);
+            WHITE_BOX.deoptimizeMethod(METHOD);
+            if (!WHITE_BOX.isMethodCompilable(METHOD)) {
+                madeNotCompilable = true;
+                break;
+            }
+        }
+        if (!madeNotCompilable) {
+            throw new RuntimeException(METHOD + " is still compilable after "
+                    + PER_METHOD_RECOMPILATION_CUTOFF + " iterations");
+        }
+        compile();
+        if (WHITE_BOX.isMethodCompiled(METHOD)) {
+            printInfo(METHOD);
+            throw new RuntimeException(
+                    METHOD + " is not compilable but compiled");
+        }
+    }
+}

hotspot/test/compiler/whitebox/MakeMethodNotCompilableTest.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test MakeMethodNotCompilableTest
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI CompilerWhiteBoxTest.java
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI MakeMethodNotCompilableTest.java
+ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI MakeMethodNotCompilableTest
+ * @author igor.ignatyev@oracle.com
+ */
+public class MakeMethodNotCompilableTest extends CompilerWhiteBoxTest {
+
+    public static void main(String[] args) throws Exception {
+        new MakeMethodNotCompilableTest().runTest();
+    }
+
+    protected void test() throws Exception  {
+        if (!WHITE_BOX.isMethodCompilable(METHOD)) {
+            throw new RuntimeException(METHOD + " must be compilable");
+        }
+        WHITE_BOX.makeMethodNotCompilable(METHOD);
+        if (WHITE_BOX.isMethodCompilable(METHOD)) {
+            throw new RuntimeException(METHOD + " must be not compilable");
+        }
+        compile();
+        if (WHITE_BOX.isMethodQueuedForCompilation(METHOD)) {
+            throw new RuntimeException(METHOD + " must not be in queue");
+        }
+        checkNotCompiled(METHOD);
+        if (WHITE_BOX.isMethodCompilable(METHOD)) {
+            throw new RuntimeException(METHOD + " must be not compilable");
+        }
+    }
+}

hotspot/test/compiler/whitebox/SetDontInlineMethodTest.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test SetDontInlineMethodTest
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI CompilerWhiteBoxTest.java
+ * @compile -J-XX:+UnlockDiagnosticVMOptions -J-XX:+WhiteBoxAPI SetDontInlineMethodTest.java
+ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI SetDontInlineMethodTest
+ * @author igor.ignatyev@oracle.com
+ */
+public class SetDontInlineMethodTest extends CompilerWhiteBoxTest {
+
+    public static void main(String[] args) throws Exception {
+        new SetDontInlineMethodTest().runTest();
+    }
+
+    protected void test() throws Exception {
+        if (WHITE_BOX.setDontInlineMethod(METHOD, true)) {
+            throw new RuntimeException("on start " + METHOD
+                    + " must be inlineable");
+        }
+        if (!WHITE_BOX.setDontInlineMethod(METHOD, true)) {
+            throw new RuntimeException("after first change to true " + METHOD
+                    + " must be not inlineable");
+        }
+        if (!WHITE_BOX.setDontInlineMethod(METHOD, false)) {
+            throw new RuntimeException("after second change to true " + METHOD
+                    + " must be still not inlineable");
+        }
+        if (WHITE_BOX.setDontInlineMethod(METHOD, false)) {
+            throw new RuntimeException("after first change to false" + METHOD
+                    + " must be inlineable");
+        }
+        if (WHITE_BOX.setDontInlineMethod(METHOD, false)) {
+            throw new RuntimeException("after second change to false " + METHOD
+                    + " must be inlineable");
+        }
+    }
+}