8337199: Add jcmd Thread.vthread_scheduler and Thread.vthread_pollers diagnostic commands

Reviewed-by: dholmes, kevinw

src/hotspot/share/classfile/vmSymbols.hpp

@@ -739,10 +739,15 @@ class SerializeClosure;
   template(toFileURL_signature,                             "(Ljava/lang/String;)Ljava/net/URL;")                 \
   template(url_array_classloader_void_signature,            "([Ljava/net/URL;Ljava/lang/ClassLoader;)V")          \
                                                                                                                   \
-  /* Thread.dump_to_file jcmd */                                                                                  \
+  /* jcmd Thread.dump_to_file */                                                                                  \
   template(jdk_internal_vm_ThreadDumper,           "jdk/internal/vm/ThreadDumper")                                \
   template(dumpThreads_name,                       "dumpThreads")                                                 \
   template(dumpThreadsToJson_name,                 "dumpThreadsToJson")                                           \
+                                                                                                                  \
+  /* jcmd Thread.vthread_scheduler and Thread.vthread_pollers */                                                  \
+  template(jdk_internal_vm_JcmdVThreadCommands,    "jdk/internal/vm/JcmdVThreadCommands")                         \
+  template(printScheduler_name,                    "printScheduler")                                              \
+  template(printPollers_name,                      "printPollers")                                                \
 
   /*end*/
 

src/hotspot/share/services/diagnosticCommand.cpp

@@ -128,6 +128,8 @@ void DCmd::register_dcmds(){
 #endif // INCLUDE_JVMTI
   DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl<ThreadDumpDCmd>(full_export, true, false));
   DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl<ThreadDumpToFileDCmd>(full_export, true, false));
+  DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl<VThreadSchedulerDCmd>(full_export, true, false));
+  DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl<VThreadPollersDCmd>(full_export, true, false));
   DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl<ClassLoaderStatsDCmd>(full_export, true, false));
   DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl<ClassLoaderHierarchyDCmd>(full_export, true, false));
   DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl<CompileQueueDCmd>(full_export, true, false));
@@ -1073,13 +1075,6 @@ void ThreadDumpToFileDCmd::dumpToFile(Symbol* name, Symbol* signature, const cha
 
   Symbol* sym = vmSymbols::jdk_internal_vm_ThreadDumper();
   Klass* k = SystemDictionary::resolve_or_fail(sym, true, CHECK);
-  InstanceKlass* ik = InstanceKlass::cast(k);
-  if (HAS_PENDING_EXCEPTION) {
-    java_lang_Throwable::print(PENDING_EXCEPTION, output());
-    output()->cr();
-    CLEAR_PENDING_EXCEPTION;
-    return;
-  }
 
   // invoke the ThreadDump method to dump to file
   JavaValue result(T_OBJECT);
@@ -1110,6 +1105,45 @@ void ThreadDumpToFileDCmd::dumpToFile(Symbol* name, Symbol* signature, const cha
   output()->print_raw((const char*)addr, ba->length());
 }
 
+// Calls a static no-arg method on jdk.internal.vm.JcmdVThreadCommands that returns a byte[] with
+// the output. If the method completes successfully then the bytes are copied to the output stream.
+// If the method fails then the exception is printed to the output stream.
+static void execute_vthread_command(Symbol* method_name, outputStream* output, TRAPS) {
+  ResourceMark rm(THREAD);
+  HandleMark hm(THREAD);
+
+  Klass* k = SystemDictionary::resolve_or_fail(vmSymbols::jdk_internal_vm_JcmdVThreadCommands(), true, CHECK);
+
+  JavaValue result(T_OBJECT);
+  JavaCallArguments args;
+  JavaCalls::call_static(&result,
+                         k,
+                         method_name,
+                         vmSymbols::void_byte_array_signature(),
+                         &args,
+                         THREAD);
+  if (HAS_PENDING_EXCEPTION) {
+    java_lang_Throwable::print(PENDING_EXCEPTION, output);
+    output->cr();
+    CLEAR_PENDING_EXCEPTION;
+    return;
+  }
+
+  // copy the bytes to the output stream
+  oop res = cast_to_oop(result.get_jobject());
+  typeArrayOop ba = typeArrayOop(res);
+  jbyte* addr = typeArrayOop(res)->byte_at_addr(0);
+  output->print_raw((const char*)addr, ba->length());
+}
+
+void VThreadSchedulerDCmd::execute(DCmdSource source, TRAPS) {
+  execute_vthread_command(vmSymbols::printScheduler_name(), output(), CHECK);
+}
+
+void VThreadPollersDCmd::execute(DCmdSource source, TRAPS) {
+  execute_vthread_command(vmSymbols::printPollers_name(), output(), CHECK);
+}
+
 CompilationMemoryStatisticDCmd::CompilationMemoryStatisticDCmd(outputStream* output, bool heap) :
     DCmdWithParser(output, heap),
   _human_readable("-H", "Human readable format", "BOOLEAN", false, "false"),

src/hotspot/share/services/diagnosticCommand.hpp

@@ -776,6 +776,33 @@ public:
   virtual void execute(DCmdSource source, TRAPS);
 };
 
+class VThreadSchedulerDCmd : public DCmd {
+public:
+  VThreadSchedulerDCmd(outputStream* output, bool heap) : DCmd(output, heap) { }
+  static const char* name() {
+    return "Thread.vthread_scheduler";
+  }
+  static const char* description() {
+    return "Print the virtual thread scheduler, and the delayed task schedulers that support "
+           "virtual threads doing timed operations.";
+  }
+  static const char* impact() { return "Low"; }
+  virtual void execute(DCmdSource source, TRAPS);
+};
+
+class VThreadPollersDCmd : public DCmd {
+public:
+  VThreadPollersDCmd(outputStream* output, bool heap) : DCmd(output, heap) { }
+  static const char* name() {
+    return "Thread.vthread_pollers";
+  }
+  static const char* description() {
+    return "Print the I/O pollers that support virtual threads doing blocking network I/O operations.";
+  }
+  static const char* impact() { return "Low"; }
+  virtual void execute(DCmdSource source, TRAPS);
+};
+
 class CompilationMemoryStatisticDCmd: public DCmdWithParser {
 protected:
   DCmdArgument<bool> _human_readable;

src/java.base/share/classes/java/lang/System.java

@@ -55,6 +55,7 @@ import java.util.Properties;
 import java.util.ResourceBundle;
 import java.util.Set;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.function.Supplier;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Stream;
@@ -2304,6 +2305,10 @@ public final class System {
                 return VirtualThread.defaultScheduler();
             }
 
+            public Stream<ScheduledExecutorService> virtualThreadDelayedTaskSchedulers() {
+                return VirtualThread.delayedTaskSchedulers();
+            }
+
             public StackWalker newStackWalkerInstance(Set<StackWalker.Option> options,
                                                       ContinuationScope contScope,
                                                       Continuation continuation) {

src/java.base/share/classes/java/lang/VirtualThread.java

@@ -24,6 +24,7 @@
  */
 package java.lang;
 
+import java.util.Arrays;
 import java.util.Locale;
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
@@ -32,12 +33,12 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory;
 import java.util.concurrent.ForkJoinTask;
-import java.util.concurrent.ForkJoinWorkerThread;
 import java.util.concurrent.Future;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 import jdk.internal.event.VirtualThreadEndEvent;
 import jdk.internal.event.VirtualThreadStartEvent;
 import jdk.internal.event.VirtualThreadSubmitFailedEvent;
@@ -192,6 +193,13 @@ final class VirtualThread extends BaseVirtualThread {
         return DEFAULT_SCHEDULER;
     }
 
+    /**
+     * Returns a stream of the delayed task schedulers used to support timed operations.
+     */
+    static Stream<ScheduledExecutorService> delayedTaskSchedulers() {
+        return Arrays.stream(DELAYED_TASK_SCHEDULERS);
+    }
+
     /**
      * Returns the continuation scope used for virtual threads.
      */

src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java

@@ -45,6 +45,7 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.stream.Stream;
 
 import jdk.internal.loader.NativeLibraries;
@@ -595,6 +596,11 @@ public interface JavaLangAccess {
      */
     Executor virtualThreadDefaultScheduler();
 
+    /**
+     * Returns a stream of the delayed task schedulers used for virtual threads.
+     */
+    Stream<ScheduledExecutorService> virtualThreadDelayedTaskSchedulers();
+
     /**
      * Creates a new StackWalker
      */

src/java.base/share/classes/jdk/internal/vm/JcmdVThreadCommands.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package jdk.internal.vm;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.stream.IntStream;
+import jdk.internal.access.JavaLangAccess;
+import jdk.internal.access.SharedSecrets;
+import sun.nio.ch.Poller;
+
+/**
+ * The implementation for the jcmd Thread.vthread_* diagnostic commands. These methods are
+ * called from the "Attach Listener" thread.
+ */
+public class JcmdVThreadCommands {
+    private static final JavaLangAccess JLA = SharedSecrets.getJavaLangAccess();
+
+    private JcmdVThreadCommands() { }
+
+    /**
+     * Invoked by the VM to print the virtual scheduler to a byte[].
+     */
+    private static byte[] printScheduler() {
+        StringBuilder sb = new StringBuilder();
+
+        // virtual thread scheduler
+        sb.append(JLA.virtualThreadDefaultScheduler())
+          .append(System.lineSeparator());
+
+        // break
+        sb.append(System.lineSeparator());
+
+        // delayed task schedulers
+        sb.append("Delayed task schedulers:").append(System.lineSeparator());
+        var delayedTaskSchedulers = JLA.virtualThreadDelayedTaskSchedulers().toList();
+        IntStream.range(0, delayedTaskSchedulers.size())
+                .forEach(i -> sb.append('[')
+                                .append(i)
+                                .append("] ")
+                                .append(delayedTaskSchedulers.get(i))
+                                .append(System.lineSeparator()));
+
+        return sb.toString().getBytes(StandardCharsets.UTF_8);
+    }
+
+    /**
+     * Invoked by the VM to print the I/O pollers to a byte[].
+     */
+    private static byte[] printPollers() {
+        StringBuilder sb = new StringBuilder();
+
+        Poller masterPoller = Poller.masterPoller();
+        List<Poller> readPollers = Poller.readPollers();
+        List<Poller> writePollers = Poller.writePollers();
+
+        if (masterPoller != null) {
+            sb.append("Master I/O poller:")
+              .append(System.lineSeparator())
+              .append(masterPoller)
+              .append(System.lineSeparator());
+
+            // break
+            sb.append(System.lineSeparator());
+        }
+
+        sb.append("Read I/O pollers:");
+        sb.append(System.lineSeparator());
+        IntStream.range(0, readPollers.size())
+                .forEach(i -> sb.append('[')
+                                .append(i)
+                                .append("] ")
+                                .append(readPollers.get(i))
+                                .append(System.lineSeparator()));
+
+        // break
+        sb.append(System.lineSeparator());
+
+        sb.append("Write I/O pollers:");
+        sb.append(System.lineSeparator());
+        IntStream.range(0, writePollers.size())
+                .forEach(i -> sb.append('[')
+                                .append(i)
+                                .append("] ")
+                                .append(writePollers.get(i))
+                                .append(System.lineSeparator()));
+
+        return sb.toString().getBytes(StandardCharsets.UTF_8);
+    }
+}

src/java.base/share/classes/sun/nio/ch/Poller.java

@@ -36,6 +36,7 @@ import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.locks.LockSupport;
 import java.util.function.BooleanSupplier;
 import jdk.internal.misc.InnocuousThread;
+import jdk.internal.vm.annotation.Stable;
 
 /**
  * Polls file descriptors. Virtual threads invoke the poll method to park
@@ -53,6 +54,9 @@ public abstract class Poller {
         }
     }
 
+    // the poller or sub-poller thread
+    private @Stable Thread owner;
+
     // maps file descriptors to parked Thread
     private final Map<Integer, Thread> map = new ConcurrentHashMap<>();
 
@@ -238,6 +242,7 @@ public abstract class Poller {
      * descriptor that is polled.
      */
     private void pollerLoop() {
+        owner = Thread.currentThread();
         try {
             for (;;) {
                 poll(-1);
@@ -258,6 +263,7 @@ public abstract class Poller {
      */
     private void subPollerLoop(Poller masterPoller) {
         assert Thread.currentThread().isVirtual();
+        owner = Thread.currentThread();
         try {
             int polled = 0;
             for (;;) {
@@ -282,7 +288,8 @@ public abstract class Poller {
 
     @Override
     public String toString() {
-        return Objects.toIdentityString(this) + " [registered = " + registered() + "]";
+        return String.format("%s [registered = %d, owner = %s]",
+                Objects.toIdentityString(this), registered(), owner);
     }
 
     /**
@@ -442,4 +449,25 @@ public abstract class Poller {
             }
         }
     }
+
+    /**
+     * Return the master poller or null if there is no master poller.
+     */
+    public static Poller masterPoller() {
+        return POLLERS.masterPoller();
+    }
+
+    /**
+     * Return the list of read pollers.
+     */
+    public static List<Poller> readPollers() {
+        return POLLERS.readPollers();
+    }
+
+    /**
+     * Return the list of write pollers.
+     */
+    public static List<Poller> writePollers() {
+        return POLLERS.writePollers();
+    }
 }

src/jdk.jcmd/share/man/jcmd.md

@@ -737,6 +737,17 @@ The following commands are available:
     -   `-e`: (Optional) Print extended thread information (BOOLEAN, false)
     -   `-l`: (Optional) Prints `java.util.concurrent` locks (BOOLEAN, false)
 
+`Thread.vthread_scheduler`
+:   Print the virtual thread scheduler, and the delayed task schedulers that support
+    virtual threads doing timed operations.
+
+    Impact: Low
+
+`Thread.vthread_pollers`
+:   Print the I/O pollers that support virtual threads doing blocking network I/O operations.
+
+    Impact: Low
+
 `VM.cds` \[*arguments*\]
 :   Dump a static or dynamic shared archive that includes all currently loaded classes.
 

test/failure_handler/src/share/conf/common.properties

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -34,7 +34,7 @@ onTimeout=\
   jcmd.vm.classloader_stats jcmd.vm.stringtable \
         jcmd.vm.symboltable jcmd.vm.uptime jcmd.vm.dynlibs \
         jcmd.vm.system_properties jcmd.vm.info \
-  jcmd.gc.heap_info jcmd.gc.class_histogram jcmd.gc.finalizer_info jcmd.thread.dump_to_file \
+  jcmd.gc.heap_info jcmd.gc.class_histogram jcmd.gc.finalizer_info jcmd.thread.dump_to_file jcmd.thread.vthread_scheduler \
   jstack jhsdb.jstack.live.default jhsdb.jstack.live.mixed
 
 jinfo.app=jinfo
@@ -61,6 +61,8 @@ jcmd.thread.dump_to_file.args=%p Thread.dump_to_file -format=json JavaThread.dum
 jcmd.thread.dump_to_file.params.repeat=6
 jcmd.thread.dump_to_file.params.successArtifacts=JavaThread.dump.%p.%iterCount
 
+jcmd.thread.vthread_scheduler.args=%p Thread.vthread_scheduler
+
 jstack.app=jstack
 jstack.args=-e -l %p
 jstack.params.repeat=6

test/hotspot/jtreg/serviceability/dcmd/thread/VThreadCommandsTest.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8337199
+ * @summary Basic test for jcmd Thread.vthread_scheduler and Thread.vthread_pollers
+ * @requires vm.continuations
+ * @modules jdk.jcmd
+ * @library /test/lib
+ * @run junit/othervm VThreadCommandsTest
+ */
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinWorkerThread;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.lang.management.ManagementFactory;
+import jdk.management.VirtualThreadSchedulerMXBean;
+
+import jdk.test.lib.dcmd.PidJcmdExecutor;
+import jdk.test.lib.process.OutputAnalyzer;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.*;
+
+class VThreadCommandsTest {
+
+    /**
+     * Thread.vthread_scheduler
+     */
+    @Test
+    void testVThreadScheduler() {
+        // ensure default scheduler and timeout schedulers are initialized
+        Thread.startVirtualThread(() -> { });
+
+        jcmd("Thread.vthread_scheduler")
+                .shouldContain(Objects.toIdentityString(defaultScheduler()))
+                .shouldContain("Delayed task schedulers:")
+                .shouldContain("[0] " + ScheduledThreadPoolExecutor.class.getName());
+    }
+
+    /**
+     * Thread.vthread_pollers
+     */
+    @Test
+    void testVThreadPollers() throws Exception {
+        // do blocking I/O op on a virtual thread to ensure poller mechanism is initialized
+        try (var executor = Executors.newVirtualThreadPerTaskExecutor()) {
+            executor.submit(() -> {
+                try (var listener = new ServerSocket()) {
+                    InetAddress lb = InetAddress.getLoopbackAddress();
+                    listener.bind(new InetSocketAddress(lb, 0));
+                    listener.setSoTimeout(200);
+                    try (Socket s = listener.accept()) {
+                        System.err.format("Connection from %s ??%n", s.getRemoteSocketAddress());
+                    } catch (SocketTimeoutException e) {
+                        // expected
+                    }
+                }
+                return null;
+            }).get();
+        }
+
+        jcmd("Thread.vthread_pollers")
+                .shouldContain("Read I/O pollers:")
+                .shouldContain("Write I/O pollers:")
+                .shouldMatch("^\\[0\\] sun\\.nio\\.ch\\..+ \\[registered = [\\d]+, owner = .+\\]$");
+    }
+
+    private OutputAnalyzer jcmd(String cmd) {
+        return new PidJcmdExecutor().execute(cmd);
+    }
+
+    /**
+     * Returns the virtual thread default scheduler. This implementation works by finding
+     * all FJ worker threads and mapping them to their pool. VirtualThreadSchedulerMXBean
+     * is used to temporarily changing target parallelism to an "unique" value, make it
+     * possbile to find the right pool.
+     */
+    private ForkJoinPool defaultScheduler() {
+        var done = new AtomicBoolean();
+        Thread vthread = Thread.startVirtualThread(() -> {
+            while (!done.get()) {
+                Thread.onSpinWait();
+            }
+        });
+        var bean = ManagementFactory.getPlatformMXBean(VirtualThreadSchedulerMXBean.class);
+        int parallelism = bean.getParallelism();
+        try {
+            bean.setParallelism(133);
+            return Thread.getAllStackTraces()
+                    .keySet()
+                    .stream()
+                    .filter(ForkJoinWorkerThread.class::isInstance)
+                    .map(t -> ((ForkJoinWorkerThread) t).getPool())
+                    .filter(p -> p.getParallelism() == 133)
+                    .findAny()
+                    .orElseThrow();
+        } finally {
+            bean.setParallelism(parallelism);
+            done.set(true);
+        }
+    }
+}