8088132: [Swing, singleThread] ClassCastException in nested event loop when showing multiple message dialogs in SwingNode

Reviewed-by: serb

src/java.desktop/share/classes/java/awt/DefaultKeyboardFocusManager.java

@@ -30,6 +30,8 @@ import java.awt.event.WindowEvent;
 import java.awt.peer.ComponentPeer;
 import java.awt.peer.LightweightPeer;
 import java.lang.ref.WeakReference;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.LinkedList;
 import java.util.Iterator;
 import java.util.ListIterator;
@@ -77,6 +79,8 @@ public class DefaultKeyboardFocusManager extends KeyboardFocusManager {
     private boolean consumeNextKeyTyped;
     private Component restoreFocusTo;
 
+    private static boolean fxAppThreadIsDispatchThread;
+
     static {
         AWTAccessor.setDefaultKeyboardFocusManagerAccessor(
             new AWTAccessor.DefaultKeyboardFocusManagerAccessor() {
@@ -84,6 +88,13 @@ public class DefaultKeyboardFocusManager extends KeyboardFocusManager {
                     dkfm.consumeNextKeyTyped(e);
                 }
             });
+        AccessController.doPrivileged(new PrivilegedAction<Object>() {
+            public Object run() {
+                fxAppThreadIsDispatchThread =
+                        "true".equals(System.getProperty("javafx.embed.singleThread"));
+                return null;
+            }
+        });
     }
 
     private static class TypeAheadMarker {
@@ -264,6 +275,7 @@ public class DefaultKeyboardFocusManager extends KeyboardFocusManager {
             }
             SunToolkit.postEvent(targetAppContext, se);
             if (EventQueue.isDispatchThread()) {
+                if (Thread.currentThread() instanceof EventDispatchThread) {
                     EventDispatchThread edt = (EventDispatchThread)
                             Thread.currentThread();
                     edt.pumpEvents(SentEvent.ID, new Conditional() {
@@ -271,6 +283,33 @@ public class DefaultKeyboardFocusManager extends KeyboardFocusManager {
                             return !se.dispatched && !targetAppContext.isDisposed();
                         }
                     });
+                } else {
+                    if (fxAppThreadIsDispatchThread) {
+                        Thread fxCheckDispatchThread = new Thread() {
+                            @Override
+                            public void run() {
+                                while (!se.dispatched && !targetAppContext.isDisposed()) {
+                                    try {
+                                        Thread.sleep(100);
+                                    } catch (InterruptedException e) {
+                                        break;
+                                    }
+                                }
+                            }
+                        };
+                        fxCheckDispatchThread.start();
+                        try {
+                            // check if event is dispatched or disposed
+                            // but since this user app thread is same as
+                            // dispatch thread in fx when run with
+                            // javafx.embed.singleThread=true
+                            // we do not wait infinitely to avoid deadlock
+                            // as dispatch will ultimately be done by this thread
+                            fxCheckDispatchThread.join(500);
+                        } catch (InterruptedException ex) {
+                        }
+                    }
+                }
             } else {
                 synchronized (se) {
                     while (!se.dispatched && !targetAppContext.isDisposed()) {

src/java.desktop/share/classes/java/awt/SequencedEvent.java

@@ -25,6 +25,8 @@
 
 package java.awt;
 
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.LinkedList;
 import sun.awt.AWTAccessor;
 import sun.awt.AppContext;
@@ -55,6 +57,8 @@ class SequencedEvent extends AWTEvent implements ActiveEvent {
     private AppContext appContext;
     private boolean disposed;
 
+    private static boolean fxAppThreadIsDispatchThread;
+    private Thread fxCheckSequenceThread;
     static {
         AWTAccessor.setSequencedEventAccessor(new AWTAccessor.SequencedEventAccessor() {
             public AWTEvent getNested(AWTEvent sequencedEvent) {
@@ -68,6 +72,13 @@ class SequencedEvent extends AWTEvent implements ActiveEvent {
                 return new SequencedEvent(event);
             }
         });
+        AccessController.doPrivileged(new PrivilegedAction<Object>() {
+            public Object run() {
+                fxAppThreadIsDispatchThread =
+                        "true".equals(System.getProperty("javafx.embed.singleThread"));
+                return null;
+            }
+        });
     }
 
     /**
@@ -83,6 +94,21 @@ class SequencedEvent extends AWTEvent implements ActiveEvent {
         // All AWTEvents that are wrapped in SequencedEvents are (at
         // least currently) implicitly generated by the system
         SunToolkit.setSystemGenerated(nested);
+
+        if (fxAppThreadIsDispatchThread) {
+            fxCheckSequenceThread = new Thread() {
+                @Override
+                public void run() {
+                    while(!isFirstOrDisposed()) {
+                        try {
+                            Thread.sleep(100);
+                        } catch (InterruptedException e) {
+                            break;
+                        }
+                    }
+                }
+            };
+        }
         synchronized (SequencedEvent.class) {
             list.add(this);
         }
@@ -106,6 +132,7 @@ class SequencedEvent extends AWTEvent implements ActiveEvent {
 
             if (getFirst() != this) {
                 if (EventQueue.isDispatchThread()) {
+                    if (Thread.currentThread() instanceof EventDispatchThread) {
                         EventDispatchThread edt = (EventDispatchThread)
                                 Thread.currentThread();
                         edt.pumpEvents(SentEvent.ID, new Conditional() {
@@ -113,6 +140,21 @@ class SequencedEvent extends AWTEvent implements ActiveEvent {
                                 return !SequencedEvent.this.isFirstOrDisposed();
                             }
                         });
+                    } else {
+                        if (fxAppThreadIsDispatchThread) {
+                            fxCheckSequenceThread.start();
+                            try {
+                                // check if event is dispatched or disposed
+                                // but since this user app thread is same as
+                                // dispatch thread in fx when run with
+                                // javafx.embed.singleThread=true
+                                // we do not wait infinitely to avoid deadlock
+                                // as dispatch will ultimately be done by this thread
+                                fxCheckSequenceThread.join(500);
+                            } catch (InterruptedException e) {
+                            }
+                        }
+                    }
                 } else {
                     while(!isFirstOrDisposed()) {
                         synchronized (SequencedEvent.class) {