8180643: Illegal handshake message

Reviewed-by: weijun

jdk/src/java.base/share/classes/sun/security/ssl/HandshakeMessage.java

@@ -79,6 +79,9 @@ public abstract class HandshakeMessage {
     public static final Debug debug = Debug.getInstance("ssl");
 
     // enum HandshakeType:
+    //
+    // Please update the isUnsupported() method accordingly if the handshake
+    // types get updated in the future.
     static final byte   ht_hello_request          = 0;      // RFC 5246
     static final byte   ht_client_hello           = 1;      // RFC 5246
     static final byte   ht_server_hello           = 2;      // RFC 5246
@@ -130,6 +133,24 @@ public abstract class HandshakeMessage {
         return b;
     }
 
+    static boolean isUnsupported(byte handshakeType) {
+        return (handshakeType != ht_hello_request) &&
+               (handshakeType != ht_client_hello) &&
+               (handshakeType != ht_server_hello) &&
+               (handshakeType != ht_hello_verify_request) &&
+               (handshakeType != ht_new_session_ticket) &&
+               (handshakeType != ht_certificate) &&
+               (handshakeType != ht_server_key_exchange) &&
+               (handshakeType != ht_certificate_request) &&
+               (handshakeType != ht_server_hello_done) &&
+               (handshakeType != ht_certificate_verify) &&
+               (handshakeType != ht_client_key_exchange) &&
+               (handshakeType != ht_finished) &&
+               (handshakeType != ht_certificate_url) &&
+               (handshakeType != ht_certificate_status) &&
+               (handshakeType != ht_supplemental_data);
+    }
+
     private static byte[] genPad(int b, int count) {
         byte[] padding = new byte[count];
         Arrays.fill(padding, (byte)b);

jdk/src/java.base/share/classes/sun/security/ssl/Handshaker.java

@@ -1034,6 +1034,12 @@ abstract class Handshaker {
             input.mark(4);
 
             messageType = (byte)input.getInt8();
+            if (HandshakeMessage.isUnsupported(messageType)) {
+                throw new SSLProtocolException(
+                    "Received unsupported or unknown handshake message: " +
+                    messageType);
+            }
+
             messageLen = input.getInt24();
 
             if (input.available() < messageLen) {

jdk/test/javax/net/ssl/SSLEngine/IllegalHandshakeMessage.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+//
+// This test case relies on updated static security property, no way to re-use
+// security property in samevm/agentvm mode.
+//
+
+/*
+ * @test
+ * @bug 8180643
+ * @summary Illegal handshake message
+ *
+ * @run main/othervm IllegalHandshakeMessage
+ */
+
+import javax.net.ssl.*;
+import javax.net.ssl.SSLEngineResult.*;
+import java.io.*;
+import java.security.*;
+import java.nio.*;
+
+public class IllegalHandshakeMessage {
+
+    public static void main(String args[]) throws Exception {
+        SSLContext context = SSLContext.getDefault();
+
+        SSLEngine cliEngine = context.createSSLEngine();
+        cliEngine.setUseClientMode(true);
+        SSLEngine srvEngine = context.createSSLEngine();
+        srvEngine.setUseClientMode(false);
+
+        SSLSession session = cliEngine.getSession();
+        int netBufferMax = session.getPacketBufferSize();
+        int appBufferMax = session.getApplicationBufferSize();
+
+        ByteBuffer cliToSrv = ByteBuffer.allocateDirect(netBufferMax);
+        ByteBuffer srvToCli = ByteBuffer.allocateDirect(netBufferMax);
+        ByteBuffer srvIBuff = ByteBuffer.allocateDirect(appBufferMax + 50);
+        ByteBuffer cliOBuff = ByteBuffer.wrap("I'm client".getBytes());
+        ByteBuffer srvOBuff = ByteBuffer.wrap("I'm server".getBytes());
+
+
+        System.out.println("client hello (handshake type(0xAB))");
+        SSLEngineResult cliRes = cliEngine.wrap(cliOBuff, cliToSrv);
+        System.out.println("Client wrap result: " + cliRes);
+        cliToSrv.flip();
+        if (cliToSrv.limit() > 7) {
+            cliToSrv.put(5, (byte)0xAB);    // use illegal handshake type
+            cliToSrv.put(7, (byte)0x80);    // use illegal message length
+        } else {
+            // unlikely
+            throw new Exception("No handshage message generated.");
+        }
+
+        try {
+            SSLEngineResult srvRes = srvEngine.unwrap(cliToSrv, srvIBuff);
+            System.out.println("Server unwrap result: " + srvRes);
+            runDelegatedTasks(srvRes, srvEngine);
+
+            srvRes = srvEngine.wrap(srvOBuff, srvToCli);
+            System.out.println("Server wrap result: " + srvRes);
+
+            throw new Exception(
+                "Unsupported handshake message is not handled properly.");
+        } catch (SSLException e) {
+            // get the expected exception
+            System.out.println("Expected exception: " + e);
+        }
+    }
+
+    private static void runDelegatedTasks(SSLEngineResult result,
+            SSLEngine engine) throws Exception {
+
+        if (result.getHandshakeStatus() == HandshakeStatus.NEED_TASK) {
+            Runnable runnable;
+            while ((runnable = engine.getDelegatedTask()) != null) {
+                System.out.println("\trunning delegated task...");
+                runnable.run();
+            }
+            HandshakeStatus hsStatus = engine.getHandshakeStatus();
+            if (hsStatus == HandshakeStatus.NEED_TASK) {
+                throw new Exception(
+                    "handshake shouldn't need additional tasks");
+            }
+            System.out.println("\tnew HandshakeStatus: " + hsStatus);
+        }
+    }
+}
+