8245417: Improve certificate chain handling

Co-authored-by: Hai-may Chao <hai-may.chao@oracle.com>
Reviewed-by: mullan, jnimeh

src/java.base/share/classes/sun/security/ssl/CertificateMessage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -137,6 +137,15 @@ final class CertificateMessage {
                     byte[] encodedCert = Record.getBytes24(m);
                     listLen -= (3 + encodedCert.length);
                     encodedCerts.add(encodedCert);
+                    if (encodedCerts.size() > SSLConfiguration.maxCertificateChainLength) {
+                        throw new SSLProtocolException(
+                                "The certificate chain length ("
+                                + encodedCerts.size()
+                                + ") exceeds the maximum allowed length ("
+                                + SSLConfiguration.maxCertificateChainLength
+                                + ")");
+                    }
+
                 }
                 this.encodedCertChain = encodedCerts;
             } else {
@@ -859,6 +868,14 @@ final class CertificateMessage {
                 SSLExtensions extensions =
                         new SSLExtensions(this, m, enabledExtensions);
                 certList.add(new CertificateEntry(encodedCert, extensions));
+                if (certList.size() > SSLConfiguration.maxCertificateChainLength) {
+                    throw new SSLProtocolException(
+                            "The certificate chain length ("
+                            + certList.size()
+                            + ") exceeds the maximum allowed length ("
+                            + SSLConfiguration.maxCertificateChainLength
+                            + ")");
+                }
             }
 
             this.certEntries = Collections.unmodifiableList(certList);

src/java.base/share/classes/sun/security/ssl/DTLSInputRecord.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,7 @@ import java.util.Set;
 import java.util.TreeSet;
 import javax.crypto.BadPaddingException;
 import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLProtocolException;
 import sun.security.ssl.SSLCipher.SSLReadCipher;
 
 /**
@@ -91,7 +92,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
     }
 
     @Override
-    Plaintext acquirePlaintext() {
+    Plaintext acquirePlaintext() throws SSLProtocolException {
         if (reassembler != null) {
             return reassembler.acquirePlaintext();
         }
@@ -114,7 +115,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
         }
     }
 
-    Plaintext[] decode(ByteBuffer packet) {
+    Plaintext[] decode(ByteBuffer packet) throws SSLProtocolException {
         if (isClosed) {
             return null;
         }
@@ -346,7 +347,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
     private static HandshakeFragment parseHandshakeMessage(
             byte contentType, byte majorVersion, byte minorVersion,
             byte[] recordEnS, int recordEpoch, long recordSeq,
-            ByteBuffer plaintextFragment) {
+            ByteBuffer plaintextFragment) throws SSLProtocolException {
 
         int remaining = plaintextFragment.remaining();
         if (remaining < handshakeHeaderSize) {
@@ -376,6 +377,16 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
                 ((plaintextFragment.get() & 0xFF) << 16) |
                 ((plaintextFragment.get() & 0xFF) << 8) |
                  (plaintextFragment.get() & 0xFF);          // pos: 1-3
+
+        if (messageLength > SSLConfiguration.maxHandshakeMessageSize) {
+            throw new SSLProtocolException(
+                    "The size of the handshake message ("
+                    + messageLength
+                    + ") exceeds the maximum allowed size ("
+                    + SSLConfiguration.maxHandshakeMessageSize
+                    + ")");
+        }
+
         int messageSeq =
                 ((plaintextFragment.get() & 0xFF) << 8) |
                  (plaintextFragment.get() & 0xFF);          // pos: 4/5
@@ -968,7 +979,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
                     (needToCheckFlight && !flightIsReady()));
         }
 
-        Plaintext acquirePlaintext() {
+        Plaintext acquirePlaintext() throws SSLProtocolException {
             if (bufferedFragments.isEmpty()) {
                 if (SSLLogger.isOn && SSLLogger.isOn("verbose")) {
                     SSLLogger.fine("No received handshake messages");
@@ -1080,7 +1091,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
             needToCheckFlight = false;
         }
 
-        private Plaintext acquireCachedMessage() {
+        private Plaintext acquireCachedMessage() throws SSLProtocolException {
             RecordFragment rFrag = bufferedFragments.first();
             if (readEpoch != rFrag.recordEpoch) {
                 if (readEpoch > rFrag.recordEpoch) {

src/java.base/share/classes/sun/security/ssl/SSLConfiguration.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -43,6 +43,7 @@ import javax.net.ssl.SNIServerName;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLParameters;
 import javax.net.ssl.SSLSocket;
+import sun.security.action.GetIntegerAction;
 import sun.security.action.GetPropertyAction;
 import sun.security.ssl.SSLExtension.ClientExtensions;
 import sun.security.ssl.SSLExtension.ServerExtensions;
@@ -104,6 +105,14 @@ final class SSLConfiguration implements Cloneable {
     static final boolean acknowledgeCloseNotify  = Utilities.getBooleanProperty(
             "jdk.tls.acknowledgeCloseNotify", false);
 
+    // Set the max size limit for Handshake Message to 2^15
+    static final int maxHandshakeMessageSize = GetIntegerAction.privilegedGetProperty(
+            "jdk.tls.maxHandshakeMessageSize", 32768);
+
+    // Set the max certificate chain length to 10
+    static final int maxCertificateChainLength = GetIntegerAction.privilegedGetProperty(
+            "jdk.tls.maxCertificateChainLength", 10);
+
     // Is the extended_master_secret extension supported?
     static {
         boolean supportExtendedMasterSecret = Utilities.getBooleanProperty(

src/java.base/share/classes/sun/security/ssl/SSLEngineInputRecord.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -297,6 +297,15 @@ final class SSLEngineInputRecord extends InputRecord implements SSLRecord {
                 }
 
                 int handshakeBodyLen = Record.getInt24(handshakeFrag);
+                if (handshakeBodyLen > SSLConfiguration.maxHandshakeMessageSize) {
+                    throw new SSLProtocolException(
+                            "The size of the handshake message ("
+                            + handshakeBodyLen
+                            + ") exceeds the maximum allowed size ("
+                            + SSLConfiguration.maxHandshakeMessageSize
+                            + ")");
+                }
+
                 handshakeFrag.reset();
                 int handshakeMessageLen =
                         handshakeHeaderSize + handshakeBodyLen;

src/java.base/share/classes/sun/security/ssl/SSLSocketInputRecord.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2020, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2020, Azul Systems, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -310,6 +310,15 @@ final class SSLSocketInputRecord extends InputRecord implements SSLRecord {
                 }
 
                 int handshakeBodyLen = Record.getInt24(handshakeFrag);
+                if (handshakeBodyLen > SSLConfiguration.maxHandshakeMessageSize) {
+                    throw new SSLProtocolException(
+                            "The size of the handshake message ("
+                            + handshakeBodyLen
+                            + ") exceeds the maximum allowed size ("
+                            + SSLConfiguration.maxHandshakeMessageSize
+                            + ")");
+                }
+
                 handshakeFrag.reset();
                 int handshakeMessageLen =
                         handshakeHeaderSize + handshakeBodyLen;

test/jdk/java/net/httpclient/LargeHandshakeTest.java

@@ -87,6 +87,7 @@ import java.util.concurrent.atomic.AtomicLong;
  * @run main/othervm -Dtest.requiresHost=true
  *                   -Djdk.httpclient.HttpClient.log=headers
  *                   -Djdk.internal.httpclient.debug=true
+ *                   -Djdk.tls.maxHandshakeMessageSize=131072
  *                   LargeHandshakeTest
  *
  */