8311596: Add separate system properties for TLS server and client for maximum chain length

Reviewed-by: jnimeh, weijun, mullan

src/java.base/share/classes/sun/security/ssl/CertificateMessage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -130,12 +130,16 @@ final class CertificateMessage {
                     byte[] encodedCert = Record.getBytes24(m);
                     listLen -= (3 + encodedCert.length);
                     encodedCerts.add(encodedCert);
-                    if (encodedCerts.size() > SSLConfiguration.maxCertificateChainLength) {
+                    int maxAllowedChainLength = handshakeContext.sslConfig.isClientMode ?
+                            SSLConfiguration.maxInboundServerCertChainLen :
+                            SSLConfiguration.maxInboundClientCertChainLen;
+
+                    if (encodedCerts.size() > maxAllowedChainLength) {
                         throw new SSLProtocolException(
                                 "The certificate chain length ("
                                 + encodedCerts.size()
                                 + ") exceeds the maximum allowed length ("
-                                + SSLConfiguration.maxCertificateChainLength
+                                + maxAllowedChainLength
                                 + ")");
                     }
 
@@ -861,12 +865,16 @@ final class CertificateMessage {
                 SSLExtensions extensions =
                         new SSLExtensions(this, m, enabledExtensions);
                 certList.add(new CertificateEntry(encodedCert, extensions));
-                if (certList.size() > SSLConfiguration.maxCertificateChainLength) {
+                int maxAllowedChainLength = handshakeContext.sslConfig.isClientMode ?
+                        SSLConfiguration.maxInboundServerCertChainLen :
+                        SSLConfiguration.maxInboundClientCertChainLen;
+
+                if (certList.size() > maxAllowedChainLength) {
                     throw new SSLProtocolException(
                             "The certificate chain length ("
                             + certList.size()
                             + ") exceeds the maximum allowed length ("
-                            + SSLConfiguration.maxCertificateChainLength
+                            + maxAllowedChainLength
                             + ")");
                 }
             }

src/java.base/share/classes/sun/security/ssl/SSLConfiguration.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -108,9 +108,11 @@ final class SSLConfiguration implements Cloneable {
     static final int maxHandshakeMessageSize = GetIntegerAction.privilegedGetProperty(
             "jdk.tls.maxHandshakeMessageSize", 32768);
 
-    // Set the max certificate chain length to 10
+    // Limit the certificate chain length accepted from clients
-    static final int maxCertificateChainLength = GetIntegerAction.privilegedGetProperty(
+    static final int maxInboundClientCertChainLen;
-            "jdk.tls.maxCertificateChainLength", 10);
+
+    // Limit the certificate chain length accepted from servers
+    static final int maxInboundServerCertChainLen;
 
     // To switch off the supported_groups extension for DHE cipher suite.
     static final boolean enableFFDHE =
@@ -133,6 +135,55 @@ final class SSLConfiguration implements Cloneable {
         useExtendedMasterSecret = supportExtendedMasterSecret;
     }
 
+    static {
+        boolean globalPropSet = false;
+
+        // jdk.tls.maxCertificateChainLength property has no default
+        Integer maxCertificateChainLength = GetIntegerAction.privilegedGetProperty(
+                "jdk.tls.maxCertificateChainLength");
+        if (maxCertificateChainLength != null && maxCertificateChainLength >= 0) {
+            globalPropSet = true;
+        }
+
+        /*
+         * If either jdk.tls.server.maxInboundCertificateChainLength or
+         * jdk.tls.client.maxInboundCertificateChainLength is set, it will
+         * override jdk.tls.maxCertificateChainLength, regardless of whether
+         * jdk.tls.maxCertificateChainLength is set or not.
+         * If neither jdk.tls.server.maxInboundCertificateChainLength nor
+         * jdk.tls.client.maxInboundCertificateChainLength is set, the behavior
+         * depends on the setting of jdk.tls.maxCertificateChainLength. If
+         * jdk.tls.maxCertificateChainLength is set, it falls back to that
+         * value; otherwise, it defaults to 8 for
+         * jdk.tls.server.maxInboundCertificateChainLength
+         * and 10 for jdk.tls.client.maxInboundCertificateChainLength.
+         * Users can independently set either
+         * jdk.tls.server.maxInboundCertificateChainLength or
+         * jdk.tls.client.maxInboundCertificateChainLength.
+         */
+        Integer inboundClientLen = GetIntegerAction.privilegedGetProperty(
+                "jdk.tls.server.maxInboundCertificateChainLength");
+
+        // Default for jdk.tls.server.maxInboundCertificateChainLength is 8
+        if (inboundClientLen == null || inboundClientLen < 0) {
+            maxInboundClientCertChainLen = globalPropSet ?
+                    maxCertificateChainLength : 8;
+        } else {
+            maxInboundClientCertChainLen = inboundClientLen;
+        }
+
+        Integer inboundServerLen = GetIntegerAction.privilegedGetProperty(
+                "jdk.tls.client.maxInboundCertificateChainLength");
+
+        // Default for jdk.tls.client.maxInboundCertificateChainLength is 10
+        if (inboundServerLen == null || inboundServerLen < 0) {
+            maxInboundServerCertChainLen = globalPropSet ?
+                    maxCertificateChainLength : 10;
+        } else {
+            maxInboundServerCertChainLen = inboundServerLen;
+        }
+    }
+
     SSLConfiguration(SSLContextImpl sslContext, boolean isClientMode) {
 
         // Configurations with SSLParameters, default values.