8295723: security/infra/wycheproof/RunWycheproof.java fails with Assertion Error

Reviewed-by: mschoene, ascarpino, coffeys, rhalade, weijun

src/java.base/share/classes/sun/security/util/SafeDHParameterSpec.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package sun.security.util;
+
+import java.math.BigInteger;
+import javax.crypto.spec.DHParameterSpec;
+
+/**
+ * Internal marker class for well-known safe DH parameters. It should
+ * only be used with trusted callers since it does not have all the needed
+ * values for validation.
+ */
+
+public final class SafeDHParameterSpec extends DHParameterSpec {
+    public SafeDHParameterSpec(BigInteger p, BigInteger g) {
+        super(p, g);
+    }
+
+    public SafeDHParameterSpec(BigInteger p, BigInteger g, int l) {
+        super(p, g, l);
+    }
+}

src/java.base/share/classes/sun/security/util/SecurityProviderConstants.java

@@ -33,6 +33,7 @@ import java.security.InvalidParameterException;
 import java.security.ProviderException;
 import java.security.NoSuchAlgorithmException;
 import javax.crypto.Cipher;
+import javax.crypto.spec.DHParameterSpec;
 import sun.security.action.GetPropertyAction;
 
 /**
@@ -102,28 +103,40 @@ public final class SecurityProviderConstants {
         }
     }
 
-    public static final int getDefDHPrivateExpSize(int dhGroupSize) {
-        // use 2*security strength as default private exponent size
-        // as in table 2 of NIST SP 800-57 part 1 rev 5, sec 5.6.1.1
-        // and table 25 of NIST SP 800-56A rev 3, appendix D.
-        if (dhGroupSize >= 15360) {
-            return 512;
-        } else if (dhGroupSize >= 8192) {
-            return 400;
-        } else if (dhGroupSize >= 7680) {
-            return 384;
-        } else if (dhGroupSize >= 6144) {
-            return 352;
-        } else if (dhGroupSize >= 4096) {
-            return 304;
-        } else if (dhGroupSize >= 3072) {
-            return 256;
-        } else if (dhGroupSize >= 2048) {
-            return 224;
+    public static final int getDefDHPrivateExpSize(DHParameterSpec spec) {
+
+        int dhGroupSize = spec.getP().bitLength();
+
+        if (spec instanceof SafeDHParameterSpec) {
+            // Known safe primes
+            // use 2*security strength as default private exponent size
+            // as in table 2 of NIST SP 800-57 part 1 rev 5, sec 5.6.1.1
+            // and table 25 of NIST SP 800-56A rev 3, appendix D.
+            if (dhGroupSize >= 15360) {
+                return 512;
+            } else if (dhGroupSize >= 8192) {
+                return 400;
+            } else if (dhGroupSize >= 7680) {
+                return 384;
+            } else if (dhGroupSize >= 6144) {
+                return 352;
+            } else if (dhGroupSize >= 4096) {
+                return 304;
+            } else if (dhGroupSize >= 3072) {
+                return 256;
+            } else if (dhGroupSize >= 2048) {
+                return 224;
+            } else {
+                // min value for legacy key sizes
+                return 160;
+            }
         } else {
-            // min value for legacy key sizes
-            return 160;
+            // assume the worst and use groupSize/2 as private exp length
+            // up to 1024-bit and use the same minimum 384 as before
+            return Math.max((dhGroupSize >= 2048 ? 1024 : dhGroupSize >> 1),
+                    384);
         }
+
     }
 
     public static final int getDefAESKeySize() {