8285404: RSA signature verification should reject non-DER OCTET STRING

Reviewed-by: valeriep

src/java.base/share/classes/sun/security/rsa/RSASignature.java

@@ -215,6 +215,10 @@ abstract class RSASignature extends SignatureSpi {
             byte[] digest = getDigestValue();
             byte[] decrypted = RSACore.rsa(sigBytes, publicKey);
             byte[] unpadded = padding.unpad(decrypted);
+            // https://www.rfc-editor.org/rfc/rfc8017.html#section-8.2.2
+            // Step 4 suggests comparing the encoded message instead of the
+            // decoded, but some vendors might omit the NULL params in
+            // digest algorithm identifier.
             byte[] decodedDigest = RSAUtil.decodeSignature(digestOID, unpadded);
             return MessageDigest.isEqual(digest, decodedDigest);
         } catch (javax.crypto.BadPaddingException e) {

src/java.base/share/classes/sun/security/rsa/RSAUtil.java

@@ -200,6 +200,9 @@ public class RSAUtil {
         if (algId.getEncodedParams() != null) {
             throw new IOException("Unexpected AlgorithmId parameters");
         }
+        if (values[1].isConstructed()) {
+            throw new IOException("Unexpected constructed digest value");
+        }
         byte[] digest = values[1].getOctetString();
         return digest;
     }