8317547: Enhance TLS connection support

Reviewed-by: ahgross, rhalade, weijun, valeriep
This commit is contained in:
Ferenc Rakoczi 2023-11-14 17:00:30 +00:00 committed by Henry Jen
parent c1a568c9c4
commit bf7bd9a16c
4 changed files with 142 additions and 90 deletions

View file

@ -98,6 +98,7 @@ public final class RSACipher extends CipherSpi {
// cipher parameter for OAEP padding and TLS RSA premaster secret
private AlgorithmParameterSpec spec = null;
private boolean forTlsPremasterSecret = false;
// buffer for the data
private byte[] buffer;
@ -286,6 +287,7 @@ public final class RSACipher extends CipherSpi {
}
spec = params;
forTlsPremasterSecret = true;
this.random = random; // for TLS RSA premaster secret
}
int blockType = (mode <= MODE_DECRYPT) ? RSAPadding.PAD_BLOCKTYPE_2
@ -377,7 +379,7 @@ public final class RSACipher extends CipherSpi {
byte[] decryptBuffer = RSACore.convert(buffer, 0, bufOfs);
paddingCopy = RSACore.rsa(decryptBuffer, privateKey, false);
result = padding.unpad(paddingCopy);
if (result == null) {
if (result == null && !forTlsPremasterSecret) {
throw new BadPaddingException
("Padding error in decryption");
}
@ -466,26 +468,22 @@ public final class RSACipher extends CipherSpi {
boolean isTlsRsaPremasterSecret =
algorithm.equals("TlsRsaPremasterSecret");
Exception failover = null;
byte[] encoded = null;
update(wrappedKey, 0, wrappedKey.length);
try {
encoded = doFinal();
} catch (BadPaddingException e) {
if (isTlsRsaPremasterSecret) {
failover = e;
} else {
throw new InvalidKeyException("Unwrapping failed", e);
}
} catch (IllegalBlockSizeException e) {
// should not occur, handled with length check above
} catch (BadPaddingException | IllegalBlockSizeException e) {
// BadPaddingException cannot happen for TLS RSA unwrap.
// In that case, padding error is indicated by returning null.
// IllegalBlockSizeException cannot happen in any case,
// because of the length check above.
throw new InvalidKeyException("Unwrapping failed", e);
}
try {
if (isTlsRsaPremasterSecret) {
if (!(spec instanceof TlsRsaPremasterSecretParameterSpec)) {
if (!forTlsPremasterSecret) {
throw new IllegalStateException(
"No TlsRsaPremasterSecretParameterSpec specified");
}
@ -494,7 +492,7 @@ public final class RSACipher extends CipherSpi {
encoded = KeyUtil.checkTlsPreMasterSecretKey(
((TlsRsaPremasterSecretParameterSpec) spec).getClientVersion(),
((TlsRsaPremasterSecretParameterSpec) spec).getServerVersion(),
random, encoded, (failover != null));
random, encoded, encoded == null);
}
return ConstructKeys.constructKey(encoded, algorithm, type);

View file

@ -291,13 +291,14 @@ public final class KeyUtil {
* contains the lower of that suggested by the client in the client
* hello and the highest supported by the server.
* @param encoded the encoded key in its "RAW" encoding format
* @param isFailOver whether the previous decryption of the
* encrypted PreMasterSecret message run into problem
* @param failure true if encoded is incorrect according to previous checks
* @return the polished PreMasterSecret key in its "RAW" encoding format
*/
public static byte[] checkTlsPreMasterSecretKey(
int clientVersion, int serverVersion, SecureRandom random,
byte[] encoded, boolean isFailOver) {
byte[] encoded, boolean failure) {
byte[] tmp;
if (random == null) {
random = JCAUtil.getSecureRandom();
@ -305,30 +306,38 @@ public final class KeyUtil {
byte[] replacer = new byte[48];
random.nextBytes(replacer);
if (!isFailOver && (encoded != null)) {
// check the length
if (encoded.length != 48) {
// private, don't need to clone the byte array.
return replacer;
}
int encodedVersion =
((encoded[0] & 0xFF) << 8) | (encoded[1] & 0xFF);
if (clientVersion != encodedVersion) {
if (clientVersion > 0x0301 || // 0x0301: TLSv1
serverVersion != encodedVersion) {
encoded = replacer;
} // Otherwise, For compatibility, we maintain the behavior
// that the version in pre_master_secret can be the
// negotiated version for TLS v1.0 and SSL v3.0.
}
// private, don't need to clone the byte array.
return encoded;
if (failure) {
tmp = replacer;
} else {
tmp = encoded;
}
// private, don't need to clone the byte array.
return replacer;
if (tmp == null) {
encoded = replacer;
} else {
encoded = tmp;
}
// check the length
if (encoded.length != 48) {
// private, don't need to clone the byte array.
tmp = replacer;
} else {
tmp = encoded;
}
int encodedVersion =
((tmp[0] & 0xFF) << 8) | (tmp[1] & 0xFF);
int check1 = 0;
int check2 = 0;
int check3 = 0;
if (clientVersion != encodedVersion) check1 = 1;
if (clientVersion > 0x0301) check2 = 1;
if (serverVersion != encodedVersion) check3 = 1;
if ((check1 & (check2 | check3)) == 1) {
return replacer;
} else {
return tmp;
}
}
/**