6913047: Long term memory leak when using PKCS11 and JCE exceeds 32 bit process address space

Extract cryptographic keys within NSS PKCS11 software tokens for memory management purposes.

Reviewed-by: valeriep
This commit is contained in:
Martin Balao 2018-11-29 13:36:23 -03:00
parent 6627b03332
commit dfcab1b85a
22 changed files with 1107 additions and 323 deletions

View file

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -1485,6 +1485,7 @@ final class P11KeyStore extends KeyStoreSpi {
}
}
// retrieves the native key handle and either update it directly or make a copy
private void updateP11Pkey(String alias, CK_ATTRIBUTE attribute, P11Key key)
throws PKCS11Exception {
@ -1492,23 +1493,22 @@ final class P11KeyStore extends KeyStoreSpi {
// if session key, convert to token key.
Session session = null;
long keyID = key.getKeyID();
try {
session = token.getOpSession();
if (key.tokenObject == true) {
// token key - set new CKA_ID
CK_ATTRIBUTE[] attrs = new CK_ATTRIBUTE[] {
new CK_ATTRIBUTE(CKA_ID, alias) };
token.p11.C_SetAttributeValue
(session.id(), key.keyID, attrs);
(session.id(), keyID, attrs);
if (debug != null) {
debug.println("updateP11Pkey set new alias [" +
alias +
"] for key entry");
}
} else {
// session key - convert to token key and set CKA_ID
CK_ATTRIBUTE[] attrs = new CK_ATTRIBUTE[] {
@ -1518,7 +1518,8 @@ final class P11KeyStore extends KeyStoreSpi {
if (attribute != null) {
attrs = addAttribute(attrs, attribute);
}
token.p11.C_CopyObject(session.id(), key.keyID, attrs);
// creates a new token key with the desired CKA_ID
token.p11.C_CopyObject(session.id(), keyID, attrs);
if (debug != null) {
debug.println("updateP11Pkey copied private session key " +
"for [" +
@ -1528,6 +1529,7 @@ final class P11KeyStore extends KeyStoreSpi {
}
} finally {
token.releaseSession(session);
key.releaseKeyID();
}
}
@ -1894,10 +1896,12 @@ final class P11KeyStore extends KeyStoreSpi {
return attrs;
}
String alg = privateKey.getAlgorithm();
if (id && alg.equals("RSA") && (publicKey instanceof RSAPublicKey)) {
if (alg.equals("RSA") && (publicKey instanceof RSAPublicKey)) {
if (id) {
BigInteger n = ((RSAPublicKey)publicKey).getModulus();
attrs[0] = new CK_ATTRIBUTE(CKA_ID, sha1(getMagnitude(n)));
}
// CKA_NETSCAPE_DB not needed for RSA public keys
BigInteger n = ((RSAPublicKey)publicKey).getModulus();
attrs[0] = new CK_ATTRIBUTE(CKA_ID, sha1(getMagnitude(n)));
} else if (alg.equals("DSA") && (publicKey instanceof DSAPublicKey)) {
BigInteger y = ((DSAPublicKey)publicKey).getY();
if (id) {