8206240: java.lang.Class.newInstance() is causing caller to leak

Reviewed-by: alanb

src/java.base/share/classes/java/lang/reflect/AccessibleObject.java

@@ -27,6 +27,7 @@ package java.lang.reflect;
 
 import java.lang.annotation.Annotation;
 import java.lang.invoke.MethodHandle;
+import java.lang.ref.WeakReference;
 import java.security.AccessController;
 
 import jdk.internal.misc.VM;
@@ -567,21 +568,68 @@ public class AccessibleObject implements AnnotatedElement {
     // Shared access checking logic.
 
     // For non-public members or members in package-private classes,
-    // it is necessary to perform somewhat expensive security checks.
-    // If the security check succeeds for a given class, it will
+    // it is necessary to perform somewhat expensive access checks.
+    // If the access check succeeds for a given class, it will
     // always succeed (it is not affected by the granting or revoking
     // of permissions); we speed up the check in the common case by
     // remembering the last Class for which the check succeeded.
     //
-    // The simple security check for Constructor is to see if
+    // The simple access check for Constructor is to see if
     // the caller has already been seen, verified, and cached.
-    // (See also Class.newInstance(), which uses a similar method.)
     //
-    // A more complicated security check cache is needed for Method and Field
-    // The cache can be either null (empty cache), a 2-array of {caller,targetClass},
+    // A more complicated access check cache is needed for Method and Field
+    // The cache can be either null (empty cache), {caller,targetClass} pair,
     // or a caller (with targetClass implicitly equal to memberClass).
-    // In the 2-array case, the targetClass is always different from the memberClass.
-    volatile Object securityCheckCache;
+    // In the {caller,targetClass} case, the targetClass is always different
+    // from the memberClass.
+    volatile Object accessCheckCache;
+
+    private static class Cache {
+        final WeakReference<Class<?>> callerRef;
+        final WeakReference<Class<?>> targetRef;
+
+        Cache(Class<?> caller, Class<?> target) {
+            this.callerRef = new WeakReference<>(caller);
+            this.targetRef = new WeakReference<>(target);
+        }
+
+        boolean isCacheFor(Class<?> caller, Class<?> refc) {
+            return callerRef.get() == caller && targetRef.get() == refc;
+        }
+
+        static Object protectedMemberCallerCache(Class<?> caller, Class<?> refc) {
+            return new Cache(caller, refc);
+        }
+    }
+
+    /*
+     * Returns true if the previous access check was verified for the
+     * given caller accessing a protected member with an instance of
+     * the given targetClass where the target class is different than
+     * the declaring member class.
+     */
+    private boolean isAccessChecked(Class<?> caller, Class<?> targetClass) {
+        Object cache = accessCheckCache;  // read volatile
+        if (cache instanceof Cache) {
+            return ((Cache) cache).isCacheFor(caller, targetClass);
+        }
+        return false;
+    }
+
+    /*
+     * Returns true if the previous access check was verified for the
+     * given caller accessing a static member or an instance member of
+     * the target class that is the same as the declaring member class.
+     */
+    private boolean isAccessChecked(Class<?> caller) {
+        Object cache = accessCheckCache;  // read volatile
+        if (cache instanceof WeakReference) {
+            @SuppressWarnings("unchecked")
+            WeakReference<Class<?>> ref = (WeakReference<Class<?>>) cache;
+            return ref.get() == caller;
+        }
+        return false;
+    }
 
     final void checkAccess(Class<?> caller, Class<?> memberClass,
                            Class<?> targetClass, int modifiers)
@@ -603,21 +651,13 @@ public class AccessibleObject implements AnnotatedElement {
         if (caller == memberClass) {  // quick check
             return true;             // ACCESS IS OK
         }
-        Object cache = securityCheckCache;  // read volatile
         if (targetClass != null // instance member or constructor
             && Modifier.isProtected(modifiers)
             && targetClass != memberClass) {
-            // Must match a 2-list of { caller, targetClass }.
-            if (cache instanceof Class[]) {
-                Class<?>[] cache2 = (Class<?>[]) cache;
-                if (cache2[1] == targetClass &&
-                    cache2[0] == caller) {
-                    return true;     // ACCESS IS OK
-                }
-                // (Test cache[1] first since range check for [1]
-                // subsumes range check for [0].)
+            if (isAccessChecked(caller, targetClass)) {
+                return true;         // ACCESS IS OK
             }
-        } else if (cache == caller) {
+        } else if (isAccessChecked(caller)) {
             // Non-protected case (or targetClass == memberClass or static member).
             return true;             // ACCESS IS OK
         }
@@ -642,14 +682,9 @@ public class AccessibleObject implements AnnotatedElement {
         Object cache = (targetClass != null
                         && Modifier.isProtected(modifiers)
                         && targetClass != memberClass)
-                        ? new Class<?>[] { caller, targetClass }
-                        : caller;
-
-        // Note:  The two cache elements are not volatile,
-        // but they are effectively final.  The Java memory model
-        // guarantees that the initializing stores for the cache
-        // elements will occur before the volatile write.
-        securityCheckCache = cache;         // write volatile
+                        ? Cache.protectedMemberCallerCache(caller, targetClass)
+                        : new WeakReference<>(caller);
+        accessCheckCache = cache;         // write volatile
         return true;
     }