8325255: jdk.internal.util.ReferencedKeySet::add using wrong test

Reviewed-by: rriggs, liach

src/java.base/share/classes/jdk/internal/util/ReferencedKeyMap.java

@@ -439,4 +439,30 @@ public final class ReferencedKeyMap<K, V> implements Map<K, V> {
         return interned;
     }
 
+
+    /**
+     * Attempt to add key to map if absent.
+     *
+     * @param setMap    {@link ReferencedKeyMap} where interning takes place
+     * @param key       key to add
+     *
+     * @param <T> type of key
+     *
+     * @return true if the key was added
+     */
+    static <T> boolean internAddKey(ReferencedKeyMap<T, ReferenceKey<T>> setMap, T key) {
+        ReferenceKey<T> entryKey = setMap.entryKey(key);
+        setMap.removeStaleReferences();
+        ReferenceKey<T> existing = setMap.map.putIfAbsent(entryKey, entryKey);
+        if (existing == null) {
+            return true;
+        } else {
+            // If {@code putIfAbsent} returns non-null then was actually a
+            // {@code replace} and older key was used. In that case the new
+            // key was not used and the reference marked stale.
+            entryKey.unused();
+            return false;
+        }
+     }
+
 }

src/java.base/share/classes/jdk/internal/util/ReferencedKeySet.java

@@ -148,7 +148,7 @@ public final class ReferencedKeySet<T> extends AbstractSet<T> {
 
     @Override
     public boolean add(T e) {
-        return intern(e) == null;
+        return ReferencedKeyMap.internAddKey(map, e);
     }
 
     @Override