8043725: javac fails with StackOverflowException

Loose type comparison test should be protected against infinite recursion with fbounds

Reviewed-by: vromero

langtools/src/share/classes/com/sun/tools/javac/code/Types.java

@@ -1212,15 +1212,36 @@ public class Types {
         TypeRelation isSameTypeLoose = new LooseSameTypeVisitor();
 
         private class LooseSameTypeVisitor extends SameTypeVisitor {
+
+            /** cache of the type-variable pairs being (recursively) tested. */
+            private Set<TypePair> cache = new HashSet<>();
+
             @Override
             boolean sameTypeVars(TypeVar tv1, TypeVar tv2) {
-                return tv1.tsym == tv2.tsym && visit(tv1.getUpperBound(), tv2.getUpperBound());
+                return tv1.tsym == tv2.tsym && checkSameBounds(tv1, tv2);
             }
             @Override
             protected boolean containsTypes(List<Type> ts1, List<Type> ts2) {
                 return containsTypeEquivalent(ts1, ts2);
             }
+
+            /**
+             * Since type-variable bounds can be recursive, we need to protect against
+             * infinite loops - where the same bounds are checked over and over recursively.
+             */
+            private boolean checkSameBounds(TypeVar tv1, TypeVar tv2) {
+                TypePair p = new TypePair(tv1, tv2, true);
+                if (cache.add(p)) {
+                    try {
+                        return visit(tv1.getUpperBound(), tv2.getUpperBound());
+                    } finally {
+                        cache.remove(p);
                     }
+                } else {
+                    return false;
+                }
+            }
+        };
 
         /**
          * Strict type-equality relation - type variables are considered
@@ -3396,9 +3417,16 @@ public class Types {
         class TypePair {
             final Type t1;
             final Type t2;
+            boolean strict;
+
             TypePair(Type t1, Type t2) {
+                this(t1, t2, false);
+            }
+
+            TypePair(Type t1, Type t2, boolean strict) {
                 this.t1 = t1;
                 this.t2 = t2;
+                this.strict = strict;
             }
             @Override
             public int hashCode() {
@@ -3409,8 +3437,8 @@ public class Types {
                 if (!(obj instanceof TypePair))
                     return false;
                 TypePair typePair = (TypePair)obj;
-                return isSameType(t1, typePair.t1)
+                return isSameType(t1, typePair.t1, strict)
-                    && isSameType(t2, typePair.t2);
+                    && isSameType(t2, typePair.t2, strict);
             }
         }
         Set<TypePair> mergeCache = new HashSet<>();

langtools/test/tools/javac/generics/inference/8043725/T8043725.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8043725
+ * @summary javac fails with StackOverflowException
+ * @compile T8043725.java
+ */
+class T8043725 {
+    <T extends Comparable<T>> T m(T v) {
+        //this will generate two upper bounds, T and Comparable<T'> respectively
+        //causing infinite recursion in lub (because of JLS 18.3.1).
+        return m(v);
+    }
+}