8029852: Bad code generated (VerifyError) when lambda instantiates enclosing local class and has captured variables

8029725: Lambda reference to containing local class causes javac infinite recursion

Reviewed-by: vromero, jlahoda, dlsmith

langtools/src/share/classes/com/sun/tools/javac/comp/LambdaToMethod.java

@@ -36,6 +36,7 @@ import com.sun.tools.javac.code.Symbol;
 import com.sun.tools.javac.code.Symbol.ClassSymbol;
 import com.sun.tools.javac.code.Symbol.DynamicMethodSymbol;
 import com.sun.tools.javac.code.Symbol.MethodSymbol;
+import com.sun.tools.javac.code.Symbol.TypeSymbol;
 import com.sun.tools.javac.code.Symbol.VarSymbol;
 import com.sun.tools.javac.code.Symtab;
 import com.sun.tools.javac.code.Type;
@@ -50,8 +51,10 @@ import com.sun.source.tree.MemberReferenceTree.ReferenceMode;
 
 import java.util.EnumMap;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Set;
 
 import static com.sun.tools.javac.comp.LambdaToMethod.LambdaSymbolKind.*;
 import static com.sun.tools.javac.code.Flags.*;
@@ -1280,7 +1283,10 @@ public class LambdaToMethod extends TreeTranslator {
 
         @Override
         public void visitNewClass(JCNewClass tree) {
-            if (lambdaNewClassFilter(context(), tree)) {
+            TypeSymbol def = tree.type.tsym;
+            boolean inReferencedClass = currentlyInClass(def);
+            boolean isLocal = def.isLocal();
+            if ((inReferencedClass && isLocal || lambdaNewClassFilter(context(), tree))) {
                 TranslationContext<?> localContext = context();
                 while (localContext != null) {
                     if (localContext.tree.getTag() == LAMBDA) {
@@ -1290,16 +1296,16 @@ public class LambdaToMethod extends TreeTranslator {
                     localContext = localContext.prev;
                 }
             }
-            if (context() != null && tree.type.tsym.owner.kind == MTH) {
+            if (context() != null && !inReferencedClass && isLocal) {
                 LambdaTranslationContext lambdaContext = (LambdaTranslationContext)context();
-                captureLocalClassDefs(tree.type.tsym, lambdaContext);
+                captureLocalClassDefs(def, lambdaContext);
             }
             super.visitNewClass(tree);
         }
         //where
             void captureLocalClassDefs(Symbol csym, final LambdaTranslationContext lambdaContext) {
                 JCClassDecl localCDef = localClassDefs.get(csym);
-                if (localCDef != null && localCDef.pos < lambdaContext.tree.pos) {
+                if (localCDef != null && lambdaContext.freeVarProcessedLocalClasses.add(csym)) {
                     BasicFreeVarCollector fvc = lower.new BasicFreeVarCollector() {
                         @Override
                         void addFreeVars(ClassSymbol c) {
@@ -1325,6 +1331,18 @@ public class LambdaToMethod extends TreeTranslator {
                     fvc.scan(localCDef);
                 }
         }
+        //where
+        boolean currentlyInClass(Symbol csym) {
+            for (Frame frame : frameStack) {
+                if (frame.tree.hasTag(JCTree.Tag.CLASSDEF)) {
+                    JCClassDecl cdef = (JCClassDecl) frame.tree;
+                    if (cdef.sym == csym) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
 
         /**
          * Method references to local class constructors, may, if the local
@@ -1750,6 +1768,11 @@ public class LambdaToMethod extends TreeTranslator {
 
             List<JCVariableDecl> syntheticParams;
 
+            /**
+             * to prevent recursion, track local classes processed
+             */
+            final Set<Symbol> freeVarProcessedLocalClasses;
+
             LambdaTranslationContext(JCLambda tree) {
                 super(tree);
                 Frame frame = frameStack.head;
@@ -1779,6 +1802,8 @@ public class LambdaToMethod extends TreeTranslator {
                 translatedSymbols.put(CAPTURED_VAR, new LinkedHashMap<Symbol, Symbol>());
                 translatedSymbols.put(CAPTURED_THIS, new LinkedHashMap<Symbol, Symbol>());
                 translatedSymbols.put(TYPE_VAR, new LinkedHashMap<Symbol, Symbol>());
+
+                freeVarProcessedLocalClasses = new HashSet<>();
             }
 
              /**

langtools/test/tools/javac/lambda/LambdaLocalTest.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8029725
+ * @summary Lambda reference to containing local class causes javac infinite recursion
+ * @author  Robert Field
+ * @run main LambdaLocalTest
+ */
+
+public class LambdaLocalTest {
+    interface F {void f();}
+
+    static F f;
+    static StringBuffer sb = new StringBuffer();
+
+    static void assertEquals(Object val, Object expected) {
+        if (!val.equals(expected)) {
+            throw new AssertionError("expected '" + expected + "' got '" + val + "'");
+        }
+    }
+
+    public static void main(String[] args) {
+        class Local {
+            public Local() {
+                f = () -> new Local();
+                sb.append("+");
+            }
+        }
+        new Local();
+        f.f();
+        assertEquals(sb.toString(), "++");
+    }
+}

langtools/test/tools/javac/lambda/LambdaOuterLocalTest.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8029725
+ * @summary Lambda reference to containing local class causes javac infinite recursion
+ * @author  Robert Field
+ * @run main LambdaOuterLocalTest
+ */
+
+public class LambdaOuterLocalTest {
+    interface F {void f();}
+
+    static F f;
+    static StringBuffer sb = new StringBuffer();
+
+    static void assertEquals(Object val, Object expected) {
+        if (!val.equals(expected)) {
+            throw new AssertionError("expected '" + expected + "' got '" + val + "'");
+        }
+    }
+
+    public static void main(String[] args) {
+        class Local1 {
+            public Local1() {
+                class Local2 {
+                    public Local2() {
+                        f = () -> new Local1();
+                        sb.append("2");
+                    }
+                }
+                sb.append("1");
+                new Local2();
+            }
+        }
+        new Local1();
+        f.f();
+        assertEquals(sb.toString(), "1212");
+    }
+}

langtools/test/tools/javac/lambda/SingleLocalTest.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8029852
+ * @summary Bad code generated (VerifyError) when lambda instantiates
+ *          enclosing local class and has captured variables
+ */
+public class SingleLocalTest {
+    interface F {void f();}
+
+    static F f;
+
+    public static void main(String[] args) {
+        StringBuffer sb = new StringBuffer();
+        class Local1 {
+            public Local1() {
+                f = () -> new Local1();
+                sb.append("1");
+            }
+        }
+        new Local1();
+        f.f();
+        String s = sb.toString();
+        if (!s.equals("11")) {
+            throw new AssertionError("Expected '11' got '" + s + "'");
+        }
+    }
+}