6746184: javac fails to compile call to public varargs method

Javac's resolution process should go through all steps described in JLS 15.12.2.2

Reviewed-by: jjg

langtools/src/share/classes/com/sun/tools/javac/comp/Resolve.java

@@ -30,6 +30,7 @@ import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition;
 import com.sun.tools.javac.code.*;
 import com.sun.tools.javac.jvm.*;
 import com.sun.tools.javac.tree.*;
+import static com.sun.tools.javac.comp.Resolve.MethodResolutionPhase.*;
 
 import com.sun.tools.javac.code.Type.*;
 import com.sun.tools.javac.code.Symbol.*;
@@ -40,6 +41,9 @@ import static com.sun.tools.javac.code.Kinds.*;
 import static com.sun.tools.javac.code.TypeTags.*;
 import javax.lang.model.element.ElementVisitor;
 
+import java.util.Map;
+import java.util.HashMap;
+
 /** Helper class for name resolution, used mostly by the attribution phase.
  *
  *  <p><b>This is NOT part of any API supported by Sun Microsystems.  If
@@ -1192,15 +1196,23 @@ public class Resolve {
                          Name name,
                          List<Type> argtypes,
                          List<Type> typeargtypes) {
-        Symbol sym = findFun(env, name, argtypes, typeargtypes, false, env.info.varArgs=false);
+        Symbol sym = methodNotFound;
-        if (varargsEnabled && sym.kind >= WRONG_MTHS) {
+        List<MethodResolutionPhase> steps = methodResolutionSteps;
-            sym = findFun(env, name, argtypes, typeargtypes, true, false);
+        while (steps.nonEmpty() &&
-            if (sym.kind >= WRONG_MTHS)
+               steps.head.isApplicable(boxingEnabled, varargsEnabled) &&
-                sym = findFun(env, name, argtypes, typeargtypes, true, env.info.varArgs=true);
+               sym.kind >= ERRONEOUS) {
+            sym = findFun(env, name, argtypes, typeargtypes,
+                    steps.head.isBoxingRequired,
+                    env.info.varArgs = steps.head.isVarargsRequired);
+            methodResolutionCache.put(steps.head, sym);
+            steps = steps.tail;
         }
-        if (sym.kind >= AMBIGUOUS) {
+        if (sym.kind >= AMBIGUOUS) {//if nothing is found return the 'first' error
-            sym = access(
+            MethodResolutionPhase errPhase =
-                sym, pos, env.enclClass.sym.type, name, false, argtypes, typeargtypes);
+                    firstErroneousResolutionPhase();
+            sym = access(methodResolutionCache.get(errPhase),
+                    pos, env.enclClass.sym.type, name, false, argtypes, typeargtypes);
+            env.info.varArgs = errPhase.isVarargsRequired;
         }
         return sym;
     }
@@ -1217,17 +1229,23 @@ public class Resolve {
     Symbol resolveQualifiedMethod(DiagnosticPosition pos, Env<AttrContext> env,
                                   Type site, Name name, List<Type> argtypes,
                                   List<Type> typeargtypes) {
-        Symbol sym = findMethod(env, site, name, argtypes, typeargtypes, false,
+        Symbol sym = methodNotFound;
-                                env.info.varArgs=false, false);
+        List<MethodResolutionPhase> steps = methodResolutionSteps;
-        if (varargsEnabled && sym.kind >= WRONG_MTHS) {
+        while (steps.nonEmpty() &&
-            sym = findMethod(env, site, name, argtypes, typeargtypes, true,
+               steps.head.isApplicable(boxingEnabled, varargsEnabled) &&
-                             false, false);
+               sym.kind >= ERRONEOUS) {
-            if (sym.kind >= WRONG_MTHS)
+            sym = findMethod(env, site, name, argtypes, typeargtypes,
-                sym = findMethod(env, site, name, argtypes, typeargtypes, true,
+                    steps.head.isBoxingRequired(),
-                                 env.info.varArgs=true, false);
+                    env.info.varArgs = steps.head.isVarargsRequired(), false);
+            methodResolutionCache.put(steps.head, sym);
+            steps = steps.tail;
         }
-        if (sym.kind >= AMBIGUOUS) {
+        if (sym.kind >= AMBIGUOUS) {//if nothing is found return the 'first' error
-            sym = access(sym, pos, site, name, true, argtypes, typeargtypes);
+            MethodResolutionPhase errPhase =
+                    firstErroneousResolutionPhase();
+            sym = access(methodResolutionCache.get(errPhase),
+                    pos, site, name, true, argtypes, typeargtypes);
+            env.info.varArgs = errPhase.isVarargsRequired;
         }
         return sym;
     }
@@ -1268,14 +1286,22 @@ public class Resolve {
                               Type site,
                               List<Type> argtypes,
                               List<Type> typeargtypes) {
-        Symbol sym = resolveConstructor(pos, env, site, argtypes, typeargtypes, false, env.info.varArgs=false);
+        Symbol sym = methodNotFound;
-        if (varargsEnabled && sym.kind >= WRONG_MTHS) {
+        List<MethodResolutionPhase> steps = methodResolutionSteps;
-            sym = resolveConstructor(pos, env, site, argtypes, typeargtypes, true, false);
+        while (steps.nonEmpty() &&
-            if (sym.kind >= WRONG_MTHS)
+               steps.head.isApplicable(boxingEnabled, varargsEnabled) &&
-                sym = resolveConstructor(pos, env, site, argtypes, typeargtypes, true, env.info.varArgs=true);
+               sym.kind >= ERRONEOUS) {
+            sym = resolveConstructor(pos, env, site, argtypes, typeargtypes,
+                    steps.head.isBoxingRequired(),
+                    env.info.varArgs = steps.head.isVarargsRequired());
+            methodResolutionCache.put(steps.head, sym);
+            steps = steps.tail;
         }
-        if (sym.kind >= AMBIGUOUS) {
+        if (sym.kind >= AMBIGUOUS) {//if nothing is found return the 'first' error
-            sym = access(sym, pos, site, names.init, true, argtypes, typeargtypes);
+            MethodResolutionPhase errPhase = firstErroneousResolutionPhase();
+            sym = access(methodResolutionCache.get(errPhase),
+                    pos, site, names.init, true, argtypes, typeargtypes);
+            env.info.varArgs = errPhase.isVarargsRequired();
         }
         return sym;
     }
@@ -1733,4 +1759,50 @@ public class Resolve {
                       pair.sym2.location(site, types));
         }
     }
+
+    enum MethodResolutionPhase {
+        BASIC(false, false),
+        BOX(true, false),
+        VARARITY(true, true);
+
+        boolean isBoxingRequired;
+        boolean isVarargsRequired;
+
+        MethodResolutionPhase(boolean isBoxingRequired, boolean isVarargsRequired) {
+           this.isBoxingRequired = isBoxingRequired;
+           this.isVarargsRequired = isVarargsRequired;
+        }
+
+        public boolean isBoxingRequired() {
+            return isBoxingRequired;
+        }
+
+        public boolean isVarargsRequired() {
+            return isVarargsRequired;
+        }
+
+        public boolean isApplicable(boolean boxingEnabled, boolean varargsEnabled) {
+            return (varargsEnabled || !isVarargsRequired) &&
+                   (boxingEnabled || !isBoxingRequired);
+        }
+    }
+
+    private Map<MethodResolutionPhase, Symbol> methodResolutionCache =
+        new HashMap<MethodResolutionPhase, Symbol>(MethodResolutionPhase.values().length);
+
+    final List<MethodResolutionPhase> methodResolutionSteps = List.of(BASIC, BOX, VARARITY);
+
+    private MethodResolutionPhase firstErroneousResolutionPhase() {
+        MethodResolutionPhase bestSoFar = BASIC;
+        Symbol sym = methodNotFound;
+        List<MethodResolutionPhase> steps = methodResolutionSteps;
+        while (steps.nonEmpty() &&
+               steps.head.isApplicable(boxingEnabled, varargsEnabled) &&
+               sym.kind >= WRONG_MTHS) {
+            sym = methodResolutionCache.get(steps.head);
+            bestSoFar = steps.head;
+            steps = steps.tail;
+        }
+        return bestSoFar;
+    }
 }

langtools/test/tools/javac/varargs/T6746184.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug     6746184
+ * @summary javac fails to compile call to public varargs method
+ */
+
+public class T6746184 {
+    public static void main(String[] args) {
+        A.m(new Object());
+    }
+}
+
+class A {
+    public static void m(final Object... varargs) {}
+    private static void m(final Object singleArg) {}
+}