8056897: Improve error recovery for empty binary and hexadecimal literals

Reviewed-by: mcimadamore

langtools/src/jdk.compiler/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -301,15 +301,16 @@ public class JavaTokenizer {
     }
 
     /** Read a number.
-     *  @param radix  The radix of the number; one of 2, j8, 10, 16.
+     *  @param radix  The radix of the number; one of 2, 8, 10, 16.
      */
     private void scanNumber(int pos, int radix) {
         // for octal, allow base-10 digit in case it's a float literal
         this.radix = radix;
         int digitRadix = (radix == 8 ? 10 : radix);
-        boolean seendigit = false;
+        int firstDigit = reader.digit(pos, Math.max(10, digitRadix));
-        if (reader.digit(pos, digitRadix) >= 0) {
+        boolean seendigit = firstDigit >= 0;
-            seendigit = true;
+        boolean seenValidDigit = firstDigit >= 0 && firstDigit < digitRadix;
+        if (seendigit) {
             scanDigits(pos, digitRadix);
         }
         if (radix == 16 && reader.ch == '.') {
@@ -325,6 +326,16 @@ public class JavaTokenizer {
                     reader.ch == 'd' || reader.ch == 'D')) {
             scanFractionAndSuffix(pos);
         } else {
+            if (!seenValidDigit) {
+                switch (radix) {
+                case 2:
+                    lexError(pos, "invalid.binary.number");
+                    break;
+                case 16:
+                    lexError(pos, "invalid.hex.number");
+                    break;
+                }
+            }
             if (reader.ch == 'l' || reader.ch == 'L') {
                 reader.scanChar();
                 tk = TokenKind.LONGLITERAL;
@@ -491,13 +502,7 @@ public class JavaTokenizer {
                     if (reader.ch == 'x' || reader.ch == 'X') {
                         reader.scanChar();
                         skipIllegalUnderscores();
-                        if (reader.ch == '.') {
+                        scanNumber(pos, 16);
-                            scanHexFractionAndSuffix(pos, false);
-                        } else if (reader.digit(pos, 16) < 0) {
-                            lexError(pos, "invalid.hex.number");
-                        } else {
-                            scanNumber(pos, 16);
-                        }
                     } else if (reader.ch == 'b' || reader.ch == 'B') {
                         if (!allowBinaryLiterals) {
                             lexError(pos, "unsupported.binary.lit", source.name);
@@ -505,11 +510,7 @@ public class JavaTokenizer {
                         }
                         reader.scanChar();
                         skipIllegalUnderscores();
-                        if (reader.digit(pos, 2) < 0) {
+                        scanNumber(pos, 2);
-                            lexError(pos, "invalid.binary.number");
-                        } else {
-                            scanNumber(pos, 2);
-                        }
                     } else {
                         reader.putChar('0');
                         if (reader.ch == '_') {

langtools/test/tools/javac/BadHexConstant.java

@@ -1,6 +1,6 @@
 /*
  * @test /nodynamiccopyright/
- * @bug 4049982
+ * @bug 4049982 8056897
  * @summary Compiler permitted invalid hex literal.
  * @author turnidge
  *

langtools/test/tools/javac/BadHexConstant.out

@@ -1,3 +1,2 @@
 BadHexConstant.java:12:14: compiler.err.invalid.hex.number
-BadHexConstant.java:12:17: compiler.err.expected: token.identifier
+1 error
-2 errors

langtools/test/tools/javac/diags/examples/IdentifierExpected.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,11 +23,9 @@
 
 // key: compiler.misc.token.identifier
 // key: compiler.err.expected
-// key: compiler.err.invalid.binary.number
+// key: compiler.misc.count.error
-// key: compiler.misc.count.error.plural
 // key: compiler.err.error
 // run: backdoor
 
-class IdentifierExpected {
+class {
-    long bl = 0BL;
 }

langtools/test/tools/javac/lexer/JavaLexerTest.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8056897
+ * @summary Proper lexing of integer literals.
+ */
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Objects;
+
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+
+import com.sun.tools.javac.parser.JavaTokenizer;
+import com.sun.tools.javac.parser.ScannerFactory;
+import com.sun.tools.javac.parser.Tokens.Token;
+import com.sun.tools.javac.parser.Tokens.TokenKind;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Log;
+
+public class JavaLexerTest {
+    public static void main(String... args) throws Exception {
+        new JavaLexerTest().run();
+    }
+
+    void run() throws Exception {
+        Context ctx = new Context();
+        Log log = Log.instance(ctx);
+        String input = "0bL 0b20L 0xL ";
+        log.useSource(new SimpleJavaFileObject(new URI("mem://Test.java"), JavaFileObject.Kind.SOURCE) {
+            @Override
+            public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {
+                return input;
+            }
+        });
+        char[] inputArr = input.toCharArray();
+        JavaTokenizer tokenizer = new JavaTokenizer(ScannerFactory.instance(ctx), inputArr, inputArr.length) {
+        };
+
+        assertKind(input, tokenizer, TokenKind.LONGLITERAL, "0bL");
+        assertKind(input, tokenizer, TokenKind.LONGLITERAL, "0b20L");
+        assertKind(input, tokenizer, TokenKind.LONGLITERAL, "0xL");
+    }
+
+    void assertKind(String input, JavaTokenizer tokenizer, TokenKind kind, String expectedText) {
+        Token token = tokenizer.readToken();
+
+        if (token.kind != kind) {
+            throw new AssertionError("Unexpected token kind: " + token.kind);
+        }
+
+        String actualText = input.substring(token.pos, token.endPos);
+
+        if (!Objects.equals(actualText, expectedText)) {
+            throw new AssertionError("Unexpected token text: " + actualText);
+        }
+    }
+}

langtools/test/tools/javac/literals/T6891079.java

@@ -1,5 +1,5 @@
 /* @test /nodynamiccopyright/
- * @bug 6891079
+ * @bug 6891079 8056897
  * @summary Compiler allows invalid binary literals 0b and oBL
  * @compile/fail/ref=T6891079.out -XDrawDiagnostics T6891079.java
  */

langtools/test/tools/javac/literals/T6891079.out

@@ -1,7 +1,5 @@
 T6891079.java:8:14: compiler.err.invalid.binary.number
 T6891079.java:9:15: compiler.err.invalid.binary.number
-T6891079.java:9:18: compiler.err.expected: token.identifier
 T6891079.java:10:14: compiler.err.invalid.hex.number
 T6891079.java:11:15: compiler.err.invalid.hex.number
-T6891079.java:11:18: compiler.err.expected: token.identifier
+4 errors
-6 errors