8048805: Request to investigate and update lexer error recovery in javac

8046620: Further investigation needed for few error messages for negative unicode tests in langtools regression ws
8048803: javac should report complete character code in the error messages

Improving error reporting and recovery in the lexer

Co-authored-by: Sonali Goel <sonali.goel@oracle.com>
Reviewed-by: jjg, mcimadamore

langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java

@@ -213,7 +213,7 @@ public class JavaTokenizer {
                 reader.putChar(true);
             }
             skipIllegalUnderscores();
-            if ('0' <= reader.ch && reader.ch <= '9') {
+            if (reader.digit(pos, 10) >= 0) {
                 scanDigits(pos, 10);
                 if (!hexFloatsWork)
                     lexError(pos, "unsupported.cross.fp.lit");
@@ -239,7 +239,7 @@ public class JavaTokenizer {
      */
     private void scanFraction(int pos) {
         skipIllegalUnderscores();
-        if ('0' <= reader.ch && reader.ch <= '9') {
+        if (reader.digit(pos, 10) >= 0) {
             scanDigits(pos, 10);
         }
         int sp1 = reader.sp;
@@ -250,7 +250,7 @@ public class JavaTokenizer {
                 reader.putChar(true);
             }
             skipIllegalUnderscores();
-            if ('0' <= reader.ch && reader.ch <= '9') {
+            if (reader.digit(pos, 10) >= 0) {
                 scanDigits(pos, 10);
                 return;
             }
@@ -384,11 +384,11 @@ public class JavaTokenizer {
                         reader.scanChar();
                         continue;
                     } else {
-                        high = reader.scanSurrogates();
+                        int codePoint = reader.peekSurrogates();
-                        if (high != 0) {
+                        if (codePoint >= 0) {
-                            reader.putChar(high);
+                            if (isJavaIdentifierPart = Character.isJavaIdentifierPart(codePoint)) {
-                            isJavaIdentifierPart = Character.isJavaIdentifierPart(
+                                reader.putChar(true);
-                                Character.toCodePoint(high, reader.ch));
+                            }
                         } else {
                             isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
                         }
@@ -530,7 +530,7 @@ public class JavaTokenizer {
                     break loop;
                 case '.':
                     reader.scanChar();
-                    if ('0' <= reader.ch && reader.ch <= '9') {
+                    if (reader.digit(pos, 10) >= 0) {
                         reader.putChar('.');
                         scanFractionAndSuffix(pos);
                     } else if (reader.ch == '.') {
@@ -613,11 +613,11 @@ public class JavaTokenizer {
                     reader.scanChar();
                     if (reader.ch == '\'') {
                         lexError(pos, "empty.char.lit");
+                        reader.scanChar();
                     } else {
                         if (reader.ch == CR || reader.ch == LF)
                             lexError(pos, "illegal.line.end.in.char.lit");
                         scanLitChar(pos);
-                        char ch2 = reader.ch;
                         if (reader.ch == '\'') {
                             reader.scanChar();
                             tk = TokenKind.CHARLITERAL;
@@ -642,29 +642,39 @@ public class JavaTokenizer {
                         scanOperator();
                     } else {
                         boolean isJavaIdentifierStart;
+                        int codePoint = -1;
                         if (reader.ch < '\u0080') {
                             // all ASCII range chars already handled, above
                             isJavaIdentifierStart = false;
                         } else {
-                            char high = reader.scanSurrogates();
+                            codePoint = reader.peekSurrogates();
-                            if (high != 0) {
+                            if (codePoint >= 0) {
-                                reader.putChar(high);
+                                if (isJavaIdentifierStart = Character.isJavaIdentifierStart(codePoint)) {
-
+                                    reader.putChar(true);
-                                isJavaIdentifierStart = Character.isJavaIdentifierStart(
+                                }
-                                    Character.toCodePoint(high, reader.ch));
                             } else {
                                 isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
                             }
                         }
                         if (isJavaIdentifierStart) {
                             scanIdent();
+                        } else if (reader.digit(pos, 10) >= 0) {
+                            scanNumber(pos, 10);
                         } else if (reader.bp == reader.buflen || reader.ch == EOI && reader.bp + 1 == reader.buflen) { // JLS 3.5
                             tk = TokenKind.EOF;
                             pos = reader.buflen;
                         } else {
-                            String arg = (32 < reader.ch && reader.ch < 127) ?
+                            String arg;
-                                            String.format("%s", reader.ch) :
+
-                                            String.format("\\u%04x", (int)reader.ch);
+                            if (codePoint >= 0) {
+                                char high = reader.ch;
+                                reader.scanChar();
+                                arg = String.format("\\u%04x\\u%04x", (int) high, (int)reader.ch);
+                            } else {
+                                arg = (32 < reader.ch && reader.ch < 127) ?
+                                                String.format("%s", reader.ch) :
+                                                String.format("\\u%04x", (int)reader.ch);
+                            }
                             lexError(pos, "illegal.char", arg);
                             reader.scanChar();
                         }

langtools/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java

@@ -197,24 +197,28 @@ public class UnicodeReader {
     }
 
     /** Scan surrogate pairs.  If 'ch' is a high surrogate and
-     *  the next character is a low surrogate, then put the low
+     *  the next character is a low surrogate, returns the code point
-     *  surrogate in 'ch', and return the high surrogate.
+     *  constructed from these surrogates. Otherwise, returns -1.
-     *  otherwise, just return 0.
+     *  This method will not consume any of the characters.
      */
-    protected char scanSurrogates() {
+    protected int peekSurrogates() {
         if (surrogatesSupported && Character.isHighSurrogate(ch)) {
             char high = ch;
+            int prevBP = bp;
 
             scanChar();
 
-            if (Character.isLowSurrogate(ch)) {
+            char low = ch;
-                return high;
-            }
 
             ch = high;
+            bp = prevBP;
+
+            if (Character.isLowSurrogate(low)) {
+                return Character.toCodePoint(high, low);
+            }
         }
 
-        return 0;
+        return -1;
     }
 
     /** Convert an ASCII digit from its base (8, 10, or 16)
@@ -222,9 +226,14 @@ public class UnicodeReader {
      */
     protected int digit(int pos, int base) {
         char c = ch;
-        int result = Character.digit(c, base);
+        if ('0' <= c && c <= '9')
+            return Character.digit(c, base); //a fast common case
+        int codePoint = peekSurrogates();
+        int result = codePoint >= 0 ? Character.digit(codePoint, base) : Character.digit(c, base);
         if (result >= 0 && c > 0x7f) {
             log.error(pos + 1, "illegal.nonascii.digit");
+            if (codePoint >= 0)
+                scanChar();
             ch = "0123456789abcdef".charAt(result);
         }
         return result;

langtools/test/tools/javac/diags/examples/EmptyCharLiteral.java

@@ -22,7 +22,6 @@
  */
 
 // key: compiler.err.empty.char.lit
-// key: compiler.err.unclosed.char.lit
 
 class X {
     char c = '';

langtools/test/tools/javac/unicode/NonasciiDigit.java

@@ -1,6 +1,6 @@
 /*
  * @test /nodynamiccopyright/
- * @bug 4707960 6183529
+ * @bug 4707960 6183529 8046620
  * @summary javac accepts unicode digits - sometimes crashing
  * @author gafter
  *
@@ -8,7 +8,16 @@
  */
 public class NonasciiDigit {
     public static void main(String[] args) {
+        // error: only ASCII allowed in constants
+        int i1 = \uff11;
+        int i2 = 1\uff11;
+        int i3 = \ud835\udfff;
         // error: floating literals use ascii only
-        float f = 0.\uff11;
+        double d1 = \uff11.0;
+        double d2 = 0.\uff11;
+        double d3 = 0x0P\uff11;
+        double d4 = 0E\uff11;
+        double d5 = .\uff11;
+        double d6 = \ud835\udfff.0;
     }
 }

langtools/test/tools/javac/unicode/NonasciiDigit.out

@@ -1,2 +1,10 @@
-NonasciiDigit.java:12:26: compiler.err.illegal.char: \uff11
+NonasciiDigit.java:12:24: compiler.err.illegal.nonascii.digit
-1 error
+NonasciiDigit.java:13:19: compiler.err.illegal.nonascii.digit
+NonasciiDigit.java:14:24: compiler.err.illegal.nonascii.digit
+NonasciiDigit.java:16:27: compiler.err.illegal.nonascii.digit
+NonasciiDigit.java:17:22: compiler.err.illegal.nonascii.digit
+NonasciiDigit.java:18:22: compiler.err.illegal.nonascii.digit
+NonasciiDigit.java:19:22: compiler.err.illegal.nonascii.digit
+NonasciiDigit.java:20:22: compiler.err.illegal.nonascii.digit
+NonasciiDigit.java:21:27: compiler.err.illegal.nonascii.digit
+9 errors

langtools/test/tools/javac/unicode/NonasciiDigit2.java

@@ -1,14 +0,0 @@
-/*
- * @test /nodynamiccopyright/
- * @bug 4707960 6183529
- * @summary javac accepts unicode digits - sometimes crashing
- * @author gafter
- *
- * @compile/fail/ref=NonasciiDigit2.out -XDrawDiagnostics  NonasciiDigit2.java
- */
-public class NonasciiDigit2 {
-    public static void main(String[] args) {
-        // error: only ASCII allowed in constants
-        int i = 1\uff11;
-    }
-}

langtools/test/tools/javac/unicode/NonasciiDigit2.out

@@ -1,2 +0,0 @@
-NonasciiDigit2.java:12:18: compiler.err.illegal.nonascii.digit
-1 error

langtools/test/tools/javac/unicode/SupplementaryJavaID2.out

@@ -1,3 +1,4 @@
 SupplementaryJavaID2.java:12:14: compiler.err.illegal.char: \ud801
+SupplementaryJavaID2.java:12:20: compiler.err.illegal.char: \ud801
 SupplementaryJavaID2.java:12:24: compiler.err.expected: token.identifier
-2 errors
+3 errors

langtools/test/tools/javac/unicode/SupplementaryJavaID3.out

@@ -1,2 +1,3 @@
+SupplementaryJavaID3.java:12:17: compiler.err.illegal.char: \ud801
 SupplementaryJavaID3.java:12:23: compiler.err.illegal.char: \ud801
-1 error
+2 errors

langtools/test/tools/javac/unicode/SupplementaryJavaID4.java

@@ -1,35 +1,12 @@
 /*
- * Copyright (c) 2003, Oracle and/or its affiliates. All rights reserved.
+ * @test /nodynamiccopyright/
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * @bug 4914724 8048803
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-/*
- * @test
- * @bug 4914724
  * @summary Ensure that a supplementary character that cannot be the start of a Java
  *          identifier causes a compilation failure, if it is used as the start of an
  *          identifier
  * @author Naoto Sato
  *
- * @compile/fail SupplementaryJavaID4.java
+ * @compile/fail/ref=SupplementaryJavaID4.out -XDrawDiagnostics  SupplementaryJavaID4.java
  */
 
 public class SupplementaryJavaID4 {

langtools/test/tools/javac/unicode/SupplementaryJavaID4.out

@@ -0,0 +1,2 @@
+SupplementaryJavaID4.java:14:14: compiler.err.illegal.char: \ud834\udd7b
+1 error

langtools/test/tools/javac/unicode/SupplementaryJavaID5.java

@@ -1,35 +1,12 @@
 /*
- * Copyright (c) 2003, Oracle and/or its affiliates. All rights reserved.
+ * @test /nodynamiccopyright/
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * @bug 4914724 8048803
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-/*
- * @test
- * @bug 4914724
  * @summary Ensure that a supplementary character that cannot be the part of a Java
  *          identifier causes a compilation failure, if it is used as the part of an
  *          identifier
  * @author Naoto Sato
  *
- * @compile/fail SupplementaryJavaID5.java
+ * @compile/fail/ref=SupplementaryJavaID5.out -XDrawDiagnostics  SupplementaryJavaID5.java
  */
 
 public class SupplementaryJavaID5 {

langtools/test/tools/javac/unicode/SupplementaryJavaID5.out

@@ -0,0 +1,2 @@
+SupplementaryJavaID5.java:14:17: compiler.err.illegal.char: \ud834\udd00
+1 error

langtools/test/tools/javac/unicode/TripleQuote.java

@@ -1,6 +1,6 @@
 /*
  * @test /nodynamiccopyright/
- * @bug 1265387
+ * @bug 1265387 8048805
  * @summary ''' and '\u0027' are not legal char literals.
  * @author turnidge
  *

langtools/test/tools/javac/unicode/TripleQuote.out

@@ -1,7 +1,5 @@
 TripleQuote.java:12:14: compiler.err.empty.char.lit
-TripleQuote.java:12:20: compiler.err.empty.char.lit
 TripleQuote.java:12:21: compiler.err.unclosed.char.lit
 TripleQuote.java:13:14: compiler.err.empty.char.lit
-TripleQuote.java:13:15: compiler.err.empty.char.lit
 TripleQuote.java:13:16: compiler.err.unclosed.char.lit
-6 errors
+4 errors