8259050: Error recovery in lexer could be improved

Reviewed-by: vromero

src/jdk.compiler/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java

@@ -205,7 +205,9 @@ public class JavaTokenizer extends UnicodeReader {
      */
     protected void lexError(DiagnosticFlag flags, int pos, JCDiagnostic.Error key) {
         log.error(flags, pos, key);
-        tk = TokenKind.ERROR;
+        if (flags != DiagnosticFlag.SOURCE_LEVEL) {
+            tk = TokenKind.ERROR;
+        }
         errPos = pos;
     }
 

src/jdk.compiler/share/classes/com/sun/tools/javac/parser/UnicodeReader.java

@@ -167,7 +167,11 @@ public class UnicodeReader {
             wasBackslash = false;
         } else if (character == '\\') {
             // May be an unicode escape.
-            wasBackslash = !unicodeEscape();
+            switch (unicodeEscape()) {
+                case BACKSLASH -> wasBackslash = true;
+                case VALID_ESCAPE -> wasBackslash = false;
+                case BROKEN_ESCAPE -> nextUnicodeInputCharacter(); //skip broken unicode escapes
+            }
         }
 
         // Codepoint and character match if not surrogate.
@@ -218,7 +222,7 @@ public class UnicodeReader {
      *
      * @return true if was an unicode escape.
      */
-    private boolean unicodeEscape() {
+    private UnicodeEscapeResult unicodeEscape() {
         // Start of unicode escape (past backslash.)
         int start = position + width;
 
@@ -236,7 +240,7 @@ public class UnicodeReader {
 
         // Needs to have been at least one u.
         if (index == start) {
-            return false;
+            return UnicodeEscapeResult.BACKSLASH;
         }
 
         int code = 0;
@@ -261,12 +265,17 @@ public class UnicodeReader {
         // If all digits are good.
         if (code >= 0) {
             character = (char)code;
+            return UnicodeEscapeResult.VALID_ESCAPE;
         } else {
-            log.error(position, Errors.IllegalUnicodeEsc);
+            log.error(index, Errors.IllegalUnicodeEsc);
+            return UnicodeEscapeResult.BROKEN_ESCAPE;
         }
+    }
 
-        // Return true even if error so that the invalid unicode escape is skipped.
+    private enum UnicodeEscapeResult {
-        return true;
+        BACKSLASH,
+        VALID_ESCAPE,
+        BROKEN_ESCAPE;
     }
 
     /**

test/langtools/tools/javac/lexer/JavaLexerTest.java

@@ -94,11 +94,12 @@ public class JavaLexerTest {
             new TestTuple(DOUBLELITERAL, "0x8pd", "0x8pd"),
             new TestTuple(INTLITERAL,    "0xpd", "0x"),
 
-            new TestTuple(ERROR,         "\"\\u20\""),
+            new TestTuple(STRINGLITERAL, "\"\\u20\""),
-            new TestTuple(ERROR,         "\"\\u\""),
+            new TestTuple(STRINGLITERAL, "\"\\u\""),
-            new TestTuple(ERROR,         "\"\\uG000\""),
+            new TestTuple(STRINGLITERAL, "\"\\uG000\""),
-            new TestTuple(ERROR,         "\"\\u \""),
+            new TestTuple(STRINGLITERAL, "\"\\u \""),
             new TestTuple(ERROR,         "\"\\q\""),
+            new TestTuple(EOF,           "\\u", ""),
 
             new TestTuple(ERROR,         "\'\'"),
             new TestTuple(ERROR,         "\'\\q\'", "\'\\"),

test/langtools/tools/javac/parser/JavacParserTest.java

@@ -23,7 +23,7 @@
 
 /*
  * @test
- * @bug 7073631 7159445 7156633 8028235 8065753 8205418 8205913 8228451 8237041 8253584 8246774 8256411 8256149
+ * @bug 7073631 7159445 7156633 8028235 8065753 8205418 8205913 8228451 8237041 8253584 8246774 8256411 8256149 8259050
  * @summary tests error and diagnostics positions
  * @author  Jan Lahoda
  * @modules jdk.compiler/com.sun.tools.javac.api
@@ -1691,6 +1691,73 @@ public class JavacParserTest extends TestCase {
         }
     }
 
+    @Test //JDK-8259050
+    void testBrokenUnicodeEscape() throws IOException {
+        String code = "package t;\n" +
+                      "class Test {\n" +
+                      "    private String s1 = \"\\" + "uaaa\";\n" +
+                      "    private String s2 = \\" + "uaaa;\n" +
+                      "}\n";
+        DiagnosticCollector<JavaFileObject> coll =
+                new DiagnosticCollector<>();
+        JavacTaskImpl ct = (JavacTaskImpl) tool.getTask(null, fm, coll, null,
+                null, Arrays.asList(new MyFileObject(code)));
+        CompilationUnitTree cut = ct.parse().iterator().next();
+        Trees trees = Trees.instance(ct);
+        String ast = cut.toString().replaceAll("\\R", "\n");
+        String expected = """
+                          package t;
+
+                          class Test {
+                              private String s1 = "";
+                              private String s2 = (ERROR);
+                          } """;
+        assertEquals("Unexpected AST, got:\n" + ast, expected, ast);
+        List<String> codes = new LinkedList<>();
+
+        for (Diagnostic<? extends JavaFileObject> d : coll.getDiagnostics()) {
+            codes.add(d.getCode());
+        }
+
+        assertEquals("testBrokenUnicodeEscape: " + codes,
+                Arrays.<String>asList("compiler.err.illegal.unicode.esc",
+                                      "compiler.err.illegal.unicode.esc"),
+                codes);
+    }
+
+    @Test //JDK-8259050
+    void testUsupportedTextBlock() throws IOException {
+        String code = """
+                      package t;
+                      class Test {
+                          private String s = \"""
+                                             \""";
+                      }""";
+        DiagnosticCollector<JavaFileObject> coll =
+                new DiagnosticCollector<>();
+        JavacTaskImpl ct = (JavacTaskImpl) tool.getTask(null, fm, coll, List.of("--release", "14"),
+                null, Arrays.asList(new MyFileObject(code)));
+        CompilationUnitTree cut = ct.parse().iterator().next();
+        Trees trees = Trees.instance(ct);
+        String ast = cut.toString().replaceAll("\\R", "\n");
+        String expected = """
+                          package t;
+
+                          class Test {
+                              private String s = "";
+                          } """;
+        assertEquals("Unexpected AST, got:\n" + ast, expected, ast);
+        List<String> codes = new LinkedList<>();
+
+        for (Diagnostic<? extends JavaFileObject> d : coll.getDiagnostics()) {
+            codes.add(d.getCode());
+        }
+
+        assertEquals("testUsupportedTextBlock: " + codes,
+                Arrays.<String>asList("compiler.err.feature.not.supported.in.source.plural"),
+                codes);
+    }
+
     void run(String[] args) throws Exception {
         int passed = 0, failed = 0;
         final Pattern p = (args != null && args.length > 0)