7104201: Refactor DocCommentScanner

Add new Comment helper class to parse contents of comments in source code

Reviewed-by: jjg

langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java

@@ -25,10 +25,11 @@
 
 package com.sun.tools.javac.parser;
 
-import java.nio.CharBuffer;
 import com.sun.tools.javac.code.Source;
+import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
 import com.sun.tools.javac.util.*;
 
+import java.nio.CharBuffer;
 
 import static com.sun.tools.javac.parser.Tokens.*;
 import static com.sun.tools.javac.util.LayoutCharacters.*;
@@ -65,9 +66,6 @@ public class JavaTokenizer {
      */
     private final Log log;
 
-    /** The name table. */
-    private final Names names;
-
     /** The token factory. */
     private final Tokens tokens;
 
@@ -87,18 +85,12 @@ public class JavaTokenizer {
      */
     protected int errPos = Position.NOPOS;
 
-    /** Has a @deprecated been encountered in last doc comment?
+    /** The Unicode reader (low-level stream reader).
-     *  this needs to be reset by client.
      */
-    protected boolean deprecatedFlag = false;
-
-    /** A character buffer for saved chars.
-     */
-    protected char[] sbuf = new char[128];
-    protected int sp;
-
     protected UnicodeReader reader;
 
+    protected ScannerFactory fac;
+
     private static final boolean hexFloatsWork = hexFloatsWork();
     private static boolean hexFloatsWork() {
         try {
@@ -129,14 +121,14 @@ public class JavaTokenizer {
     }
 
     protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
-        log = fac.log;
+        this.fac = fac;
-        names = fac.names;
+        this.log = fac.log;
-        tokens = fac.tokens;
+        this.tokens = fac.tokens;
-        source = fac.source;
+        this.source = fac.source;
         this.reader = reader;
-        allowBinaryLiterals = source.allowBinaryLiterals();
+        this.allowBinaryLiterals = source.allowBinaryLiterals();
-        allowHexFloats = source.allowHexFloats();
+        this.allowHexFloats = source.allowHexFloats();
-        allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
+        this.allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
     }
 
     /** Report an error at the given position using the provided arguments.
@@ -147,38 +139,13 @@ public class JavaTokenizer {
         errPos = pos;
     }
 
-    /** Read next character in comment, skipping over double '\' characters.
-     */
-    protected void scanCommentChar() {
-        reader.scanChar();
-        if (reader.ch == '\\') {
-            if (reader.peekChar() == '\\' && !reader.isUnicode()) {
-                reader.skipChar();
-            } else {
-                reader.convertUnicode();
-            }
-        }
-    }
-
-    /** Append a character to sbuf.
-     */
-    private void putChar(char ch) {
-        if (sp == sbuf.length) {
-            char[] newsbuf = new char[sbuf.length * 2];
-            System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
-            sbuf = newsbuf;
-        }
-        sbuf[sp++] = ch;
-    }
-
     /** Read next character in character or string literal and copy into sbuf.
      */
     private void scanLitChar(int pos) {
         if (reader.ch == '\\') {
             if (reader.peekChar() == '\\' && !reader.isUnicode()) {
                 reader.skipChar();
-                putChar('\\');
+                reader.putChar('\\', true);
-                reader.scanChar();
             } else {
                 reader.scanChar();
                 switch (reader.ch) {
@@ -195,30 +162,30 @@ public class JavaTokenizer {
                             reader.scanChar();
                         }
                     }
-                    putChar((char)oct);
+                    reader.putChar((char)oct);
                     break;
                 case 'b':
-                    putChar('\b'); reader.scanChar(); break;
+                    reader.putChar('\b', true); break;
                 case 't':
-                    putChar('\t'); reader.scanChar(); break;
+                    reader.putChar('\t', true); break;
                 case 'n':
-                    putChar('\n'); reader.scanChar(); break;
+                    reader.putChar('\n', true); break;
                 case 'f':
-                    putChar('\f'); reader.scanChar(); break;
+                    reader.putChar('\f', true); break;
                 case 'r':
-                    putChar('\r'); reader.scanChar(); break;
+                    reader.putChar('\r', true); break;
                 case '\'':
-                    putChar('\''); reader.scanChar(); break;
+                    reader.putChar('\'', true); break;
                 case '\"':
-                    putChar('\"'); reader.scanChar(); break;
+                    reader.putChar('\"', true); break;
                 case '\\':
-                    putChar('\\'); reader.scanChar(); break;
+                    reader.putChar('\\', true); break;
                 default:
                     lexError(reader.bp, "illegal.esc.char");
                 }
             }
         } else if (reader.bp != reader.buflen) {
-            putChar(reader.ch); reader.scanChar();
+            reader.putChar(true);
         }
     }
 
@@ -227,7 +194,7 @@ public class JavaTokenizer {
         int savePos;
         do {
             if (reader.ch != '_') {
-                putChar(reader.ch);
+                reader.putChar(false);
             } else {
                 if (!allowUnderscoresInLiterals) {
                     lexError(pos, "unsupported.underscore.lit", source.name);
@@ -246,12 +213,10 @@ public class JavaTokenizer {
      */
     private void scanHexExponentAndSuffix(int pos) {
         if (reader.ch == 'p' || reader.ch == 'P') {
-            putChar(reader.ch);
+            reader.putChar(true);
-            reader.scanChar();
             skipIllegalUnderscores();
             if (reader.ch == '+' || reader.ch == '-') {
-                putChar(reader.ch);
+                reader.putChar(true);
-                reader.scanChar();
             }
             skipIllegalUnderscores();
             if ('0' <= reader.ch && reader.ch <= '9') {
@@ -268,14 +233,12 @@ public class JavaTokenizer {
             lexError(pos, "malformed.fp.lit");
         }
         if (reader.ch == 'f' || reader.ch == 'F') {
-            putChar(reader.ch);
+            reader.putChar(true);
-            reader.scanChar();
             tk = TokenKind.FLOATLITERAL;
             radix = 16;
         } else {
             if (reader.ch == 'd' || reader.ch == 'D') {
-                putChar(reader.ch);
+                reader.putChar(true);
-                reader.scanChar();
             }
             tk = TokenKind.DOUBLELITERAL;
             radix = 16;
@@ -289,14 +252,12 @@ public class JavaTokenizer {
         if ('0' <= reader.ch && reader.ch <= '9') {
             scanDigits(pos, 10);
         }
-        int sp1 = sp;
+        int sp1 = reader.sp;
         if (reader.ch == 'e' || reader.ch == 'E') {
-            putChar(reader.ch);
+            reader.putChar(true);
-            reader.scanChar();
             skipIllegalUnderscores();
             if (reader.ch == '+' || reader.ch == '-') {
-                putChar(reader.ch);
+                reader.putChar(true);
-                reader.scanChar();
             }
             skipIllegalUnderscores();
             if ('0' <= reader.ch && reader.ch <= '9') {
@@ -304,7 +265,7 @@ public class JavaTokenizer {
                 return;
             }
             lexError(pos, "malformed.fp.lit");
-            sp = sp1;
+            reader.sp = sp1;
         }
     }
 
@@ -314,13 +275,11 @@ public class JavaTokenizer {
         radix = 10;
         scanFraction(pos);
         if (reader.ch == 'f' || reader.ch == 'F') {
-            putChar(reader.ch);
+            reader.putChar(true);
-            reader.scanChar();
             tk = TokenKind.FLOATLITERAL;
         } else {
             if (reader.ch == 'd' || reader.ch == 'D') {
-                putChar(reader.ch);
+                reader.putChar(true);
-                reader.scanChar();
             }
             tk = TokenKind.DOUBLELITERAL;
         }
@@ -331,8 +290,7 @@ public class JavaTokenizer {
     private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
         radix = 16;
         Assert.check(reader.ch == '.');
-        putChar(reader.ch);
+        reader.putChar(true);
-        reader.scanChar();
         skipIllegalUnderscores();
         if (reader.digit(pos, 16) >= 0) {
             seendigit = true;
@@ -369,8 +327,7 @@ public class JavaTokenizer {
         } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
             scanHexExponentAndSuffix(pos);
         } else if (digitRadix == 10 && reader.ch == '.') {
-            putChar(reader.ch);
+            reader.putChar(true);
-            reader.scanChar();
             scanFractionAndSuffix(pos);
         } else if (digitRadix == 10 &&
                    (reader.ch == 'e' || reader.ch == 'E' ||
@@ -393,10 +350,7 @@ public class JavaTokenizer {
         boolean isJavaIdentifierPart;
         char high;
         do {
-            if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch;
+            reader.putChar(true);
-            // optimization, was: putChar(reader.ch);
-
-            reader.scanChar();
             switch (reader.ch) {
             case 'A': case 'B': case 'C': case 'D': case 'E':
             case 'F': case 'G': case 'H': case 'I': case 'J':
@@ -423,7 +377,7 @@ public class JavaTokenizer {
                 break;
             case '\u001A': // EOI is also a legal identifier part
                 if (reader.bp >= reader.buflen) {
-                    name = names.fromChars(sbuf, 0, sp);
+                    name = reader.name();
                     tk = tokens.lookupKind(name);
                     return;
                 }
@@ -435,11 +389,7 @@ public class JavaTokenizer {
                 } else {
                     high = reader.scanSurrogates();
                     if (high != 0) {
-                        if (sp == sbuf.length) {
+                        reader.putChar(high);
-                            putChar(high);
-                        } else {
-                            sbuf[sp++] = high;
-                        }
                         isJavaIdentifierPart = Character.isJavaIdentifierPart(
                             Character.toCodePoint(high, reader.ch));
                     } else {
@@ -447,7 +397,7 @@ public class JavaTokenizer {
                     }
                 }
                 if (!isJavaIdentifierPart) {
-                    name = names.fromChars(sbuf, 0, sp);
+                    name = reader.name();
                     tk = tokens.lookupKind(name);
                     return;
                 }
@@ -474,11 +424,11 @@ public class JavaTokenizer {
      */
     private void scanOperator() {
         while (true) {
-            putChar(reader.ch);
+            reader.putChar(false);
-            Name newname = names.fromChars(sbuf, 0, sp);
+            Name newname = reader.name();
             TokenKind tk1 = tokens.lookupKind(newname);
             if (tk1 == TokenKind.IDENTIFIER) {
-                sp--;
+                reader.sp--;
                 break;
             }
             tk = tk1;
@@ -487,111 +437,17 @@ public class JavaTokenizer {
         }
     }
 
-    /**
-     * Scan a documentation comment; determine if a deprecated tag is present.
-     * Called once the initial /, * have been skipped, positioned at the second *
-     * (which is treated as the beginning of the first line).
-     * Stops positioned at the closing '/'.
-     */
-    @SuppressWarnings("fallthrough")
-    private void scanDocComment() {
-        boolean deprecatedPrefix = false;
-
-        forEachLine:
-        while (reader.bp < reader.buflen) {
-
-            // Skip optional WhiteSpace at beginning of line
-            while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
-                scanCommentChar();
-            }
-
-            // Skip optional consecutive Stars
-            while (reader.bp < reader.buflen && reader.ch == '*') {
-                scanCommentChar();
-                if (reader.ch == '/') {
-                    return;
-                }
-            }
-
-            // Skip optional WhiteSpace after Stars
-            while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
-                scanCommentChar();
-            }
-
-            deprecatedPrefix = false;
-            // At beginning of line in the JavaDoc sense.
-            if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) {
-                scanCommentChar();
-                if (reader.bp < reader.buflen && reader.ch == 'd') {
-                    scanCommentChar();
-                    if (reader.bp < reader.buflen && reader.ch == 'e') {
-                        scanCommentChar();
-                        if (reader.bp < reader.buflen && reader.ch == 'p') {
-                            scanCommentChar();
-                            if (reader.bp < reader.buflen && reader.ch == 'r') {
-                                scanCommentChar();
-                                if (reader.bp < reader.buflen && reader.ch == 'e') {
-                                    scanCommentChar();
-                                    if (reader.bp < reader.buflen && reader.ch == 'c') {
-                                        scanCommentChar();
-                                        if (reader.bp < reader.buflen && reader.ch == 'a') {
-                                            scanCommentChar();
-                                            if (reader.bp < reader.buflen && reader.ch == 't') {
-                                                scanCommentChar();
-                                                if (reader.bp < reader.buflen && reader.ch == 'e') {
-                                                    scanCommentChar();
-                                                    if (reader.bp < reader.buflen && reader.ch == 'd') {
-                                                        deprecatedPrefix = true;
-                                                        scanCommentChar();
-                                                    }}}}}}}}}}}
-            if (deprecatedPrefix && reader.bp < reader.buflen) {
-                if (Character.isWhitespace(reader.ch)) {
-                    deprecatedFlag = true;
-                } else if (reader.ch == '*') {
-                    scanCommentChar();
-                    if (reader.ch == '/') {
-                        deprecatedFlag = true;
-                        return;
-                    }
-                }
-            }
-
-            // Skip rest of line
-            while (reader.bp < reader.buflen) {
-                switch (reader.ch) {
-                case '*':
-                    scanCommentChar();
-                    if (reader.ch == '/') {
-                        return;
-                    }
-                    break;
-                case CR: // (Spec 3.4)
-                    scanCommentChar();
-                    if (reader.ch != LF) {
-                        continue forEachLine;
-                    }
-                    /* fall through to LF case */
-                case LF: // (Spec 3.4)
-                    scanCommentChar();
-                    continue forEachLine;
-                default:
-                    scanCommentChar();
-                }
-            } // rest of line
-        } // forEachLine
-        return;
-    }
-
     /** Read token.
      */
     public Token readToken() {
 
-        sp = 0;
+        reader.sp = 0;
         name = null;
-        deprecatedFlag = false;
         radix = 0;
+
         int pos = 0;
         int endPos = 0;
+        List<Comment> comments = null;
 
         try {
             loop: while (true) {
@@ -656,7 +512,7 @@ public class JavaTokenizer {
                             scanNumber(pos, 2);
                         }
                     } else {
-                        putChar('0');
+                        reader.putChar('0');
                         if (reader.ch == '_') {
                             int savePos = reader.bp;
                             do {
@@ -676,14 +532,13 @@ public class JavaTokenizer {
                 case '.':
                     reader.scanChar();
                     if ('0' <= reader.ch && reader.ch <= '9') {
-                        putChar('.');
+                        reader.putChar('.');
                         scanFractionAndSuffix(pos);
                     } else if (reader.ch == '.') {
-                        putChar('.'); putChar('.');
+                        reader.putChar('.'); reader.putChar('.', true);
-                        reader.scanChar();
                         if (reader.ch == '.') {
                             reader.scanChar();
-                            putChar('.');
+                            reader.putChar('.');
                             tk = TokenKind.ELLIPSIS;
                         } else {
                             lexError(pos, "malformed.fp.lit");
@@ -712,32 +567,36 @@ public class JavaTokenizer {
                     reader.scanChar();
                     if (reader.ch == '/') {
                         do {
-                            scanCommentChar();
+                            reader.scanCommentChar();
                         } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
                         if (reader.bp < reader.buflen) {
-                            processComment(pos, reader.bp, CommentStyle.LINE);
+                            comments = addDocReader(comments, processComment(pos, reader.bp, CommentStyle.LINE));
                         }
                         break;
                     } else if (reader.ch == '*') {
+                        boolean isEmpty = false;
                         reader.scanChar();
                         CommentStyle style;
                         if (reader.ch == '*') {
                             style = CommentStyle.JAVADOC;
-                            scanDocComment();
+                            reader.scanCommentChar();
+                            if (reader.ch == '/') {
+                                isEmpty = true;
+                            }
                         } else {
                             style = CommentStyle.BLOCK;
-                            while (reader.bp < reader.buflen) {
+                        }
-                                if (reader.ch == '*') {
+                        while (!isEmpty && reader.bp < reader.buflen) {
-                                    reader.scanChar();
+                            if (reader.ch == '*') {
-                                    if (reader.ch == '/') break;
+                                reader.scanChar();
-                                } else {
+                                if (reader.ch == '/') break;
-                                    scanCommentChar();
+                            } else {
-                                }
+                                reader.scanCommentChar();
                             }
                         }
                         if (reader.ch == '/') {
                             reader.scanChar();
-                            processComment(pos, reader.bp, style);
+                            comments = addDocReader(comments, processComment(pos, reader.bp, style));
                             break;
                         } else {
                             lexError(pos, "unclosed.comment");
@@ -789,11 +648,7 @@ public class JavaTokenizer {
                         } else {
                             char high = reader.scanSurrogates();
                             if (high != 0) {
-                                if (sp == sbuf.length) {
+                                reader.putChar(high);
-                                    putChar(high);
-                                } else {
-                                    sbuf[sp++] = high;
-                                }
 
                                 isJavaIdentifierStart = Character.isJavaIdentifierStart(
                                     Character.toCodePoint(high, reader.ch));
@@ -816,10 +671,10 @@ public class JavaTokenizer {
             }
             endPos = reader.bp;
             switch (tk.tag) {
-                case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag);
+                case DEFAULT: return new Token(tk, pos, endPos, comments);
-                case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag);
+                case NAMED: return new NamedToken(tk, pos, endPos, name, comments);
-                case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag);
+                case STRING: return new StringToken(tk, pos, endPos, reader.chars(), comments);
-                case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag);
+                case NUMERIC: return new NumericToken(tk, pos, endPos, reader.chars(), radix, comments);
                 default: throw new AssertionError();
             }
         }
@@ -832,6 +687,12 @@ public class JavaTokenizer {
             }
         }
     }
+    //where
+        List<Comment> addDocReader(List<Comment> docReaders, Comment docReader) {
+            return docReaders == null ?
+                    List.of(docReader) :
+                    docReaders.prepend(docReader);
+        }
 
     /** Return the position where a lexical error occurred;
      */
@@ -845,22 +706,18 @@ public class JavaTokenizer {
         errPos = pos;
     }
 
-    public enum CommentStyle {
-        LINE,
-        BLOCK,
-        JAVADOC,
-    }
-
     /**
      * Called when a complete comment has been scanned. pos and endPos
      * will mark the comment boundary.
      */
-    protected void processComment(int pos, int endPos, CommentStyle style) {
+    protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) {
         if (scannerDebug)
             System.out.println("processComment(" + pos
                                + "," + endPos + "," + style + ")=|"
                                + new String(reader.getRawCharacters(pos, endPos))
                                + "|");
+        char[] buf = reader.getRawCharacters(pos, endPos);
+        return new BasicComment<UnicodeReader>(new UnicodeReader(fac, buf, buf.length), style);
     }
 
     /**
@@ -893,4 +750,125 @@ public class JavaTokenizer {
     public Position.LineMap getLineMap() {
         return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
     }
+
+
+    /**
+    * Scan a documentation comment; determine if a deprecated tag is present.
+    * Called once the initial /, * have been skipped, positioned at the second *
+    * (which is treated as the beginning of the first line).
+    * Stops positioned at the closing '/'.
+    */
+    protected class BasicComment<U extends UnicodeReader> implements Comment {
+
+        CommentStyle cs;
+        U comment_reader;
+
+        protected boolean deprecatedFlag = false;
+        protected boolean scanned = false;
+
+        protected BasicComment(U comment_reader, CommentStyle cs) {
+            this.comment_reader = comment_reader;
+            this.cs = cs;
+        }
+
+        public String getText() {
+            return null;
+        }
+
+        public CommentStyle getStyle() {
+            return cs;
+        }
+
+        public boolean isDeprecated() {
+            if (!scanned && cs == CommentStyle.JAVADOC) {
+                scanDocComment();
+            }
+            return deprecatedFlag;
+        }
+
+        @SuppressWarnings("fallthrough")
+        protected void scanDocComment() {
+            try {
+                boolean deprecatedPrefix = false;
+
+                comment_reader.bp += 3; // '/**'
+                comment_reader.ch = comment_reader.buf[comment_reader.bp];
+
+                forEachLine:
+                while (comment_reader.bp < comment_reader.buflen) {
+
+                    // Skip optional WhiteSpace at beginning of line
+                    while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
+                        comment_reader.scanCommentChar();
+                    }
+
+                    // Skip optional consecutive Stars
+                    while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
+                        comment_reader.scanCommentChar();
+                        if (comment_reader.ch == '/') {
+                            return;
+                        }
+                    }
+
+                    // Skip optional WhiteSpace after Stars
+                    while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
+                        comment_reader.scanCommentChar();
+                    }
+
+                    deprecatedPrefix = false;
+                    // At beginning of line in the JavaDoc sense.
+                    if (!deprecatedFlag) {
+                        String deprecated = "@deprecated";
+                        int i = 0;
+                        while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == deprecated.charAt(i)) {
+                            comment_reader.scanCommentChar();
+                            i++;
+                            if (i == deprecated.length()) {
+                                deprecatedPrefix = true;
+                                break;
+                            }
+                        }
+                    }
+
+                    if (deprecatedPrefix && comment_reader.bp < comment_reader.buflen) {
+                        if (Character.isWhitespace(comment_reader.ch)) {
+                            deprecatedFlag = true;
+                        } else if (comment_reader.ch == '*') {
+                            comment_reader.scanCommentChar();
+                            if (comment_reader.ch == '/') {
+                                deprecatedFlag = true;
+                                return;
+                            }
+                        }
+                    }
+
+                    // Skip rest of line
+                    while (comment_reader.bp < comment_reader.buflen) {
+                        switch (comment_reader.ch) {
+                            case '*':
+                                comment_reader.scanCommentChar();
+                                if (comment_reader.ch == '/') {
+                                    return;
+                                }
+                                break;
+                            case CR: // (Spec 3.4)
+                                comment_reader.scanCommentChar();
+                                if (comment_reader.ch != LF) {
+                                    continue forEachLine;
+                                }
+                            /* fall through to LF case */
+                            case LF: // (Spec 3.4)
+                                comment_reader.scanCommentChar();
+                                continue forEachLine;
+                            default:
+                                comment_reader.scanCommentChar();
+                        }
+                    } // rest of line
+                } // forEachLine
+                return;
+            } finally {
+                scanned = true;
+            }
+        }
+    }
 }

langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java

@@ -29,6 +29,7 @@ import java.util.*;
 
 import com.sun.tools.javac.code.*;
 import com.sun.tools.javac.parser.Tokens.*;
+import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
 import com.sun.tools.javac.tree.*;
 import com.sun.tools.javac.tree.JCTree.*;
 import com.sun.tools.javac.util.*;
@@ -1584,7 +1585,7 @@ public class JavacParser implements Parser {
                 break;
             case MONKEYS_AT:
             case FINAL: {
-                String dc = token.docComment;
+                String dc = token.comment(CommentStyle.JAVADOC);
                 JCModifiers mods = modifiersOpt();
                 if (token.kind == INTERFACE ||
                     token.kind == CLASS ||
@@ -1601,21 +1602,21 @@ public class JavacParser implements Parser {
                 break;
             }
             case ABSTRACT: case STRICTFP: {
-                String dc = token.docComment;
+                String dc = token.comment(CommentStyle.JAVADOC);
                 JCModifiers mods = modifiersOpt();
                 stats.append(classOrInterfaceOrEnumDeclaration(mods, dc));
                 break;
             }
             case INTERFACE:
             case CLASS:
-                String dc = token.docComment;
+                String dc = token.comment(CommentStyle.JAVADOC);
                 stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
                 break;
             case ENUM:
             case ASSERT:
                 if (allowEnums && token.kind == ENUM) {
                     error(token.pos, "local.enum");
-                    dc = token.docComment;
+                    dc = token.comment(CommentStyle.JAVADOC);
                     stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
                     break;
                 } else if (allowAsserts && token.kind == ASSERT) {
@@ -1991,7 +1992,7 @@ public class JavacParser implements Parser {
             annotations.appendList(partial.annotations);
             pos = partial.pos;
         }
-        if (token.deprecatedFlag) {
+        if (token.deprecatedFlag()) {
             flags |= Flags.DEPRECATED;
         }
         int lastPos = Position.NOPOS;
@@ -2271,9 +2272,9 @@ public class JavacParser implements Parser {
                 seenImport = true;
                 defs.append(importDeclaration());
             } else {
-                String docComment = token.docComment;
+                String docComment = token.comment(CommentStyle.JAVADOC);
                 if (firstTypeDecl && !seenImport && !seenPackage) {
-                    docComment = firstToken.docComment;
+                    docComment = firstToken.comment(CommentStyle.JAVADOC);
                     consumedToplevelDoc = true;
                 }
                 JCTree def = typeDeclaration(mods, docComment);
@@ -2288,7 +2289,7 @@ public class JavacParser implements Parser {
         }
         JCTree.JCCompilationUnit toplevel = F.at(firstToken.pos).TopLevel(packageAnnotations, pid, defs.toList());
         if (!consumedToplevelDoc)
-            attach(toplevel, firstToken.docComment);
+            attach(toplevel, firstToken.comment(CommentStyle.JAVADOC));
         if (defs.elems.isEmpty())
             storeEnd(toplevel, S.prevToken().endPos);
         if (keepDocComments)
@@ -2498,9 +2499,9 @@ public class JavacParser implements Parser {
     /** EnumeratorDeclaration = AnnotationsOpt [TypeArguments] IDENTIFIER [ Arguments ] [ "{" ClassBody "}" ]
      */
     JCTree enumeratorDeclaration(Name enumName) {
-        String dc = token.docComment;
+        String dc = token.comment(CommentStyle.JAVADOC);
         int flags = Flags.PUBLIC|Flags.STATIC|Flags.FINAL|Flags.ENUM;
-        if (token.deprecatedFlag) {
+        if (token.deprecatedFlag()) {
             flags |= Flags.DEPRECATED;
         }
         int pos = token.pos;
@@ -2587,7 +2588,7 @@ public class JavacParser implements Parser {
             nextToken();
             return List.<JCTree>nil();
         } else {
-            String dc = token.docComment;
+            String dc = token.comment(CommentStyle.JAVADOC);
             int pos = token.pos;
             JCModifiers mods = modifiersOpt();
             if (token.kind == CLASS ||

langtools/src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java

@@ -25,8 +25,8 @@
 
 package com.sun.tools.javac.parser;
 
-import com.sun.tools.javac.file.JavacFileManager;
+import com.sun.tools.javac.parser.Tokens.Comment;
-import com.sun.tools.javac.parser.Tokens.Token;
+import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
 import com.sun.tools.javac.util.*;
 
 import java.nio.*;
@@ -59,352 +59,295 @@ public class JavadocTokenizer extends JavaTokenizer {
         super(fac, input, inputLength);
     }
 
-    /** The comment input buffer, index of next chacter to be read,
+    @Override
-     *  index of one past last character in buffer.
+    protected Comment processComment(int pos, int endPos, CommentStyle style) {
-     */
+        char[] buf = reader.getRawCharacters(pos, endPos);
-    private char[] buf;
+        return new JavadocComment(new ColReader(fac, buf, buf.length), style);
-    private int bp;
-    private int buflen;
-
-    /** The current character.
-     */
-    private char ch;
-
-    /** The column number position of the current character.
-     */
-    private int col;
-
-    /** The buffer index of the last converted Unicode character
-     */
-    private int unicodeConversionBp = 0;
-
-    /**
-     * Buffer for doc comment.
-     */
-    private char[] docCommentBuffer = new char[1024];
-
-    /**
-     * Number of characters in doc comment buffer.
-     */
-    private int docCommentCount;
-
-    /**
-     * Translated and stripped contents of doc comment
-     */
-    private String docComment = null;
-
-
-    /** Unconditionally expand the comment buffer.
-     */
-    private void expandCommentBuffer() {
-        char[] newBuffer = new char[docCommentBuffer.length * 2];
-        System.arraycopy(docCommentBuffer, 0, newBuffer,
-                         0, docCommentBuffer.length);
-        docCommentBuffer = newBuffer;
     }
 
-    /** Convert an ASCII digit from its base (8, 10, or 16)
+    /**
-     *  to its value.
+     * This is a specialized version of UnicodeReader that keeps track of the
+     * column position within a given character stream (used for Javadoc processing).
      */
-    private int digit(int base) {
+    static class ColReader extends UnicodeReader {
-        char c = ch;
+
-        int result = Character.digit(c, base);
+         int col;
-        if (result >= 0 && c > 0x7f) {
+
-            ch = "0123456789abcdef".charAt(result);
+         ColReader(ScannerFactory fac, char[] input, int inputLength) {
+             super(fac, input, inputLength);
+         }
+
+         @Override
+         protected void convertUnicode() {
+             if (ch == '\\' && unicodeConversionBp != bp) {
+                 bp++; ch = buf[bp]; col++;
+                 if (ch == 'u') {
+                     do {
+                         bp++; ch = buf[bp]; col++;
+                     } while (ch == 'u');
+                     int limit = bp + 3;
+                     if (limit < buflen) {
+                         int d = digit(bp, 16);
+                         int code = d;
+                         while (bp < limit && d >= 0) {
+                             bp++; ch = buf[bp]; col++;
+                             d = digit(bp, 16);
+                             code = (code << 4) + d;
+                         }
+                         if (d >= 0) {
+                             ch = (char)code;
+                             unicodeConversionBp = bp;
+                             return;
+                         }
+                     }
+                     // "illegal.Unicode.esc", reported by base scanner
+                 } else {
+                     bp--;
+                     ch = '\\';
+                     col--;
+                 }
+             }
+         }
+
+         @Override
+         protected void scanCommentChar() {
+             scanChar();
+             if (ch == '\\') {
+                 if (peekChar() == '\\' && !isUnicode()) {
+                     putChar(ch, false);
+                     bp++; col++;
+                 } else {
+                     convertUnicode();
+                 }
+             }
+         }
+
+         @Override
+         protected void scanChar() {
+             bp++;
+             ch = buf[bp];
+             switch (ch) {
+             case '\r': // return
+                 col = 0;
+                 break;
+             case '\n': // newline
+                 if (bp == 0 || buf[bp-1] != '\r') {
+                     col = 0;
+                 }
+                 break;
+             case '\t': // tab
+                 col = (col / TabInc * TabInc) + TabInc;
+                 break;
+             case '\\': // possible Unicode
+                 col++;
+                 convertUnicode();
+                 break;
+             default:
+                 col++;
+                 break;
+             }
+         }
+     }
+
+     protected class JavadocComment extends JavaTokenizer.BasicComment<ColReader> {
+
+        /**
+        * Translated and stripped contents of doc comment
+        */
+        private String docComment = null;
+
+        JavadocComment(ColReader comment_reader, CommentStyle cs) {
+            super(comment_reader, cs);
         }
-        return result;
-    }
 
-    /** Convert Unicode escape; bp points to initial '\' character
+        public String getText() {
-     *  (Spec 3.3).
+            if (!scanned && cs == CommentStyle.JAVADOC) {
-     */
+                scanDocComment();
-    private void convertUnicode() {
+            }
-        if (ch == '\\' && unicodeConversionBp != bp) {
+            return docComment;
-            bp++; ch = buf[bp]; col++;
+        }
-            if (ch == 'u') {
+
-                do {
+        @Override
-                    bp++; ch = buf[bp]; col++;
+        @SuppressWarnings("fallthrough")
-                } while (ch == 'u');
+        protected void scanDocComment() {
-                int limit = bp + 3;
+             try {
-                if (limit < buflen) {
+                 boolean firstLine = true;
-                    int d = digit(16);
+
-                    int code = d;
+                 // Skip over first slash
-                    while (bp < limit && d >= 0) {
+                 comment_reader.scanCommentChar();
-                        bp++; ch = buf[bp]; col++;
+                 // Skip over first star
-                        d = digit(16);
+                 comment_reader.scanCommentChar();
-                        code = (code << 4) + d;
+
-                    }
+                 // consume any number of stars
-                    if (d >= 0) {
+                 while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
-                        ch = (char)code;
+                     comment_reader.scanCommentChar();
-                        unicodeConversionBp = bp;
+                 }
-                        return;
+                 // is the comment in the form /**/, /***/, /****/, etc. ?
-                    }
+                 if (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '/') {
+                     docComment = "";
+                     return;
+                 }
+
+                 // skip a newline on the first line of the comment.
+                 if (comment_reader.bp < comment_reader.buflen) {
+                     if (comment_reader.ch == LF) {
+                         comment_reader.scanCommentChar();
+                         firstLine = false;
+                     } else if (comment_reader.ch == CR) {
+                         comment_reader.scanCommentChar();
+                         if (comment_reader.ch == LF) {
+                             comment_reader.scanCommentChar();
+                             firstLine = false;
+                         }
+                     }
+                 }
+
+             outerLoop:
+
+                 // The outerLoop processes the doc comment, looping once
+                 // for each line.  For each line, it first strips off
+                 // whitespace, then it consumes any stars, then it
+                 // puts the rest of the line into our buffer.
+                 while (comment_reader.bp < comment_reader.buflen) {
+
+                     // The wsLoop consumes whitespace from the beginning
+                     // of each line.
+                 wsLoop:
+
+                     while (comment_reader.bp < comment_reader.buflen) {
+                         switch(comment_reader.ch) {
+                         case ' ':
+                             comment_reader.scanCommentChar();
+                             break;
+                         case '\t':
+                             comment_reader.col = ((comment_reader.col - 1) / TabInc * TabInc) + TabInc;
+                             comment_reader.scanCommentChar();
+                             break;
+                         case FF:
+                             comment_reader.col = 0;
+                             comment_reader.scanCommentChar();
+                             break;
+         // Treat newline at beginning of line (blank line, no star)
+         // as comment text.  Old Javadoc compatibility requires this.
+         /*---------------------------------*
+                         case CR: // (Spec 3.4)
+                             doc_reader.scanCommentChar();
+                             if (ch == LF) {
+                                 col = 0;
+                                 doc_reader.scanCommentChar();
+                             }
+                             break;
+                         case LF: // (Spec 3.4)
+                             doc_reader.scanCommentChar();
+                             break;
+         *---------------------------------*/
+                         default:
+                             // we've seen something that isn't whitespace;
+                             // jump out.
+                             break wsLoop;
+                         }
+                     }
+
+                     // Are there stars here?  If so, consume them all
+                     // and check for the end of comment.
+                     if (comment_reader.ch == '*') {
+                         // skip all of the stars
+                         do {
+                             comment_reader.scanCommentChar();
+                         } while (comment_reader.ch == '*');
+
+                         // check for the closing slash.
+                         if (comment_reader.ch == '/') {
+                             // We're done with the doc comment
+                             // scanChar() and breakout.
+                             break outerLoop;
+                         }
+                     } else if (! firstLine) {
+                         //The current line does not begin with a '*' so we will indent it.
+                         for (int i = 1; i < comment_reader.col; i++) {
+                             comment_reader.putChar(' ', false);
+                         }
+                     }
+                     // The textLoop processes the rest of the characters
+                     // on the line, adding them to our buffer.
+                 textLoop:
+                     while (comment_reader.bp < comment_reader.buflen) {
+                         switch (comment_reader.ch) {
+                         case '*':
+                             // Is this just a star?  Or is this the
+                             // end of a comment?
+                             comment_reader.scanCommentChar();
+                             if (comment_reader.ch == '/') {
+                                 // This is the end of the comment,
+                                 // set ch and return our buffer.
+                                 break outerLoop;
+                             }
+                             // This is just an ordinary star.  Add it to
+                             // the buffer.
+                             comment_reader.putChar('*', false);
+                             break;
+                         case ' ':
+                         case '\t':
+                             comment_reader.putChar(comment_reader.ch, false);
+                             comment_reader.scanCommentChar();
+                             break;
+                         case FF:
+                             comment_reader.scanCommentChar();
+                             break textLoop; // treat as end of line
+                         case CR: // (Spec 3.4)
+                             comment_reader.scanCommentChar();
+                             if (comment_reader.ch != LF) {
+                                 // Canonicalize CR-only line terminator to LF
+                                 comment_reader.putChar((char)LF, false);
+                                 break textLoop;
+                             }
+                             /* fall through to LF case */
+                         case LF: // (Spec 3.4)
+                             // We've seen a newline.  Add it to our
+                             // buffer and break out of this loop,
+                             // starting fresh on a new line.
+                             comment_reader.putChar(comment_reader.ch, false);
+                             comment_reader.scanCommentChar();
+                             break textLoop;
+                         default:
+                             // Add the character to our buffer.
+                             comment_reader.putChar(comment_reader.ch, false);
+                             comment_reader.scanCommentChar();
+                         }
+                     } // end textLoop
+                     firstLine = false;
+                 } // end outerLoop
+
+                 if (comment_reader.sp > 0) {
+                     int i = comment_reader.sp - 1;
+                 trailLoop:
+                     while (i > -1) {
+                         switch (comment_reader.sbuf[i]) {
+                         case '*':
+                             i--;
+                             break;
+                         default:
+                             break trailLoop;
+                         }
+                     }
+                     comment_reader.sp = i + 1;
+
+                     // Store the text of the doc comment
+                    docComment = comment_reader.chars();
+                } else {
+                    docComment = "";
+                }
+            } finally {
+                scanned = true;
+                if (docComment != null &&
+                        docComment.matches("(?sm).*^\\s*@deprecated( |$).*")) {
+                    deprecatedFlag = true;
                 }
-                // "illegal.Unicode.esc", reported by base scanner
-            } else {
-                bp--;
-                ch = '\\';
-                col--;
             }
         }
     }
 
-
-    /** Read next character.
-     */
-    private void scanChar() {
-        bp++;
-        ch = buf[bp];
-        switch (ch) {
-        case '\r': // return
-            col = 0;
-            break;
-        case '\n': // newline
-            if (bp == 0 || buf[bp-1] != '\r') {
-                col = 0;
-            }
-            break;
-        case '\t': // tab
-            col = (col / TabInc * TabInc) + TabInc;
-            break;
-        case '\\': // possible Unicode
-            col++;
-            convertUnicode();
-            break;
-        default:
-            col++;
-            break;
-        }
-    }
-
     @Override
-    public Token readToken() {
-        docComment = null;
-        Token tk = super.readToken();
-        tk.docComment = docComment;
-        return tk;
-    }
-
-    /**
-     * Read next character in doc comment, skipping over double '\' characters.
-     * If a double '\' is skipped, put in the buffer and update buffer count.
-     */
-    private void scanDocCommentChar() {
-        scanChar();
-        if (ch == '\\') {
-            if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
-                if (docCommentCount == docCommentBuffer.length)
-                    expandCommentBuffer();
-                docCommentBuffer[docCommentCount++] = ch;
-                bp++; col++;
-            } else {
-                convertUnicode();
-            }
-        }
-    }
-
-    /**
-     * Process a doc comment and make the string content available.
-     * Strips leading whitespace and stars.
-     */
-    @SuppressWarnings("fallthrough")
-    protected void processComment(int pos, int endPos, CommentStyle style) {
-        if (style != CommentStyle.JAVADOC) {
-            return;
-        }
-
-        buf = reader.getRawCharacters(pos, endPos);
-        buflen = buf.length;
-        bp = 0;
-        col = 0;
-
-        docCommentCount = 0;
-
-        boolean firstLine = true;
-
-        // Skip over first slash
-        scanDocCommentChar();
-        // Skip over first star
-        scanDocCommentChar();
-
-        // consume any number of stars
-        while (bp < buflen && ch == '*') {
-            scanDocCommentChar();
-        }
-        // is the comment in the form /**/, /***/, /****/, etc. ?
-        if (bp < buflen && ch == '/') {
-            docComment = "";
-            return;
-        }
-
-        // skip a newline on the first line of the comment.
-        if (bp < buflen) {
-            if (ch == LF) {
-                scanDocCommentChar();
-                firstLine = false;
-            } else if (ch == CR) {
-                scanDocCommentChar();
-                if (ch == LF) {
-                    scanDocCommentChar();
-                    firstLine = false;
-                }
-            }
-        }
-
-    outerLoop:
-
-        // The outerLoop processes the doc comment, looping once
-        // for each line.  For each line, it first strips off
-        // whitespace, then it consumes any stars, then it
-        // puts the rest of the line into our buffer.
-        while (bp < buflen) {
-
-            // The wsLoop consumes whitespace from the beginning
-            // of each line.
-        wsLoop:
-
-            while (bp < buflen) {
-                switch(ch) {
-                case ' ':
-                    scanDocCommentChar();
-                    break;
-                case '\t':
-                    col = ((col - 1) / TabInc * TabInc) + TabInc;
-                    scanDocCommentChar();
-                    break;
-                case FF:
-                    col = 0;
-                    scanDocCommentChar();
-                    break;
-// Treat newline at beginning of line (blank line, no star)
-// as comment text.  Old Javadoc compatibility requires this.
-/*---------------------------------*
-                case CR: // (Spec 3.4)
-                    scanDocCommentChar();
-                    if (ch == LF) {
-                        col = 0;
-                        scanDocCommentChar();
-                    }
-                    break;
-                case LF: // (Spec 3.4)
-                    scanDocCommentChar();
-                    break;
-*---------------------------------*/
-                default:
-                    // we've seen something that isn't whitespace;
-                    // jump out.
-                    break wsLoop;
-                }
-            }
-
-            // Are there stars here?  If so, consume them all
-            // and check for the end of comment.
-            if (ch == '*') {
-                // skip all of the stars
-                do {
-                    scanDocCommentChar();
-                } while (ch == '*');
-
-                // check for the closing slash.
-                if (ch == '/') {
-                    // We're done with the doc comment
-                    // scanChar() and breakout.
-                    break outerLoop;
-                }
-            } else if (! firstLine) {
-                //The current line does not begin with a '*' so we will indent it.
-                for (int i = 1; i < col; i++) {
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = ' ';
-                }
-            }
-
-            // The textLoop processes the rest of the characters
-            // on the line, adding them to our buffer.
-        textLoop:
-            while (bp < buflen) {
-                switch (ch) {
-                case '*':
-                    // Is this just a star?  Or is this the
-                    // end of a comment?
-                    scanDocCommentChar();
-                    if (ch == '/') {
-                        // This is the end of the comment,
-                        // set ch and return our buffer.
-                        break outerLoop;
-                    }
-                    // This is just an ordinary star.  Add it to
-                    // the buffer.
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = '*';
-                    break;
-                case ' ':
-                case '\t':
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = ch;
-                    scanDocCommentChar();
-                    break;
-                case FF:
-                    scanDocCommentChar();
-                    break textLoop; // treat as end of line
-                case CR: // (Spec 3.4)
-                    scanDocCommentChar();
-                    if (ch != LF) {
-                        // Canonicalize CR-only line terminator to LF
-                        if (docCommentCount == docCommentBuffer.length)
-                            expandCommentBuffer();
-                        docCommentBuffer[docCommentCount++] = (char)LF;
-                        break textLoop;
-                    }
-                    /* fall through to LF case */
-                case LF: // (Spec 3.4)
-                    // We've seen a newline.  Add it to our
-                    // buffer and break out of this loop,
-                    // starting fresh on a new line.
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = ch;
-                    scanDocCommentChar();
-                    break textLoop;
-                default:
-                    // Add the character to our buffer.
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = ch;
-                    scanDocCommentChar();
-                }
-            } // end textLoop
-            firstLine = false;
-        } // end outerLoop
-
-        if (docCommentCount > 0) {
-            int i = docCommentCount - 1;
-        trailLoop:
-            while (i > -1) {
-                switch (docCommentBuffer[i]) {
-                case '*':
-                    i--;
-                    break;
-                default:
-                    break trailLoop;
-                }
-            }
-            docCommentCount = i + 1;
-
-            // Store the text of the doc comment
-            docComment = new String(docCommentBuffer, 0 , docCommentCount);
-        } else {
-            docComment = "";
-        }
-    }
-
-    /** Build a map for translating between line numbers and
-     * positions in the input.
-     *
-     * @return a LineMap */
     public Position.LineMap getLineMap() {
         char[] buf = reader.getRawCharacters();
         return Position.makeLineMap(buf, buf.length, true);

langtools/src/share/classes/com/sun/tools/javac/parser/Tokens.java

@@ -30,8 +30,10 @@ import java.util.Locale;
 import com.sun.tools.javac.api.Formattable;
 import com.sun.tools.javac.api.Messages;
 import com.sun.tools.javac.parser.Tokens.Token.Tag;
+import com.sun.tools.javac.util.List;
 import com.sun.tools.javac.util.Name;
 import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.ListBuffer;
 import com.sun.tools.javac.util.Names;
 
 /** A class that defines codes/utilities for Java source tokens
@@ -281,6 +283,19 @@ public class Tokens {
         }
     }
 
+    public interface Comment {
+
+        enum CommentStyle {
+            LINE,
+            BLOCK,
+            JAVADOC,
+        }
+
+        String getText();
+        CommentStyle getStyle();
+        boolean isDeprecated();
+    }
+
     /**
      * This is the class representing a javac token. Each token has several fields
      * that are set by the javac lexer (i.e. start/end position, string value, etc).
@@ -304,18 +319,14 @@ public class Tokens {
         /** The end position of this token */
         public final int endPos;
 
-        /** Is this token preceeded by a deprecated comment? */
+        /** Comment reader associated with this token */
-        public final boolean deprecatedFlag;
+        public final List<Comment> comments;
 
-        /** Is this token preceeded by a deprecated comment? */
+        Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
-        public String docComment;
-
-        Token(TokenKind kind, int pos, int endPos,
-                boolean deprecatedFlag) {
             this.kind = kind;
             this.pos = pos;
             this.endPos = endPos;
-            this.deprecatedFlag = deprecatedFlag;
+            this.comments = comments;
             checkKind();
         }
 
@@ -331,8 +342,8 @@ public class Tokens {
                 throw new AssertionError("Cant split - bad subtokens");
             }
             return new Token[] {
-                new Token(t1, pos, pos + t1.name.length(), deprecatedFlag),
+                new Token(t1, pos, pos + t1.name.length(), comments),
-                new Token(t2, pos + t1.name.length(), endPos, false)
+                new Token(t2, pos + t1.name.length(), endPos, null)
             };
         }
 
@@ -353,14 +364,52 @@ public class Tokens {
         public int radix() {
             throw new UnsupportedOperationException();
         }
+
+        /**
+         * Preserve classic semantics - if multiple javadocs are found on the token
+         * the last one is returned
+         */
+        public String comment(Comment.CommentStyle style) {
+            List<Comment> readers = getReaders(Comment.CommentStyle.JAVADOC);
+            return readers.isEmpty() ?
+                    null :
+                    readers.head.getText();
+        }
+
+        /**
+         * Preserve classic semantics - deprecated should be set if at least one
+         * javadoc comment attached to this token contains the '@deprecated' string
+         */
+        public boolean deprecatedFlag() {
+            for (Comment r : getReaders(Comment.CommentStyle.JAVADOC)) {
+                if (r.isDeprecated()) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        private List<Comment> getReaders(Comment.CommentStyle style) {
+            if (comments == null) {
+                return List.nil();
+            } else {
+                ListBuffer<Comment> buf = ListBuffer.lb();
+                for (Comment r : comments) {
+                    if (r.getStyle() == style) {
+                        buf.add(r);
+                    }
+                }
+                return buf.toList();
+            }
+        }
     }
 
     final static class NamedToken extends Token {
         /** The name of this token */
         public final Name name;
 
-        public NamedToken(TokenKind kind, int pos, int endPos, Name name, boolean deprecatedFlag) {
+        public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
-            super(kind, pos, endPos, deprecatedFlag);
+            super(kind, pos, endPos, comments);
             this.name = name;
         }
 
@@ -380,8 +429,8 @@ public class Tokens {
         /** The string value of this token */
         public final String stringVal;
 
-        public StringToken(TokenKind kind, int pos, int endPos, String stringVal, boolean deprecatedFlag) {
+        public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
-            super(kind, pos, endPos, deprecatedFlag);
+            super(kind, pos, endPos, comments);
             this.stringVal = stringVal;
         }
 
@@ -401,8 +450,8 @@ public class Tokens {
         /** The 'radix' value of this token */
         public final int radix;
 
-        public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, boolean deprecatedFlag) {
+        public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
-            super(kind, pos, endPos, stringVal, deprecatedFlag);
+            super(kind, pos, endPos, stringVal, comments);
             this.radix = radix;
         }
 
@@ -419,5 +468,5 @@ public class Tokens {
     }
 
     public static final Token DUMMY =
-                new Token(TokenKind.ERROR, 0, 0, false);
+                new Token(TokenKind.ERROR, 0, 0, null);
 }

langtools/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java

@@ -26,8 +26,12 @@
 package com.sun.tools.javac.parser;
 
 import com.sun.tools.javac.file.JavacFileManager;
-import java.nio.CharBuffer;
 import com.sun.tools.javac.util.Log;
+import com.sun.tools.javac.util.Name;
+import com.sun.tools.javac.util.Names;
+
+import java.nio.CharBuffer;
+
 import static com.sun.tools.javac.util.LayoutCharacters.*;
 
 /** The char reader used by the javac lexer/tokenizer. Returns the sequence of
@@ -58,6 +62,12 @@ public class UnicodeReader {
     protected int unicodeConversionBp = -1;
 
     protected Log log;
+    protected Names names;
+
+    /** A character buffer for saved chars.
+     */
+    protected char[] sbuf = new char[128];
+    protected int sp;
 
     /**
      * Create a scanner from the input array.  This method might
@@ -76,6 +86,7 @@ public class UnicodeReader {
 
     protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) {
         log = sf.log;
+        names = sf.names;
         if (inputLength == input.length) {
             if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
                 inputLength--;
@@ -103,6 +114,48 @@ public class UnicodeReader {
         }
     }
 
+    /** Read next character in comment, skipping over double '\' characters.
+     */
+    protected void scanCommentChar() {
+        scanChar();
+        if (ch == '\\') {
+            if (peekChar() == '\\' && !isUnicode()) {
+                skipChar();
+            } else {
+                convertUnicode();
+            }
+        }
+    }
+
+    /** Append a character to sbuf.
+     */
+    protected void putChar(char ch, boolean scan) {
+        if (sp == sbuf.length) {
+            char[] newsbuf = new char[sbuf.length * 2];
+            System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
+            sbuf = newsbuf;
+        }
+        sbuf[sp++] = ch;
+        if (scan)
+            scanChar();
+    }
+
+    protected void putChar(char ch) {
+        putChar(ch, false);
+    }
+
+    protected void putChar(boolean scan) {
+        putChar(ch, scan);
+    }
+
+    Name name() {
+        return names.fromChars(sbuf, 0, sp);
+    }
+
+    String chars() {
+        return new String(sbuf, 0, sp);
+    }
+
     /** Convert unicode escape; bp points to initial '\' character
      *  (Spec 3.3).
      */

langtools/test/tools/javac/depDocComment/DeprecatedDocComment4.java

@@ -0,0 +1,20 @@
+/**
+ * @test  /nodynamiccopyright/
+ * @bug 7104201
+ * @summary Refactor DocCommentScanner
+ * @compile/fail/ref=DeprecatedDocComment4.out -XDrawDiagnostics -Werror -Xlint:dep-ann DeprecatedDocComment4.java
+ */
+
+class DeprecatedDocComment4 {
+    /** @deprecated **/
+    /* block */
+    void test1() {};
+
+    /** @deprecated **/
+    /** double javadoc */
+    void test2() {};
+
+    /** @deprecated **/
+    //line comment
+    void test3() {};
+}

langtools/test/tools/javac/depDocComment/DeprecatedDocComment4.out

@@ -0,0 +1,6 @@
+DeprecatedDocComment4.java:11:10: compiler.warn.missing.deprecated.annotation
+DeprecatedDocComment4.java:15:10: compiler.warn.missing.deprecated.annotation
+DeprecatedDocComment4.java:19:10: compiler.warn.missing.deprecated.annotation
+- compiler.err.warnings.and.werror
+1 error
+3 warnings