This is NOT part of any supported API.
diff --git a/langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java b/langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java
index 715839bd776..e428f6e0b54 100644
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java
@@ -67,14 +67,14 @@ public class EndPosParser extends JavacParser {
/** {@inheritDoc} */
@Override
protected This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.
+ */
+public class JavaTokenizer {
+
+ private static boolean scannerDebug = false;
+
+ /** Allow hex floating-point literals.
+ */
+ private boolean allowHexFloats;
+
+ /** Allow binary literals.
+ */
+ private boolean allowBinaryLiterals;
+
+ /** Allow underscores in literals.
+ */
+ private boolean allowUnderscoresInLiterals;
+
+ /** The source language setting.
+ */
+ private Source source;
+
+ /** The log to be used for error reporting.
+ */
+ private final Log log;
+
+ /** The name table. */
+ private final Names names;
+
+ /** The token factory. */
+ private final Tokens tokens;
+
+ /** The token kind, set by nextToken().
+ */
+ protected TokenKind tk;
+
+ /** The token's radix, set by nextToken().
+ */
+ protected int radix;
+
+ /** The token's name, set by nextToken().
+ */
+ protected Name name;
+
+ /** The position where a lexical error occurred;
+ */
+ protected int errPos = Position.NOPOS;
+
+ /** Has a @deprecated been encountered in last doc comment?
+ * this needs to be reset by client.
+ */
+ protected boolean deprecatedFlag = false;
+
+ /** A character buffer for saved chars.
+ */
+ protected char[] sbuf = new char[128];
+ protected int sp;
+
+ protected UnicodeReader reader;
+
+ private static final boolean hexFloatsWork = hexFloatsWork();
+ private static boolean hexFloatsWork() {
+ try {
+ Float.valueOf("0x1.0p1");
+ return true;
+ } catch (NumberFormatException ex) {
+ return false;
+ }
+ }
+
+ /**
+ * Create a scanner from the input array. This method might
+ * modify the array. To avoid copying the input array, ensure
+ * that {@code inputLength < input.length} or
+ * {@code input[input.length -1]} is a white space character.
+ *
+ * @param fac the factory which created this Scanner
+ * @param input the input, might be modified
+ * @param inputLength the size of the input.
+ * Must be positive and less than or equal to input.length.
+ */
+ protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) {
+ this(fac, new UnicodeReader(fac, buf));
+ }
+
+ protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {
+ this(fac, new UnicodeReader(fac, buf, inputLength));
+ }
+
+ protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
+ log = fac.log;
+ names = fac.names;
+ tokens = fac.tokens;
+ source = fac.source;
+ this.reader = reader;
+ allowBinaryLiterals = source.allowBinaryLiterals();
+ allowHexFloats = source.allowHexFloats();
+ allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
+ }
+
+ /** Report an error at the given position using the provided arguments.
+ */
+ protected void lexError(int pos, String key, Object... args) {
+ log.error(pos, key, args);
+ tk = TokenKind.ERROR;
+ errPos = pos;
+ }
+
+ /** Read next character in comment, skipping over double '\' characters.
+ */
+ protected void scanCommentChar() {
+ reader.scanChar();
+ if (reader.ch == '\\') {
+ if (reader.peekChar() == '\\' && !reader.isUnicode()) {
+ reader.skipChar();
+ } else {
+ reader.convertUnicode();
+ }
+ }
+ }
+
+ /** Append a character to sbuf.
+ */
+ private void putChar(char ch) {
+ if (sp == sbuf.length) {
+ char[] newsbuf = new char[sbuf.length * 2];
+ System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
+ sbuf = newsbuf;
+ }
+ sbuf[sp++] = ch;
+ }
+
+ /** Read next character in character or string literal and copy into sbuf.
+ */
+ private void scanLitChar(int pos) {
+ if (reader.ch == '\\') {
+ if (reader.peekChar() == '\\' && !reader.isUnicode()) {
+ reader.skipChar();
+ putChar('\\');
+ reader.scanChar();
+ } else {
+ reader.scanChar();
+ switch (reader.ch) {
+ case '0': case '1': case '2': case '3':
+ case '4': case '5': case '6': case '7':
+ char leadch = reader.ch;
+ int oct = reader.digit(pos, 8);
+ reader.scanChar();
+ if ('0' <= reader.ch && reader.ch <= '7') {
+ oct = oct * 8 + reader.digit(pos, 8);
+ reader.scanChar();
+ if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
+ oct = oct * 8 + reader.digit(pos, 8);
+ reader.scanChar();
+ }
+ }
+ putChar((char)oct);
+ break;
+ case 'b':
+ putChar('\b'); reader.scanChar(); break;
+ case 't':
+ putChar('\t'); reader.scanChar(); break;
+ case 'n':
+ putChar('\n'); reader.scanChar(); break;
+ case 'f':
+ putChar('\f'); reader.scanChar(); break;
+ case 'r':
+ putChar('\r'); reader.scanChar(); break;
+ case '\'':
+ putChar('\''); reader.scanChar(); break;
+ case '\"':
+ putChar('\"'); reader.scanChar(); break;
+ case '\\':
+ putChar('\\'); reader.scanChar(); break;
+ default:
+ lexError(reader.bp, "illegal.esc.char");
+ }
+ }
+ } else if (reader.bp != reader.buflen) {
+ putChar(reader.ch); reader.scanChar();
+ }
+ }
+
+ private void scanDigits(int pos, int digitRadix) {
+ char saveCh;
+ int savePos;
+ do {
+ if (reader.ch != '_') {
+ putChar(reader.ch);
+ } else {
+ if (!allowUnderscoresInLiterals) {
+ lexError(pos, "unsupported.underscore.lit", source.name);
+ allowUnderscoresInLiterals = true;
+ }
+ }
+ saveCh = reader.ch;
+ savePos = reader.bp;
+ reader.scanChar();
+ } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_');
+ if (saveCh == '_')
+ lexError(savePos, "illegal.underscore");
+ }
+
+ /** Read fractional part of hexadecimal floating point number.
+ */
+ private void scanHexExponentAndSuffix(int pos) {
+ if (reader.ch == 'p' || reader.ch == 'P') {
+ putChar(reader.ch);
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.ch == '+' || reader.ch == '-') {
+ putChar(reader.ch);
+ reader.scanChar();
+ }
+ skipIllegalUnderscores();
+ if ('0' <= reader.ch && reader.ch <= '9') {
+ scanDigits(pos, 10);
+ if (!allowHexFloats) {
+ lexError(pos, "unsupported.fp.lit", source.name);
+ allowHexFloats = true;
+ }
+ else if (!hexFloatsWork)
+ lexError(pos, "unsupported.cross.fp.lit");
+ } else
+ lexError(pos, "malformed.fp.lit");
+ } else {
+ lexError(pos, "malformed.fp.lit");
+ }
+ if (reader.ch == 'f' || reader.ch == 'F') {
+ putChar(reader.ch);
+ reader.scanChar();
+ tk = TokenKind.FLOATLITERAL;
+ radix = 16;
+ } else {
+ if (reader.ch == 'd' || reader.ch == 'D') {
+ putChar(reader.ch);
+ reader.scanChar();
+ }
+ tk = TokenKind.DOUBLELITERAL;
+ radix = 16;
+ }
+ }
+
+ /** Read fractional part of floating point number.
+ */
+ private void scanFraction(int pos) {
+ skipIllegalUnderscores();
+ if ('0' <= reader.ch && reader.ch <= '9') {
+ scanDigits(pos, 10);
+ }
+ int sp1 = sp;
+ if (reader.ch == 'e' || reader.ch == 'E') {
+ putChar(reader.ch);
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.ch == '+' || reader.ch == '-') {
+ putChar(reader.ch);
+ reader.scanChar();
+ }
+ skipIllegalUnderscores();
+ if ('0' <= reader.ch && reader.ch <= '9') {
+ scanDigits(pos, 10);
+ return;
+ }
+ lexError(pos, "malformed.fp.lit");
+ sp = sp1;
+ }
+ }
+
+ /** Read fractional part and 'd' or 'f' suffix of floating point number.
+ */
+ private void scanFractionAndSuffix(int pos) {
+ radix = 10;
+ scanFraction(pos);
+ if (reader.ch == 'f' || reader.ch == 'F') {
+ putChar(reader.ch);
+ reader.scanChar();
+ tk = TokenKind.FLOATLITERAL;
+ } else {
+ if (reader.ch == 'd' || reader.ch == 'D') {
+ putChar(reader.ch);
+ reader.scanChar();
+ }
+ tk = TokenKind.DOUBLELITERAL;
+ }
+ }
+
+ /** Read fractional part and 'd' or 'f' suffix of floating point number.
+ */
+ private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
+ radix = 16;
+ Assert.check(reader.ch == '.');
+ putChar(reader.ch);
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.digit(pos, 16) >= 0) {
+ seendigit = true;
+ scanDigits(pos, 16);
+ }
+ if (!seendigit)
+ lexError(pos, "invalid.hex.number");
+ else
+ scanHexExponentAndSuffix(pos);
+ }
+
+ private void skipIllegalUnderscores() {
+ if (reader.ch == '_') {
+ lexError(reader.bp, "illegal.underscore");
+ while (reader.ch == '_')
+ reader.scanChar();
+ }
+ }
+
+ /** Read a number.
+ * @param radix The radix of the number; one of 2, j8, 10, 16.
+ */
+ private void scanNumber(int pos, int radix) {
+ // for octal, allow base-10 digit in case it's a float literal
+ this.radix = radix;
+ int digitRadix = (radix == 8 ? 10 : radix);
+ boolean seendigit = false;
+ if (reader.digit(pos, digitRadix) >= 0) {
+ seendigit = true;
+ scanDigits(pos, digitRadix);
+ }
+ if (radix == 16 && reader.ch == '.') {
+ scanHexFractionAndSuffix(pos, seendigit);
+ } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
+ scanHexExponentAndSuffix(pos);
+ } else if (digitRadix == 10 && reader.ch == '.') {
+ putChar(reader.ch);
+ reader.scanChar();
+ scanFractionAndSuffix(pos);
+ } else if (digitRadix == 10 &&
+ (reader.ch == 'e' || reader.ch == 'E' ||
+ reader.ch == 'f' || reader.ch == 'F' ||
+ reader.ch == 'd' || reader.ch == 'D')) {
+ scanFractionAndSuffix(pos);
+ } else {
+ if (reader.ch == 'l' || reader.ch == 'L') {
+ reader.scanChar();
+ tk = TokenKind.LONGLITERAL;
+ } else {
+ tk = TokenKind.INTLITERAL;
+ }
+ }
+ }
+
+ /** Read an identifier.
+ */
+ private void scanIdent() {
+ boolean isJavaIdentifierPart;
+ char high;
+ do {
+ if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch;
+ // optimization, was: putChar(reader.ch);
+
+ reader.scanChar();
+ switch (reader.ch) {
+ case 'A': case 'B': case 'C': case 'D': case 'E':
+ case 'F': case 'G': case 'H': case 'I': case 'J':
+ case 'K': case 'L': case 'M': case 'N': case 'O':
+ case 'P': case 'Q': case 'R': case 'S': case 'T':
+ case 'U': case 'V': case 'W': case 'X': case 'Y':
+ case 'Z':
+ case 'a': case 'b': case 'c': case 'd': case 'e':
+ case 'f': case 'g': case 'h': case 'i': case 'j':
+ case 'k': case 'l': case 'm': case 'n': case 'o':
+ case 'p': case 'q': case 'r': case 's': case 't':
+ case 'u': case 'v': case 'w': case 'x': case 'y':
+ case 'z':
+ case '$': case '_':
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ case '\u0000': case '\u0001': case '\u0002': case '\u0003':
+ case '\u0004': case '\u0005': case '\u0006': case '\u0007':
+ case '\u0008': case '\u000E': case '\u000F': case '\u0010':
+ case '\u0011': case '\u0012': case '\u0013': case '\u0014':
+ case '\u0015': case '\u0016': case '\u0017':
+ case '\u0018': case '\u0019': case '\u001B':
+ case '\u007F':
+ break;
+ case '\u001A': // EOI is also a legal identifier part
+ if (reader.bp >= reader.buflen) {
+ name = names.fromChars(sbuf, 0, sp);
+ tk = tokens.lookupKind(name);
+ return;
+ }
+ break;
+ default:
+ if (reader.ch < '\u0080') {
+ // all ASCII range chars already handled, above
+ isJavaIdentifierPart = false;
+ } else {
+ high = reader.scanSurrogates();
+ if (high != 0) {
+ if (sp == sbuf.length) {
+ putChar(high);
+ } else {
+ sbuf[sp++] = high;
+ }
+ isJavaIdentifierPart = Character.isJavaIdentifierPart(
+ Character.toCodePoint(high, reader.ch));
+ } else {
+ isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+ }
+ }
+ if (!isJavaIdentifierPart) {
+ name = names.fromChars(sbuf, 0, sp);
+ tk = tokens.lookupKind(name);
+ return;
+ }
+ }
+ } while (true);
+ }
+
+ /** Return true if reader.ch can be part of an operator.
+ */
+ private boolean isSpecial(char ch) {
+ switch (ch) {
+ case '!': case '%': case '&': case '*': case '?':
+ case '+': case '-': case ':': case '<': case '=':
+ case '>': case '^': case '|': case '~':
+ case '@':
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /** Read longest possible sequence of special characters and convert
+ * to token.
+ */
+ private void scanOperator() {
+ while (true) {
+ putChar(reader.ch);
+ Name newname = names.fromChars(sbuf, 0, sp);
+ TokenKind tk1 = tokens.lookupKind(newname);
+ if (tk1 == TokenKind.IDENTIFIER) {
+ sp--;
+ break;
+ }
+ tk = tk1;
+ reader.scanChar();
+ if (!isSpecial(reader.ch)) break;
+ }
+ }
+
+ /**
+ * Scan a documentation comment; determine if a deprecated tag is present.
+ * Called once the initial /, * have been skipped, positioned at the second *
+ * (which is treated as the beginning of the first line).
+ * Stops positioned at the closing '/'.
+ */
+ @SuppressWarnings("fallthrough")
+ private void scanDocComment() {
+ boolean deprecatedPrefix = false;
+
+ forEachLine:
+ while (reader.bp < reader.buflen) {
+
+ // Skip optional WhiteSpace at beginning of line
+ while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
+ scanCommentChar();
+ }
+
+ // Skip optional consecutive Stars
+ while (reader.bp < reader.buflen && reader.ch == '*') {
+ scanCommentChar();
+ if (reader.ch == '/') {
+ return;
+ }
+ }
+
+ // Skip optional WhiteSpace after Stars
+ while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
+ scanCommentChar();
+ }
+
+ deprecatedPrefix = false;
+ // At beginning of line in the JavaDoc sense.
+ if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'd') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'e') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'p') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'r') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'e') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'c') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'a') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 't') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'e') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'd') {
+ deprecatedPrefix = true;
+ scanCommentChar();
+ }}}}}}}}}}}
+ if (deprecatedPrefix && reader.bp < reader.buflen) {
+ if (Character.isWhitespace(reader.ch)) {
+ deprecatedFlag = true;
+ } else if (reader.ch == '*') {
+ scanCommentChar();
+ if (reader.ch == '/') {
+ deprecatedFlag = true;
+ return;
+ }
+ }
+ }
+
+ // Skip rest of line
+ while (reader.bp < reader.buflen) {
+ switch (reader.ch) {
+ case '*':
+ scanCommentChar();
+ if (reader.ch == '/') {
+ return;
+ }
+ break;
+ case CR: // (Spec 3.4)
+ scanCommentChar();
+ if (reader.ch != LF) {
+ continue forEachLine;
+ }
+ /* fall through to LF case */
+ case LF: // (Spec 3.4)
+ scanCommentChar();
+ continue forEachLine;
+ default:
+ scanCommentChar();
+ }
+ } // rest of line
+ } // forEachLine
+ return;
+ }
+
+ /** Read token.
+ */
+ public Token readToken() {
+
+ sp = 0;
+ name = null;
+ deprecatedFlag = false;
+ radix = 0;
+ int pos = 0;
+ int endPos = 0;
+
+ try {
+ loop: while (true) {
+ pos = reader.bp;
+ switch (reader.ch) {
+ case ' ': // (Spec 3.6)
+ case '\t': // (Spec 3.6)
+ case FF: // (Spec 3.6)
+ do {
+ reader.scanChar();
+ } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF);
+ processWhiteSpace(pos, reader.bp);
+ break;
+ case LF: // (Spec 3.4)
+ reader.scanChar();
+ processLineTerminator(pos, reader.bp);
+ break;
+ case CR: // (Spec 3.4)
+ reader.scanChar();
+ if (reader.ch == LF) {
+ reader.scanChar();
+ }
+ processLineTerminator(pos, reader.bp);
+ break;
+ case 'A': case 'B': case 'C': case 'D': case 'E':
+ case 'F': case 'G': case 'H': case 'I': case 'J':
+ case 'K': case 'L': case 'M': case 'N': case 'O':
+ case 'P': case 'Q': case 'R': case 'S': case 'T':
+ case 'U': case 'V': case 'W': case 'X': case 'Y':
+ case 'Z':
+ case 'a': case 'b': case 'c': case 'd': case 'e':
+ case 'f': case 'g': case 'h': case 'i': case 'j':
+ case 'k': case 'l': case 'm': case 'n': case 'o':
+ case 'p': case 'q': case 'r': case 's': case 't':
+ case 'u': case 'v': case 'w': case 'x': case 'y':
+ case 'z':
+ case '$': case '_':
+ scanIdent();
+ break loop;
+ case '0':
+ reader.scanChar();
+ if (reader.ch == 'x' || reader.ch == 'X') {
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.ch == '.') {
+ scanHexFractionAndSuffix(pos, false);
+ } else if (reader.digit(pos, 16) < 0) {
+ lexError(pos, "invalid.hex.number");
+ } else {
+ scanNumber(pos, 16);
+ }
+ } else if (reader.ch == 'b' || reader.ch == 'B') {
+ if (!allowBinaryLiterals) {
+ lexError(pos, "unsupported.binary.lit", source.name);
+ allowBinaryLiterals = true;
+ }
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.digit(pos, 2) < 0) {
+ lexError(pos, "invalid.binary.number");
+ } else {
+ scanNumber(pos, 2);
+ }
+ } else {
+ putChar('0');
+ if (reader.ch == '_') {
+ int savePos = reader.bp;
+ do {
+ reader.scanChar();
+ } while (reader.ch == '_');
+ if (reader.digit(pos, 10) < 0) {
+ lexError(savePos, "illegal.underscore");
+ }
+ }
+ scanNumber(pos, 8);
+ }
+ break loop;
+ case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ scanNumber(pos, 10);
+ break loop;
+ case '.':
+ reader.scanChar();
+ if ('0' <= reader.ch && reader.ch <= '9') {
+ putChar('.');
+ scanFractionAndSuffix(pos);
+ } else if (reader.ch == '.') {
+ putChar('.'); putChar('.');
+ reader.scanChar();
+ if (reader.ch == '.') {
+ reader.scanChar();
+ putChar('.');
+ tk = TokenKind.ELLIPSIS;
+ } else {
+ lexError(pos, "malformed.fp.lit");
+ }
+ } else {
+ tk = TokenKind.DOT;
+ }
+ break loop;
+ case ',':
+ reader.scanChar(); tk = TokenKind.COMMA; break loop;
+ case ';':
+ reader.scanChar(); tk = TokenKind.SEMI; break loop;
+ case '(':
+ reader.scanChar(); tk = TokenKind.LPAREN; break loop;
+ case ')':
+ reader.scanChar(); tk = TokenKind.RPAREN; break loop;
+ case '[':
+ reader.scanChar(); tk = TokenKind.LBRACKET; break loop;
+ case ']':
+ reader.scanChar(); tk = TokenKind.RBRACKET; break loop;
+ case '{':
+ reader.scanChar(); tk = TokenKind.LBRACE; break loop;
+ case '}':
+ reader.scanChar(); tk = TokenKind.RBRACE; break loop;
+ case '/':
+ reader.scanChar();
+ if (reader.ch == '/') {
+ do {
+ scanCommentChar();
+ } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
+ if (reader.bp < reader.buflen) {
+ processComment(pos, reader.bp, CommentStyle.LINE);
+ }
+ break;
+ } else if (reader.ch == '*') {
+ reader.scanChar();
+ CommentStyle style;
+ if (reader.ch == '*') {
+ style = CommentStyle.JAVADOC;
+ scanDocComment();
+ } else {
+ style = CommentStyle.BLOCK;
+ while (reader.bp < reader.buflen) {
+ if (reader.ch == '*') {
+ reader.scanChar();
+ if (reader.ch == '/') break;
+ } else {
+ scanCommentChar();
+ }
+ }
+ }
+ if (reader.ch == '/') {
+ reader.scanChar();
+ processComment(pos, reader.bp, style);
+ break;
+ } else {
+ lexError(pos, "unclosed.comment");
+ break loop;
+ }
+ } else if (reader.ch == '=') {
+ tk = TokenKind.SLASHEQ;
+ reader.scanChar();
+ } else {
+ tk = TokenKind.SLASH;
+ }
+ break loop;
+ case '\'':
+ reader.scanChar();
+ if (reader.ch == '\'') {
+ lexError(pos, "empty.char.lit");
+ } else {
+ if (reader.ch == CR || reader.ch == LF)
+ lexError(pos, "illegal.line.end.in.char.lit");
+ scanLitChar(pos);
+ char ch2 = reader.ch;
+ if (reader.ch == '\'') {
+ reader.scanChar();
+ tk = TokenKind.CHARLITERAL;
+ } else {
+ lexError(pos, "unclosed.char.lit");
+ }
+ }
+ break loop;
+ case '\"':
+ reader.scanChar();
+ while (reader.ch != '\"' && reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen)
+ scanLitChar(pos);
+ if (reader.ch == '\"') {
+ tk = TokenKind.STRINGLITERAL;
+ reader.scanChar();
+ } else {
+ lexError(pos, "unclosed.str.lit");
+ }
+ break loop;
+ default:
+ if (isSpecial(reader.ch)) {
+ scanOperator();
+ } else {
+ boolean isJavaIdentifierStart;
+ if (reader.ch < '\u0080') {
+ // all ASCII range chars already handled, above
+ isJavaIdentifierStart = false;
+ } else {
+ char high = reader.scanSurrogates();
+ if (high != 0) {
+ if (sp == sbuf.length) {
+ putChar(high);
+ } else {
+ sbuf[sp++] = high;
+ }
+
+ isJavaIdentifierStart = Character.isJavaIdentifierStart(
+ Character.toCodePoint(high, reader.ch));
+ } else {
+ isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
+ }
+ }
+ if (isJavaIdentifierStart) {
+ scanIdent();
+ } else if (reader.bp == reader.buflen || reader.ch == EOI && reader.bp + 1 == reader.buflen) { // JLS 3.5
+ tk = TokenKind.EOF;
+ pos = reader.buflen;
+ } else {
+ lexError(pos, "illegal.char", String.valueOf((int)reader.ch));
+ reader.scanChar();
+ }
+ }
+ break loop;
+ }
+ }
+ endPos = reader.bp;
+ switch (tk.tag) {
+ case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag);
+ case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag);
+ case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag);
+ case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag);
+ default: throw new AssertionError();
+ }
+ }
+ finally {
+ if (scannerDebug) {
+ System.out.println("nextToken(" + pos
+ + "," + endPos + ")=|" +
+ new String(reader.getRawCharacters(pos, endPos))
+ + "|");
+ }
+ }
+ }
+
+ /** Return the position where a lexical error occurred;
+ */
+ public int errPos() {
+ return errPos;
+ }
+
+ /** Set the position where a lexical error occurred;
+ */
+ public void errPos(int pos) {
+ errPos = pos;
+ }
+
+ public enum CommentStyle {
+ LINE,
+ BLOCK,
+ JAVADOC,
+ }
+
+ /**
+ * Called when a complete comment has been scanned. pos and endPos
+ * will mark the comment boundary.
+ */
+ protected void processComment(int pos, int endPos, CommentStyle style) {
+ if (scannerDebug)
+ System.out.println("processComment(" + pos
+ + "," + endPos + "," + style + ")=|"
+ + new String(reader.getRawCharacters(pos, endPos))
+ + "|");
+ }
+
+ /**
+ * Called when a complete whitespace run has been scanned. pos and endPos
+ * will mark the whitespace boundary.
+ */
+ protected void processWhiteSpace(int pos, int endPos) {
+ if (scannerDebug)
+ System.out.println("processWhitespace(" + pos
+ + "," + endPos + ")=|" +
+ new String(reader.getRawCharacters(pos, endPos))
+ + "|");
+ }
+
+ /**
+ * Called when a line terminator has been processed.
+ */
+ protected void processLineTerminator(int pos, int endPos) {
+ if (scannerDebug)
+ System.out.println("processTerminator(" + pos
+ + "," + endPos + ")=|" +
+ new String(reader.getRawCharacters(pos, endPos))
+ + "|");
+ }
+
+ /** Build a map for translating between line numbers and
+ * positions in the input.
+ *
+ * @return a LineMap */
+ public Position.LineMap getLineMap() {
+ return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
+ }
+}
diff --git a/langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java b/langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java
index 2151a67cdd5..9c754978a56 100644
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java
@@ -28,6 +28,7 @@ package com.sun.tools.javac.parser;
import java.util.*;
import com.sun.tools.javac.code.*;
+import com.sun.tools.javac.parser.Tokens.*;
import com.sun.tools.javac.tree.*;
import com.sun.tools.javac.tree.JCTree.*;
import com.sun.tools.javac.util.*;
@@ -36,7 +37,7 @@ import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition;
import com.sun.tools.javac.util.List;
import static com.sun.tools.javac.util.ListBuffer.lb;
-import static com.sun.tools.javac.parser.Token.*;
+import static com.sun.tools.javac.parser.Tokens.TokenKind.*;
/** The parser maps a token sequence into an abstract syntax
* tree. It operates by recursive descent, with code derived
@@ -67,9 +68,6 @@ public class JavacParser implements Parser {
*/
private Log log;
- /** The keyword table. */
- private Keywords keywords;
-
/** The Source language setting. */
private Source source;
@@ -83,11 +81,10 @@ public class JavacParser implements Parser {
boolean keepDocComments,
boolean keepLineMap) {
this.S = S;
- S.nextToken(); // prime the pump
+ nextToken(); // prime the pump
this.F = fac.F;
this.log = fac.log;
this.names = fac.names;
- this.keywords = fac.keywords;
this.source = fac.source;
this.allowGenerics = source.allowGenerics();
this.allowVarargs = source.allowVarargs();
@@ -178,7 +175,16 @@ public class JavacParser implements Parser {
*/
private int lastmode = 0;
-/* ---------- error recovery -------------- */
+ /* ---------- token management -------------- */
+
+ protected Token token;
+
+ protected void nextToken() {
+ S.nextToken();
+ token = S.token();
+ }
+
+ /* ---------- error recovery -------------- */
private JCErroneous errorTree;
@@ -186,9 +192,9 @@ public class JavacParser implements Parser {
*/
private void skip(boolean stopAtImport, boolean stopAtMemberDecl, boolean stopAtIdentifier, boolean stopAtStatement) {
while (true) {
- switch (S.token()) {
+ switch (token.kind) {
case SEMI:
- S.nextToken();
+ nextToken();
return;
case PUBLIC:
case FINAL:
@@ -249,15 +255,15 @@ public class JavacParser implements Parser {
return;
break;
}
- S.nextToken();
+ nextToken();
}
}
- private JCErroneous syntaxError(int pos, String key, Token... args) {
+ private JCErroneous syntaxError(int pos, String key, TokenKind... args) {
return syntaxError(pos, List.
This is NOT part of any supported API.
- * If you write code that depends on this, you do so at your own risk.
- * This code and its internal interfaces are subject to change or
- * deletion without notice.
- */
-public class Keywords {
- public static final Context.Key This is NOT part of any supported API.
- * If you write code that depends on this, you do so at your own risk.
- * This code and its internal interfaces are subject to change or
- * deletion without notice.
- */
-public enum Token implements Formattable {
- EOF,
- ERROR,
- IDENTIFIER,
- ABSTRACT("abstract"),
- ASSERT("assert"),
- BOOLEAN("boolean"),
- BREAK("break"),
- BYTE("byte"),
- CASE("case"),
- CATCH("catch"),
- CHAR("char"),
- CLASS("class"),
- CONST("const"),
- CONTINUE("continue"),
- DEFAULT("default"),
- DO("do"),
- DOUBLE("double"),
- ELSE("else"),
- ENUM("enum"),
- EXTENDS("extends"),
- FINAL("final"),
- FINALLY("finally"),
- FLOAT("float"),
- FOR("for"),
- GOTO("goto"),
- IF("if"),
- IMPLEMENTS("implements"),
- IMPORT("import"),
- INSTANCEOF("instanceof"),
- INT("int"),
- INTERFACE("interface"),
- LONG("long"),
- NATIVE("native"),
- NEW("new"),
- PACKAGE("package"),
- PRIVATE("private"),
- PROTECTED("protected"),
- PUBLIC("public"),
- RETURN("return"),
- SHORT("short"),
- STATIC("static"),
- STRICTFP("strictfp"),
- SUPER("super"),
- SWITCH("switch"),
- SYNCHRONIZED("synchronized"),
- THIS("this"),
- THROW("throw"),
- THROWS("throws"),
- TRANSIENT("transient"),
- TRY("try"),
- VOID("void"),
- VOLATILE("volatile"),
- WHILE("while"),
- INTLITERAL,
- LONGLITERAL,
- FLOATLITERAL,
- DOUBLELITERAL,
- CHARLITERAL,
- STRINGLITERAL,
- TRUE("true"),
- FALSE("false"),
- NULL("null"),
- LPAREN("("),
- RPAREN(")"),
- LBRACE("{"),
- RBRACE("}"),
- LBRACKET("["),
- RBRACKET("]"),
- SEMI(";"),
- COMMA(","),
- DOT("."),
- ELLIPSIS("..."),
- EQ("="),
- GT(">"),
- LT("<"),
- BANG("!"),
- TILDE("~"),
- QUES("?"),
- COLON(":"),
- EQEQ("=="),
- LTEQ("<="),
- GTEQ(">="),
- BANGEQ("!="),
- AMPAMP("&&"),
- BARBAR("||"),
- PLUSPLUS("++"),
- SUBSUB("--"),
- PLUS("+"),
- SUB("-"),
- STAR("*"),
- SLASH("/"),
- AMP("&"),
- BAR("|"),
- CARET("^"),
- PERCENT("%"),
- LTLT("<<"),
- GTGT(">>"),
- GTGTGT(">>>"),
- PLUSEQ("+="),
- SUBEQ("-="),
- STAREQ("*="),
- SLASHEQ("/="),
- AMPEQ("&="),
- BAREQ("|="),
- CARETEQ("^="),
- PERCENTEQ("%="),
- LTLTEQ("<<="),
- GTGTEQ(">>="),
- GTGTGTEQ(">>>="),
- MONKEYS_AT("@"),
- CUSTOM;
-
- Token() {
- this(null);
- }
- Token(String name) {
- this.name = name;
- }
-
- public final String name;
-
- public String toString() {
- switch (this) {
- case IDENTIFIER:
- return "token.identifier";
- case CHARLITERAL:
- return "token.character";
- case STRINGLITERAL:
- return "token.string";
- case INTLITERAL:
- return "token.integer";
- case LONGLITERAL:
- return "token.long-integer";
- case FLOATLITERAL:
- return "token.float";
- case DOUBLELITERAL:
- return "token.double";
- case ERROR:
- return "token.bad-symbol";
- case EOF:
- return "token.end-of-input";
- case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
- case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
- return "'" + name + "'";
- default:
- return name;
- }
- }
-
- public String getKind() {
- return "Token";
- }
-
- public String toString(Locale locale, Messages messages) {
- return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
- }
-}
diff --git a/langtools/src/share/classes/com/sun/tools/javac/parser/Tokens.java b/langtools/src/share/classes/com/sun/tools/javac/parser/Tokens.java
new file mode 100644
index 00000000000..934dfb90520
--- /dev/null
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/Tokens.java
@@ -0,0 +1,423 @@
+/*
+ * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.util.Locale;
+
+import com.sun.tools.javac.api.Formattable;
+import com.sun.tools.javac.api.Messages;
+import com.sun.tools.javac.parser.Tokens.Token.Tag;
+import com.sun.tools.javac.util.Name;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Names;
+
+/** A class that defines codes/utilities for Java source tokens
+ * returned from lexical analysis.
+ *
+ * This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.
+ */
+public class Tokens {
+
+ private final Names names;
+
+ /**
+ * Keyword array. Maps name indices to Token.
+ */
+ private final TokenKind[] key;
+
+ /** The number of the last entered keyword.
+ */
+ private int maxKey = 0;
+
+ /** The names of all tokens.
+ */
+ private Name[] tokenName = new Name[TokenKind.values().length];
+
+ public static final Context.Key This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.
+ */
+public class UnicodeReader {
+
+ /** The input buffer, index of next character to be read,
+ * index of one past last character in buffer.
+ */
+ protected char[] buf;
+ protected int bp;
+ protected final int buflen;
+
+ /** The current character.
+ */
+ protected char ch;
+
+ /** The buffer index of the last converted unicode character
+ */
+ protected int unicodeConversionBp = -1;
+
+ protected Log log;
+
+ /**
+ * Create a scanner from the input array. This method might
+ * modify the array. To avoid copying the input array, ensure
+ * that {@code inputLength < input.length} or
+ * {@code input[input.length -1]} is a white space character.
+ *
+ * @param fac the factory which created this Scanner
+ * @param input the input, might be modified
+ * @param inputLength the size of the input.
+ * Must be positive and less than or equal to input.length.
+ */
+ protected UnicodeReader(ScannerFactory sf, CharBuffer buffer) {
+ this(sf, JavacFileManager.toArray(buffer), buffer.limit());
+ }
+
+ protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) {
+ log = sf.log;
+ if (inputLength == input.length) {
+ if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
+ inputLength--;
+ } else {
+ char[] newInput = new char[inputLength + 1];
+ System.arraycopy(input, 0, newInput, 0, input.length);
+ input = newInput;
+ }
+ }
+ buf = input;
+ buflen = inputLength;
+ buf[buflen] = EOI;
+ bp = -1;
+ scanChar();
+ }
+
+ /** Read next character.
+ */
+ protected void scanChar() {
+ if (bp < buflen) {
+ ch = buf[++bp];
+ if (ch == '\\') {
+ convertUnicode();
+ }
+ }
+ }
+
+ /** Convert unicode escape; bp points to initial '\' character
+ * (Spec 3.3).
+ */
+ protected void convertUnicode() {
+ if (ch == '\\' && unicodeConversionBp != bp) {
+ bp++; ch = buf[bp];
+ if (ch == 'u') {
+ do {
+ bp++; ch = buf[bp];
+ } while (ch == 'u');
+ int limit = bp + 3;
+ if (limit < buflen) {
+ int d = digit(bp, 16);
+ int code = d;
+ while (bp < limit && d >= 0) {
+ bp++; ch = buf[bp];
+ d = digit(bp, 16);
+ code = (code << 4) + d;
+ }
+ if (d >= 0) {
+ ch = (char)code;
+ unicodeConversionBp = bp;
+ return;
+ }
+ }
+ log.error(bp, "illegal.unicode.esc");
+ } else {
+ bp--;
+ ch = '\\';
+ }
+ }
+ }
+
+ /** Are surrogates supported?
+ */
+ final static boolean surrogatesSupported = surrogatesSupported();
+ private static boolean surrogatesSupported() {
+ try {
+ Character.isHighSurrogate('a');
+ return true;
+ } catch (NoSuchMethodError ex) {
+ return false;
+ }
+ }
+
+ /** Scan surrogate pairs. If 'ch' is a high surrogate and
+ * the next character is a low surrogate, then put the low
+ * surrogate in 'ch', and return the high surrogate.
+ * otherwise, just return 0.
+ */
+ protected char scanSurrogates() {
+ if (surrogatesSupported && Character.isHighSurrogate(ch)) {
+ char high = ch;
+
+ scanChar();
+
+ if (Character.isLowSurrogate(ch)) {
+ return high;
+ }
+
+ ch = high;
+ }
+
+ return 0;
+ }
+
+ /** Convert an ASCII digit from its base (8, 10, or 16)
+ * to its value.
+ */
+ protected int digit(int pos, int base) {
+ char c = ch;
+ int result = Character.digit(c, base);
+ if (result >= 0 && c > 0x7f) {
+ log.error(pos + 1, "illegal.nonascii.digit");
+ ch = "0123456789abcdef".charAt(result);
+ }
+ return result;
+ }
+
+ protected boolean isUnicode() {
+ return unicodeConversionBp == bp;
+ }
+
+ protected void skipChar() {
+ bp++;
+ }
+
+ protected char peekChar() {
+ return buf[bp + 1];
+ }
+
+ /**
+ * Returns a copy of the input buffer, up to its inputLength.
+ * Unicode escape sequences are not translated.
+ */
+ public char[] getRawCharacters() {
+ char[] chars = new char[buflen];
+ System.arraycopy(buf, 0, chars, 0, buflen);
+ return chars;
+ }
+
+ /**
+ * Returns a copy of a character array subset of the input buffer.
+ * The returned array begins at the beginIndex and
- * extends to the character at index endIndex - 1.
- * Thus the length of the substring is endIndex-beginIndex.
- * This behavior is like
- * String.substring(beginIndex, endIndex).
- * Unicode escape sequences are not translated.
- *
- * @param beginIndex the beginning index, inclusive.
- * @param endIndex the ending index, exclusive.
- * @throws IndexOutOfBounds if either offset is outside of the
- * array bounds
- */
- char[] getRawCharacters(int beginIndex, int endIndex);
-
- /**
- * Return the name of an identifier or token for the current token.
- */
- Name name();
-
- /**
- * Read token.
- */
- void nextToken();
-
- /**
- * Return the current token's position: a 0-based
- * offset from beginning of the raw input stream
- * (before unicode translation)
- */
- int pos();
-
- /**
- * Return the last character position of the previous token.
- */
- int prevEndPos();
-
- /**
- * Return the radix of a numeric literal token.
- */
- int radix();
-
- /**
- * The value of a literal token, recorded as a string.
- * For integers, leading 0x and 'l' suffixes are suppressed.
- */
- String stringVal();
-
- /**
- * Return the current token, set by nextToken().
- */
- Token token();
-
- /**
- * Sets the current token.
- */
- void token(Token token);
}
diff --git a/langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java b/langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java
index 709bd5f1018..ed70d0b27df 100644
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java
@@ -55,7 +55,7 @@ public class ParserFactory {
final TreeMaker F;
final Log log;
- final Keywords keywords;
+ final Tokens tokens;
final Source source;
final Names names;
final Options options;
@@ -67,7 +67,7 @@ public class ParserFactory {
this.F = TreeMaker.instance(context);
this.log = Log.instance(context);
this.names = Names.instance(context);
- this.keywords = Keywords.instance(context);
+ this.tokens = Tokens.instance(context);
this.source = Source.instance(context);
this.options = Options.instance(context);
this.scannerFactory = ScannerFactory.instance(context);
diff --git a/langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java b/langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java
index f2d62db7fe8..41f50cb0110 100644
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java
@@ -27,13 +27,11 @@ package com.sun.tools.javac.parser;
import java.nio.*;
-import com.sun.tools.javac.code.Source;
-import com.sun.tools.javac.file.JavacFileManager;
import com.sun.tools.javac.util.*;
+import com.sun.tools.javac.util.Position.LineMap;
+import com.sun.tools.javac.parser.JavaTokenizer.*;
-
-import static com.sun.tools.javac.parser.Token.*;
-import static com.sun.tools.javac.util.LayoutCharacters.*;
+import static com.sun.tools.javac.parser.Tokens.*;
/** The lexical analyzer maps an input stream consisting of
* ASCII characters and Unicode escapes into a token sequence.
@@ -45,119 +43,17 @@ import static com.sun.tools.javac.util.LayoutCharacters.*;
*/
public class Scanner implements Lexer {
- private static boolean scannerDebug = false;
-
- /* Output variables; set by nextToken():
- */
+ private Tokens tokens;
/** The token, set by nextToken().
*/
private Token token;
- /** Allow hex floating-point literals.
+ /** The previous token, set by nextToken().
*/
- private boolean allowHexFloats;
-
- /** Allow binary literals.
- */
- private boolean allowBinaryLiterals;
-
- /** Allow underscores in literals.
- */
- private boolean allowUnderscoresInLiterals;
-
- /** The source language setting.
- */
- private Source source;
-
- /** The token's position, 0-based offset from beginning of text.
- */
- private int pos;
-
- /** Character position just after the last character of the token.
- */
- private int endPos;
-
- /** The last character position of the previous token.
- */
- private int prevEndPos;
-
- /** The position where a lexical error occurred;
- */
- private int errPos = Position.NOPOS;
-
- /** The name of an identifier or token:
- */
- private Name name;
-
- /** The radix of a numeric literal token.
- */
- private int radix;
-
- /** Has a @deprecated been encountered in last doc comment?
- * this needs to be reset by client.
- */
- protected boolean deprecatedFlag = false;
-
- /** A character buffer for literals.
- */
- private char[] sbuf = new char[128];
- private int sp;
-
- /** The input buffer, index of next chacter to be read,
- * index of one past last character in buffer.
- */
- private char[] buf;
- private int bp;
- private int buflen;
- private int eofPos;
-
- /** The current character.
- */
- private char ch;
-
- /** The buffer index of the last converted unicode character
- */
- private int unicodeConversionBp = -1;
-
- /** The log to be used for error reporting.
- */
- private final Log log;
-
- /** The name table. */
- private final Names names;
-
- /** The keyword table. */
- private final Keywords keywords;
-
- /** Common code for constructors. */
- private Scanner(ScannerFactory fac) {
- log = fac.log;
- names = fac.names;
- keywords = fac.keywords;
- source = fac.source;
- allowBinaryLiterals = source.allowBinaryLiterals();
- allowHexFloats = source.allowHexFloats();
- allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
- }
-
- private static final boolean hexFloatsWork = hexFloatsWork();
- private static boolean hexFloatsWork() {
- try {
- Float.valueOf("0x1.0p1");
- return true;
- } catch (NumberFormatException ex) {
- return false;
- }
- }
-
- /** Create a scanner from the input buffer. buffer must implement
- * array() and compact(), and remaining() must be less than limit().
- */
- protected Scanner(ScannerFactory fac, CharBuffer buffer) {
- this(fac, JavacFileManager.toArray(buffer), buffer.limit());
- }
+ private Token prevToken;
+ private JavaTokenizer tokenizer;
/**
* Create a scanner from the input array. This method might
* modify the array. To avoid copying the input array, ensure
@@ -169,972 +65,49 @@ public class Scanner implements Lexer {
* @param inputLength the size of the input.
* Must be positive and less than or equal to input.length.
*/
- protected Scanner(ScannerFactory fac, char[] input, int inputLength) {
- this(fac);
- eofPos = inputLength;
- if (inputLength == input.length) {
- if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
- inputLength--;
- } else {
- char[] newInput = new char[inputLength + 1];
- System.arraycopy(input, 0, newInput, 0, input.length);
- input = newInput;
- }
- }
- buf = input;
- buflen = inputLength;
- buf[buflen] = EOI;
- bp = -1;
- scanChar();
+ protected Scanner(ScannerFactory fac, CharBuffer buf) {
+ this(fac, new JavaTokenizer(fac, buf));
}
- /** Report an error at the given position using the provided arguments.
- */
- private void lexError(int pos, String key, Object... args) {
- log.error(pos, key, args);
- token = ERROR;
- errPos = pos;
+ protected Scanner(ScannerFactory fac, char[] buf, int inputLength) {
+ this(fac, new JavaTokenizer(fac, buf, inputLength));
}
- /** Report an error at the current token position using the provided
- * arguments.
- */
- private void lexError(String key, Object... args) {
- lexError(pos, key, args);
+ protected Scanner(ScannerFactory fac, JavaTokenizer tokenizer) {
+ this.tokenizer = tokenizer;
+ tokens = fac.tokens;
+ token = prevToken = DUMMY;
}
- /** Convert an ASCII digit from its base (8, 10, or 16)
- * to its value.
- */
- private int digit(int base) {
- char c = ch;
- int result = Character.digit(c, base);
- if (result >= 0 && c > 0x7f) {
- lexError(pos+1, "illegal.nonascii.digit");
- ch = "0123456789abcdef".charAt(result);
- }
- return result;
- }
-
- /** Convert unicode escape; bp points to initial '\' character
- * (Spec 3.3).
- */
- private void convertUnicode() {
- if (ch == '\\' && unicodeConversionBp != bp) {
- bp++; ch = buf[bp];
- if (ch == 'u') {
- do {
- bp++; ch = buf[bp];
- } while (ch == 'u');
- int limit = bp + 3;
- if (limit < buflen) {
- int d = digit(16);
- int code = d;
- while (bp < limit && d >= 0) {
- bp++; ch = buf[bp];
- d = digit(16);
- code = (code << 4) + d;
- }
- if (d >= 0) {
- ch = (char)code;
- unicodeConversionBp = bp;
- return;
- }
- }
- lexError(bp, "illegal.unicode.esc");
- } else {
- bp--;
- ch = '\\';
- }
- }
- }
-
- /** Read next character.
- */
- private void scanChar() {
- ch = buf[++bp];
- if (ch == '\\') {
- convertUnicode();
- }
- }
-
- /** Read next character in comment, skipping over double '\' characters.
- */
- private void scanCommentChar() {
- scanChar();
- if (ch == '\\') {
- if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
- bp++;
- } else {
- convertUnicode();
- }
- }
- }
-
- /** Append a character to sbuf.
- */
- private void putChar(char ch) {
- if (sp == sbuf.length) {
- char[] newsbuf = new char[sbuf.length * 2];
- System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
- sbuf = newsbuf;
- }
- sbuf[sp++] = ch;
- }
-
- /** Read next character in character or string literal and copy into sbuf.
- */
- private void scanLitChar() {
- if (ch == '\\') {
- if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
- bp++;
- putChar('\\');
- scanChar();
- } else {
- scanChar();
- switch (ch) {
- case '0': case '1': case '2': case '3':
- case '4': case '5': case '6': case '7':
- char leadch = ch;
- int oct = digit(8);
- scanChar();
- if ('0' <= ch && ch <= '7') {
- oct = oct * 8 + digit(8);
- scanChar();
- if (leadch <= '3' && '0' <= ch && ch <= '7') {
- oct = oct * 8 + digit(8);
- scanChar();
- }
- }
- putChar((char)oct);
- break;
- case 'b':
- putChar('\b'); scanChar(); break;
- case 't':
- putChar('\t'); scanChar(); break;
- case 'n':
- putChar('\n'); scanChar(); break;
- case 'f':
- putChar('\f'); scanChar(); break;
- case 'r':
- putChar('\r'); scanChar(); break;
- case '\'':
- putChar('\''); scanChar(); break;
- case '\"':
- putChar('\"'); scanChar(); break;
- case '\\':
- putChar('\\'); scanChar(); break;
- default:
- lexError(bp, "illegal.esc.char");
- }
- }
- } else if (bp != buflen) {
- putChar(ch); scanChar();
- }
- }
-
- private void scanDigits(int digitRadix) {
- char saveCh;
- int savePos;
- do {
- if (ch != '_') {
- putChar(ch);
- } else {
- if (!allowUnderscoresInLiterals) {
- lexError("unsupported.underscore.lit", source.name);
- allowUnderscoresInLiterals = true;
- }
- }
- saveCh = ch;
- savePos = bp;
- scanChar();
- } while (digit(digitRadix) >= 0 || ch == '_');
- if (saveCh == '_')
- lexError(savePos, "illegal.underscore");
- }
-
- /** Read fractional part of hexadecimal floating point number.
- */
- private void scanHexExponentAndSuffix() {
- if (ch == 'p' || ch == 'P') {
- putChar(ch);
- scanChar();
- skipIllegalUnderscores();
- if (ch == '+' || ch == '-') {
- putChar(ch);
- scanChar();
- }
- skipIllegalUnderscores();
- if ('0' <= ch && ch <= '9') {
- scanDigits(10);
- if (!allowHexFloats) {
- lexError("unsupported.fp.lit", source.name);
- allowHexFloats = true;
- }
- else if (!hexFloatsWork)
- lexError("unsupported.cross.fp.lit");
- } else
- lexError("malformed.fp.lit");
- } else {
- lexError("malformed.fp.lit");
- }
- if (ch == 'f' || ch == 'F') {
- putChar(ch);
- scanChar();
- token = FLOATLITERAL;
- } else {
- if (ch == 'd' || ch == 'D') {
- putChar(ch);
- scanChar();
- }
- token = DOUBLELITERAL;
- }
- }
-
- /** Read fractional part of floating point number.
- */
- private void scanFraction() {
- skipIllegalUnderscores();
- if ('0' <= ch && ch <= '9') {
- scanDigits(10);
- }
- int sp1 = sp;
- if (ch == 'e' || ch == 'E') {
- putChar(ch);
- scanChar();
- skipIllegalUnderscores();
- if (ch == '+' || ch == '-') {
- putChar(ch);
- scanChar();
- }
- skipIllegalUnderscores();
- if ('0' <= ch && ch <= '9') {
- scanDigits(10);
- return;
- }
- lexError("malformed.fp.lit");
- sp = sp1;
- }
- }
-
- /** Read fractional part and 'd' or 'f' suffix of floating point number.
- */
- private void scanFractionAndSuffix() {
- this.radix = 10;
- scanFraction();
- if (ch == 'f' || ch == 'F') {
- putChar(ch);
- scanChar();
- token = FLOATLITERAL;
- } else {
- if (ch == 'd' || ch == 'D') {
- putChar(ch);
- scanChar();
- }
- token = DOUBLELITERAL;
- }
- }
-
- /** Read fractional part and 'd' or 'f' suffix of floating point number.
- */
- private void scanHexFractionAndSuffix(boolean seendigit) {
- this.radix = 16;
- Assert.check(ch == '.');
- putChar(ch);
- scanChar();
- skipIllegalUnderscores();
- if (digit(16) >= 0) {
- seendigit = true;
- scanDigits(16);
- }
- if (!seendigit)
- lexError("invalid.hex.number");
- else
- scanHexExponentAndSuffix();
- }
-
- private void skipIllegalUnderscores() {
- if (ch == '_') {
- lexError(bp, "illegal.underscore");
- while (ch == '_')
- scanChar();
- }
- }
-
- /** Read a number.
- * @param radix The radix of the number; one of 2, j8, 10, 16.
- */
- private void scanNumber(int radix) {
- this.radix = radix;
- // for octal, allow base-10 digit in case it's a float literal
- int digitRadix = (radix == 8 ? 10 : radix);
- boolean seendigit = false;
- if (digit(digitRadix) >= 0) {
- seendigit = true;
- scanDigits(digitRadix);
- }
- if (radix == 16 && ch == '.') {
- scanHexFractionAndSuffix(seendigit);
- } else if (seendigit && radix == 16 && (ch == 'p' || ch == 'P')) {
- scanHexExponentAndSuffix();
- } else if (digitRadix == 10 && ch == '.') {
- putChar(ch);
- scanChar();
- scanFractionAndSuffix();
- } else if (digitRadix == 10 &&
- (ch == 'e' || ch == 'E' ||
- ch == 'f' || ch == 'F' ||
- ch == 'd' || ch == 'D')) {
- scanFractionAndSuffix();
- } else {
- if (ch == 'l' || ch == 'L') {
- scanChar();
- token = LONGLITERAL;
- } else {
- token = INTLITERAL;
- }
- }
- }
-
- /** Read an identifier.
- */
- private void scanIdent() {
- boolean isJavaIdentifierPart;
- char high;
- do {
- if (sp == sbuf.length) putChar(ch); else sbuf[sp++] = ch;
- // optimization, was: putChar(ch);
-
- scanChar();
- switch (ch) {
- case 'A': case 'B': case 'C': case 'D': case 'E':
- case 'F': case 'G': case 'H': case 'I': case 'J':
- case 'K': case 'L': case 'M': case 'N': case 'O':
- case 'P': case 'Q': case 'R': case 'S': case 'T':
- case 'U': case 'V': case 'W': case 'X': case 'Y':
- case 'Z':
- case 'a': case 'b': case 'c': case 'd': case 'e':
- case 'f': case 'g': case 'h': case 'i': case 'j':
- case 'k': case 'l': case 'm': case 'n': case 'o':
- case 'p': case 'q': case 'r': case 's': case 't':
- case 'u': case 'v': case 'w': case 'x': case 'y':
- case 'z':
- case '$': case '_':
- case '0': case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- case '\u0000': case '\u0001': case '\u0002': case '\u0003':
- case '\u0004': case '\u0005': case '\u0006': case '\u0007':
- case '\u0008': case '\u000E': case '\u000F': case '\u0010':
- case '\u0011': case '\u0012': case '\u0013': case '\u0014':
- case '\u0015': case '\u0016': case '\u0017':
- case '\u0018': case '\u0019': case '\u001B':
- case '\u007F':
- break;
- case '\u001A': // EOI is also a legal identifier part
- if (bp >= buflen) {
- name = names.fromChars(sbuf, 0, sp);
- token = keywords.key(name);
- return;
- }
- break;
- default:
- if (ch < '\u0080') {
- // all ASCII range chars already handled, above
- isJavaIdentifierPart = false;
- } else {
- high = scanSurrogates();
- if (high != 0) {
- if (sp == sbuf.length) {
- putChar(high);
- } else {
- sbuf[sp++] = high;
- }
- isJavaIdentifierPart = Character.isJavaIdentifierPart(
- Character.toCodePoint(high, ch));
- } else {
- isJavaIdentifierPart = Character.isJavaIdentifierPart(ch);
- }
- }
- if (!isJavaIdentifierPart) {
- name = names.fromChars(sbuf, 0, sp);
- token = keywords.key(name);
- return;
- }
- }
- } while (true);
- }
-
- /** Are surrogates supported?
- */
- final static boolean surrogatesSupported = surrogatesSupported();
- private static boolean surrogatesSupported() {
- try {
- Character.isHighSurrogate('a');
- return true;
- } catch (NoSuchMethodError ex) {
- return false;
- }
- }
-
- /** Scan surrogate pairs. If 'ch' is a high surrogate and
- * the next character is a low surrogate, then put the low
- * surrogate in 'ch', and return the high surrogate.
- * otherwise, just return 0.
- */
- private char scanSurrogates() {
- if (surrogatesSupported && Character.isHighSurrogate(ch)) {
- char high = ch;
-
- scanChar();
-
- if (Character.isLowSurrogate(ch)) {
- return high;
- }
-
- ch = high;
- }
-
- return 0;
- }
-
- /** Return true if ch can be part of an operator.
- */
- private boolean isSpecial(char ch) {
- switch (ch) {
- case '!': case '%': case '&': case '*': case '?':
- case '+': case '-': case ':': case '<': case '=':
- case '>': case '^': case '|': case '~':
- case '@':
- return true;
- default:
- return false;
- }
- }
-
- /** Read longest possible sequence of special characters and convert
- * to token.
- */
- private void scanOperator() {
- while (true) {
- putChar(ch);
- Name newname = names.fromChars(sbuf, 0, sp);
- if (keywords.key(newname) == IDENTIFIER) {
- sp--;
- break;
- }
- name = newname;
- token = keywords.key(newname);
- scanChar();
- if (!isSpecial(ch)) break;
- }
- }
-
- /**
- * Scan a documention comment; determine if a deprecated tag is present.
- * Called once the initial /, * have been skipped, positioned at the second *
- * (which is treated as the beginning of the first line).
- * Stops positioned at the closing '/'.
- */
- @SuppressWarnings("fallthrough")
- private void scanDocComment() {
- boolean deprecatedPrefix = false;
-
- forEachLine:
- while (bp < buflen) {
-
- // Skip optional WhiteSpace at beginning of line
- while (bp < buflen && (ch == ' ' || ch == '\t' || ch == FF)) {
- scanCommentChar();
- }
-
- // Skip optional consecutive Stars
- while (bp < buflen && ch == '*') {
- scanCommentChar();
- if (ch == '/') {
- return;
- }
- }
-
- // Skip optional WhiteSpace after Stars
- while (bp < buflen && (ch == ' ' || ch == '\t' || ch == FF)) {
- scanCommentChar();
- }
-
- deprecatedPrefix = false;
- // At beginning of line in the JavaDoc sense.
- if (bp < buflen && ch == '@' && !deprecatedFlag) {
- scanCommentChar();
- if (bp < buflen && ch == 'd') {
- scanCommentChar();
- if (bp < buflen && ch == 'e') {
- scanCommentChar();
- if (bp < buflen && ch == 'p') {
- scanCommentChar();
- if (bp < buflen && ch == 'r') {
- scanCommentChar();
- if (bp < buflen && ch == 'e') {
- scanCommentChar();
- if (bp < buflen && ch == 'c') {
- scanCommentChar();
- if (bp < buflen && ch == 'a') {
- scanCommentChar();
- if (bp < buflen && ch == 't') {
- scanCommentChar();
- if (bp < buflen && ch == 'e') {
- scanCommentChar();
- if (bp < buflen && ch == 'd') {
- deprecatedPrefix = true;
- scanCommentChar();
- }}}}}}}}}}}
- if (deprecatedPrefix && bp < buflen) {
- if (Character.isWhitespace(ch)) {
- deprecatedFlag = true;
- } else if (ch == '*') {
- scanCommentChar();
- if (ch == '/') {
- deprecatedFlag = true;
- return;
- }
- }
- }
-
- // Skip rest of line
- while (bp < buflen) {
- switch (ch) {
- case '*':
- scanCommentChar();
- if (ch == '/') {
- return;
- }
- break;
- case CR: // (Spec 3.4)
- scanCommentChar();
- if (ch != LF) {
- continue forEachLine;
- }
- /* fall through to LF case */
- case LF: // (Spec 3.4)
- scanCommentChar();
- continue forEachLine;
- default:
- scanCommentChar();
- }
- } // rest of line
- } // forEachLine
- return;
- }
-
- /** The value of a literal token, recorded as a string.
- * For integers, leading 0x and 'l' suffixes are suppressed.
- */
- public String stringVal() {
- return new String(sbuf, 0, sp);
- }
-
- /** Read token.
- */
- public void nextToken() {
-
- try {
- prevEndPos = endPos;
- sp = 0;
-
- while (true) {
- pos = bp;
- switch (ch) {
- case ' ': // (Spec 3.6)
- case '\t': // (Spec 3.6)
- case FF: // (Spec 3.6)
- do {
- scanChar();
- } while (ch == ' ' || ch == '\t' || ch == FF);
- endPos = bp;
- processWhiteSpace();
- break;
- case LF: // (Spec 3.4)
- scanChar();
- endPos = bp;
- processLineTerminator();
- break;
- case CR: // (Spec 3.4)
- scanChar();
- if (ch == LF) {
- scanChar();
- }
- endPos = bp;
- processLineTerminator();
- break;
- case 'A': case 'B': case 'C': case 'D': case 'E':
- case 'F': case 'G': case 'H': case 'I': case 'J':
- case 'K': case 'L': case 'M': case 'N': case 'O':
- case 'P': case 'Q': case 'R': case 'S': case 'T':
- case 'U': case 'V': case 'W': case 'X': case 'Y':
- case 'Z':
- case 'a': case 'b': case 'c': case 'd': case 'e':
- case 'f': case 'g': case 'h': case 'i': case 'j':
- case 'k': case 'l': case 'm': case 'n': case 'o':
- case 'p': case 'q': case 'r': case 's': case 't':
- case 'u': case 'v': case 'w': case 'x': case 'y':
- case 'z':
- case '$': case '_':
- scanIdent();
- return;
- case '0':
- scanChar();
- if (ch == 'x' || ch == 'X') {
- scanChar();
- skipIllegalUnderscores();
- if (ch == '.') {
- scanHexFractionAndSuffix(false);
- } else if (digit(16) < 0) {
- lexError("invalid.hex.number");
- } else {
- scanNumber(16);
- }
- } else if (ch == 'b' || ch == 'B') {
- if (!allowBinaryLiterals) {
- lexError("unsupported.binary.lit", source.name);
- allowBinaryLiterals = true;
- }
- scanChar();
- skipIllegalUnderscores();
- if (digit(2) < 0) {
- lexError("invalid.binary.number");
- } else {
- scanNumber(2);
- }
- } else {
- putChar('0');
- if (ch == '_') {
- int savePos = bp;
- do {
- scanChar();
- } while (ch == '_');
- if (digit(10) < 0) {
- lexError(savePos, "illegal.underscore");
- }
- }
- scanNumber(8);
- }
- return;
- case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- scanNumber(10);
- return;
- case '.':
- scanChar();
- if ('0' <= ch && ch <= '9') {
- putChar('.');
- scanFractionAndSuffix();
- } else if (ch == '.') {
- putChar('.'); putChar('.');
- scanChar();
- if (ch == '.') {
- scanChar();
- putChar('.');
- token = ELLIPSIS;
- } else {
- lexError("malformed.fp.lit");
- }
- } else {
- token = DOT;
- }
- return;
- case ',':
- scanChar(); token = COMMA; return;
- case ';':
- scanChar(); token = SEMI; return;
- case '(':
- scanChar(); token = LPAREN; return;
- case ')':
- scanChar(); token = RPAREN; return;
- case '[':
- scanChar(); token = LBRACKET; return;
- case ']':
- scanChar(); token = RBRACKET; return;
- case '{':
- scanChar(); token = LBRACE; return;
- case '}':
- scanChar(); token = RBRACE; return;
- case '/':
- scanChar();
- if (ch == '/') {
- do {
- scanCommentChar();
- } while (ch != CR && ch != LF && bp < buflen);
- if (bp < buflen) {
- endPos = bp;
- processComment(CommentStyle.LINE);
- }
- break;
- } else if (ch == '*') {
- scanChar();
- CommentStyle style;
- if (ch == '*') {
- style = CommentStyle.JAVADOC;
- scanDocComment();
- } else {
- style = CommentStyle.BLOCK;
- while (bp < buflen) {
- if (ch == '*') {
- scanChar();
- if (ch == '/') break;
- } else {
- scanCommentChar();
- }
- }
- }
- if (ch == '/') {
- scanChar();
- endPos = bp;
- processComment(style);
- break;
- } else {
- lexError("unclosed.comment");
- return;
- }
- } else if (ch == '=') {
- name = names.slashequals;
- token = SLASHEQ;
- scanChar();
- } else {
- name = names.slash;
- token = SLASH;
- }
- return;
- case '\'':
- scanChar();
- if (ch == '\'') {
- lexError("empty.char.lit");
- } else {
- if (ch == CR || ch == LF)
- lexError(pos, "illegal.line.end.in.char.lit");
- scanLitChar();
- if (ch == '\'') {
- scanChar();
- token = CHARLITERAL;
- } else {
- lexError(pos, "unclosed.char.lit");
- }
- }
- return;
- case '\"':
- scanChar();
- while (ch != '\"' && ch != CR && ch != LF && bp < buflen)
- scanLitChar();
- if (ch == '\"') {
- token = STRINGLITERAL;
- scanChar();
- } else {
- lexError(pos, "unclosed.str.lit");
- }
- return;
- default:
- if (isSpecial(ch)) {
- scanOperator();
- } else {
- boolean isJavaIdentifierStart;
- if (ch < '\u0080') {
- // all ASCII range chars already handled, above
- isJavaIdentifierStart = false;
- } else {
- char high = scanSurrogates();
- if (high != 0) {
- if (sp == sbuf.length) {
- putChar(high);
- } else {
- sbuf[sp++] = high;
- }
-
- isJavaIdentifierStart = Character.isJavaIdentifierStart(
- Character.toCodePoint(high, ch));
- } else {
- isJavaIdentifierStart = Character.isJavaIdentifierStart(ch);
- }
- }
- if (isJavaIdentifierStart) {
- scanIdent();
- } else if (bp == buflen || ch == EOI && bp+1 == buflen) { // JLS 3.5
- token = EOF;
- pos = bp = eofPos;
- } else {
- lexError("illegal.char", String.valueOf((int)ch));
- scanChar();
- }
- }
- return;
- }
- }
- } finally {
- endPos = bp;
- if (scannerDebug)
- System.out.println("nextToken(" + pos
- + "," + endPos + ")=|" +
- new String(getRawCharacters(pos, endPos))
- + "|");
- }
- }
-
- /** Return the current token, set by nextToken().
- */
public Token token() {
return token;
}
- /** Sets the current token.
- * This method is primarily used to update the token stream when the
- * parser is handling the end of nested type arguments such as
- * {@code ListbeginIndex and
- * extends to the character at index endIndex - 1.
- * Thus the length of the substring is endIndex-beginIndex.
- * This behavior is like
- * String.substring(beginIndex, endIndex).
- * Unicode escape sequences are not translated.
- *
- * @param beginIndex the beginning index, inclusive.
- * @param endIndex the ending index, exclusive.
- * @throws IndexOutOfBounds if either offset is outside of the
- * array bounds
- */
- public char[] getRawCharacters(int beginIndex, int endIndex) {
- int length = endIndex - beginIndex;
- char[] chars = new char[length];
- System.arraycopy(buf, beginIndex, chars, 0, length);
- return chars;
- }
-
- public enum CommentStyle {
- LINE,
- BLOCK,
- JAVADOC,
- }
-
- /**
- * Called when a complete comment has been scanned. pos and endPos
- * will mark the comment boundary.
- */
- protected void processComment(CommentStyle style) {
- if (scannerDebug)
- System.out.println("processComment(" + pos
- + "," + endPos + "," + style + ")=|"
- + new String(getRawCharacters(pos, endPos))
- + "|");
- }
-
- /**
- * Called when a complete whitespace run has been scanned. pos and endPos
- * will mark the whitespace boundary.
- */
- protected void processWhiteSpace() {
- if (scannerDebug)
- System.out.println("processWhitespace(" + pos
- + "," + endPos + ")=|" +
- new String(getRawCharacters(pos, endPos))
- + "|");
- }
-
- /**
- * Called when a line terminator has been processed.
- */
- protected void processLineTerminator() {
- if (scannerDebug)
- System.out.println("processTerminator(" + pos
- + "," + endPos + ")=|" +
- new String(getRawCharacters(pos, endPos))
- + "|");
- }
-
- /** Build a map for translating between line numbers and
- * positions in the input.
- *
- * @return a LineMap */
- public Position.LineMap getLineMap() {
- return Position.makeLineMap(buf, buflen, false);
- }
-
}
diff --git a/langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java b/langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java
index 86c9bb2b11c..d03d77b62cd 100644
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java
@@ -57,7 +57,7 @@ public class ScannerFactory {
final Log log;
final Names names;
final Source source;
- final Keywords keywords;
+ final Tokens tokens;
/** Create a new scanner factory. */
protected ScannerFactory(Context context) {
@@ -65,14 +65,14 @@ public class ScannerFactory {
this.log = Log.instance(context);
this.names = Names.instance(context);
this.source = Source.instance(context);
- this.keywords = Keywords.instance(context);
+ this.tokens = Tokens.instance(context);
}
public Scanner newScanner(CharSequence input, boolean keepDocComments) {
if (input instanceof CharBuffer) {
CharBuffer buf = (CharBuffer) input;
if (keepDocComments)
- return new DocCommentScanner(this, buf);
+ return new Scanner(this, new JavadocTokenizer(this, buf));
else
return new Scanner(this, buf);
} else {
@@ -83,7 +83,7 @@ public class ScannerFactory {
public Scanner newScanner(char[] input, int inputLength, boolean keepDocComments) {
if (keepDocComments)
- return new DocCommentScanner(this, input, inputLength);
+ return new Scanner(this, new JavadocTokenizer(this, input, inputLength));
else
return new Scanner(this, input, inputLength);
}
diff --git a/langtools/src/share/classes/com/sun/tools/javac/parser/Token.java b/langtools/src/share/classes/com/sun/tools/javac/parser/Token.java
deleted file mode 100644
index 8fec6f8aeb7..00000000000
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Token.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Copyright (c) 1999, 2008, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation. Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-package com.sun.tools.javac.parser;
-
-import java.util.Locale;
-
-import com.sun.tools.javac.api.Formattable;
-import com.sun.tools.javac.api.Messages;
-
-/** An interface that defines codes for Java source tokens
- * returned from lexical analysis.
- *
- * beginIndex and
+ * extends to the character at index endIndex - 1.
+ * Thus the length of the substring is endIndex-beginIndex.
+ * This behavior is like
+ * String.substring(beginIndex, endIndex).
+ * Unicode escape sequences are not translated.
+ *
+ * @param beginIndex the beginning index, inclusive.
+ * @param endIndex the ending index, exclusive.
+ * @throws IndexOutOfBounds if either offset is outside of the
+ * array bounds
+ */
+ public char[] getRawCharacters(int beginIndex, int endIndex) {
+ int length = endIndex - beginIndex;
+ char[] chars = new char[length];
+ System.arraycopy(buf, beginIndex, chars, 0, length);
+ return chars;
+ }
+}
diff --git a/langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java b/langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java
index e8040766280..ce16646eea1 100644
--- a/langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java
+++ b/langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java
@@ -1072,9 +1072,9 @@ public class JavacProcessingEnvironment implements ProcessingEnvironment, Closea
Assert.checkNonNull(names);
next.put(Names.namesKey, names);
- Keywords keywords = Keywords.instance(context);
- Assert.checkNonNull(keywords);
- next.put(Keywords.keywordsKey, keywords);
+ Tokens tokens = Tokens.instance(context);
+ Assert.checkNonNull(tokens);
+ next.put(Tokens.tokensKey, tokens);
JavaCompiler oldCompiler = JavaCompiler.instance(context);
JavaCompiler nextCompiler = JavaCompiler.instance(next);
diff --git a/langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java b/langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java
index 12096869f4f..a5592acf8ad 100644
--- a/langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java
+++ b/langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java
@@ -39,7 +39,6 @@ import javax.tools.StandardLocation;
import com.sun.tools.javac.code.Symbol.CompletionFailure;
import com.sun.tools.javac.comp.Annotate;
-import com.sun.tools.javac.parser.DocCommentScanner;
import com.sun.tools.javac.tree.JCTree;
import com.sun.tools.javac.tree.JCTree.JCClassDecl;
import com.sun.tools.javac.tree.JCTree.JCCompilationUnit;
diff --git a/langtools/test/tools/javac/api/TestJavacTaskScanner.java b/langtools/test/tools/javac/api/TestJavacTaskScanner.java
index 40bde1679e5..ba50138ecc1 100644
--- a/langtools/test/tools/javac/api/TestJavacTaskScanner.java
+++ b/langtools/test/tools/javac/api/TestJavacTaskScanner.java
@@ -32,6 +32,7 @@
import com.sun.tools.javac.api.JavacTaskImpl;
import com.sun.tools.javac.parser.*;
+import com.sun.tools.javac.parser.Tokens.Token;
import com.sun.tools.javac.util.*;
import java.io.*;
import java.net.*;
@@ -93,7 +94,7 @@ public class TestJavacTaskScanner extends ToolTester {
check(numTokens, "#Tokens", 1222);
check(numParseTypeElements, "#parseTypeElements", 136);
- check(numAllMembers, "#allMembers", 67);
+ check(numAllMembers, "#allMembers", 52);
}
void check(int value, String name, int expected) {
@@ -206,7 +207,8 @@ class MyScanner extends Scanner {
public void nextToken() {
super.nextToken();
- System.err.format("Saw token %s (%s)%n", token(), name());
+ Token tk = token();
+ System.err.format("Saw token %s %n", tk.kind);
test.numTokens++;
}
diff --git a/langtools/test/tools/javac/depDocComment/DeprecatedDocComment3.java b/langtools/test/tools/javac/depDocComment/DeprecatedDocComment3.java
new file mode 100644
index 00000000000..d7f4dbc0f2c
--- /dev/null
+++ b/langtools/test/tools/javac/depDocComment/DeprecatedDocComment3.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 7096014
+ * @summary Javac tokens should retain state
+ * @compile -Xlint -Werror DeprecatedDocComment3.java
+ */
+
+class DeprecatedDocComment3 {
+ static class Foo { }
+
+ ; /** @deprecated */ ;
+
+ static class A {}
+
+ static class B {
+ A a; //not deprecated!
+ }
+}
diff --git a/langtools/test/tools/javac/tree/DocCommentToplevelTest.java b/langtools/test/tools/javac/tree/DocCommentToplevelTest.java
new file mode 100644
index 00000000000..ff236024f0f
--- /dev/null
+++ b/langtools/test/tools/javac/tree/DocCommentToplevelTest.java
@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 7096014
+ * @summary Javac tokens should retain state
+ */
+
+import com.sun.source.tree.*;
+import com.sun.source.util.*;
+import com.sun.tools.javac.tree.JCTree;
+
+import java.net.URI;
+import java.util.*;
+import javax.tools.*;
+
+
+public class DocCommentToplevelTest {
+
+ enum PackageKind {
+ HAS_PKG("package pkg;"),
+ NO_PKG("");
+
+ String pkgStr;
+
+ PackageKind(String pkgStr) {
+ this.pkgStr = pkgStr;
+ }
+ }
+
+ enum ImportKind {
+ ZERO(""),
+ ONE("import java.lang.*;"),
+ TWO("import java.lang.*; import java.util.*;");
+
+ String importStr;
+
+ ImportKind(String importStr) {
+ this.importStr = importStr;
+ }
+ }
+
+ enum ModifierKind {
+ DEFAULT(""),
+ PUBLIC("public");
+
+ String modStr;
+
+ ModifierKind(String modStr) {
+ this.modStr = modStr;
+ }
+ }
+
+ enum ToplevelDocKind {
+ HAS_DOC("/** Toplevel! */"),
+ NO_DOC("");
+
+ String docStr;
+
+ ToplevelDocKind(String docStr) {
+ this.docStr = docStr;
+ }
+ }
+
+ static int errors;
+ static int checks;
+
+ public static void main(String... args) throws Exception {
+ //create default shared JavaCompiler - reused across multiple compilations
+ JavaCompiler comp = ToolProvider.getSystemJavaCompiler();
+ StandardJavaFileManager fm = comp.getStandardFileManager(null, null, null);
+
+ for (PackageKind pk : PackageKind.values()) {
+ for (ImportKind ik : ImportKind.values()) {
+ for (ModifierKind mk1 : ModifierKind.values()) {
+ for (ModifierKind mk2 : ModifierKind.values()) {
+ for (ToplevelDocKind tdk : ToplevelDocKind.values()) {
+ new DocCommentToplevelTest(pk, ik, mk1, mk2, tdk).run(comp, fm);
+ }
+ }
+ }
+ }
+ }
+
+ if (errors > 0)
+ throw new AssertionError(errors + " errors found");
+
+ System.out.println(checks + " checks were made");
+ }
+
+ PackageKind pk;
+ ImportKind ik;
+ ModifierKind mk1;
+ ModifierKind mk2;
+ ToplevelDocKind tdk;
+ JavaSource source;
+
+ DocCommentToplevelTest(PackageKind pk, ImportKind ik, ModifierKind mk1, ModifierKind mk2, ToplevelDocKind tdk) {
+ this.pk = pk;
+ this.ik = ik;
+ this.mk1 = mk1;
+ this.mk2 = mk2;
+ this.tdk = tdk;
+ source = new JavaSource();
+ }
+
+ void run(JavaCompiler comp, JavaFileManager fm) throws Exception {
+ JavacTask task = (JavacTask)comp.getTask(null, fm, null, Arrays.asList("-printsource"), null, Arrays.asList(source));
+ for (CompilationUnitTree cu: task.parse()) {
+ check(cu);
+ }
+ }
+
+ void check(CompilationUnitTree cu) {
+ checks++;
+
+ new TreeScanner