archive/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2025-09-20 11:04:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

7096014: Javac tokens should retain state

Browse source 
Refactor javac tokens from enum constants to stateful instances (to keep track of position, comments, etc.)

Reviewed-by: jjg
This commit is contained in:
Maurizio Cimadamore 2011-10-24 13:00:20 +01:00
parent 5955806228
commit 063b622daf
18 changed files with 2370 additions and 1994 deletions
Download patch file Download diff file Expand all files Collapse all files

1
langtools/src/share/classes/com/sun/tools/apt/main/AptJavaCompiler.java
View file

@ -42,7 +42,6 @@ import com.sun.tools.javac.tree.JCTree.*;
import com.sun.tools.apt.comp.*;
import com.sun.tools.apt.util.Bark;
import com.sun.mirror.apt.AnnotationProcessorFactory;
import com.sun.tools.javac.parser.DocCommentScanner;
/**
* <p><b>This is NOT part of any supported API.

6
langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java
View file

@ -67,14 +67,14 @@ public class EndPosParser extends JavacParser {
/** {@inheritDoc} */
@Override
protected <T extends JCTree> T to(T t) {
storeEnd(t, S.endPos());
storeEnd(t, token.endPos);
return t;
}
/** {@inheritDoc} */
@Override
protected <T extends JCTree> T toP(T t) {
storeEnd(t, S.prevEndPos());
storeEnd(t, S.prevToken().endPos);
return t;
}
@ -88,7 +88,7 @@ public class EndPosParser extends JavacParser {
/** {@inheritDoc} */
@Override
JCExpression parExpression() {
int pos = S.pos();
int pos = token.pos;
JCExpression t = super.parExpression();
return toP(F.at(pos).Parens(t));
}

896
langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java Normal file
View file

@ -0,0 +1,896 @@
/*
* Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package com.sun.tools.javac.parser;
import java.nio.CharBuffer;
import com.sun.tools.javac.code.Source;
import com.sun.tools.javac.util.*;
import static com.sun.tools.javac.parser.Tokens.*;
import static com.sun.tools.javac.util.LayoutCharacters.*;
/** The lexical analyzer maps an input stream consisting of
* ASCII characters and Unicode escapes into a token sequence.
*
* <p><b>This is NOT part of any supported API.
* If you write code that depends on this, you do so at your own risk.
* This code and its internal interfaces are subject to change or
* deletion without notice.</b>
*/
public class JavaTokenizer {
private static boolean scannerDebug = false;
/** Allow hex floating-point literals.
*/
private boolean allowHexFloats;
/** Allow binary literals.
*/
private boolean allowBinaryLiterals;
/** Allow underscores in literals.
*/
private boolean allowUnderscoresInLiterals;
/** The source language setting.
*/
private Source source;
/** The log to be used for error reporting.
*/
private final Log log;
/** The name table. */
private final Names names;
/** The token factory. */
private final Tokens tokens;
/** The token kind, set by nextToken().
*/
protected TokenKind tk;
/** The token's radix, set by nextToken().
*/
protected int radix;
/** The token's name, set by nextToken().
*/
protected Name name;
/** The position where a lexical error occurred;
*/
protected int errPos = Position.NOPOS;
/** Has a @deprecated been encountered in last doc comment?
* this needs to be reset by client.
*/
protected boolean deprecatedFlag = false;
/** A character buffer for saved chars.
*/
protected char[] sbuf = new char[128];
protected int sp;
protected UnicodeReader reader;
private static final boolean hexFloatsWork = hexFloatsWork();
private static boolean hexFloatsWork() {
try {
Float.valueOf("0x1.0p1");
return true;
} catch (NumberFormatException ex) {
return false;
}
}
/**
* Create a scanner from the input array. This method might
* modify the array. To avoid copying the input array, ensure
* that {@code inputLength < input.length} or
* {@code input[input.length -1]} is a white space character.
*
* @param fac the factory which created this Scanner
* @param input the input, might be modified
* @param inputLength the size of the input.
* Must be positive and less than or equal to input.length.
*/
protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) {
this(fac, new UnicodeReader(fac, buf));
}
protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {
this(fac, new UnicodeReader(fac, buf, inputLength));
}
protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
log = fac.log;
names = fac.names;
tokens = fac.tokens;
source = fac.source;
this.reader = reader;
allowBinaryLiterals = source.allowBinaryLiterals();
allowHexFloats = source.allowHexFloats();
allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
}
/** Report an error at the given position using the provided arguments.
*/
protected void lexError(int pos, String key, Object... args) {
log.error(pos, key, args);
tk = TokenKind.ERROR;
errPos = pos;
}
/** Read next character in comment, skipping over double '\' characters.
*/
protected void scanCommentChar() {
reader.scanChar();
if (reader.ch == '\\') {
if (reader.peekChar() == '\\' && !reader.isUnicode()) {
reader.skipChar();
} else {
reader.convertUnicode();
}
}
}
/** Append a character to sbuf.
*/
private void putChar(char ch) {
if (sp == sbuf.length) {
char[] newsbuf = new char[sbuf.length * 2];
System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
sbuf = newsbuf;
}
sbuf[sp++] = ch;
}
/** Read next character in character or string literal and copy into sbuf.
*/
private void scanLitChar(int pos) {
if (reader.ch == '\\') {
if (reader.peekChar() == '\\' && !reader.isUnicode()) {
reader.skipChar();
putChar('\\');
reader.scanChar();
} else {
reader.scanChar();
switch (reader.ch) {
case '0': case '1': case '2': case '3':
case '4': case '5': case '6': case '7':
char leadch = reader.ch;
int oct = reader.digit(pos, 8);
reader.scanChar();
if ('0' <= reader.ch && reader.ch <= '7') {
oct = oct * 8 + reader.digit(pos, 8);
reader.scanChar();
if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
oct = oct * 8 + reader.digit(pos, 8);
reader.scanChar();
}
}
putChar((char)oct);
break;
case 'b':
putChar('\b'); reader.scanChar(); break;
case 't':
putChar('\t'); reader.scanChar(); break;
case 'n':
putChar('\n'); reader.scanChar(); break;
case 'f':
putChar('\f'); reader.scanChar(); break;
case 'r':
putChar('\r'); reader.scanChar(); break;
case '\'':
putChar('\''); reader.scanChar(); break;
case '\"':
putChar('\"'); reader.scanChar(); break;
case '\\':
putChar('\\'); reader.scanChar(); break;
default:
lexError(reader.bp, "illegal.esc.char");
}
}
} else if (reader.bp != reader.buflen) {
putChar(reader.ch); reader.scanChar();
}
}
private void scanDigits(int pos, int digitRadix) {
char saveCh;
int savePos;
do {
if (reader.ch != '_') {
putChar(reader.ch);
} else {
if (!allowUnderscoresInLiterals) {
lexError(pos, "unsupported.underscore.lit", source.name);
allowUnderscoresInLiterals = true;
}
}
saveCh = reader.ch;
savePos = reader.bp;
reader.scanChar();
} while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_');
if (saveCh == '_')
lexError(savePos, "illegal.underscore");
}
/** Read fractional part of hexadecimal floating point number.
*/
private void scanHexExponentAndSuffix(int pos) {
if (reader.ch == 'p' || reader.ch == 'P') {
putChar(reader.ch);
reader.scanChar();
skipIllegalUnderscores();
if (reader.ch == '+' || reader.ch == '-') {
putChar(reader.ch);
reader.scanChar();
}
skipIllegalUnderscores();
if ('0' <= reader.ch && reader.ch <= '9') {
scanDigits(pos, 10);
if (!allowHexFloats) {
lexError(pos, "unsupported.fp.lit", source.name);
allowHexFloats = true;
}
else if (!hexFloatsWork)
lexError(pos, "unsupported.cross.fp.lit");
} else
lexError(pos, "malformed.fp.lit");
} else {
lexError(pos, "malformed.fp.lit");
}
if (reader.ch == 'f' || reader.ch == 'F') {
putChar(reader.ch);
reader.scanChar();
tk = TokenKind.FLOATLITERAL;
radix = 16;
} else {
if (reader.ch == 'd' || reader.ch == 'D') {
putChar(reader.ch);
reader.scanChar();
}
tk = TokenKind.DOUBLELITERAL;
radix = 16;
}
}
/** Read fractional part of floating point number.
*/
private void scanFraction(int pos) {
skipIllegalUnderscores();
if ('0' <= reader.ch && reader.ch <= '9') {
scanDigits(pos, 10);
}
int sp1 = sp;
if (reader.ch == 'e' || reader.ch == 'E') {
putChar(reader.ch);
reader.scanChar();
skipIllegalUnderscores();
if (reader.ch == '+' || reader.ch == '-') {
putChar(reader.ch);
reader.scanChar();
}
skipIllegalUnderscores();
if ('0' <= reader.ch && reader.ch <= '9') {
scanDigits(pos, 10);
return;
}
lexError(pos, "malformed.fp.lit");
sp = sp1;
}
}
/** Read fractional part and 'd' or 'f' suffix of floating point number.
*/
private void scanFractionAndSuffix(int pos) {
radix = 10;
scanFraction(pos);
if (reader.ch == 'f' || reader.ch == 'F') {
putChar(reader.ch);
reader.scanChar();
tk = TokenKind.FLOATLITERAL;
} else {
if (reader.ch == 'd' || reader.ch == 'D') {
putChar(reader.ch);
reader.scanChar();
}
tk = TokenKind.DOUBLELITERAL;
}
}
/** Read fractional part and 'd' or 'f' suffix of floating point number.
*/
private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
radix = 16;
Assert.check(reader.ch == '.');
putChar(reader.ch);
reader.scanChar();
skipIllegalUnderscores();
if (reader.digit(pos, 16) >= 0) {
seendigit = true;
scanDigits(pos, 16);
}
if (!seendigit)
lexError(pos, "invalid.hex.number");
else
scanHexExponentAndSuffix(pos);
}
private void skipIllegalUnderscores() {
if (reader.ch == '_') {
lexError(reader.bp, "illegal.underscore");
while (reader.ch == '_')
reader.scanChar();
}
}
/** Read a number.
* @param radix The radix of the number; one of 2, j8, 10, 16.
*/
private void scanNumber(int pos, int radix) {
// for octal, allow base-10 digit in case it's a float literal
this.radix = radix;
int digitRadix = (radix == 8 ? 10 : radix);
boolean seendigit = false;
if (reader.digit(pos, digitRadix) >= 0) {
seendigit = true;
scanDigits(pos, digitRadix);
}
if (radix == 16 && reader.ch == '.') {
scanHexFractionAndSuffix(pos, seendigit);
} else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
scanHexExponentAndSuffix(pos);
} else if (digitRadix == 10 && reader.ch == '.') {
putChar(reader.ch);
reader.scanChar();
scanFractionAndSuffix(pos);
} else if (digitRadix == 10 &&
(reader.ch == 'e' || reader.ch == 'E' ||
reader.ch == 'f' || reader.ch == 'F' ||
reader.ch == 'd' || reader.ch == 'D')) {
scanFractionAndSuffix(pos);
} else {
if (reader.ch == 'l' || reader.ch == 'L') {
reader.scanChar();
tk = TokenKind.LONGLITERAL;
} else {
tk = TokenKind.INTLITERAL;
}
}
}
/** Read an identifier.
*/
private void scanIdent() {
boolean isJavaIdentifierPart;
char high;
do {
if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch;
// optimization, was: putChar(reader.ch);
reader.scanChar();
switch (reader.ch) {
case 'A': case 'B': case 'C': case 'D': case 'E':
case 'F': case 'G': case 'H': case 'I': case 'J':
case 'K': case 'L': case 'M': case 'N': case 'O':
case 'P': case 'Q': case 'R': case 'S': case 'T':
case 'U': case 'V': case 'W': case 'X': case 'Y':
case 'Z':
case 'a': case 'b': case 'c': case 'd': case 'e':
case 'f': case 'g': case 'h': case 'i': case 'j':
case 'k': case 'l': case 'm': case 'n': case 'o':
case 'p': case 'q': case 'r': case 's': case 't':
case 'u': case 'v': case 'w': case 'x': case 'y':
case 'z':
case '$': case '_':
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
case '\u0000': case '\u0001': case '\u0002': case '\u0003':
case '\u0004': case '\u0005': case '\u0006': case '\u0007':
case '\u0008': case '\u000E': case '\u000F': case '\u0010':
case '\u0011': case '\u0012': case '\u0013': case '\u0014':
case '\u0015': case '\u0016': case '\u0017':
case '\u0018': case '\u0019': case '\u001B':
case '\u007F':
break;
case '\u001A': // EOI is also a legal identifier part
if (reader.bp >= reader.buflen) {
name = names.fromChars(sbuf, 0, sp);
tk = tokens.lookupKind(name);
return;
}
break;
default:
if (reader.ch < '\u0080') {
// all ASCII range chars already handled, above
isJavaIdentifierPart = false;
} else {
high = reader.scanSurrogates();
if (high != 0) {
if (sp == sbuf.length) {
putChar(high);
} else {
sbuf[sp++] = high;
}
isJavaIdentifierPart = Character.isJavaIdentifierPart(
Character.toCodePoint(high, reader.ch));
} else {
isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
}
}
if (!isJavaIdentifierPart) {
name = names.fromChars(sbuf, 0, sp);
tk = tokens.lookupKind(name);
return;
}
}
} while (true);
}
/** Return true if reader.ch can be part of an operator.
*/
private boolean isSpecial(char ch) {
switch (ch) {
case '!': case '%': case '&': case '*': case '?':
case '+': case '-': case ':': case '<': case '=':
case '>': case '^': case '|': case '~':
case '@':
return true;
default:
return false;
}
}
/** Read longest possible sequence of special characters and convert
* to token.
*/
private void scanOperator() {
while (true) {
putChar(reader.ch);
Name newname = names.fromChars(sbuf, 0, sp);
TokenKind tk1 = tokens.lookupKind(newname);
if (tk1 == TokenKind.IDENTIFIER) {
sp--;
break;
}
tk = tk1;
reader.scanChar();
if (!isSpecial(reader.ch)) break;
}
}
/**
* Scan a documentation comment; determine if a deprecated tag is present.
* Called once the initial /, * have been skipped, positioned at the second *
* (which is treated as the beginning of the first line).
* Stops positioned at the closing '/'.
*/
@SuppressWarnings("fallthrough")
private void scanDocComment() {
boolean deprecatedPrefix = false;
forEachLine:
while (reader.bp < reader.buflen) {
// Skip optional WhiteSpace at beginning of line
while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
scanCommentChar();
}
// Skip optional consecutive Stars
while (reader.bp < reader.buflen && reader.ch == '*') {
scanCommentChar();
if (reader.ch == '/') {
return;
}
}
// Skip optional WhiteSpace after Stars
while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
scanCommentChar();
}
deprecatedPrefix = false;
// At beginning of line in the JavaDoc sense.
if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'd') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'e') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'p') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'r') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'e') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'c') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'a') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 't') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'e') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'd') {
deprecatedPrefix = true;
scanCommentChar();
}}}}}}}}}}}
if (deprecatedPrefix && reader.bp < reader.buflen) {
if (Character.isWhitespace(reader.ch)) {
deprecatedFlag = true;
} else if (reader.ch == '*') {
scanCommentChar();
if (reader.ch == '/') {
deprecatedFlag = true;
return;
}
}
}
// Skip rest of line
while (reader.bp < reader.buflen) {
switch (reader.ch) {
case '*':
scanCommentChar();
if (reader.ch == '/') {
return;
}
break;
case CR: // (Spec 3.4)
scanCommentChar();
if (reader.ch != LF) {
continue forEachLine;
}
/* fall through to LF case */
case LF: // (Spec 3.4)
scanCommentChar();
continue forEachLine;
default:
scanCommentChar();
}
} // rest of line
} // forEachLine
return;
}
/** Read token.
*/
public Token readToken() {
sp = 0;
name = null;
deprecatedFlag = false;
radix = 0;
int pos = 0;
int endPos = 0;
try {
loop: while (true) {
pos = reader.bp;
switch (reader.ch) {
case ' ': // (Spec 3.6)
case '\t': // (Spec 3.6)
case FF: // (Spec 3.6)
do {
reader.scanChar();
} while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF);
processWhiteSpace(pos, reader.bp);
break;
case LF: // (Spec 3.4)
reader.scanChar();
processLineTerminator(pos, reader.bp);
break;
case CR: // (Spec 3.4)
reader.scanChar();
if (reader.ch == LF) {
reader.scanChar();
}
processLineTerminator(pos, reader.bp);
break;
case 'A': case 'B': case 'C': case 'D': case 'E':
case 'F': case 'G': case 'H': case 'I': case 'J':
case 'K': case 'L': case 'M': case 'N': case 'O':
case 'P': case 'Q': case 'R': case 'S': case 'T':
case 'U': case 'V': case 'W': case 'X': case 'Y':
case 'Z':
case 'a': case 'b': case 'c': case 'd': case 'e':
case 'f': case 'g': case 'h': case 'i': case 'j':
case 'k': case 'l': case 'm': case 'n': case 'o':
case 'p': case 'q': case 'r': case 's': case 't':
case 'u': case 'v': case 'w': case 'x': case 'y':
case 'z':
case '$': case '_':
scanIdent();
break loop;
case '0':
reader.scanChar();
if (reader.ch == 'x' || reader.ch == 'X') {
reader.scanChar();
skipIllegalUnderscores();
if (reader.ch == '.') {
scanHexFractionAndSuffix(pos, false);
} else if (reader.digit(pos, 16) < 0) {
lexError(pos, "invalid.hex.number");
} else {
scanNumber(pos, 16);
}
} else if (reader.ch == 'b' || reader.ch == 'B') {
if (!allowBinaryLiterals) {
lexError(pos, "unsupported.binary.lit", source.name);
allowBinaryLiterals = true;
}
reader.scanChar();
skipIllegalUnderscores();
if (reader.digit(pos, 2) < 0) {
lexError(pos, "invalid.binary.number");
} else {
scanNumber(pos, 2);
}
} else {
putChar('0');
if (reader.ch == '_') {
int savePos = reader.bp;
do {
reader.scanChar();
} while (reader.ch == '_');
if (reader.digit(pos, 10) < 0) {
lexError(savePos, "illegal.underscore");
}
}
scanNumber(pos, 8);
}
break loop;
case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
scanNumber(pos, 10);
break loop;
case '.':
reader.scanChar();
if ('0' <= reader.ch && reader.ch <= '9') {
putChar('.');
scanFractionAndSuffix(pos);
} else if (reader.ch == '.') {
putChar('.'); putChar('.');
reader.scanChar();
if (reader.ch == '.') {
reader.scanChar();
putChar('.');
tk = TokenKind.ELLIPSIS;
} else {
lexError(pos, "malformed.fp.lit");
}
} else {
tk = TokenKind.DOT;
}
break loop;
case ',':
reader.scanChar(); tk = TokenKind.COMMA; break loop;
case ';':
reader.scanChar(); tk = TokenKind.SEMI; break loop;
case '(':
reader.scanChar(); tk = TokenKind.LPAREN; break loop;
case ')':
reader.scanChar(); tk = TokenKind.RPAREN; break loop;
case '[':
reader.scanChar(); tk = TokenKind.LBRACKET; break loop;
case ']':
reader.scanChar(); tk = TokenKind.RBRACKET; break loop;
case '{':
reader.scanChar(); tk = TokenKind.LBRACE; break loop;
case '}':
reader.scanChar(); tk = TokenKind.RBRACE; break loop;
case '/':
reader.scanChar();
if (reader.ch == '/') {
do {
scanCommentChar();
} while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
if (reader.bp < reader.buflen) {
processComment(pos, reader.bp, CommentStyle.LINE);
}
break;
} else if (reader.ch == '*') {
reader.scanChar();
CommentStyle style;
if (reader.ch == '*') {
style = CommentStyle.JAVADOC;
scanDocComment();
} else {
style = CommentStyle.BLOCK;
while (reader.bp < reader.buflen) {
if (reader.ch == '*') {
reader.scanChar();
if (reader.ch == '/') break;
} else {
scanCommentChar();
}
}
}
if (reader.ch == '/') {
reader.scanChar();
processComment(pos, reader.bp, style);
break;
} else {
lexError(pos, "unclosed.comment");
break loop;
}
} else if (reader.ch == '=') {
tk = TokenKind.SLASHEQ;
reader.scanChar();
} else {
tk = TokenKind.SLASH;
}
break loop;
case '\'':
reader.scanChar();
if (reader.ch == '\'') {
lexError(pos, "empty.char.lit");
} else {
if (reader.ch == CR || reader.ch == LF)
lexError(pos, "illegal.line.end.in.char.lit");
scanLitChar(pos);
char ch2 = reader.ch;
if (reader.ch == '\'') {
reader.scanChar();
tk = TokenKind.CHARLITERAL;
} else {
lexError(pos, "unclosed.char.lit");
}
}
break loop;
case '\"':
reader.scanChar();
while (reader.ch != '\"' && reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen)
scanLitChar(pos);
if (reader.ch == '\"') {
tk = TokenKind.STRINGLITERAL;
reader.scanChar();
} else {
lexError(pos, "unclosed.str.lit");
}
break loop;
default:
if (isSpecial(reader.ch)) {
scanOperator();
} else {
boolean isJavaIdentifierStart;
if (reader.ch < '\u0080') {
// all ASCII range chars already handled, above
isJavaIdentifierStart = false;
} else {
char high = reader.scanSurrogates();
if (high != 0) {
if (sp == sbuf.length) {
putChar(high);
} else {
sbuf[sp++] = high;
}
isJavaIdentifierStart = Character.isJavaIdentifierStart(
Character.toCodePoint(high, reader.ch));
} else {
isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
}
}
if (isJavaIdentifierStart) {
scanIdent();
} else if (reader.bp == reader.buflen || reader.ch == EOI && reader.bp + 1 == reader.buflen) { // JLS 3.5
tk = TokenKind.EOF;
pos = reader.buflen;
} else {
lexError(pos, "illegal.char", String.valueOf((int)reader.ch));
reader.scanChar();
}
}
break loop;
}
}
endPos = reader.bp;
switch (tk.tag) {
case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag);
case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag);
case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag);
case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag);
default: throw new AssertionError();
}
}
finally {
if (scannerDebug) {
System.out.println("nextToken(" + pos
+ "," + endPos + ")=|" +
new String(reader.getRawCharacters(pos, endPos))
+ "|");
}
}
}
/** Return the position where a lexical error occurred;
*/
public int errPos() {
return errPos;
}
/** Set the position where a lexical error occurred;
*/
public void errPos(int pos) {
errPos = pos;
}
public enum CommentStyle {
LINE,
BLOCK,
JAVADOC,
}
/**
* Called when a complete comment has been scanned. pos and endPos
* will mark the comment boundary.
*/
protected void processComment(int pos, int endPos, CommentStyle style) {
if (scannerDebug)
System.out.println("processComment(" + pos
+ "," + endPos + "," + style + ")=|"
+ new String(reader.getRawCharacters(pos, endPos))
+ "|");
}
/**
* Called when a complete whitespace run has been scanned. pos and endPos
* will mark the whitespace boundary.
*/
protected void processWhiteSpace(int pos, int endPos) {
if (scannerDebug)
System.out.println("processWhitespace(" + pos
+ "," + endPos + ")=|" +
new String(reader.getRawCharacters(pos, endPos))
+ "|");
}
/**
* Called when a line terminator has been processed.
*/
protected void processLineTerminator(int pos, int endPos) {
if (scannerDebug)
System.out.println("processTerminator(" + pos
+ "," + endPos + ")=|" +
new String(reader.getRawCharacters(pos, endPos))
+ "|");
}
/** Build a map for translating between line numbers and
* positions in the input.
*
* @return a LineMap */
public Position.LineMap getLineMap() {
return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
}
}

1033
langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java
View file

File diff suppressed because it is too large Load diff

46
langtools/src/share/classes/com/sun/tools/javac/parser/DocCommentScanner.java → langtools/src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,9 +25,12 @@
package com.sun.tools.javac.parser;
import com.sun.tools.javac.file.JavacFileManager;
import com.sun.tools.javac.parser.Tokens.Token;
import com.sun.tools.javac.util.*;
import java.nio.*;
import com.sun.tools.javac.util.*;
import static com.sun.tools.javac.util.LayoutCharacters.*;
/** An extension to the base lexical analyzer that captures
@ -40,26 +43,22 @@ import static com.sun.tools.javac.util.LayoutCharacters.*;
* This code and its internal interfaces are subject to change or
* deletion without notice.</b>
*/
public class DocCommentScanner extends Scanner {
public class JavadocTokenizer extends JavaTokenizer {
/** Create a scanner from the input buffer. buffer must implement
* array() and compact(), and remaining() must be less than limit().
*/
protected DocCommentScanner(ScannerFactory fac, CharBuffer buffer) {
protected JavadocTokenizer(ScannerFactory fac, CharBuffer buffer) {
super(fac, buffer);
}
/** Create a scanner from the input array. The array must have at
* least a single character of extra space.
*/
protected DocCommentScanner(ScannerFactory fac, char[] input, int inputLength) {
protected JavadocTokenizer(ScannerFactory fac, char[] input, int inputLength) {
super(fac, input, inputLength);
}
/** Starting position of the comment in original source
*/
private int pos;
/** The comment input buffer, index of next chacter to be read,
* index of one past last character in buffer.
*/
@ -178,6 +177,14 @@ public class DocCommentScanner extends Scanner {
}
}
@Override
public Token readToken() {
docComment = null;
Token tk = super.readToken();
tk.docComment = docComment;
return tk;
}
/**
* Read next character in doc comment, skipping over double '\' characters.
* If a double '\' is skipped, put in the buffer and update buffer count.
@ -196,32 +203,17 @@ public class DocCommentScanner extends Scanner {
}
}
/* Reset doc comment before reading each new token
*/
public void nextToken() {
docComment = null;
super.nextToken();
}
/**
* Returns the documentation string of the current token.
*/
public String docComment() {
return docComment;
}
/**
* Process a doc comment and make the string content available.
* Strips leading whitespace and stars.
*/
@SuppressWarnings("fallthrough")
protected void processComment(CommentStyle style) {
protected void processComment(int pos, int endPos, CommentStyle style) {
if (style != CommentStyle.JAVADOC) {
return;
}
pos = pos();
buf = getRawCharacters(pos, endPos());
buf = reader.getRawCharacters(pos, endPos);
buflen = buf.length;
bp = 0;
col = 0;
@ -414,7 +406,7 @@ public class DocCommentScanner extends Scanner {
*
* @return a LineMap */
public Position.LineMap getLineMap() {
char[] buf = getRawCharacters();
char[] buf = reader.getRawCharacters();
return Position.makeLineMap(buf, buf.length, true);
}
}

98
langtools/src/share/classes/com/sun/tools/javac/parser/Keywords.java
View file

@ -1,98 +0,0 @@
/*
* Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package com.sun.tools.javac.parser;
import com.sun.tools.javac.util.Context;
import com.sun.tools.javac.util.Log;
import com.sun.tools.javac.util.Name;
import com.sun.tools.javac.util.Names;
import static com.sun.tools.javac.parser.Token.*;
/**
* Map from Name to Token and Token to String.
*
* <p><b>This is NOT part of any supported API.
* If you write code that depends on this, you do so at your own risk.
* This code and its internal interfaces are subject to change or
* deletion without notice.</b>
*/
public class Keywords {
public static final Context.Key<Keywords> keywordsKey =
new Context.Key<Keywords>();
public static Keywords instance(Context context) {
Keywords instance = context.get(keywordsKey);
if (instance == null)
instance = new Keywords(context);
return instance;
}
private final Names names;
protected Keywords(Context context) {
context.put(keywordsKey, this);
names = Names.instance(context);
for (Token t : Token.values()) {
if (t.name != null)
enterKeyword(t.name, t);
else
tokenName[t.ordinal()] = null;
}
key = new Token[maxKey+1];
for (int i = 0; i <= maxKey; i++) key[i] = IDENTIFIER;
for (Token t : Token.values()) {
if (t.name != null)
key[tokenName[t.ordinal()].getIndex()] = t;
}
}
public Token key(Name name) {
return (name.getIndex() > maxKey) ? IDENTIFIER : key[name.getIndex()];
}
/**
* Keyword array. Maps name indices to Token.
*/
private final Token[] key;
/** The number of the last entered keyword.
*/
private int maxKey = 0;
/** The names of all tokens.
*/
private Name[] tokenName = new Name[Token.values().length];
private void enterKeyword(String s, Token token) {
Name n = names.fromString(s);
tokenName[token.ordinal()] = n;
if (n.getIndex() > maxKey) maxKey = n.getIndex();
}
}

89
langtools/src/share/classes/com/sun/tools/javac/parser/Lexer.java
View file

@ -25,7 +25,7 @@
package com.sun.tools.javac.parser;
import com.sun.tools.javac.util.*;
import com.sun.tools.javac.parser.Tokens.*;
import com.sun.tools.javac.util.Position.LineMap;
/**
@ -40,22 +40,26 @@ import com.sun.tools.javac.util.Position.LineMap;
public interface Lexer {
/**
* Has a @deprecated been encountered in last doc comment?
* This needs to be reset by client with resetDeprecatedFlag.
* Consume the next token.
*/
boolean deprecatedFlag();
void resetDeprecatedFlag();
void nextToken();
/**
* Returns the documentation string of the current token.
* Return current token.
*/
String docComment();
Token token();
/**
* Return the last character position of the current token.
* Return the last character position of the previous token.
*/
int endPos();
Token prevToken();
/**
* Splits the current token in two and return the first (splitted) token.
* For instance '<<<' is splitted into two tokens '<' and '<<' respectively,
* and the latter is returned.
*/
Token split();
/**
* Return the position where a lexical error occurred;
@ -74,69 +78,4 @@ public interface Lexer {
* @return a LineMap
*/
LineMap getLineMap();
/**
* Returns a copy of the input buffer, up to its inputLength.
* Unicode escape sequences are not translated.
*/
char[] getRawCharacters();
/**
* Returns a copy of a character array subset of the input buffer.
* The returned array begins at the <code>beginIndex</code> and
* extends to the character at index <code>endIndex - 1</code>.
* Thus the length of the substring is <code>endIndex-beginIndex</code>.
* This behavior is like
* <code>String.substring(beginIndex, endIndex)</code>.
* Unicode escape sequences are not translated.
*
* @param beginIndex the beginning index, inclusive.
* @param endIndex the ending index, exclusive.
* @throws IndexOutOfBounds if either offset is outside of the
* array bounds
*/
char[] getRawCharacters(int beginIndex, int endIndex);
/**
* Return the name of an identifier or token for the current token.
*/
Name name();
/**
* Read token.
*/
void nextToken();
/**
* Return the current token's position: a 0-based
* offset from beginning of the raw input stream
* (before unicode translation)
*/
int pos();
/**
* Return the last character position of the previous token.
*/
int prevEndPos();
/**
* Return the radix of a numeric literal token.
*/
int radix();
/**
* The value of a literal token, recorded as a string.
* For integers, leading 0x and 'l' suffixes are suppressed.
*/
String stringVal();
/**
* Return the current token, set by nextToken().
*/
Token token();
/**
* Sets the current token.
*/
void token(Token token);
}

4
langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java
View file

@ -55,7 +55,7 @@ public class ParserFactory {
final TreeMaker F;
final Log log;
final Keywords keywords;
final Tokens tokens;
final Source source;
final Names names;
final Options options;
@ -67,7 +67,7 @@ public class ParserFactory {
this.F = TreeMaker.instance(context);
this.log = Log.instance(context);
this.names = Names.instance(context);
this.keywords = Keywords.instance(context);
this.tokens = Tokens.instance(context);
this.source = Source.instance(context);
this.options = Options.instance(context);
this.scannerFactory = ScannerFactory.instance(context);

1085
langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java
View file

File diff suppressed because it is too large Load diff

8
langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java
View file

@ -57,7 +57,7 @@ public class ScannerFactory {
final Log log;
final Names names;
final Source source;
final Keywords keywords;
final Tokens tokens;
/** Create a new scanner factory. */
protected ScannerFactory(Context context) {
@ -65,14 +65,14 @@ public class ScannerFactory {
this.log = Log.instance(context);
this.names = Names.instance(context);
this.source = Source.instance(context);
this.keywords = Keywords.instance(context);
this.tokens = Tokens.instance(context);
}
public Scanner newScanner(CharSequence input, boolean keepDocComments) {
if (input instanceof CharBuffer) {
CharBuffer buf = (CharBuffer) input;
if (keepDocComments)
return new DocCommentScanner(this, buf);
return new Scanner(this, new JavadocTokenizer(this, buf));
else
return new Scanner(this, buf);
} else {
@ -83,7 +83,7 @@ public class ScannerFactory {
public Scanner newScanner(char[] input, int inputLength, boolean keepDocComments) {
if (keepDocComments)
return new DocCommentScanner(this, input, inputLength);
return new Scanner(this, new JavadocTokenizer(this, input, inputLength));
else
return new Scanner(this, input, inputLength);
}

198
langtools/src/share/classes/com/sun/tools/javac/parser/Token.java
View file

@ -1,198 +0,0 @@
/*
* Copyright (c) 1999, 2008, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package com.sun.tools.javac.parser;
import java.util.Locale;
import com.sun.tools.javac.api.Formattable;
import com.sun.tools.javac.api.Messages;
/** An interface that defines codes for Java source tokens
* returned from lexical analysis.
*
* <p><b>This is NOT part of any supported API.
* If you write code that depends on this, you do so at your own risk.
* This code and its internal interfaces are subject to change or
* deletion without notice.</b>
*/
public enum Token implements Formattable {
EOF,
ERROR,
IDENTIFIER,
ABSTRACT("abstract"),
ASSERT("assert"),
BOOLEAN("boolean"),
BREAK("break"),
BYTE("byte"),
CASE("case"),
CATCH("catch"),
CHAR("char"),
CLASS("class"),
CONST("const"),
CONTINUE("continue"),
DEFAULT("default"),
DO("do"),
DOUBLE("double"),
ELSE("else"),
ENUM("enum"),
EXTENDS("extends"),
FINAL("final"),
FINALLY("finally"),
FLOAT("float"),
FOR("for"),
GOTO("goto"),
IF("if"),
IMPLEMENTS("implements"),
IMPORT("import"),
INSTANCEOF("instanceof"),
INT("int"),
INTERFACE("interface"),
LONG("long"),
NATIVE("native"),
NEW("new"),
PACKAGE("package"),
PRIVATE("private"),
PROTECTED("protected"),
PUBLIC("public"),
RETURN("return"),
SHORT("short"),
STATIC("static"),
STRICTFP("strictfp"),
SUPER("super"),
SWITCH("switch"),
SYNCHRONIZED("synchronized"),
THIS("this"),
THROW("throw"),
THROWS("throws"),
TRANSIENT("transient"),
TRY("try"),
VOID("void"),
VOLATILE("volatile"),
WHILE("while"),
INTLITERAL,
LONGLITERAL,
FLOATLITERAL,
DOUBLELITERAL,
CHARLITERAL,
STRINGLITERAL,
TRUE("true"),
FALSE("false"),
NULL("null"),
LPAREN("("),
RPAREN(")"),
LBRACE("{"),
RBRACE("}"),
LBRACKET("["),
RBRACKET("]"),
SEMI(";"),
COMMA(","),
DOT("."),
ELLIPSIS("..."),
EQ("="),
GT(">"),
LT("<"),
BANG("!"),
TILDE("~"),
QUES("?"),
COLON(":"),
EQEQ("=="),
LTEQ("<="),
GTEQ(">="),
BANGEQ("!="),
AMPAMP("&&"),
BARBAR("||"),
PLUSPLUS("++"),
SUBSUB("--"),
PLUS("+"),
SUB("-"),
STAR("*"),
SLASH("/"),
AMP("&"),
BAR("|"),
CARET("^"),
PERCENT("%"),
LTLT("<<"),
GTGT(">>"),
GTGTGT(">>>"),
PLUSEQ("+="),
SUBEQ("-="),
STAREQ("*="),
SLASHEQ("/="),
AMPEQ("&="),
BAREQ("|="),
CARETEQ("^="),
PERCENTEQ("%="),
LTLTEQ("<<="),
GTGTEQ(">>="),
GTGTGTEQ(">>>="),
MONKEYS_AT("@"),
CUSTOM;
Token() {
this(null);
}
Token(String name) {
this.name = name;
}
public final String name;
public String toString() {
switch (this) {
case IDENTIFIER:
return "token.identifier";
case CHARLITERAL:
return "token.character";
case STRINGLITERAL:
return "token.string";
case INTLITERAL:
return "token.integer";
case LONGLITERAL:
return "token.long-integer";
case FLOATLITERAL:
return "token.float";
case DOUBLELITERAL:
return "token.double";
case ERROR:
return "token.bad-symbol";
case EOF:
return "token.end-of-input";
case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
return "'" + name + "'";
default:
return name;
}
}
public String getKind() {
return "Token";
}
public String toString(Locale locale, Messages messages) {
return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
}
}

423
langtools/src/share/classes/com/sun/tools/javac/parser/Tokens.java Normal file
View file

@ -0,0 +1,423 @@
/*
* Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package com.sun.tools.javac.parser;
import java.util.Locale;
import com.sun.tools.javac.api.Formattable;
import com.sun.tools.javac.api.Messages;
import com.sun.tools.javac.parser.Tokens.Token.Tag;
import com.sun.tools.javac.util.Name;
import com.sun.tools.javac.util.Context;
import com.sun.tools.javac.util.Names;
/** A class that defines codes/utilities for Java source tokens
* returned from lexical analysis.
*
* <p><b>This is NOT part of any supported API.
* If you write code that depends on this, you do so at your own risk.
* This code and its internal interfaces are subject to change or
* deletion without notice.</b>
*/
public class Tokens {
private final Names names;
/**
* Keyword array. Maps name indices to Token.
*/
private final TokenKind[] key;
/** The number of the last entered keyword.
*/
private int maxKey = 0;
/** The names of all tokens.
*/
private Name[] tokenName = new Name[TokenKind.values().length];
public static final Context.Key<Tokens> tokensKey =
new Context.Key<Tokens>();
public static Tokens instance(Context context) {
Tokens instance = context.get(tokensKey);
if (instance == null)
instance = new Tokens(context);
return instance;
}
protected Tokens(Context context) {
context.put(tokensKey, this);
names = Names.instance(context);
for (TokenKind t : TokenKind.values()) {
if (t.name != null)
enterKeyword(t.name, t);
else
tokenName[t.ordinal()] = null;
}
key = new TokenKind[maxKey+1];
for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
for (TokenKind t : TokenKind.values()) {
if (t.name != null)
key[tokenName[t.ordinal()].getIndex()] = t;
}
}
private void enterKeyword(String s, TokenKind token) {
Name n = names.fromString(s);
tokenName[token.ordinal()] = n;
if (n.getIndex() > maxKey) maxKey = n.getIndex();
}
/**
* Create a new token given a name; if the name corresponds to a token name,
* a new token of the corresponding kind is returned; otherwise, an
* identifier token is returned.
*/
TokenKind lookupKind(Name name) {
return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
}
TokenKind lookupKind(String name) {
return lookupKind(names.fromString(name));
}
/**
* This enum defines all tokens used by the javac scanner. A token is
* optionally associated with a name.
*/
public enum TokenKind implements Formattable {
EOF(),
ERROR(),
IDENTIFIER(Tag.NAMED),
ABSTRACT("abstract"),
ASSERT("assert", Tag.NAMED),
BOOLEAN("boolean", Tag.NAMED),
BREAK("break"),
BYTE("byte", Tag.NAMED),
CASE("case"),
CATCH("catch"),
CHAR("char", Tag.NAMED),
CLASS("class"),
CONST("const"),
CONTINUE("continue"),
DEFAULT("default"),
DO("do"),
DOUBLE("double", Tag.NAMED),
ELSE("else"),
ENUM("enum", Tag.NAMED),
EXTENDS("extends"),
FINAL("final"),
FINALLY("finally"),
FLOAT("float", Tag.NAMED),
FOR("for"),
GOTO("goto"),
IF("if"),
IMPLEMENTS("implements"),
IMPORT("import"),
INSTANCEOF("instanceof"),
INT("int", Tag.NAMED),
INTERFACE("interface"),
LONG("long", Tag.NAMED),
NATIVE("native"),
NEW("new"),
PACKAGE("package"),
PRIVATE("private"),
PROTECTED("protected"),
PUBLIC("public"),
RETURN("return"),
SHORT("short", Tag.NAMED),
STATIC("static"),
STRICTFP("strictfp"),
SUPER("super", Tag.NAMED),
SWITCH("switch"),
SYNCHRONIZED("synchronized"),
THIS("this", Tag.NAMED),
THROW("throw"),
THROWS("throws"),
TRANSIENT("transient"),
TRY("try"),
VOID("void", Tag.NAMED),
VOLATILE("volatile"),
WHILE("while"),
INTLITERAL(Tag.NUMERIC),
LONGLITERAL(Tag.NUMERIC),
FLOATLITERAL(Tag.NUMERIC),
DOUBLELITERAL(Tag.NUMERIC),
CHARLITERAL(Tag.NUMERIC),
STRINGLITERAL(Tag.STRING),
TRUE("true", Tag.NAMED),
FALSE("false", Tag.NAMED),
NULL("null", Tag.NAMED),
LPAREN("("),
RPAREN(")"),
LBRACE("{"),
RBRACE("}"),
LBRACKET("["),
RBRACKET("]"),
SEMI(";"),
COMMA(","),
DOT("."),
ELLIPSIS("..."),
EQ("="),
GT(">"),
LT("<"),
BANG("!"),
TILDE("~"),
QUES("?"),
COLON(":"),
EQEQ("=="),
LTEQ("<="),
GTEQ(">="),
BANGEQ("!="),
AMPAMP("&&"),
BARBAR("||"),
PLUSPLUS("++"),
SUBSUB("--"),
PLUS("+"),
SUB("-"),
STAR("*"),
SLASH("/"),
AMP("&"),
BAR("|"),
CARET("^"),
PERCENT("%"),
LTLT("<<"),
GTGT(">>"),
GTGTGT(">>>"),
PLUSEQ("+="),
SUBEQ("-="),
STAREQ("*="),
SLASHEQ("/="),
AMPEQ("&="),
BAREQ("|="),
CARETEQ("^="),
PERCENTEQ("%="),
LTLTEQ("<<="),
GTGTEQ(">>="),
GTGTGTEQ(">>>="),
MONKEYS_AT("@"),
CUSTOM;
public final String name;
public final Tag tag;
TokenKind() {
this(null, Tag.DEFAULT);
}
TokenKind(String name) {
this(name, Tag.DEFAULT);
}
TokenKind(Tag tag) {
this(null, tag);
}
TokenKind(String name, Tag tag) {
this.name = name;
this.tag = tag;
}
public String toString() {
switch (this) {
case IDENTIFIER:
return "token.identifier";
case CHARLITERAL:
return "token.character";
case STRINGLITERAL:
return "token.string";
case INTLITERAL:
return "token.integer";
case LONGLITERAL:
return "token.long-integer";
case FLOATLITERAL:
return "token.float";
case DOUBLELITERAL:
return "token.double";
case ERROR:
return "token.bad-symbol";
case EOF:
return "token.end-of-input";
case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
return "'" + name + "'";
default:
return name;
}
}
public String getKind() {
return "Token";
}
public String toString(Locale locale, Messages messages) {
return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
}
}
/**
* This is the class representing a javac token. Each token has several fields
* that are set by the javac lexer (i.e. start/end position, string value, etc).
*/
public static class Token {
/** tags constants **/
enum Tag {
DEFAULT,
NAMED,
STRING,
NUMERIC;
}
/** The token kind */
public final TokenKind kind;
/** The start position of this token */
public final int pos;
/** The end position of this token */
public final int endPos;
/** Is this token preceeded by a deprecated comment? */
public final boolean deprecatedFlag;
/** Is this token preceeded by a deprecated comment? */
public String docComment;
Token(TokenKind kind, int pos, int endPos,
boolean deprecatedFlag) {
this.kind = kind;
this.pos = pos;
this.endPos = endPos;
this.deprecatedFlag = deprecatedFlag;
checkKind();
}
Token[] split(Tokens tokens) {
if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
throw new AssertionError("Cant split" + kind);
}
TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
if (t1 == null || t2 == null) {
throw new AssertionError("Cant split - bad subtokens");
}
return new Token[] {
new Token(t1, pos, pos + t1.name.length(), deprecatedFlag),
new Token(t2, pos + t1.name.length(), endPos, false)
};
}
protected void checkKind() {
if (kind.tag != Tag.DEFAULT) {
throw new AssertionError("Bad token kind - expected " + Tag.STRING);
}
}
public Name name() {
throw new UnsupportedOperationException();
}
public String stringVal() {
throw new UnsupportedOperationException();
}
public int radix() {
throw new UnsupportedOperationException();
}
}
final static class NamedToken extends Token {
/** The name of this token */
public final Name name;
public NamedToken(TokenKind kind, int pos, int endPos, Name name, boolean deprecatedFlag) {
super(kind, pos, endPos, deprecatedFlag);
this.name = name;
}
protected void checkKind() {
if (kind.tag != Tag.NAMED) {
throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
}
}
@Override
public Name name() {
return name;
}
}
static class StringToken extends Token {
/** The string value of this token */
public final String stringVal;
public StringToken(TokenKind kind, int pos, int endPos, String stringVal, boolean deprecatedFlag) {
super(kind, pos, endPos, deprecatedFlag);
this.stringVal = stringVal;
}
protected void checkKind() {
if (kind.tag != Tag.STRING) {
throw new AssertionError("Bad token kind - expected " + Tag.STRING);
}
}
@Override
public String stringVal() {
return stringVal;
}
}
final static class NumericToken extends StringToken {
/** The 'radix' value of this token */
public final int radix;
public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, boolean deprecatedFlag) {
super(kind, pos, endPos, stringVal, deprecatedFlag);
this.radix = radix;
}
protected void checkKind() {
if (kind.tag != Tag.NUMERIC) {
throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
}
}
@Override
public int radix() {
return radix;
}
}
public static final Token DUMMY =
new Token(TokenKind.ERROR, 0, 0, false);
}

227
langtools/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java Normal file
View file

@ -0,0 +1,227 @@
/*
* Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package com.sun.tools.javac.parser;
import com.sun.tools.javac.file.JavacFileManager;
import java.nio.CharBuffer;
import com.sun.tools.javac.util.Log;
import static com.sun.tools.javac.util.LayoutCharacters.*;
/** The char reader used by the javac lexer/tokenizer. Returns the sequence of
* characters contained in the input stream, handling unicode escape accordingly.
* Additionally, it provide features for saving chars into a buffer and to retrieve
* them at a later stage.
*
* <p><b>This is NOT part of any supported API.
* If you write code that depends on this, you do so at your own risk.
* This code and its internal interfaces are subject to change or
* deletion without notice.</b>
*/
public class UnicodeReader {
/** The input buffer, index of next character to be read,
* index of one past last character in buffer.
*/
protected char[] buf;
protected int bp;
protected final int buflen;
/** The current character.
*/
protected char ch;
/** The buffer index of the last converted unicode character
*/
protected int unicodeConversionBp = -1;
protected Log log;
/**
* Create a scanner from the input array. This method might
* modify the array. To avoid copying the input array, ensure
* that {@code inputLength < input.length} or
* {@code input[input.length -1]} is a white space character.
*
* @param fac the factory which created this Scanner
* @param input the input, might be modified
* @param inputLength the size of the input.
* Must be positive and less than or equal to input.length.
*/
protected UnicodeReader(ScannerFactory sf, CharBuffer buffer) {
this(sf, JavacFileManager.toArray(buffer), buffer.limit());
}
protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) {
log = sf.log;
if (inputLength == input.length) {
if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
inputLength--;
} else {
char[] newInput = new char[inputLength + 1];
System.arraycopy(input, 0, newInput, 0, input.length);
input = newInput;
}
}
buf = input;
buflen = inputLength;
buf[buflen] = EOI;
bp = -1;
scanChar();
}
/** Read next character.
*/
protected void scanChar() {
if (bp < buflen) {
ch = buf[++bp];
if (ch == '\\') {
convertUnicode();
}
}
}
/** Convert unicode escape; bp points to initial '\' character
* (Spec 3.3).
*/
protected void convertUnicode() {
if (ch == '\\' && unicodeConversionBp != bp) {
bp++; ch = buf[bp];
if (ch == 'u') {
do {
bp++; ch = buf[bp];
} while (ch == 'u');
int limit = bp + 3;
if (limit < buflen) {
int d = digit(bp, 16);
int code = d;
while (bp < limit && d >= 0) {
bp++; ch = buf[bp];
d = digit(bp, 16);
code = (code << 4) + d;
}
if (d >= 0) {
ch = (char)code;
unicodeConversionBp = bp;
return;
}
}
log.error(bp, "illegal.unicode.esc");
} else {
bp--;
ch = '\\';
}
}
}
/** Are surrogates supported?
*/
final static boolean surrogatesSupported = surrogatesSupported();
private static boolean surrogatesSupported() {
try {
Character.isHighSurrogate('a');
return true;
} catch (NoSuchMethodError ex) {
return false;
}
}
/** Scan surrogate pairs. If 'ch' is a high surrogate and
* the next character is a low surrogate, then put the low
* surrogate in 'ch', and return the high surrogate.
* otherwise, just return 0.
*/
protected char scanSurrogates() {
if (surrogatesSupported && Character.isHighSurrogate(ch)) {
char high = ch;
scanChar();
if (Character.isLowSurrogate(ch)) {
return high;
}
ch = high;
}
return 0;
}
/** Convert an ASCII digit from its base (8, 10, or 16)
* to its value.
*/
protected int digit(int pos, int base) {
char c = ch;
int result = Character.digit(c, base);
if (result >= 0 && c > 0x7f) {
log.error(pos + 1, "illegal.nonascii.digit");
ch = "0123456789abcdef".charAt(result);
}
return result;
}
protected boolean isUnicode() {
return unicodeConversionBp == bp;
}
protected void skipChar() {
bp++;
}
protected char peekChar() {
return buf[bp + 1];
}
/**
* Returns a copy of the input buffer, up to its inputLength.
* Unicode escape sequences are not translated.
*/
public char[] getRawCharacters() {
char[] chars = new char[buflen];
System.arraycopy(buf, 0, chars, 0, buflen);
return chars;
}
/**
* Returns a copy of a character array subset of the input buffer.
* The returned array begins at the <code>beginIndex</code> and
* extends to the character at index <code>endIndex - 1</code>.
* Thus the length of the substring is <code>endIndex-beginIndex</code>.
* This behavior is like
* <code>String.substring(beginIndex, endIndex)</code>.
* Unicode escape sequences are not translated.
*
* @param beginIndex the beginning index, inclusive.
* @param endIndex the ending index, exclusive.
* @throws IndexOutOfBounds if either offset is outside of the
* array bounds
*/
public char[] getRawCharacters(int beginIndex, int endIndex) {
int length = endIndex - beginIndex;
char[] chars = new char[length];
System.arraycopy(buf, beginIndex, chars, 0, length);
return chars;
}
}

6
langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java
View file

@ -1072,9 +1072,9 @@ public class JavacProcessingEnvironment implements ProcessingEnvironment, Closea
Assert.checkNonNull(names);
next.put(Names.namesKey, names);
Keywords keywords = Keywords.instance(context);
Assert.checkNonNull(keywords);
next.put(Keywords.keywordsKey, keywords);
Tokens tokens = Tokens.instance(context);
Assert.checkNonNull(tokens);
next.put(Tokens.tokensKey, tokens);
JavaCompiler oldCompiler = JavaCompiler.instance(context);
JavaCompiler nextCompiler = JavaCompiler.instance(next);

1
langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java
View file

@ -39,7 +39,6 @@ import javax.tools.StandardLocation;
import com.sun.tools.javac.code.Symbol.CompletionFailure;
import com.sun.tools.javac.comp.Annotate;
import com.sun.tools.javac.parser.DocCommentScanner;
import com.sun.tools.javac.tree.JCTree;
import com.sun.tools.javac.tree.JCTree.JCClassDecl;
import com.sun.tools.javac.tree.JCTree.JCCompilationUnit;

6
langtools/test/tools/javac/api/TestJavacTaskScanner.java
View file

@ -32,6 +32,7 @@
import com.sun.tools.javac.api.JavacTaskImpl;
import com.sun.tools.javac.parser.*;
import com.sun.tools.javac.parser.Tokens.Token;
import com.sun.tools.javac.util.*;
import java.io.*;
import java.net.*;
@ -93,7 +94,7 @@ public class TestJavacTaskScanner extends ToolTester {
check(numTokens, "#Tokens", 1222);
check(numParseTypeElements, "#parseTypeElements", 136);
check(numAllMembers, "#allMembers", 67);
check(numAllMembers, "#allMembers", 52);
}
void check(int value, String name, int expected) {
@ -206,7 +207,8 @@ class MyScanner extends Scanner {
public void nextToken() {
super.nextToken();
System.err.format("Saw token %s (%s)%n", token(), name());
Token tk = token();
System.err.format("Saw token %s %n", tk.kind);
test.numTokens++;
}

41
langtools/test/tools/javac/depDocComment/DeprecatedDocComment3.java Normal file
View file

@ -0,0 +1,41 @@
/*
* Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/**
* @test
* @bug 7096014
* @summary Javac tokens should retain state
* @compile -Xlint -Werror DeprecatedDocComment3.java
*/
class DeprecatedDocComment3 {
static class Foo { }
; /** @deprecated */ ;
static class A {}
static class B {
A a; //not deprecated!
}
}

196
langtools/test/tools/javac/tree/DocCommentToplevelTest.java Normal file
View file

@ -0,0 +1,196 @@
/*
* Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/*
* @test
* @bug 7096014
* @summary Javac tokens should retain state
*/
import com.sun.source.tree.*;
import com.sun.source.util.*;
import com.sun.tools.javac.tree.JCTree;
import java.net.URI;
import java.util.*;
import javax.tools.*;
public class DocCommentToplevelTest {
enum PackageKind {
HAS_PKG("package pkg;"),
NO_PKG("");
String pkgStr;
PackageKind(String pkgStr) {
this.pkgStr = pkgStr;
}
}
enum ImportKind {
ZERO(""),
ONE("import java.lang.*;"),
TWO("import java.lang.*; import java.util.*;");
String importStr;
ImportKind(String importStr) {
this.importStr = importStr;
}
}
enum ModifierKind {
DEFAULT(""),
PUBLIC("public");
String modStr;
ModifierKind(String modStr) {
this.modStr = modStr;
}
}
enum ToplevelDocKind {
HAS_DOC("/** Toplevel! */"),
NO_DOC("");
String docStr;
ToplevelDocKind(String docStr) {
this.docStr = docStr;
}
}
static int errors;
static int checks;
public static void main(String... args) throws Exception {
//create default shared JavaCompiler - reused across multiple compilations
JavaCompiler comp = ToolProvider.getSystemJavaCompiler();
StandardJavaFileManager fm = comp.getStandardFileManager(null, null, null);
for (PackageKind pk : PackageKind.values()) {
for (ImportKind ik : ImportKind.values()) {
for (ModifierKind mk1 : ModifierKind.values()) {
for (ModifierKind mk2 : ModifierKind.values()) {
for (ToplevelDocKind tdk : ToplevelDocKind.values()) {
new DocCommentToplevelTest(pk, ik, mk1, mk2, tdk).run(comp, fm);
}
}
}
}
}
if (errors > 0)
throw new AssertionError(errors + " errors found");
System.out.println(checks + " checks were made");
}
PackageKind pk;
ImportKind ik;
ModifierKind mk1;
ModifierKind mk2;
ToplevelDocKind tdk;
JavaSource source;
DocCommentToplevelTest(PackageKind pk, ImportKind ik, ModifierKind mk1, ModifierKind mk2, ToplevelDocKind tdk) {
this.pk = pk;
this.ik = ik;
this.mk1 = mk1;
this.mk2 = mk2;
this.tdk = tdk;
source = new JavaSource();
}
void run(JavaCompiler comp, JavaFileManager fm) throws Exception {
JavacTask task = (JavacTask)comp.getTask(null, fm, null, Arrays.asList("-printsource"), null, Arrays.asList(source));
for (CompilationUnitTree cu: task.parse()) {
check(cu);
}
}
void check(CompilationUnitTree cu) {
checks++;
new TreeScanner<ClassTree,Void>() {
Map<JCTree, String> docComments;
@Override
public ClassTree visitCompilationUnit(CompilationUnitTree node, Void unused) {
docComments = ((JCTree.JCCompilationUnit)node).docComments;
boolean expectedComment = tdk == ToplevelDocKind.HAS_DOC &&
(pk != PackageKind.NO_PKG || ik != ImportKind.ZERO);
boolean foundComment = docComments.get(node) != null;
if (expectedComment != foundComment) {
error("Unexpected comment " + docComments.get(node) + " on toplevel");
}
return super.visitCompilationUnit(node, null);
}
@Override
public ClassTree visitClass(ClassTree node, Void unused) {
boolean expectedComment = tdk == ToplevelDocKind.HAS_DOC &&
pk == PackageKind.NO_PKG && ik == ImportKind.ZERO &&
node.getSimpleName().toString().equals("First");
boolean foundComment = docComments.get(node) != null;
if (expectedComment != foundComment) {
error("Unexpected comment " + docComments.get(node) + " on class " + node.getSimpleName());
}
return super.visitClass(node, unused);
}
}.scan(cu, null);
}
void error(String msg) {
System.err.println("Error: " + msg);
System.err.println("Source: " + source.source);
errors++;
}
class JavaSource extends SimpleJavaFileObject {
String template = "#D\n#P\n#I\n" +
"#M1 class First { }\n" +
"#M2 class Second { }\n";
String source;
public JavaSource() {
super(URI.create("myfo:/Test.java"), JavaFileObject.Kind.SOURCE);
source = template.replace("#P", pk.pkgStr)
.replace("#I", ik.importStr)
.replace("#M1", mk1.modStr)
.replace("#M2", mk2.modStr)
.replace("#D", tdk.docStr);
}
@Override
public CharSequence getCharContent(boolean ignoreEncodingErrors) {
return source;
}
}
}