archive/php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2025-08-15 21:48:51 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Don't return T_ERROR from token_get_all()

Browse source 
This turned out to be rather inconvenient after all. Instead just
return the same output we did on PHP 5. If people want to have an
error, use TOKEN_PARSE.
This commit is contained in:
Nikita Popov 2015-07-09 22:59:31 +02:00
parent fc5c7e633b
commit a49ce7bb91
7 changed files with 212 additions and 226 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Zend/zend_compile.c
View file

@ -1536,6 +1536,10 @@ int zendlex(zend_parser_stack_elem *elem) /* {{{ */
again:
ZVAL_UNDEF(&zv);
retval = lex_scan(&zv);
if (EG(exception)) {
return T_ERROR;
}
switch (retval) {
case T_COMMENT:
case T_DOC_COMMENT:

390
Zend/zend_language_scanner.c
View file

File diff suppressed because it is too large Load diff

22
Zend/zend_language_scanner.l
View file

@ -1662,7 +1662,7 @@ NEWLINE ("\r"|"\n"|"\r\n")
if (end != yytext + yyleng) {
zend_throw_exception(zend_ce_parse_error, "Invalid numeric literal", 0);
ZVAL_UNDEF(zendlval);
RETURN_TOKEN(T_ERROR);
RETURN_TOKEN(T_LNUMBER);
}
} else {
errno = 0;
@ -1680,7 +1680,7 @@ NEWLINE ("\r"|"\n"|"\r\n")
zend_throw_exception(zend_ce_parse_error,
"Invalid numeric literal", 0);
ZVAL_UNDEF(zendlval);
RETURN_TOKEN(T_ERROR);
RETURN_TOKEN(T_DNUMBER);
}
ZEND_ASSERT(!errno);
RETURN_TOKEN(T_DNUMBER);
@ -1689,7 +1689,7 @@ NEWLINE ("\r"|"\n"|"\r\n")
if (end != yytext + yyleng) {
zend_throw_exception(zend_ce_parse_error, "Invalid numeric literal", 0);
ZVAL_UNDEF(zendlval);
RETURN_TOKEN(T_ERROR);
RETURN_TOKEN(T_DNUMBER);
}
}
ZEND_ASSERT(!errno);
@ -2047,9 +2047,7 @@ inline_char_handler:
switch (*YYCURSOR++) {
case '"':
yyleng = YYCURSOR - SCNG(yy_text);
if (zend_scan_escape_string(zendlval, yytext+bprefix+1, yyleng-bprefix-2, '"') == FAILURE) {
RETURN_TOKEN(T_ERROR);
}
zend_scan_escape_string(zendlval, yytext+bprefix+1, yyleng-bprefix-2, '"');
RETURN_TOKEN(T_CONSTANT_ENCAPSED_STRING);
case '$':
if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
@ -2216,9 +2214,7 @@ inline_char_handler:
double_quotes_scan_done:
yyleng = YYCURSOR - SCNG(yy_text);
if (zend_scan_escape_string(zendlval, yytext, yyleng, '"') == FAILURE) {
RETURN_TOKEN(T_ERROR);
}
zend_scan_escape_string(zendlval, yytext, yyleng, '"');
RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
}
@ -2260,9 +2256,7 @@ double_quotes_scan_done:
yyleng = YYCURSOR - SCNG(yy_text);
if (zend_scan_escape_string(zendlval, yytext, yyleng, '`') == FAILURE) {
RETURN_TOKEN(T_ERROR);
}
zend_scan_escape_string(zendlval, yytext, yyleng, '`');
RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
}
@ -2336,9 +2330,7 @@ double_quotes_scan_done:
heredoc_scan_done:
yyleng = YYCURSOR - SCNG(yy_text);
if (zend_scan_escape_string(zendlval, yytext, yyleng - newline, 0) == FAILURE) {
RETURN_TOKEN(T_ERROR);
}
zend_scan_escape_string(zendlval, yytext, yyleng - newline, 0);
RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
}

10
ext/tokenizer/tests/parse_errors.phpt
View file

@ -33,7 +33,7 @@ Invalid numeric literal
T_OPEN_TAG (<?php )
T_STRING (var_dump)
(
T_ERROR (078)
T_LNUMBER (078)
)
;
@ -41,7 +41,7 @@ Invalid UTF-8 codepoint escape sequence
T_OPEN_TAG (<?php )
T_STRING (var_dump)
(
T_ERROR ("\u{xyz}")
T_CONSTANT_ENCAPSED_STRING ("\u{xyz}")
)
;
@ -49,7 +49,7 @@ Invalid UTF-8 codepoint escape sequence: Codepoint too large
T_OPEN_TAG (<?php )
T_STRING (var_dump)
(
T_ERROR ("\u{ffffff}")
T_CONSTANT_ENCAPSED_STRING ("\u{ffffff}")
)
;
@ -57,10 +57,10 @@ Invalid numeric literal
T_OPEN_TAG (<?php )
T_STRING (var_dump)
(
T_ERROR (078)
T_LNUMBER (078)
T_WHITESPACE ( )
+
T_WHITESPACE ( )
T_ERROR (078)
T_LNUMBER (078)
)
;

6
ext/tokenizer/tokenizer.c
View file

@ -191,9 +191,9 @@ void on_event(zend_php_scanner_event event, int token, int line)
HashTable *tokens_ht;
zval *token_zv;
switch(event) {
switch (event) {
case ON_TOKEN:
if (token == T_ERROR || token == END) break;
if (token == END) break;
if (token >= 256) {
array_init(&keyword);
add_next_index_long(&keyword, token);
@ -281,7 +281,7 @@ PHP_FUNCTION(token_get_all)
success = tokenize_parse(return_value, source);
} else {
success = tokenize(return_value, source);
/* Normal token_get_all() should never throw. Errors are indicated by T_ERROR tokens. */
/* Normal token_get_all() should not throw. */
zend_clear_exception();
}

2
ext/tokenizer/tokenizer_data.c
View file

@ -162,7 +162,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
REGISTER_LONG_CONSTANT("T_NS_C", T_NS_C, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NS_SEPARATOR", T_NS_SEPARATOR, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ELLIPSIS", T_ELLIPSIS, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ERROR", T_ERROR, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);
}
@ -303,7 +302,6 @@ char *get_token_type_name(int token_type)
case T_NS_C: return "T_NS_C";
case T_NS_SEPARATOR: return "T_NS_SEPARATOR";
case T_ELLIPSIS: return "T_ELLIPSIS";
case T_ERROR: return "T_ERROR";
}
return "UNKNOWN";

4
ext/tokenizer/tokenizer_data_gen.sh
View file

@ -46,7 +46,7 @@ echo '/*
echo 'void tokenizer_register_constants(INIT_FUNC_ARGS) {' >> $OUTFILE
$AWK '
/^#define T_NOELSE/ { next }
/^#define T_(NOELSE|ERROR)/ { next }
/^#define T_/ { print " REGISTER_LONG_CONSTANT(\"" $2 "\", " $2 ", CONST_CS | CONST_PERSISTENT);" }
' < $INFILE >> $OUTFILE
echo ' REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);' >> $OUTFILE
@ -64,7 +64,7 @@ $AWK '
print " case T_PAAMAYIM_NEKUDOTAYIM: return \"T_DOUBLE_COLON\";"
next
}
/^#define T_NOELSE/ { next }
/^#define T_(NOELSE|ERROR)/ { next }
/^#define T_/ {
print " case " $2 ": return \"" $2 "\";"
}