Rename PhpToken::getAll() to PhpToken::tokenize()

See https://externals.io/message/112189.
Fixes bug #80328.
This commit is contained in:
Nikita Popov 2020-11-09 09:40:31 +01:00
parent fe2cbcac70
commit b1019f46ed
11 changed files with 19 additions and 16 deletions

3
NEWS
View file

@ -27,6 +27,9 @@ PHP NEWS
message). (Nikita) message). (Nikita)
. Fixed bug #80266 (parse_url silently drops port number 0). (cmb, Nikita) . Fixed bug #80266 (parse_url silently drops port number 0). (cmb, Nikita)
- Tokenizer:
. Fixed bug #80328 (PhpToken::getAll() confusing name). (Nikita)
29 Oct 2020, PHP 8.0.0RC3 29 Oct 2020, PHP 8.0.0RC3
- Core: - Core:

View file

@ -20,7 +20,7 @@ class MyPhpToken extends PhpToken {
} }
} }
foreach (MyPhpToken::getAll($code) as $token) { foreach (MyPhpToken::tokenize($code) as $token) {
echo $token->getLoweredText(); echo $token->getLoweredText();
if ($token->extra !== 123) { if ($token->extra !== 123) {

View file

@ -10,7 +10,7 @@ class MyPhpToken1 extends PhpToken {
} }
try { try {
var_dump(MyPhpToken1::getAll("<?php foo")); var_dump(MyPhpToken1::tokenize("<?php foo"));
} catch (Error $e) { } catch (Error $e) {
echo $e->getMessage(), "\n"; echo $e->getMessage(), "\n";
} }
@ -19,7 +19,7 @@ abstract class MyPhpToken2 extends PhpToken {
} }
try { try {
var_dump(MyPhpToken2::getAll("<?php foo")); var_dump(MyPhpToken2::tokenize("<?php foo"));
} catch (Error $e) { } catch (Error $e) {
echo $e->getMessage(), "\n"; echo $e->getMessage(), "\n";
} }

View file

@ -15,7 +15,7 @@ function foo() {
PHP; PHP;
// Token names and ignorability. // Token names and ignorability.
$tokens = PhpToken::getAll($code); $tokens = PhpToken::tokenize($code);
foreach ($tokens as $i => $token) { foreach ($tokens as $i => $token) {
printf("[%2d] %-26s %s\n", $i, $token->getTokenName(), printf("[%2d] %-26s %s\n", $i, $token->getTokenName(),
$token->isIgnorable() ? "ignorable" : "meaningful"); $token->isIgnorable() ? "ignorable" : "meaningful");

View file

@ -3,7 +3,7 @@ PhpToken implements __toString()
--FILE-- --FILE--
<?php <?php
$tokens = PhpToken::getAll('<?php echo "Hello ". $what;'); $tokens = PhpToken::tokenize('<?php echo "Hello ". $what;');
var_dump(implode($tokens)); var_dump(implode($tokens));
var_dump($tokens[0] instanceof Stringable); var_dump($tokens[0] instanceof Stringable);

View file

@ -1,5 +1,5 @@
--TEST-- --TEST--
PhpToken::getAll() method PhpToken::tokenize() method
--SKIPIF-- --SKIPIF--
<?php if (!extension_loaded("tokenizer")) print "skip tokenizer extension not enabled"; ?> <?php if (!extension_loaded("tokenizer")) print "skip tokenizer extension not enabled"; ?>
--FILE-- --FILE--
@ -11,8 +11,8 @@ function foo() {
echo "bar"; echo "bar";
} }
PHP; PHP;
var_dump(PhpToken::getAll($code)); var_dump(PhpToken::tokenize($code));
var_dump(PhpToken::getAll($code, TOKEN_PARSE)); var_dump(PhpToken::tokenize($code, TOKEN_PARSE));
?> ?>
--EXPECTF-- --EXPECTF--

View file

@ -12,7 +12,7 @@ class C {
} }
CODE; CODE;
$tokens = PhpToken::getAll($code, TOKEN_PARSE); $tokens = PhpToken::tokenize($code, TOKEN_PARSE);
foreach ($tokens as $token) { foreach ($tokens as $token) {
echo "{$token->getTokenName()}: \"$token->text\"\n"; echo "{$token->getTokenName()}: \"$token->text\"\n";
} }

View file

@ -12,7 +12,7 @@ namespace\Foo
Foo \ Bar Foo \ Bar
CODE; CODE;
foreach (PhpToken::getAll($code) as $token) { foreach (PhpToken::tokenize($code) as $token) {
echo "{$token->getTokenName()}: \"$token->text\"\n"; echo "{$token->getTokenName()}: \"$token->text\"\n";
} }

View file

@ -92,7 +92,7 @@ static zend_string *php_token_get_text(zval *obj) {
static zend_bool tokenize_common( static zend_bool tokenize_common(
zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class); zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class);
PHP_METHOD(PhpToken, getAll) PHP_METHOD(PhpToken, tokenize)
{ {
zend_string *source; zend_string *source;
zend_long flags = 0; zend_long flags = 0;

View file

@ -9,7 +9,7 @@ function token_name(int $id): string {}
class PhpToken implements Stringable class PhpToken implements Stringable
{ {
/** @return static[] */ /** @return static[] */
public static function getAll(string $code, int $flags = 0): array {} public static function tokenize(string $code, int $flags = 0): array {}
final public function __construct(int $id, string $text, int $line = -1, int $pos = -1) {} final public function __construct(int $id, string $text, int $line = -1, int $pos = -1) {}

View file

@ -1,5 +1,5 @@
/* This is a generated file, edit the .stub.php file instead. /* This is a generated file, edit the .stub.php file instead.
* Stub hash: ef39f4efec05a3ebc5cf56a6b26e01369f00d129 */ * Stub hash: a06da9ea0191ed78ee7af8f0d9eaccb17dfa4b20 */
ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_token_get_all, 0, 1, IS_ARRAY, 0) ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_token_get_all, 0, 1, IS_ARRAY, 0)
ZEND_ARG_TYPE_INFO(0, code, IS_STRING, 0) ZEND_ARG_TYPE_INFO(0, code, IS_STRING, 0)
@ -10,7 +10,7 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_token_name, 0, 1, IS_STRING, 0)
ZEND_ARG_TYPE_INFO(0, id, IS_LONG, 0) ZEND_ARG_TYPE_INFO(0, id, IS_LONG, 0)
ZEND_END_ARG_INFO() ZEND_END_ARG_INFO()
#define arginfo_class_PhpToken_getAll arginfo_token_get_all #define arginfo_class_PhpToken_tokenize arginfo_token_get_all
ZEND_BEGIN_ARG_INFO_EX(arginfo_class_PhpToken___construct, 0, 0, 2) ZEND_BEGIN_ARG_INFO_EX(arginfo_class_PhpToken___construct, 0, 0, 2)
ZEND_ARG_TYPE_INFO(0, id, IS_LONG, 0) ZEND_ARG_TYPE_INFO(0, id, IS_LONG, 0)
@ -35,7 +35,7 @@ ZEND_END_ARG_INFO()
ZEND_FUNCTION(token_get_all); ZEND_FUNCTION(token_get_all);
ZEND_FUNCTION(token_name); ZEND_FUNCTION(token_name);
ZEND_METHOD(PhpToken, getAll); ZEND_METHOD(PhpToken, tokenize);
ZEND_METHOD(PhpToken, __construct); ZEND_METHOD(PhpToken, __construct);
ZEND_METHOD(PhpToken, is); ZEND_METHOD(PhpToken, is);
ZEND_METHOD(PhpToken, isIgnorable); ZEND_METHOD(PhpToken, isIgnorable);
@ -51,7 +51,7 @@ static const zend_function_entry ext_functions[] = {
static const zend_function_entry class_PhpToken_methods[] = { static const zend_function_entry class_PhpToken_methods[] = {
ZEND_ME(PhpToken, getAll, arginfo_class_PhpToken_getAll, ZEND_ACC_PUBLIC|ZEND_ACC_STATIC) ZEND_ME(PhpToken, tokenize, arginfo_class_PhpToken_tokenize, ZEND_ACC_PUBLIC|ZEND_ACC_STATIC)
ZEND_ME(PhpToken, __construct, arginfo_class_PhpToken___construct, ZEND_ACC_PUBLIC|ZEND_ACC_FINAL) ZEND_ME(PhpToken, __construct, arginfo_class_PhpToken___construct, ZEND_ACC_PUBLIC|ZEND_ACC_FINAL)
ZEND_ME(PhpToken, is, arginfo_class_PhpToken_is, ZEND_ACC_PUBLIC) ZEND_ME(PhpToken, is, arginfo_class_PhpToken_is, ZEND_ACC_PUBLIC)
ZEND_ME(PhpToken, isIgnorable, arginfo_class_PhpToken_isIgnorable, ZEND_ACC_PUBLIC) ZEND_ME(PhpToken, isIgnorable, arginfo_class_PhpToken_isIgnorable, ZEND_ACC_PUBLIC)