Fixed bug #26463 (Incorrect handling of semicolons after heredoc)

Zend/zend_highlight.c

@@ -181,13 +181,7 @@ ZEND_API void zend_highlight(zend_syntax_highlighter_ini *syntax_highlighter_ini
 					break;
 			}
 		} else if (token_type == T_END_HEREDOC) {
-			zend_bool has_semicolon=(strchr(token.value.str.val, ';')?1:0);
-
 			efree(token.value.str.val);
-			if (has_semicolon) {
-				/* the following semicolon was unput(), ignore it */
-				lex_scan(&token TSRMLS_CC);
-			}
 		}
 		token.type = 0;
 	}

Zend/zend_language_scanner.l

@@ -1702,8 +1702,8 @@ NEWLINE ("\r"|"\n"|"\r\n")
 	}
 
 	if (label_len==CG(heredoc_len) && !memcmp(yytext, CG(heredoc), label_len)) {
-		zendlval->value.str.val = estrndup(yytext, yyleng); /* unput destroys yytext */
+		zendlval->value.str.val = estrndup(yytext, label_len); /* unput destroys yytext */
-		zendlval->value.str.len = yyleng;
+		zendlval->value.str.len = label_len;
 		if (unput_semicolon) {
 			unput(';');
 		}

ext/standard/tests/general_functions/highlight_heredoc.phpt

@@ -0,0 +1,19 @@
+--TEST--
+highlight_string() handling of heredoc
+--FILE--
+<?php
+$str = '
+$x=<<<DD
+jhdsjkfhjdsh
+DD
+."";
+$a=<<<DDDD
+jhdsjkfhjdsh
+DDDD;
+';
+highlight_string($str);
+?>
+--EXPECT--
+<code><font color="#000000">
+<br />$x=&lt;&lt;&lt;DD<br />jhdsjkfhjdsh<br />DD<br />."";<br />$a=&lt;&lt;&lt;DDDD<br />jhdsjkfhjdsh<br />DDDD;<br /></font>
+</code>

ext/tokenizer/tests/bug26463.phpt

@@ -0,0 +1,118 @@
+--TEST--
+Bug #26463 (token_get_all() does not correctly handle semicolons after T_END_HEREDOC)
+--FILE--
+<?php
+$str = '<?php
+$x=<<<DD
+jhdsjkfhjdsh
+DD
+."";
+$a=<<<DDDD
+jhdsjkfhjdsh
+DDDD;
+?>';
+var_dump(token_get_all($str));
+?>
+--EXPECT--
+array(17) {
+  [0]=>
+  array(2) {
+    [0]=>
+    int(363)
+    [1]=>
+    string(6) "<?php
+"
+  }
+  [1]=>
+  array(2) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(2) "$x"
+  }
+  [2]=>
+  string(1) "="
+  [3]=>
+  array(2) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<<<DD
+"
+  }
+  [4]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(13) "jhdsjkfhjdsh
+"
+  }
+  [5]=>
+  array(2) {
+    [0]=>
+    int(368)
+    [1]=>
+    string(2) "DD"
+  }
+  [6]=>
+  string(1) "."
+  [7]=>
+  array(2) {
+    [0]=>
+    int(313)
+    [1]=>
+    string(2) """"
+  }
+  [8]=>
+  string(1) ";"
+  [9]=>
+  array(2) {
+    [0]=>
+    int(366)
+    [1]=>
+    string(1) "
+"
+  }
+  [10]=>
+  array(2) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(2) "$a"
+  }
+  [11]=>
+  string(1) "="
+  [12]=>
+  array(2) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(8) "<<<DDDD
+"
+  }
+  [13]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(13) "jhdsjkfhjdsh
+"
+  }
+  [14]=>
+  array(2) {
+    [0]=>
+    int(368)
+    [1]=>
+    string(4) "DDDD"
+  }
+  [15]=>
+  string(1) ";"
+  [16]=>
+  array(2) {
+    [0]=>
+    int(365)
+    [1]=>
+    string(2) "?>"
+  }
+}

ext/tokenizer/tokenizer.c

@@ -361,7 +361,12 @@ static void tokenize(zval *return_value TSRMLS_DC)
 			MAKE_STD_ZVAL(keyword);
 			array_init(keyword);
 			add_next_index_long(keyword, token_type);
-			add_next_index_stringl(keyword, zendtext, zendleng, 1);
+			if (token_type == T_END_HEREDOC) {
+				add_next_index_stringl(keyword, Z_STRVAL(token), Z_STRLEN(token), 1);
+				efree(Z_STRVAL(token));
+			} else {
+				add_next_index_stringl(keyword, zendtext, zendleng, 1);
+			}
 			add_next_index_zval(return_value, keyword);
 		} else {
 			add_next_index_stringl(return_value, zendtext, zendleng, 1);
@@ -369,9 +374,6 @@ static void tokenize(zval *return_value TSRMLS_DC)
 		if (destroy && Z_TYPE(token) != IS_NULL) {
 			zval_dtor(&token);
 		}
-		if (token_type == T_END_HEREDOC) {
-			efree(Z_STRVAL(token));
-		}
 		ZVAL_NULL(&token);
 	}
 }