archive/ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2025-08-25 05:55:46 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

[ruby/prism] Enable remaining heredoc unescape tests

Browse source 
c7ea4941c5
This commit is contained in:
Kevin Newton 2023-10-11 11:12:41 -04:00
parent 24768d8a57
commit d6424453db
3 changed files with 23 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

27
prism/prism.c
View file

@ -8482,6 +8482,7 @@ parser_lex(pm_parser_t *parser) {
const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end);
pm_token_buffer_t token_buffer = { 0 }; pm_token_buffer_t token_buffer = { 0 };
bool was_escaped_newline = false;
while (breakpoint != NULL) { while (breakpoint != NULL) {
switch (*breakpoint) { switch (*breakpoint) {
@ -8509,6 +8510,7 @@ parser_lex(pm_parser_t *parser) {
// content. Then, the next time a token is lexed, it will match // content. Then, the next time a token is lexed, it will match
// again and return the end of the heredoc. // again and return the end of the heredoc.
if ( if (
!was_escaped_newline &&
(start + ident_length <= parser->end) && (start + ident_length <= parser->end) &&
(memcmp(start, ident_start, ident_length) == 0) (memcmp(start, ident_start, ident_length) == 0)
) { ) {
@ -8550,6 +8552,9 @@ parser_lex(pm_parser_t *parser) {
case '\r': case '\r':
parser->current.end++; parser->current.end++;
if (peek(parser) != '\n') { if (peek(parser) != '\n') {
if (quote == PM_HEREDOC_QUOTE_SINGLE) {
pm_token_buffer_push(&token_buffer, '\\');
}
pm_token_buffer_push(&token_buffer, '\r'); pm_token_buffer_push(&token_buffer, '\r');
break; break;
} }
@ -8559,25 +8564,19 @@ parser_lex(pm_parser_t *parser) {
// to leave the escaped newline in place so that // to leave the escaped newline in place so that
// it can be removed later when we dedent the // it can be removed later when we dedent the
// heredoc. // heredoc.
if (lex_mode->as.heredoc.indent == PM_HEREDOC_INDENT_TILDE) { if (quote == PM_HEREDOC_QUOTE_SINGLE || lex_mode->as.heredoc.indent == PM_HEREDOC_INDENT_TILDE) {
pm_token_buffer_push(&token_buffer, '\\'); pm_token_buffer_push(&token_buffer, '\\');
pm_token_buffer_push(&token_buffer, '\n'); pm_token_buffer_push(&token_buffer, '\n');
} }
if (parser->heredoc_end) { token_buffer.cursor = parser->current.end + 1;
// ... if we are on the same line as a heredoc, breakpoint = parser->current.end;
// flush the heredoc and continue parsing after
// heredoc_end. if (quote != PM_HEREDOC_QUOTE_SINGLE) {
parser_flush_heredoc_end(parser); was_escaped_newline = true;
pm_token_buffer_copy(parser, &token_buffer);
LEX(PM_TOKEN_STRING_CONTENT);
} else {
// ... else track the newline.
pm_newline_list_append(&parser->newline_list, parser->current.end);
} }
parser->current.end++; continue;
break;
default: default:
if (quote == PM_HEREDOC_QUOTE_SINGLE) { if (quote == PM_HEREDOC_QUOTE_SINGLE) {
pm_token_buffer_push(&token_buffer, '\\'); pm_token_buffer_push(&token_buffer, '\\');
@ -8616,6 +8615,8 @@ parser_lex(pm_parser_t *parser) {
default: default:
assert(false && "unreachable"); assert(false && "unreachable");
} }
was_escaped_newline = false;
} }
// If we've hit the end of the string, then this is an unterminated // If we've hit the end of the string, then this is an unterminated

2
test/prism/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt
View file

@ -114,7 +114,7 @@
│ ├── opening_loc: (51,0)-(51,9) = "<<-'HERE'" │ ├── opening_loc: (51,0)-(51,9) = "<<-'HERE'"
│ ├── content_loc: (52,0)-(53,0) = "a\\\nb\n" │ ├── content_loc: (52,0)-(53,0) = "a\\\nb\n"
│ ├── closing_loc: (54,0)-(54,0) = "HERE\n" │ ├── closing_loc: (54,0)-(54,0) = "HERE\n"
│ └── unescaped: "ab\n" │ └── unescaped: "a\\\nb\n"
├── @ XStringNode (location: (56,0)-(56,9)) ├── @ XStringNode (location: (56,0)-(56,9))
│ ├── opening_loc: (56,0)-(56,9) = "<<-`HERE`" │ ├── opening_loc: (56,0)-(56,9) = "<<-`HERE`"
│ ├── content_loc: (57,0)-(58,0) = "a\\\nb\n" │ ├── content_loc: (57,0)-(58,0) = "a\\\nb\n"

8
test/prism/unescape_test.rb
View file

@ -102,6 +102,14 @@ module Prism
[Context::String.new("%[", "]"), escapes], [Context::String.new("%[", "]"), escapes],
[Context::String.new("`", "`"), escapes], [Context::String.new("`", "`"), escapes],
[Context::String.new("%x[", "]"), escapes], [Context::String.new("%x[", "]"), escapes],
[Context::String.new("<<H\n", "\nH"), escapes],
[Context::String.new("<<'H'\n", "\nH"), escapes],
[Context::String.new("<<\"H\"\n", "\nH"), escapes],
[Context::String.new("<<`H`\n", "\nH"), escapes],
[Context::String.new("<<-H\n", "\nH"), escapes],
[Context::String.new("<<-'H'\n", "\nH"), escapes],
[Context::String.new("<<-\"H\"\n", "\nH"), escapes],
[Context::String.new("<<-`H`\n", "\nH"), escapes],
# [Context::String.new("<<~H\n", "\nH"), escapes], # [Context::String.new("<<~H\n", "\nH"), escapes],
# [Context::String.new("<<~'H'\n", "\nH"), escapes], # [Context::String.new("<<~'H'\n", "\nH"), escapes],
# [Context::String.new("<<~\"H\"\n", "\nH"), escapes], # [Context::String.new("<<~\"H\"\n", "\nH"), escapes],