[Universal Parser] Reduce dependence on RArray in parse.y

- Introduce `rb_parser_ary_t` structure to partly eliminate RArray from parse.y
  - In this patch, `parser_params->tokens` and `parser_params->ast->node_buffer->tokens` are now `rb_parser_ary_t *`
  - Instead, `ast_node_all_tokens()` internally creates a Ruby Array object from the `rb_parser_ary_t`
  - Also, delete `rb_ast_tokens()` and `rb_ast_set_tokens()` in node.c

- Implement `rb_parser_str_escape()`
  - This is a port of the `rb_str_escape()` function in string.c
  - `rb_parser_str_escape()` does not depend on `VALUE` (RString)
  - Instead, it uses `rb_parser_stirng_t *`
  - This function works when --dump=y option passed

- Because WIP of the universal parser, similar functions like `rb_parser_tokens_free()` exist in both node.c and parse.y. Refactoring them may be needed in some way in the future

- Although we considered redesigning the structure: `ast->node_buffer->tokens` into `ast->tokens`, we leave it as it is because `rb_ast_t` is an imemo. (We will address it in the future)
This commit is contained in:
HASUMI Hitoshi 2024-02-16 17:45:22 +09:00 committed by Yuichiro Kaneko
parent f42164e037
commit 9a19cfd4cd
7 changed files with 428 additions and 214 deletions

View file

@ -461,6 +461,27 @@ str_coderange_scan_restartable(const char *s, const char *e, void *enc, int *cr)
return rb_str_coderange_scan_restartable(s, e, (rb_encoding *)enc, cr);
}
static int
enc_mbminlen(void *enc)
{
return rb_enc_mbminlen((rb_encoding *)enc);
}
static bool
enc_isascii(OnigCodePoint c, void *enc)
{
return rb_enc_isascii(c, (rb_encoding *)enc);
}
static OnigCodePoint
enc_mbc_to_codepoint(const char *p, const char *e, void *enc)
{
const OnigUChar *up = RBIMPL_CAST((const OnigUChar *)p);
const OnigUChar *ue = RBIMPL_CAST((const OnigUChar *)e);
return ONIGENC_MBC_TO_CODE((rb_encoding *)enc, up, ue);
}
VALUE rb_io_gets_internal(VALUE io);
extern VALUE rb_eArgError;
extern VALUE rb_mRubyVMFrozenCore;
@ -596,6 +617,10 @@ static const rb_parser_config_t rb_global_parser_config = {
.encoding_set = encoding_set,
.encoding_is_ascii8bit = encoding_is_ascii8bit,
.usascii_encoding = usascii_encoding,
.enc_coderange_broken = ENC_CODERANGE_BROKEN,
.enc_mbminlen = enc_mbminlen,
.enc_isascii = enc_isascii,
.enc_mbc_to_codepoint = enc_mbc_to_codepoint,
.ractor_make_shareable = rb_ractor_make_shareable,