From 8a8d889ca2ba1ace8784e49e02a6616dd5765d9e Mon Sep 17 00:00:00 2001 From: "U.Nakamura" Date: Tue, 5 Sep 2023 20:47:58 +0900 Subject: [PATCH] merge revision(s) 1f115f141dd17f75049a5e17107906c5bcc372e1: [Backport #19246] Speed up rebuilding the loaded feature index Rebuilding the loaded feature index slowed down with the bug fix for #17885 in 79a4484a072e9769b603e7b4fbdb15b1d7eccb15. The slowdown was extreme if realpath emulation was used, but even when not emulated, it could be about 10x slower. This adds loaded_features_realpath_map to rb_vm_struct. This is a hidden hash mapping loaded feature paths to realpaths. When rebuilding the loaded feature index, look at this hash to get cached realpath values, and skip calling rb_check_realpath if a cached value is found. Fixes [Bug #19246] --- load.c | 27 +++++++++++++++++++++++---- vm.c | 2 ++ vm_core.h | 1 + 3 files changed, 26 insertions(+), 4 deletions(-) --- .github/workflows/mingw.yml | 4 +-- load.c | 49 +++++++++++++++++++++++++------------ version.h | 2 +- vm.c | 2 ++ vm_core.h | 1 + 5 files changed, 40 insertions(+), 18 deletions(-) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 3e6b77c58f..ba0b9c25a6 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -124,16 +124,16 @@ jobs: timeout-minutes: 5 run: | make test + shell: cmd - name: test-all timeout-minutes: 45 run: | - # Actions uses UTF8, causes test failures, similar to normal OS setup - chcp.com 437 make test-all env: RUBY_TESTOPTS: -j${{env.TEST_JOBS}} --retry --job-status=normal --show-skip --timeout-scale=1.5 BUNDLER_VERSION: + shell: cmd - name: test-spec timeout-minutes: 10 diff --git a/load.c b/load.c index 48c7bc4949..ba6f0839af 100644 --- a/load.c +++ b/load.c @@ -162,6 +162,12 @@ get_loaded_features_realpaths(rb_vm_t *vm) return vm->loaded_features_realpaths; } +static VALUE +get_loaded_features_realpath_map(rb_vm_t *vm) +{ + return vm->loaded_features_realpath_map; +} + static VALUE get_LOADED_FEATURES(ID _x, VALUE *_y) { @@ -327,26 +333,34 @@ get_loaded_features_index(rb_vm_t *vm) st_foreach(vm->loaded_features_index, loaded_features_index_clear_i, 0); VALUE realpaths = vm->loaded_features_realpaths; + VALUE realpath_map = vm->loaded_features_realpath_map; + VALUE previous_realpath_map = rb_hash_dup(realpath_map); rb_hash_clear(realpaths); - features = vm->loaded_features; - for (i = 0; i < RARRAY_LEN(features); i++) { - VALUE entry, as_str; - as_str = entry = rb_ary_entry(features, i); - StringValue(as_str); - as_str = rb_fstring(rb_str_freeze(as_str)); - if (as_str != entry) - rb_ary_store(features, i, as_str); - features_index_add(vm, as_str, INT2FIX(i)); - } - reset_loaded_features_snapshot(vm); + rb_hash_clear(realpath_map); + features = vm->loaded_features; + for (i = 0; i < RARRAY_LEN(features); i++) { + VALUE entry, as_str; + as_str = entry = rb_ary_entry(features, i); + StringValue(as_str); + as_str = rb_fstring(as_str); + if (as_str != entry) + rb_ary_store(features, i, as_str); + features_index_add(vm, as_str, INT2FIX(i)); + } + reset_loaded_features_snapshot(vm); features = rb_ary_dup(vm->loaded_features_snapshot); long j = RARRAY_LEN(features); for (i = 0; i < j; i++) { VALUE as_str = rb_ary_entry(features, i); - VALUE realpath = rb_check_realpath(Qnil, as_str, NULL); - if (NIL_P(realpath)) realpath = as_str; - rb_hash_aset(realpaths, rb_fstring(realpath), Qtrue); + VALUE realpath = rb_hash_aref(previous_realpath_map, as_str); + if (NIL_P(realpath)) { + realpath = rb_check_realpath(Qnil, as_str, NULL); + if (NIL_P(realpath)) realpath = as_str; + realpath = rb_fstring(realpath); + } + rb_hash_aset(realpaths, realpath, Qtrue); + rb_hash_aset(realpath_map, as_str, realpath); } } return vm->loaded_features_index; @@ -1122,6 +1136,7 @@ require_internal(rb_execution_context_t *ec, VALUE fname, int exception, bool wa volatile VALUE saved_path; volatile VALUE realpath = 0; VALUE realpaths = get_loaded_features_realpaths(th->vm); + VALUE realpath_map = get_loaded_features_realpath_map(th->vm); volatile bool reset_ext_config = false; struct rb_ext_config prev_ext_config; @@ -1213,7 +1228,9 @@ require_internal(rb_execution_context_t *ec, VALUE fname, int exception, bool wa rb_provide_feature(th2->vm, path); VALUE real = realpath; if (real) { - rb_hash_aset(realpaths, rb_fstring(real), Qtrue); + real = rb_fstring(real); + rb_hash_aset(realpaths, real, Qtrue); + rb_hash_aset(realpath_map, path, real); } } ec->errinfo = saved.errinfo; @@ -1426,6 +1443,8 @@ Init_load(void) vm->loaded_features_index = st_init_numtable(); vm->loaded_features_realpaths = rb_hash_new(); rb_obj_hide(vm->loaded_features_realpaths); + vm->loaded_features_realpath_map = rb_hash_new(); + rb_obj_hide(vm->loaded_features_realpath_map); rb_define_global_function("load", rb_f_load, -1); rb_define_global_function("require", rb_f_require, 1); diff --git a/version.h b/version.h index e151fd9496..1b22eb2527 100644 --- a/version.h +++ b/version.h @@ -11,7 +11,7 @@ # define RUBY_VERSION_MINOR RUBY_API_VERSION_MINOR #define RUBY_VERSION_TEENY 4 #define RUBY_RELEASE_DATE RUBY_RELEASE_YEAR_STR"-"RUBY_RELEASE_MONTH_STR"-"RUBY_RELEASE_DAY_STR -#define RUBY_PATCHLEVEL 237 +#define RUBY_PATCHLEVEL 238 #define RUBY_RELEASE_YEAR 2023 #define RUBY_RELEASE_MONTH 9 diff --git a/vm.c b/vm.c index 0126f0c6de..12634d687e 100644 --- a/vm.c +++ b/vm.c @@ -2539,6 +2539,7 @@ rb_vm_update_references(void *ptr) vm->loaded_features = rb_gc_location(vm->loaded_features); vm->loaded_features_snapshot = rb_gc_location(vm->loaded_features_snapshot); vm->loaded_features_realpaths = rb_gc_location(vm->loaded_features_realpaths); + vm->loaded_features_realpath_map = rb_gc_location(vm->loaded_features_realpath_map); vm->top_self = rb_gc_location(vm->top_self); vm->orig_progname = rb_gc_location(vm->orig_progname); @@ -2630,6 +2631,7 @@ rb_vm_mark(void *ptr) rb_gc_mark_movable(vm->loaded_features); rb_gc_mark_movable(vm->loaded_features_snapshot); rb_gc_mark_movable(vm->loaded_features_realpaths); + rb_gc_mark_movable(vm->loaded_features_realpath_map); rb_gc_mark_movable(vm->top_self); rb_gc_mark_movable(vm->orig_progname); RUBY_MARK_MOVABLE_UNLESS_NULL(vm->coverages); diff --git a/vm_core.h b/vm_core.h index 1cc0659700..0b736bda02 100644 --- a/vm_core.h +++ b/vm_core.h @@ -681,6 +681,7 @@ typedef struct rb_vm_struct { VALUE loaded_features; VALUE loaded_features_snapshot; VALUE loaded_features_realpaths; + VALUE loaded_features_realpath_map; struct st_table *loaded_features_index; struct st_table *loading_table; #if EXTSTATIC