ZJIT: Start testing against /test/ruby/ and update all ZJIT related testing targets/docs (#13791)

* ZJIT: Add test exclusions for ZJIT

* ZJIT: Update test targets and documentation

- Rename `zjit-test-all` to `zjit-check`
- Add `zjit-test-all` target to zjit.mk to run all Ruby tests with ZJIT enabled
  excluding known failing tests
- Update documentation and CI workflow to reflect the new targets

.github/workflows/zjit-macos.yml

@@ -38,9 +38,8 @@ jobs:
           - test_task: 'ruby' # build test for combo build
             configure: '--enable-yjit --enable-zjit'
 
-          - test_task: 'test-all'
+          - test_task: 'zjit-test-all'
             configure: '--enable-zjit=dev'
-            tests: '../src/test/ruby/test_zjit.rb'
 
           - test_task: 'btest'
             configure: '--enable-zjit=dev'

.github/workflows/zjit-ubuntu.yml

@@ -40,9 +40,8 @@ jobs:
           - test_task: 'zjit-test'
             configure: '--enable-yjit --enable-zjit=dev'
 
-          - test_task: 'test-all'
+          - test_task: 'zjit-test-all'
             configure: '--enable-zjit=dev'
-            tests: '../src/test/ruby/test_zjit.rb'
 
           - test_task: 'btest'
             configure: '--enable-zjit=dev'

doc/zjit.md

@@ -28,12 +28,12 @@ in a way that can be easily shared with other team members.
 
 Make sure you have a `--enable-zjit=dev` build, and run `brew install cargo-nextest` first.
 
-### make zjit-test-all
+### make zjit-check
 
 This command runs all ZJIT tests: `make zjit-test` and `test/ruby/test_zjit.rb`.
 
 ```
-make zjit-test-all
+make zjit-check
 ```
 
 ### make zjit-test
@@ -78,6 +78,16 @@ use `make`.
 
 </details>
 
+### make zjit-test-all
+
+```
+make zjit-test-all
+```
+
+This command runs all Ruby tests under `/test/ruby/` with ZJIT enabled.
+
+Certain tests are excluded under `/test/.excludes-zjit`.
+
 ### test/ruby/test\_zjit.rb
 
 This command runs Ruby execution tests.

test/.excludes-zjit/TestArgf.rb

@@ -0,0 +1 @@
+exclude(:test_skip_in_each_byte, 'Test fails with ZJIT')

test/.excludes-zjit/TestArray.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Tests make ZJIT panic')

test/.excludes-zjit/TestBignum.rb

@@ -0,0 +1 @@
+exclude(:test_quad_pack, 'Test fails with ZJIT')

test/.excludes-zjit/TestEncoding.rb

@@ -0,0 +1 @@
+exclude(/test_ractor/, "Tests make ZJIT panic")

test/.excludes-zjit/TestEnumerator.rb

@@ -0,0 +1 @@
+exclude(:test_size_for_loops, 'Test crashes on Ubuntu with ZJIT')

test/.excludes-zjit/TestEnv.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Multiple tests make ZJIT panic')

test/.excludes-zjit/TestFixnum.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Tests make ZJIT panic')

test/.excludes-zjit/TestInteger.rb

@@ -0,0 +1 @@
+exclude(:test_ceildiv, 'Test crashes on Ubuntu with ZJIT')

test/.excludes-zjit/TestIntegerComb.rb

@@ -0,0 +1,2 @@
+exclude(:test_pack_utf8, 'Test crashes with ZJIT')
+exclude(:test_pack_ber, 'Test fails with ZJIT')

test/.excludes-zjit/TestKeywordArguments.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Multiple tests make ZJIT panic')

test/.excludes-zjit/TestMemoryView.rb

@@ -0,0 +1,8 @@
+exclude(:test_rb_memory_view_get_item_pointer_multiple_members, 'Test crashes with ZJIT')
+exclude(:test_rb_memory_view_extract_item_members_doble, 'Test crashes with ZJIT')
+exclude(:test_rb_memory_view_extract_item_members_float_endianness, 'Test crashes with ZJIT')
+exclude(:test_rb_memory_view_get_item_pointer_single_member, 'Test crashes with ZJIT')
+exclude(:test_rb_memory_view_extract_item_members_doble_endianness, 'Test crashes with ZJIT')
+
+exclude(:test_rb_memory_view_extract_item_members_endianness, 'Test fails with ZJIT')
+exclude(:test_rb_memory_view_extract_item_members, 'Test fails with ZJIT')

test/.excludes-zjit/TestPack.rb

@@ -0,0 +1,32 @@
+exclude(:test_ascii_incompatible, 'Test crashes with ZJIT')
+exclude(:test_pack_U, 'Test crashes with ZJIT')
+exclude(:test_unpack_garbage, 'Test crashes with ZJIT')
+exclude(:test_pack_garbage, 'Test crashes with ZJIT')
+exclude(:test_pack, 'Test crashes with ZJIT')
+
+exclude(:test_pack_unpack_lL, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_x, 'Test fails with ZJIT')
+exclude(:test_unpack_n, 'Test fails with ZJIT')
+exclude(:test_pack_N, 'Test fails with ZJIT')
+exclude(:test_integer_endian, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_m0, 'Test fails with ZJIT')
+exclude(:test_pack_p2, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_X, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_nN, 'Test fails with ZJIT')
+exclude(:test_unpack1, 'Test fails with ZJIT')
+exclude(:test_pack_n, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_percent, 'Test fails with ZJIT')
+exclude(:test_unpack_N, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_cC, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_vV, 'Test fails with ZJIT')
+exclude(:test_format_string_modified, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_atmark, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_iI, 'Test fails with ZJIT')
+exclude(:test_illegal_bang, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_U, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_fdeEgG, 'Test fails with ZJIT')
+exclude(:test_invalid_warning, 'Test fails with ZJIT')
+exclude(:test_short_string, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_sS, 'Test fails with ZJIT')
+exclude(:test_comment, 'Test fails with ZJIT')
+exclude(:test_pack_unpack_w, 'Test fails with ZJIT')

test/.excludes-zjit/TestParse.rb

@@ -0,0 +1,2 @@
+exclude(:test_flip_flop, 'Test fails with ZJIT')
+exclude(:test_rescue_in_command_assignment, 'Test fails with ZJIT')

test/.excludes-zjit/TestProc.rb

@@ -0,0 +1,6 @@
+exclude(/test_/, 'Tests make ZJIT panic')
+
+# exclude(:test_proc_args_pos_rest_block, 'Test crashes with ZJIT')
+# exclude(:test_proc_args_rest_post_block, 'Test crashes with ZJIT')
+
+# exclude(:test_binding_receiver, 'Test fails with ZJIT')

test/.excludes-zjit/TestRactor.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Tests make ZJIT panic')

test/.excludes-zjit/TestRefinement.rb

@@ -0,0 +1 @@
+exclude(:test_override_builtin_method_with_method_added, 'Test fails with ZJIT')

test/.excludes-zjit/TestRegexp.rb

@@ -0,0 +1,4 @@
+exclude(:test_inspect, 'Test fails with ZJIT')
+exclude(:test_quote, 'Test fails with ZJIT')
+exclude(:test_union, 'Test fails with ZJIT')
+exclude(:test_unescape, 'Test fails with ZJIT')

test/.excludes-zjit/TestRubyOptions.rb

@@ -0,0 +1,10 @@
+exclude(:test_verbose, 'Test crashes with ZJIT')
+exclude(:test_segv_setproctitle, 'Test crashes with ZJIT')
+exclude(:test_crash_report, 'Test crashes with ZJIT')
+exclude(:test_segv_loaded_features, 'Test crashes with ZJIT')
+exclude(:test_crash_report_executable_path, 'Test crashes with ZJIT')
+exclude(:test_segv_test, 'Test crashes with ZJIT')
+exclude(:test_crash_report_script, 'Test crashes with ZJIT')
+exclude(:test_crash_report_script_path, 'Test crashes with ZJIT')
+
+exclude(:test_version, 'Test fails with ZJIT')

test/.excludes-zjit/TestSetTraceFunc.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Test fails with ZJIT intermittently')

test/.excludes-zjit/TestSprintfComb.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Test fails with ZJIT')

test/.excludes-zjit/TestString.rb

@@ -0,0 +1 @@
+exclude(:test_to_f, 'Test crashes with ZJIT')

test/.excludes-zjit/TestString2.rb

@@ -0,0 +1 @@
+exclude(:test_to_f, 'Test crashes with ZJIT')

test/.excludes-zjit/TestThread.rb

@@ -0,0 +1,2 @@
+exclude(:test_switch_while_busy_loop, 'Test hangs with ZJIT')
+exclude(:test_handle_interrupted?, 'Test fails with ZJIT')

test/.excludes-zjit/TestTime.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Tests make ZJIT panic')

test/.excludes-zjit/TestTimeTZ.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Tests make ZJIT panic')

test/.excludes-zjit/TestVariable.rb

@@ -0,0 +1 @@
+exclude(/test_/, 'Tests make ZJIT panic')

zjit/zjit.mk

@@ -49,11 +49,15 @@ update-zjit-bench:
 		https://github.com/Shopify/zjit-bench zjit-bench $(GIT_OPTS)
 
 # Gives quick feedback about ZJIT. Not a replacement for a full test run.
-.PHONY: zjit-test-all
-zjit-test-all:
+.PHONY: zjit-check
+zjit-check:
 	$(MAKE) zjit-test
 	$(MAKE) test-all TESTS='$(top_srcdir)/test/ruby/test_zjit.rb'
 
+.PHONY: zjit-test-all
+zjit-test-all:
+	$(MAKE) test-all RUST_BACKTRACE=1 TEST_EXCLUDES='--excludes-dir=$(top_srcdir)/test/.excludes-zjit --name=!/memory_leak/' RUN_OPTS='--zjit-call-threshold=1' TESTS='$(top_srcdir)/test/ruby/'
+
 ZJIT_BINDGEN_DIFF_OPTS =
 
 # Generate Rust bindings. See source for details.