Merge 078b17da92 into 5e8f9ea495

compile.c

@@ -978,7 +978,7 @@ rb_iseq_compile_node(rb_iseq_t *iseq, const NODE *node)
 static int
 rb_iseq_translate_threaded_code(rb_iseq_t *iseq)
 {
-#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
+#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
     const void * const *table = rb_vm_get_insns_address_table();
     unsigned int i;
     VALUE *encoded = (VALUE *)ISEQ_BODY(iseq)->iseq_encoded;
@@ -1009,7 +1009,7 @@ rb_iseq_original_iseq(const rb_iseq_t *iseq) /* cold path */
     original_code = ISEQ_ORIGINAL_ISEQ_ALLOC(iseq, ISEQ_BODY(iseq)->iseq_size);
     MEMCPY(original_code, ISEQ_BODY(iseq)->iseq_encoded, VALUE, ISEQ_BODY(iseq)->iseq_size);
 
-#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
+#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
     {
         unsigned int i;
 

insns.def

@@ -1164,7 +1164,7 @@ leave
     }
 
     if (vm_pop_frame(ec, GET_CFP(), GET_EP())) {
-#if OPT_CALL_THREADED_CODE
+#if OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
         rb_ec_thread_ptr(ec)->retval = val;
         return 0;
 #else
@@ -1700,7 +1700,7 @@ opt_invokebuiltin_delegate_leave
     /* leave fastpath */
     /* TracePoint/return fallbacks this insn to opt_invokebuiltin_delegate */
     if (vm_pop_frame(ec, GET_CFP(), GET_EP())) {
-#if OPT_CALL_THREADED_CODE
+#if OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
         rb_ec_thread_ptr(ec)->retval = val;
         return 0;
 #else

iseq.c

@@ -3786,7 +3786,7 @@ rb_free_encoded_insn_data(void)
 void
 rb_vm_encoded_insn_data_table_init(void)
 {
-#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
+#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
     const void * const *table = rb_vm_get_insns_address_table();
 #define INSN_CODE(insn) ((VALUE)table[insn])
 #else
@@ -3871,7 +3871,7 @@ rb_vm_insn_addr2opcode(const void *addr)
 int
 rb_vm_insn_decode(const VALUE encoded)
 {
-#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
+#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
     int insn = rb_vm_insn_addr2insn((void *)encoded);
 #else
     int insn = (int)encoded;

vm.c

@@ -3719,7 +3719,7 @@ th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm)
 
     th->ec->storage = Qnil;
 
-#if OPT_CALL_THREADED_CODE
+#if OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
     th->retval = Qundef;
 #endif
     th->name = Qnil;
@@ -4234,7 +4234,7 @@ Init_VM(void)
     rb_ary_push(opts, rb_str_new2("direct threaded code"));
 #elif OPT_TOKEN_THREADED_CODE
     rb_ary_push(opts, rb_str_new2("token threaded code"));
-#elif OPT_CALL_THREADED_CODE
+#elif OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
     rb_ary_push(opts, rb_str_new2("call threaded code"));
 #endif
 

vm_core.h

@@ -204,7 +204,7 @@ void *rb_register_sigaltstack(void *);
 #endif
 
 /* call threaded code */
-#if    OPT_CALL_THREADED_CODE
+#if    OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
 #if    OPT_DIRECT_THREADED_CODE
 #undef OPT_DIRECT_THREADED_CODE
 #endif /* OPT_DIRECT_THREADED_CODE */
@@ -1159,7 +1159,7 @@ typedef struct rb_thread_struct {
     VALUE value;
 
     /* temporary place of retval on OPT_CALL_THREADED_CODE */
-#if OPT_CALL_THREADED_CODE
+#if OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
     VALUE retval;
 #endif
 

vm_exec.c

@@ -40,7 +40,7 @@ static void vm_analysis_insn(int insn);
 #endif
 /* #define DECL_SC_REG(r, reg) VALUE reg_##r */
 
-#if !OPT_CALL_THREADED_CODE
+#if !OPT_CALL_THREADED_CODE && !OPT_TAILCALL_THREADED_CODE
 static VALUE
 vm_exec_core(rb_execution_context_t *ec)
 {
@@ -115,7 +115,23 @@ rb_vm_get_insns_address_table(void)
     return (const void **)vm_exec_core(0);
 }
 
-#else /* OPT_CALL_THREADED_CODE */
+#else /* OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE */
+
+#if OPT_TAILCALL_THREADED_CODE
+#undef  RESTORE_REGS
+#define RESTORE_REGS() \
+{ \
+  VM_REG_CFP = ec->cfp; \
+  reg_pc  = reg_cfp->pc; \
+}
+
+#undef  VM_REG_PC
+#define VM_REG_PC reg_pc
+#undef  GET_PC
+#define GET_PC() (reg_pc)
+#undef  SET_PC
+#define SET_PC(x) (reg_cfp->pc = VM_REG_PC = (x))
+#endif
 
 #include "vm.inc"
 #include "vmtc.inc"
@@ -132,6 +148,12 @@ vm_exec_core(rb_execution_context_t *ec)
     register rb_control_frame_t *reg_cfp = ec->cfp;
     rb_thread_t *th;
 
+#ifdef OPT_TAILCALL_THREADED_CODE
+    const VALUE *reg_pc = reg_cfp->pc;
+    reg_cfp = ((rb_insn_tailcall_func_t *) (*GET_PC()))(INSN_FUNC_ARGS);
+
+    RUBY_ASSERT_ALWAYS(reg_cfp == 0);
+#else
     while (1) {
         reg_cfp = ((rb_insn_func_t) (*GET_PC()))(ec, reg_cfp);
 
@@ -139,6 +161,7 @@ vm_exec_core(rb_execution_context_t *ec)
             break;
         }
     }
+#endif
 
     if (!UNDEF_P((th = rb_ec_thread_ptr(ec))->retval)) {
         VALUE ret = th->retval;

vm_exec.h

@@ -11,6 +11,8 @@
 
 **********************************************************************/
 
+#include "ruby/internal/has/attribute.h"
+
 typedef long OFFSET;
 typedef unsigned long lindex_t;
 typedef VALUE GENTRY;
@@ -57,6 +59,55 @@ error !
 #define START_OF_ORIGINAL_INSN(x) /* ignore */
 #define DISPATCH_ORIGINAL_INSN(x) return LABEL(x)(ec, reg_cfp);
 
+/************************************************/
+#elif OPT_TAILCALL_THREADED_CODE
+
+#if !RBIMPL_HAS_ATTRIBUTE(musttail)
+#error support for musttail attribute is required for tailcall threading
+#endif
+
+/* Declares that the function call MUST be tailcall optimized */
+#define MUSTTAIL __attribute__((musttail))
+
+#define LABEL(x)  insn_func_##x
+#define ELABEL(x)
+#define LABEL_PTR(x) &LABEL(x)
+
+#if RBIMPL_HAS_ATTRIBUTE(preserve_none)
+#define ATTR_PRESERVE_NONE __attribute__((preserve_none))
+#endif
+
+#ifdef ATTR_PRESERVE_NONE
+#define INSN_FUNC_CONV ATTR_PRESERVE_NONE
+#else
+#define INSN_FUNC_CONV
+#endif
+
+#define INSN_FUNC_RET rb_control_frame_t *
+#define INSN_FUNC_PARAMS rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *reg_pc
+#define INSN_FUNC_ARGS ec, reg_cfp, reg_pc
+
+typedef INSN_FUNC_CONV INSN_FUNC_RET rb_insn_tailcall_func_t(INSN_FUNC_PARAMS);
+
+#define INSN_FUNC_ATTRIBUTES \
+    __attribute__((no_stack_protector))
+
+#define INSN_ENTRY(insn) \
+  static INSN_FUNC_CONV INSN_FUNC_ATTRIBUTES INSN_FUNC_RET \
+    FUNC_FASTCALL(LABEL(insn))(INSN_FUNC_PARAMS) {
+
+#define TC_DISPATCH(insn) \
+  MUSTTAIL return (*(rb_insn_tailcall_func_t *)GET_CURRENT_INSN())(INSN_FUNC_ARGS);
+
+//#define END_INSN(insn) return reg_cfp;}
+#define END_INSN(insn) TC_DISPATCH(__NEXT_INSN__);}
+
+//#define NEXT_INSN() return reg_cfp;
+#define NEXT_INSN() TC_DISPATCH(__NEXT_INSN__)
+
+#define START_OF_ORIGINAL_INSN(x) /* ignore */
+#define DISPATCH_ORIGINAL_INSN(x) MUSTTAIL return LABEL(x)(INSN_FUNC_ARGS);
+
 /************************************************/
 #elif OPT_TOKEN_THREADED_CODE || OPT_DIRECT_THREADED_CODE
 /* threaded code with gcc */
@@ -156,7 +207,7 @@ default:                        \
 
 #define VM_SP_CNT(ec, sp) ((sp) - (ec)->vm_stack)
 
-#if OPT_CALL_THREADED_CODE
+#if OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
 #define THROW_EXCEPTION(exc) do { \
     ec->errinfo = (VALUE)(exc); \
     return 0; \

vm_opts.h

@@ -10,6 +10,8 @@
 
 **********************************************************************/
 
+#include "ruby/internal/has/attribute.h"
+
 /* Compile options.
  * You can change these options at runtime by VM::CompileOption.
  * Following definitions are default values.
@@ -32,14 +34,20 @@
  * 0: direct (using labeled goto using GCC special)
  * 1: token (switch/case)
  * 2: call (function call for each insn dispatch)
+ * 3: call continuation (musttail attribute)
  */
 #ifndef OPT_THREADED_CODE
+#if RBIMPL_HAS_ATTRIBUTE(musttail)
+#define OPT_THREADED_CODE 3
+#else
 #define OPT_THREADED_CODE 0
 #endif
+#endif
 
-#define OPT_DIRECT_THREADED_CODE (OPT_THREADED_CODE == 0)
-#define OPT_TOKEN_THREADED_CODE  (OPT_THREADED_CODE == 1)
-#define OPT_CALL_THREADED_CODE   (OPT_THREADED_CODE == 2)
+#define OPT_DIRECT_THREADED_CODE   (OPT_THREADED_CODE == 0)
+#define OPT_TOKEN_THREADED_CODE    (OPT_THREADED_CODE == 1)
+#define OPT_CALL_THREADED_CODE     (OPT_THREADED_CODE == 2)
+#define OPT_TAILCALL_THREADED_CODE (OPT_THREADED_CODE == 3)
 
 /* VM running option */
 #define OPT_CHECKED_RUN              1