Implement duparray and expandarray

Merged: https://github.com/ruby/ruby/pull/7448

lib/ruby_vm/mjit/assembler.rb

@@ -121,6 +121,16 @@ module RubyVM::MJIT
           mod_rm: ModRM[mod: Mod11, reg: 4, rm: dst_reg],
           imm: imm8(src_imm),
         )
+      # AND r/m64, imm32 (Mod 11: reg)
+      in [Symbol => dst_reg, Integer => src_imm] if r64?(dst_reg) && imm32?(src_imm)
+        # REX.W + 81 /4 id
+        # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32
+        insn(
+          prefix: REX_W,
+          opcode: 0x81,
+          mod_rm: ModRM[mod: Mod11, reg: 4, rm: dst_reg],
+          imm: imm32(src_imm),
+        )
       # AND r64, r/m64 (Mod 01: [reg]+disp8)
       in [Symbol => dst_reg, [Symbol => src_reg, Integer => src_disp]] if r64?(dst_reg) && r64?(src_reg) && imm8?(src_disp)
         # REX.W + 23 /r
@@ -236,6 +246,48 @@ module RubyVM::MJIT
       end
     end
 
+    def cmovnz(dst, src)
+      case [dst, src]
+      # CMOVNZ r64, r/m64 (Mod 11: reg)
+      in [Symbol => dst_reg, Symbol => src_reg] if r64?(dst_reg) && r64?(src_reg)
+        # REX.W + 0F 45 /r
+        # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)
+        insn(
+          prefix: REX_W,
+          opcode: [0x0f, 0x45],
+          mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],
+        )
+      else
+        raise NotImplementedError, "cmovnz: not-implemented operands: #{dst.inspect}, #{src.inspect}"
+      end
+    end
+
+    def cmovz(dst, src)
+      case [dst, src]
+      # CMOVZ r64, r/m64 (Mod 11: reg)
+      in [Symbol => dst_reg, Symbol => src_reg] if r64?(dst_reg) && r64?(src_reg)
+        # REX.W + 0F 44 /r
+        # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)
+        insn(
+          prefix: REX_W,
+          opcode: [0x0f, 0x44],
+          mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],
+        )
+      # CMOVZ r64, r/m64 (Mod 01: [reg]+disp8)
+      in [Symbol => dst_reg, [Symbol => src_reg, Integer => src_disp]] if r64?(dst_reg) && r64?(src_reg) && imm8?(src_disp)
+        # REX.W + 0F 44 /r
+        # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)
+        insn(
+          prefix: REX_W,
+          opcode: [0x0f, 0x44],
+          mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg],
+          disp: imm8(src_disp),
+        )
+      else
+        raise NotImplementedError, "cmovz: not-implemented operands: #{dst.inspect}, #{src.inspect}"
+      end
+    end
+
     def cmp(left, right)
       case [left, right]
       # CMP r/m32, imm32 (Mod 01: [reg]+disp8)
@@ -319,6 +371,17 @@ module RubyVM::MJIT
       end
     end
 
+    def jl(dst)
+      case dst
+      # JL rel32
+      in Integer => dst_addr
+        # 0F 8C cd
+        insn(opcode: [0x0f, 0x8c], imm: rel32(dst_addr))
+      else
+        raise NotImplementedError, "jl: not-implemented operands: #{dst.inspect}"
+      end
+    end
+
     def jmp(dst)
       case dst
       # JZ rel8

lib/ruby_vm/mjit/insn_compiler.rb

@@ -23,7 +23,7 @@ module RubyVM::MJIT
       asm.incr_counter(:mjit_insns_count)
       asm.comment("Insn: #{insn.name}")
 
-      # 46/101
+      # 48/101
       case insn.name
       when :nop then nop(jit, ctx, asm)
       when :getlocal then getlocal(jit, ctx, asm)
@@ -53,9 +53,9 @@ module RubyVM::MJIT
       # intern
       # newarray
       # newarraykwsplat
-      # duparray
+      when :duparray then duparray(jit, ctx, asm)
       # duphash
-      # expandarray
+      when :expandarray then expandarray(jit, ctx, asm)
       # concatarray
       # splatarray
       # newhash
@@ -299,9 +299,97 @@ module RubyVM::MJIT
     # intern
     # newarray
     # newarraykwsplat
-    # duparray
+
+    # @param jit [RubyVM::MJIT::JITState]
+    # @param ctx [RubyVM::MJIT::Context]
+    # @param asm [RubyVM::MJIT::Assembler]
+    def duparray(jit, ctx, asm)
+      ary = jit.operand(0)
+
+      # Save the PC and SP because we are allocating
+      jit_prepare_routine_call(jit, ctx, asm)
+
+      # call rb_ary_resurrect(VALUE ary);
+      asm.comment('call rb_ary_resurrect')
+      asm.mov(C_ARGS[0], ary)
+      asm.call(C.rb_ary_resurrect)
+
+      stack_ret = ctx.stack_push
+      asm.mov(stack_ret, C_RET)
+
+      KeepCompiling
+    end
+
     # duphash
-    # expandarray
+
+    # @param jit [RubyVM::MJIT::JITState]
+    # @param ctx [RubyVM::MJIT::Context]
+    # @param asm [RubyVM::MJIT::Assembler]
+    def expandarray(jit, ctx, asm)
+      # Both arguments are rb_num_t which is unsigned
+      num = jit.operand(0)
+      flag = jit.operand(1)
+
+      # If this instruction has the splat flag, then bail out.
+      if flag & 0x01 != 0
+        asm.incr_counter(:expandarray_splat)
+        return CantCompile
+      end
+
+      # If this instruction has the postarg flag, then bail out.
+      if flag & 0x02 != 0
+        asm.incr_counter(:expandarray_postarg)
+        return CantCompile
+      end
+
+      side_exit = side_exit(jit, ctx)
+
+      array_opnd = ctx.stack_pop(1)
+
+      # num is the number of requested values. If there aren't enough in the
+      # array then we're going to push on nils.
+      # TODO: implement this
+
+      # Move the array from the stack and check that it's an array.
+      asm.mov(:rax, array_opnd)
+      guard_object_is_heap(asm, :rax, counted_exit(side_exit, :expandarray_not_array))
+      guard_object_is_array(asm, :rax, :rcx, counted_exit(side_exit, :expandarray_not_array))
+
+      # If we don't actually want any values, then just return.
+      if num == 0
+        return KeepCompiling
+      end
+
+      jit_array_len(asm, :rax, :rcx)
+
+      # Only handle the case where the number of values in the array is greater
+      # than or equal to the number of values requested.
+      asm.cmp(:rcx, num)
+      asm.jl(counted_exit(side_exit, :expandarray_rhs_too_small))
+
+      # Conditionally load the address of the heap array into REG1.
+      # (struct RArray *)(obj)->as.heap.ptr
+      #asm.mov(:rax, array_opnd)
+      asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)])
+      asm.test(:rcx, C.RARRAY_EMBED_FLAG);
+      asm.mov(:rcx, [:rax, C.RArray.offsetof(:as, :heap, :ptr)])
+
+      # Load the address of the embedded array into REG1.
+      # (struct RArray *)(obj)->as.ary
+      asm.lea(:rax, [:rax, C.RArray.offsetof(:as, :ary)])
+
+      asm.cmovnz(:rcx, :rax)
+
+      # Loop backward through the array and push each element onto the stack.
+      (num - 1).downto(0).each do |i|
+        top = ctx.stack_push
+        asm.mov(:rax, [:rcx, i * C.VALUE.size])
+        asm.mov(top, :rax)
+      end
+
+      KeepCompiling
+    end
+
     # concatarray
     # splatarray
     # newhash
@@ -1156,6 +1244,18 @@ module RubyVM::MJIT
       asm.je(side_exit)
     end
 
+    # @param asm [RubyVM::MJIT::Assembler]
+    def guard_object_is_array(asm, object_reg, flags_reg, side_exit)
+      asm.comment('guard object is array')
+      # Pull out the type mask
+      asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)])
+      asm.and(flags_reg, C.RUBY_T_MASK)
+
+      # Compare the result with T_ARRAY
+      asm.cmp(flags_reg, C.RUBY_T_ARRAY)
+      asm.jne(side_exit)
+    end
+
     # @param jit [RubyVM::MJIT::JITState]
     # @param ctx [RubyVM::MJIT::Context]
     # @param asm [RubyVM::MJIT::Assembler]
@@ -1957,6 +2057,25 @@ module RubyVM::MJIT
       end
     end
 
+    # Generate RARRAY_LEN. For array_opnd, use Opnd::Reg to reduce memory access,
+    # and use Opnd::Mem to save registers.
+    def jit_array_len(asm, array_reg, len_reg)
+      asm.comment('get array length for embedded or heap')
+
+      # Pull out the embed flag to check if it's an embedded array.
+      asm.mov(len_reg, [array_reg, C.RBasic.offsetof(:flags)])
+
+      # Get the length of the array
+      asm.and(len_reg, C.RARRAY_EMBED_LEN_MASK)
+      asm.sar(len_reg, C.RARRAY_EMBED_LEN_SHIFT)
+
+      # Conditionally move the length of the heap array
+      asm.test([array_reg, C.RBasic.offsetof(:flags)], C.RARRAY_EMBED_FLAG)
+
+      # Select the array length value
+      asm.cmovz(len_reg, [array_reg, C.RArray.offsetof(:as, :heap, :len)])
+    end
+
     def assert_equal(left, right)
       if left != right
         raise "'#{left.inspect}' was not '#{right.inspect}'"

lib/ruby_vm/mjit/stats.rb

@@ -38,6 +38,7 @@ module RubyVM::MJIT
       print_counters(stats, prefix: 'getivar_', prompt: 'getinstancevariable exit reasons')
       print_counters(stats, prefix: 'optaref_', prompt: 'opt_aref exit reasons')
       print_counters(stats, prefix: 'optgetconst_', prompt: 'opt_getconstant_path exit reasons')
+      print_counters(stats, prefix: 'expandarray_', prompt: 'expandarray exit reasons')
 
       $stderr.puts "compiled_block_count:  #{format_number(13, stats[:compiled_block_count])}"
       $stderr.puts "side_exit_count:       #{format_number(13, stats[:side_exit_count])}"

mjit_c.h

@@ -160,6 +160,11 @@ MJIT_RUNTIME_COUNTERS(
     optgetconst_not_cached,
     optgetconst_cref,
 
+    expandarray_splat,
+    expandarray_postarg,
+    expandarray_not_array,
+    expandarray_rhs_too_small,
+
     compiled_block_count
 )
 #undef MJIT_RUNTIME_COUNTERS

mjit_c.rb

@@ -168,6 +168,10 @@ module RubyVM::MJIT # :nodoc: all
       Primitive.cexpr! 'SIZET2NUM((size_t)rb_str_eql_internal)'
     end
 
+    def rb_ary_resurrect
+      Primitive.cexpr! 'SIZET2NUM((size_t)rb_ary_resurrect)'
+    end
+
     #========================================================================================
     #
     # Old stuff
@@ -397,6 +401,10 @@ module RubyVM::MJIT # :nodoc: all
     Primitive.cexpr! %q{ UINT2NUM(METHOD_VISI_PUBLIC) }
   end
 
+  def C.RARRAY_EMBED_FLAG
+    Primitive.cexpr! %q{ UINT2NUM(RARRAY_EMBED_FLAG) }
+  end
+
   def C.ROBJECT_EMBED
     Primitive.cexpr! %q{ UINT2NUM(ROBJECT_EMBED) }
   end
@@ -565,6 +573,14 @@ module RubyVM::MJIT # :nodoc: all
     Primitive.cexpr! %q{ ULONG2NUM(OBJ_TOO_COMPLEX_SHAPE_ID) }
   end
 
+  def C.RARRAY_EMBED_LEN_MASK
+    Primitive.cexpr! %q{ ULONG2NUM(RARRAY_EMBED_LEN_MASK) }
+  end
+
+  def C.RARRAY_EMBED_LEN_SHIFT
+    Primitive.cexpr! %q{ ULONG2NUM(RARRAY_EMBED_LEN_SHIFT) }
+  end
+
   def C.RUBY_FIXNUM_FLAG
     Primitive.cexpr! %q{ ULONG2NUM(RUBY_FIXNUM_FLAG) }
   end
@@ -573,6 +589,14 @@ module RubyVM::MJIT # :nodoc: all
     Primitive.cexpr! %q{ ULONG2NUM(RUBY_IMMEDIATE_MASK) }
   end
 
+  def C.RUBY_T_ARRAY
+    Primitive.cexpr! %q{ ULONG2NUM(RUBY_T_ARRAY) }
+  end
+
+  def C.RUBY_T_MASK
+    Primitive.cexpr! %q{ ULONG2NUM(RUBY_T_MASK) }
+  end
+
   def C.SHAPE_MASK
     Primitive.cexpr! %q{ ULONG2NUM(SHAPE_MASK) }
   end
@@ -617,6 +641,27 @@ module RubyVM::MJIT # :nodoc: all
     @IVC ||= self.iseq_inline_iv_cache_entry
   end
 
+  def C.RArray
+    @RArray ||= CType::Struct.new(
+      "RArray", Primitive.cexpr!("SIZEOF(struct RArray)"),
+      basic: [self.RBasic, Primitive.cexpr!("OFFSETOF((*((struct RArray *)NULL)), basic)")],
+      as: [CType::Union.new(
+        "", Primitive.cexpr!("SIZEOF(((struct RArray *)NULL)->as)"),
+        heap: CType::Struct.new(
+          "", Primitive.cexpr!("SIZEOF(((struct RArray *)NULL)->as.heap)"),
+          len: [CType::Immediate.parse("long"), Primitive.cexpr!("OFFSETOF(((struct RArray *)NULL)->as.heap, len)")],
+          aux: [CType::Union.new(
+            "", Primitive.cexpr!("SIZEOF(((struct RArray *)NULL)->as.heap.aux)"),
+            capa: CType::Immediate.parse("long"),
+            shared_root: self.VALUE,
+          ), Primitive.cexpr!("OFFSETOF(((struct RArray *)NULL)->as.heap, aux)")],
+          ptr: [CType::Pointer.new { self.VALUE }, Primitive.cexpr!("OFFSETOF(((struct RArray *)NULL)->as.heap, ptr)")],
+        ),
+        ary: CType::Pointer.new { self.VALUE },
+      ), Primitive.cexpr!("OFFSETOF((*((struct RArray *)NULL)), as)")],
+    )
+  end
+
   def C.RB_BUILTIN
     @RB_BUILTIN ||= self.rb_builtin_function
   end
@@ -1080,6 +1125,10 @@ module RubyVM::MJIT # :nodoc: all
       optaref_send: [CType::Immediate.parse("size_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_mjit_runtime_counters *)NULL)), optaref_send)")],
       optgetconst_not_cached: [CType::Immediate.parse("size_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_mjit_runtime_counters *)NULL)), optgetconst_not_cached)")],
       optgetconst_cref: [CType::Immediate.parse("size_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_mjit_runtime_counters *)NULL)), optgetconst_cref)")],
+      expandarray_splat: [CType::Immediate.parse("size_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_mjit_runtime_counters *)NULL)), expandarray_splat)")],
+      expandarray_postarg: [CType::Immediate.parse("size_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_mjit_runtime_counters *)NULL)), expandarray_postarg)")],
+      expandarray_not_array: [CType::Immediate.parse("size_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_mjit_runtime_counters *)NULL)), expandarray_not_array)")],
+      expandarray_rhs_too_small: [CType::Immediate.parse("size_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_mjit_runtime_counters *)NULL)), expandarray_rhs_too_small)")],
       compiled_block_count: [CType::Immediate.parse("size_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_mjit_runtime_counters *)NULL)), compiled_block_count)")],
     )
   end

tool/mjit/bindgen.rb

@@ -367,6 +367,7 @@ generator = BindingGenerator.new(
       METHOD_VISI_PROTECTED
       METHOD_VISI_PUBLIC
       ROBJECT_EMBED
+      RARRAY_EMBED_FLAG
       RUBY_EVENT_CLASS
       RUBY_EVENT_C_CALL
       RUBY_EVENT_C_RETURN
@@ -411,7 +412,11 @@ generator = BindingGenerator.new(
       OBJ_TOO_COMPLEX_SHAPE_ID
       RUBY_FIXNUM_FLAG
       RUBY_IMMEDIATE_MASK
+      RARRAY_EMBED_LEN_MASK
+      RARRAY_EMBED_LEN_SHIFT
       SHAPE_MASK
+      RUBY_T_ARRAY
+      RUBY_T_MASK
     ],
     PTR: %w[
       rb_cFalseClass
@@ -428,6 +433,7 @@ generator = BindingGenerator.new(
     ID
     IVC
     RB_BUILTIN
+    RArray
     RBasic
     RObject
     attr_index_t