8202713: Create a MacroAssembler::access_load/store_at wrapper for S390 and PPC

Reviewed-by: eosterlund, goetz

src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp

@@ -304,15 +304,15 @@ void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet deco
 
 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                     Register base, RegisterOrConstant ind_or_offs, Register dst,
-                                    Register tmp1, Register tmp2, bool needs_frame, Label *is_null) {
+                                    Register tmp1, Register tmp2, bool needs_frame, Label *L_handle_null) {
   bool on_oop = type == T_OBJECT || type == T_ARRAY;
   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
   bool on_reference = on_weak || on_phantom;
   Label done;
-  if (on_oop && on_reference && is_null == NULL) { is_null = &done; }
+  if (on_oop && on_reference && L_handle_null == NULL) { L_handle_null = &done; }
   // Load the value of the referent field.
-  ModRefBarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst, tmp1, tmp2, needs_frame, is_null);
+  ModRefBarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst, tmp1, tmp2, needs_frame, L_handle_null);
   if (on_oop && on_reference) {
     // Generate the G1 pre-barrier code to log the value of
     // the referent field in an SATB buffer. Note with

src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp

@@ -61,7 +61,7 @@ public:
 
   virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                        Register base, RegisterOrConstant ind_or_offs, Register dst,
-                       Register tmp1, Register tmp2, bool needs_frame, Label *is_null = NULL);
+                       Register tmp1, Register tmp2, bool needs_frame, Label *L_handle_null = NULL);
 
   virtual void resolve_jobject(MacroAssembler* masm, Register value, Register tmp1, Register tmp2, bool needs_frame);
 };

src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp

@@ -65,9 +65,10 @@ void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators
 
 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                   Register base, RegisterOrConstant ind_or_offs, Register dst,
-                                  Register tmp1, Register tmp2, bool needs_frame, Label *is_null) {
+                                  Register tmp1, Register tmp2, bool needs_frame, Label *L_handle_null) {
   bool on_heap  = (decorators & IN_HEAP) != 0;
   bool on_root  = (decorators & IN_ROOT) != 0;
+  bool not_null = (decorators & OOP_NOT_NULL) != 0;
   assert(on_heap || on_root, "where?");
   assert_different_registers(ind_or_offs.register_or_noreg(), dst, R0);
 
@@ -75,19 +76,24 @@ void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators,
   case T_ARRAY:
   case T_OBJECT: {
     if (UseCompressedOops && on_heap) {
+      if (L_handle_null != NULL) { // Label provided.
         __ lwz(dst, ind_or_offs, base);
-      if (is_null) {
         __ cmpwi(CCR0, dst, 0);
-        __ beq(CCR0, *is_null);
+        __ beq(CCR0, *L_handle_null);
         __ decode_heap_oop_not_null(dst);
-      } else {
+      } else if (not_null) { // Guaranteed to be not null.
+        Register narrowOop = (tmp1 != noreg && Universe::narrow_oop_base_disjoint()) ? tmp1 : dst;
+        __ lwz(narrowOop, ind_or_offs, base);
+        __ decode_heap_oop_not_null(dst, narrowOop);
+      } else { // Any oop.
+        __ lwz(dst, ind_or_offs, base);
         __ decode_heap_oop(dst);
       }
     } else {
       __ ld(dst, ind_or_offs, base);
-      if (is_null) {
+      if (L_handle_null != NULL) {
         __ cmpdi(CCR0, dst, 0);
-        __ beq(CCR0, *is_null);
+        __ beq(CCR0, *L_handle_null);
       }
     }
     break;

src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp

@@ -45,7 +45,7 @@ public:
 
   virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                        Register base, RegisterOrConstant ind_or_offs, Register dst,
-                       Register tmp1, Register tmp2, bool needs_frame, Label *is_null = NULL);
+                       Register tmp1, Register tmp2, bool needs_frame, Label *L_handle_null = NULL);
 
   virtual void resolve_jobject(MacroAssembler* masm, Register value, Register tmp1, Register tmp2, bool needs_frame);
 

src/hotspot/cpu/ppc/interp_masm_ppc.hpp

@@ -77,7 +77,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
                          Register tmp1, Register tmp2, Register tmp3, Label &ok_is_subtype);
 
   // Load object from cpool->resolved_references(index).
-  void load_resolved_reference_at_index(Register result, Register index, Label *is_null = NULL);
+  void load_resolved_reference_at_index(Register result, Register index, Label *L_handle_null = NULL);
 
   // load cpool->resolved_klass_at(index)
   void load_resolved_klass_at_offset(Register Rcpool, Register Roffset, Register Rklass);

src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp

@@ -471,7 +471,7 @@ void InterpreterMacroAssembler::get_u4(Register Rdst, Register Rsrc, int offset,
 }
 
 // Load object from cpool->resolved_references(index).
-void InterpreterMacroAssembler::load_resolved_reference_at_index(Register result, Register index, Label *is_null) {
+void InterpreterMacroAssembler::load_resolved_reference_at_index(Register result, Register index, Label *L_handle_null) {
   assert_different_registers(result, index);
   get_constant_pool(result);
 
@@ -494,8 +494,7 @@ void InterpreterMacroAssembler::load_resolved_reference_at_index(Register result
 #endif
   // Add in the index.
   add(result, tmp, result);
-  BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  load_heap_oop(result, arrayOopDesc::base_offset_in_bytes(T_OBJECT), result, tmp, R0, false, 0, L_handle_null);
-  bs->load_at(this, IN_HEAP, T_OBJECT, result, arrayOopDesc::base_offset_in_bytes(T_OBJECT), result, tmp, R0, false, is_null);
 }
 
 // load cpool->resolved_klass_at(index)

src/hotspot/cpu/ppc/macroAssembler_ppc.cpp

@@ -2045,7 +2045,8 @@ void MacroAssembler::check_method_handle_type(Register mtype_reg, Register mh_re
                                               Label& wrong_method_type) {
   assert_different_registers(mtype_reg, mh_reg, temp_reg);
   // Compare method type against that of the receiver.
-  load_heap_oop_not_null(temp_reg, delayed_value(java_lang_invoke_MethodHandle::type_offset_in_bytes, temp_reg), mh_reg);
+  load_heap_oop(temp_reg, delayed_value(java_lang_invoke_MethodHandle::type_offset_in_bytes, temp_reg), mh_reg,
+                noreg, noreg, false, OOP_NOT_NULL);
   cmpd(CCR0, temp_reg, mtype_reg);
   bne(CCR0, wrong_method_type);
 }

src/hotspot/cpu/ppc/macroAssembler_ppc.hpp

@@ -27,6 +27,7 @@
 #define CPU_PPC_VM_MACROASSEMBLER_PPC_HPP
 
 #include "asm/assembler.hpp"
+#include "oops/accessDecorators.hpp"
 #include "runtime/rtmLocking.hpp"
 #include "utilities/macros.hpp"
 
@@ -691,17 +692,26 @@ class MacroAssembler: public Assembler {
   inline void null_check_throw(Register a, int offset, Register temp_reg, address exception_entry);
   inline void null_check(Register a, int offset, Label *Lis_null); // implicit only if Lis_null not provided
 
-  // Load heap oop and decompress. Loaded oop may not be null.
+  // Access heap oop, handle encoding and GC barriers.
-  // Specify tmp to save one cycle.
+  // Some GC barriers call C so use needs_frame = true if an extra frame is needed at the current call site.
-  inline void load_heap_oop_not_null(Register d, RegisterOrConstant offs, Register s1 = noreg,
+ private:
-                                     Register tmp = noreg);
+  inline void access_store_at(BasicType type, DecoratorSet decorators,
-  // Store heap oop and decompress.  Decompressed oop may not be null.
+                              Register base, RegisterOrConstant ind_or_offs, Register val,
-  // Specify tmp register if d should not be changed.
+                              Register tmp1, Register tmp2, Register tmp3, bool needs_frame);
-  inline void store_heap_oop_not_null(Register d, RegisterOrConstant offs, Register s1,
+  inline void access_load_at(BasicType type, DecoratorSet decorators,
-                                      Register tmp = noreg);
+                             Register base, RegisterOrConstant ind_or_offs, Register dst,
+                             Register tmp1, Register tmp2, bool needs_frame, Label *L_handle_null = NULL);
 
-  // Null allowed.
+ public:
-  inline void load_heap_oop(Register d, RegisterOrConstant offs, Register s1 = noreg, Label *is_null = NULL);
+  // Specify tmp1 for better code in certain compressed oops cases. Specify Label to bail out on null oop.
+  // tmp1, tmp2 and needs_frame are used with decorators ON_PHANTOM_OOP_REF or ON_WEAK_OOP_REF.
+  inline void load_heap_oop(Register d, RegisterOrConstant offs, Register s1,
+                            Register tmp1, Register tmp2, bool needs_frame,
+                            DecoratorSet decorators = 0, Label *L_handle_null = NULL);
+
+  inline void store_heap_oop(Register d, RegisterOrConstant offs, Register s1,
+                             Register tmp1, Register tmp2, Register tmp3, bool needs_frame,
+                             DecoratorSet decorators = 0);
 
   // Encode/decode heap oop. Oop may not be null, else en/decoding goes wrong.
   // src == d allowed.

src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp

@@ -30,6 +30,8 @@
 #include "asm/macroAssembler.hpp"
 #include "asm/codeBuffer.hpp"
 #include "code/codeCache.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
 #include "runtime/safepointMechanism.hpp"
 
 inline bool MacroAssembler::is_ld_largeoffset(address a) {
@@ -323,45 +325,52 @@ inline void MacroAssembler::null_check(Register a, int offset, Label *Lis_null)
   }
 }
 
-inline void MacroAssembler::load_heap_oop_not_null(Register d, RegisterOrConstant offs, Register s1, Register tmp) {
+inline void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
-  if (UseCompressedOops) {
+                                            Register base, RegisterOrConstant ind_or_offs, Register val,
-    // In disjoint mode decoding can save a cycle if src != dst.
+                                            Register tmp1, Register tmp2, Register tmp3, bool needs_frame) {
-    Register narrowOop = (tmp != noreg && Universe::narrow_oop_base_disjoint()) ? tmp : d;
+  assert((decorators & ~(AS_RAW | IN_HEAP | IN_HEAP_ARRAY | IN_ROOT | OOP_NOT_NULL |
-    lwz(narrowOop, offs, s1);
+                         ON_UNKNOWN_OOP_REF)) == 0, "unsupported decorator");
-    // Attention: no null check here!
+  BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
-    Register res = decode_heap_oop_not_null(d, narrowOop);
+  bool as_raw = (decorators & AS_RAW) != 0;
-    assert(res == d, "caller will not consume loaded value");
+  if (as_raw) {
+    bs->BarrierSetAssembler::store_at(this, decorators, type,
+                                      base, ind_or_offs, val,
+                                      tmp1, tmp2, tmp3, needs_frame);
   } else {
-    ld(d, offs, s1);
+    bs->store_at(this, decorators, type,
+                 base, ind_or_offs, val,
+                 tmp1, tmp2, tmp3, needs_frame);
   }
 }
 
-inline void MacroAssembler::store_heap_oop_not_null(Register d, RegisterOrConstant offs, Register s1, Register tmp) {
+inline void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
-  if (UseCompressedOops) {
+                                           Register base, RegisterOrConstant ind_or_offs, Register dst,
-    Register compressedOop = encode_heap_oop_not_null((tmp != noreg) ? tmp : d, d);
+                                           Register tmp1, Register tmp2, bool needs_frame, Label *L_handle_null) {
-    stw(compressedOop, offs, s1);
+  assert((decorators & ~(AS_RAW | IN_HEAP | IN_HEAP_ARRAY | IN_ROOT | OOP_NOT_NULL |
+                         ON_PHANTOM_OOP_REF | ON_WEAK_OOP_REF)) == 0, "unsupported decorator");
+  BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  bool as_raw = (decorators & AS_RAW) != 0;
+  if (as_raw) {
+    bs->BarrierSetAssembler::load_at(this, decorators, type,
+                                     base, ind_or_offs, dst,
+                                     tmp1, tmp2, needs_frame, L_handle_null);
   } else {
-    std(d, offs, s1);
+    bs->load_at(this, decorators, type,
+                base, ind_or_offs, dst,
+                tmp1, tmp2, needs_frame, L_handle_null);
   }
 }
 
-inline void MacroAssembler::load_heap_oop(Register d, RegisterOrConstant offs, Register s1, Label *is_null) {
+inline void MacroAssembler::load_heap_oop(Register d, RegisterOrConstant offs, Register s1,
-  if (UseCompressedOops) {
+                                          Register tmp1, Register tmp2,
-    lwz(d, offs, s1);
+                                          bool needs_frame, DecoratorSet decorators, Label *L_handle_null) {
-    if (is_null != NULL) {
+  access_load_at(T_OBJECT, IN_HEAP | decorators, s1, offs, d, tmp1, tmp2, needs_frame, L_handle_null);
-      cmpwi(CCR0, d, 0);
-      beq(CCR0, *is_null);
-      decode_heap_oop_not_null(d);
-    } else {
-      decode_heap_oop(d);
-    }
-  } else {
-    ld(d, offs, s1);
-    if (is_null != NULL) {
-      cmpdi(CCR0, d, 0);
-      beq(CCR0, *is_null);
-    }
 }
+
+inline void MacroAssembler::store_heap_oop(Register d, RegisterOrConstant offs, Register s1,
+                                           Register tmp1, Register tmp2, Register tmp3,
+                                           bool needs_frame, DecoratorSet decorators) {
+  access_store_at(T_OBJECT, IN_HEAP | decorators, s1, offs, d, tmp1, tmp2, tmp3, needs_frame);
 }
 
 inline Register MacroAssembler::encode_heap_oop_not_null(Register d, Register src) {

src/hotspot/cpu/ppc/methodHandles_ppc.cpp

@@ -173,11 +173,14 @@ void MethodHandles::jump_to_lambda_form(MacroAssembler* _masm,
 
   // Load the invoker, as MH -> MH.form -> LF.vmentry
   __ verify_oop(recv);
-  __ load_heap_oop_not_null(method_temp, NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes()), recv, temp2);
+  __ load_heap_oop(method_temp, NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes()), recv,
+                   temp2, noreg, false, OOP_NOT_NULL);
   __ verify_oop(method_temp);
-  __ load_heap_oop_not_null(method_temp, NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes()), method_temp, temp2);
+  __ load_heap_oop(method_temp, NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes()), method_temp,
+                   temp2, noreg, false, OOP_NOT_NULL);
   __ verify_oop(method_temp);
-  __ load_heap_oop_not_null(method_temp, NONZERO(java_lang_invoke_MemberName::method_offset_in_bytes()), method_temp);
+  __ load_heap_oop(method_temp, NONZERO(java_lang_invoke_MemberName::method_offset_in_bytes()), method_temp,
+                   temp2, noreg, false, OOP_NOT_NULL);
   __ verify_oop(method_temp);
   __ ld(method_temp, NONZERO(java_lang_invoke_ResolvedMethodName::vmtarget_offset_in_bytes()), method_temp);
 
@@ -338,7 +341,8 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
       if (VerifyMethodHandles && iid != vmIntrinsics::_linkToInterface) {
         Label L_ok;
         Register temp2_defc = temp2;
-        __ load_heap_oop_not_null(temp2_defc, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes()), member_reg, temp3);
+        __ load_heap_oop(temp2_defc, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes()), member_reg,
+                         temp3, noreg, false, OOP_NOT_NULL);
         load_klass_from_Class(_masm, temp2_defc, temp3, temp4);
         __ verify_klass_ptr(temp2_defc);
         __ check_klass_subtype(temp1_recv_klass, temp2_defc, temp3, temp4, L_ok);
@@ -365,7 +369,8 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
       if (VerifyMethodHandles) {
         verify_ref_kind(_masm, JVM_REF_invokeSpecial, member_reg, temp2);
       }
-      __ load_heap_oop(R19_method, NONZERO(java_lang_invoke_MemberName::method_offset_in_bytes()), member_reg);
+      __ load_heap_oop(R19_method, NONZERO(java_lang_invoke_MemberName::method_offset_in_bytes()), member_reg,
+                       temp3, noreg, false, OOP_NOT_NULL);
       __ ld(R19_method, NONZERO(java_lang_invoke_ResolvedMethodName::vmtarget_offset_in_bytes()), R19_method);
       break;
 
@@ -373,7 +378,8 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
       if (VerifyMethodHandles) {
         verify_ref_kind(_masm, JVM_REF_invokeStatic, member_reg, temp2);
       }
-      __ load_heap_oop(R19_method, NONZERO(java_lang_invoke_MemberName::method_offset_in_bytes()), member_reg);
+      __ load_heap_oop(R19_method, NONZERO(java_lang_invoke_MemberName::method_offset_in_bytes()), member_reg,
+                       temp3, noreg, false, OOP_NOT_NULL);
       __ ld(R19_method, NONZERO(java_lang_invoke_ResolvedMethodName::vmtarget_offset_in_bytes()), R19_method);
       break;
 
@@ -415,7 +421,8 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
       }
 
       Register temp2_intf = temp2;
-      __ load_heap_oop_not_null(temp2_intf, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes()), member_reg, temp3);
+      __ load_heap_oop(temp2_intf, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes()), member_reg,
+                       temp3, noreg, false, OOP_NOT_NULL);
       load_klass_from_Class(_masm, temp2_intf, temp3, temp4);
       __ verify_klass_ptr(temp2_intf);
 

src/hotspot/cpu/ppc/stubGenerator_ppc.cpp

@@ -2198,7 +2198,7 @@ class StubGenerator: public StubCodeGenerator {
 
     // ======== loop entry is here ========
     __ bind(load_element);
-    __ load_heap_oop(R10_oop, R8_offset, R3_from, &store_null);  // Load the oop.
+    __ load_heap_oop(R10_oop, R8_offset, R3_from, R12_tmp, noreg, false, AS_RAW, &store_null);
 
     __ load_klass(R11_klass, R10_oop); // Query the object klass.
 

src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp

@@ -524,11 +524,8 @@ address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
   __ cmpdi(CCR0, R3_RET, 0);
   __ beq(CCR0, slow_path);
 
-  // Load the value of the referent field.
+  __ load_heap_oop(R3_RET, referent_offset, R3_RET,
-  BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
+                   /* non-volatile temp */ R31, R11_scratch1, true, ON_WEAK_OOP_REF);
-  bs->load_at(_masm, IN_HEAP | ON_WEAK_OOP_REF, T_OBJECT,
-                    R3_RET, referent_offset, R3_RET,
-                    /* non-volatile temp */ R31, R11_scratch1, true);
 
   // Generate the G1 pre-barrier code to log the value of
   // the referent field in an SATB buffer. Note with

src/hotspot/cpu/ppc/templateTable_ppc_64.cpp

@@ -62,8 +62,7 @@ static void do_oop_store(InterpreterMacroAssembler* _masm,
                          Register           tmp3,
                          DecoratorSet       decorators) {
   assert_different_registers(tmp1, tmp2, tmp3, val, base);
-  BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  __ store_heap_oop(val, offset, base, tmp1, tmp2, tmp3, false, decorators);
-  bs->store_at(_masm, decorators, T_OBJECT, base, offset, val, tmp1, tmp2, tmp3, false);
 }
 
 static void do_oop_load(InterpreterMacroAssembler* _masm,
@@ -75,8 +74,7 @@ static void do_oop_load(InterpreterMacroAssembler* _masm,
                         DecoratorSet decorators) {
   assert_different_registers(base, tmp1, tmp2);
   assert_different_registers(dst, tmp1, tmp2);
-  BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  __ load_heap_oop(dst, offset, base, tmp1, tmp2, false, decorators);
-  bs->load_at(_masm, decorators, T_OBJECT, base, offset, dst, tmp1, tmp2, false);
 }
 
 // ============================================================================

src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp

@@ -96,14 +96,14 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas
 }
 
 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
-                                    const Address& src, Register dst, Register tmp1, Register tmp2, Label *is_null) {
+                                    const Address& src, Register dst, Register tmp1, Register tmp2, Label *L_handle_null) {
   bool on_oop = type == T_OBJECT || type == T_ARRAY;
   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
   bool on_reference = on_weak || on_phantom;
   Label done;
-  if (on_oop && on_reference && is_null == NULL) { is_null = &done; }
+  if (on_oop && on_reference && L_handle_null == NULL) { L_handle_null = &done; }
-  ModRefBarrierSetAssembler::load_at(masm, decorators, type, src, dst, tmp1, tmp2, is_null);
+  ModRefBarrierSetAssembler::load_at(masm, decorators, type, src, dst, tmp1, tmp2, L_handle_null);
   if (on_oop && on_reference) {
     // Generate the G1 pre-barrier code to log the value of
     // the referent field in an SATB buffer.

src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp

@@ -65,7 +65,7 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
 #endif
 
   virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
-                       const Address& src, Register dst, Register tmp1, Register tmp2, Label *is_null = NULL);
+                       const Address& src, Register dst, Register tmp1, Register tmp2, Label *L_handle_null = NULL);
 
   virtual void resolve_jobject(MacroAssembler* masm, Register value, Register tmp1, Register tmp2);
 };

src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp

@@ -36,9 +36,10 @@ void BarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet
 }
 
 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
-                                  const Address& addr, Register dst, Register tmp1, Register tmp2, Label *is_null) {
+                                  const Address& addr, Register dst, Register tmp1, Register tmp2, Label *L_handle_null) {
   bool on_heap  = (decorators & IN_HEAP) != 0;
   bool on_root  = (decorators & IN_ROOT) != 0;
+  bool not_null = (decorators & OOP_NOT_NULL) != 0;
   assert(on_heap || on_root, "where?");
 
   switch (type) {
@@ -46,16 +47,16 @@ void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators,
   case T_OBJECT: {
     if (UseCompressedOops && on_heap) {
       __ z_llgf(dst, addr);
-      if (is_null) {
+      if (L_handle_null != NULL) { // Label provided.
-        __ compareU32_and_branch(dst, (intptr_t)0, Assembler::bcondEqual, *is_null);
+        __ compareU32_and_branch(dst, (intptr_t)0, Assembler::bcondEqual, *L_handle_null);
         __ oop_decoder(dst, dst, false);
       } else {
-        __ oop_decoder(dst, dst, true);
+        __ oop_decoder(dst, dst, !not_null);
       }
     } else {
       __ z_lg(dst, addr);
-      if (is_null) {
+      if (L_handle_null != NULL) {
-        __ compareU64_and_branch(dst, (intptr_t)0, Assembler::bcondEqual, *is_null);
+        __ compareU64_and_branch(dst, (intptr_t)0, Assembler::bcondEqual, *L_handle_null);
       }
     }
     break;

src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp

@@ -40,7 +40,7 @@ public:
                                   Register dst, Register count, bool do_return = false);
 
   virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
-                       const Address& addr, Register dst, Register tmp1, Register tmp2, Label *is_null = NULL);
+                       const Address& addr, Register dst, Register tmp1, Register tmp2, Label *L_handle_null = NULL);
   virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                         const Address& addr, Register val, Register tmp1, Register tmp2, Register tmp3);
 

src/hotspot/cpu/s390/interp_masm_s390.cpp

@@ -391,8 +391,7 @@ void InterpreterMacroAssembler::load_resolved_reference_at_index(Register result
   bind(index_ok);
 #endif
   z_agr(result, index);    // Address of indexed array element.
-  BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  load_heap_oop(result, Address(result, arrayOopDesc::base_offset_in_bytes(T_OBJECT)), tmp, noreg);
-  bs->load_at(this, IN_HEAP, T_OBJECT, Address(result, arrayOopDesc::base_offset_in_bytes(T_OBJECT)), result, tmp, noreg);
 }
 
 // load cpool->resolved_klass_at(index)

src/hotspot/cpu/s390/macroAssembler_s390.cpp

@@ -4047,68 +4047,52 @@ void MacroAssembler::compare_heap_oop(Register Rop1, Address mem, bool maybeNULL
   BLOCK_COMMENT("} compare heap oop");
 }
 
-// Load heap oop and decompress, if necessary.
+void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
-void  MacroAssembler::load_heap_oop(Register dest, const Address &a) {
+                                     const Address& addr, Register val,
-  if (UseCompressedOops) {
+                                     Register tmp1, Register tmp2, Register tmp3) {
-    z_llgf(dest, a.disp(), a.indexOrR0(), a.baseOrR0());
+  assert((decorators & ~(AS_RAW | IN_HEAP | IN_HEAP_ARRAY | IN_ROOT | OOP_NOT_NULL |
-    oop_decoder(dest, dest, true);
+                         ON_UNKNOWN_OOP_REF)) == 0, "unsupported decorator");
+  BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  bool as_raw = (decorators & AS_RAW) != 0;
+  if (as_raw) {
+    bs->BarrierSetAssembler::store_at(this, decorators, type,
+                                      addr, val,
+                                      tmp1, tmp2, tmp3);
   } else {
-    z_lg(dest, a.disp(), a.indexOrR0(), a.baseOrR0());
+    bs->store_at(this, decorators, type,
+                 addr, val,
+                 tmp1, tmp2, tmp3);
   }
 }
 
-// Load heap oop and decompress, if necessary.
+void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
-void MacroAssembler::load_heap_oop(Register dest, int64_t disp, Register base) {
+                                    const Address& addr, Register dst,
-  if (UseCompressedOops) {
+                                    Register tmp1, Register tmp2, Label *is_null) {
-    z_llgf(dest, disp, base);
+  assert((decorators & ~(AS_RAW | IN_HEAP | IN_HEAP_ARRAY | IN_ROOT | OOP_NOT_NULL |
-    oop_decoder(dest, dest, true);
+                         ON_PHANTOM_OOP_REF | ON_WEAK_OOP_REF)) == 0, "unsupported decorator");
+  BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  bool as_raw = (decorators & AS_RAW) != 0;
+  if (as_raw) {
+    bs->BarrierSetAssembler::load_at(this, decorators, type,
+                                     addr, dst,
+                                     tmp1, tmp2, is_null);
   } else {
-    z_lg(dest, disp, base);
+    bs->load_at(this, decorators, type,
+                addr, dst,
+                tmp1, tmp2, is_null);
   }
 }
 
-// Load heap oop and decompress, if necessary.
+void MacroAssembler::load_heap_oop(Register dest, const Address &a,
-void MacroAssembler::load_heap_oop_not_null(Register dest, int64_t disp, Register base) {
+                                   Register tmp1, Register tmp2,
-  if (UseCompressedOops) {
+                                   DecoratorSet decorators, Label *is_null) {
-    z_llgf(dest, disp, base);
+  access_load_at(T_OBJECT, IN_HEAP | decorators, a, dest, tmp1, tmp2, is_null);
-    oop_decoder(dest, dest, false);
-  } else {
-    z_lg(dest, disp, base);
-  }
 }
 
-// Compress, if necessary, and store oop to heap.
+void MacroAssembler::store_heap_oop(Register Roop, const Address &a,
-void MacroAssembler::store_heap_oop(Register Roop, RegisterOrConstant offset, Register base) {
+                                    Register tmp1, Register tmp2, Register tmp3,
-  Register Ridx = offset.is_register() ? offset.register_or_noreg() : Z_R0;
+                                    DecoratorSet decorators) {
-  if (UseCompressedOops) {
+  access_store_at(T_OBJECT, IN_HEAP | decorators, a, Roop, tmp1, tmp2, tmp3);
-    assert_different_registers(Roop, offset.register_or_noreg(), base);
-    encode_heap_oop(Roop);
-    z_st(Roop, offset.constant_or_zero(), Ridx, base);
-  } else {
-    z_stg(Roop, offset.constant_or_zero(), Ridx, base);
-  }
-}
-
-// Compress, if necessary, and store oop to heap. Oop is guaranteed to be not NULL.
-void MacroAssembler::store_heap_oop_not_null(Register Roop, RegisterOrConstant offset, Register base) {
-  Register Ridx = offset.is_register() ? offset.register_or_noreg() : Z_R0;
-  if (UseCompressedOops) {
-    assert_different_registers(Roop, offset.register_or_noreg(), base);
-    encode_heap_oop_not_null(Roop);
-    z_st(Roop, offset.constant_or_zero(), Ridx, base);
-  } else {
-    z_stg(Roop, offset.constant_or_zero(), Ridx, base);
-  }
-}
-
-// Store NULL oop to heap.
-void MacroAssembler::store_heap_oop_null(Register zero, RegisterOrConstant offset, Register base) {
-  Register Ridx = offset.is_register() ? offset.register_or_noreg() : Z_R0;
-  if (UseCompressedOops) {
-    z_st(zero, offset.constant_or_zero(), Ridx, base);
-  } else {
-    z_stg(zero, offset.constant_or_zero(), Ridx, base);
-  }
 }
 
 //-------------------------------------------------

src/hotspot/cpu/s390/macroAssembler_s390.hpp

@@ -27,6 +27,7 @@
 #define CPU_S390_VM_MACROASSEMBLER_S390_HPP
 
 #include "asm/assembler.hpp"
+#include "oops/accessDecorators.hpp"
 
 #define MODERN_IFUN(name)  ((void (MacroAssembler::*)(Register, int64_t, Register, Register))&MacroAssembler::name)
 #define CLASSIC_IFUN(name) ((void (MacroAssembler::*)(Register, int64_t, Register, Register))&MacroAssembler::name)
@@ -804,12 +805,25 @@ class MacroAssembler: public Assembler {
   int  get_oop_base_complement(Register Rbase, uint64_t oop_base);
   void compare_heap_oop(Register Rop1, Address mem, bool maybeNULL);
   void compare_klass_ptr(Register Rop1, int64_t disp, Register Rbase, bool maybeNULL);
-  void load_heap_oop(Register dest, const Address &a);
+
-  void load_heap_oop(Register d, int64_t si16, Register s1);
+  // Access heap oop, handle encoding and GC barriers.
-  void load_heap_oop_not_null(Register d, int64_t si16, Register s1);
+ private:
-  void store_heap_oop(Register Roop, RegisterOrConstant offset, Register base);
+  void access_store_at(BasicType type, DecoratorSet decorators,
-  void store_heap_oop_not_null(Register Roop, RegisterOrConstant offset, Register base);
+                       const Address& addr, Register val,
-  void store_heap_oop_null(Register zero, RegisterOrConstant offset, Register base);
+                       Register tmp1, Register tmp2, Register tmp3);
+  void access_load_at(BasicType type, DecoratorSet decorators,
+                      const Address& addr, Register dst,
+                      Register tmp1, Register tmp2, Label *is_null = NULL);
+
+ public:
+  // tmp1 and tmp2 are used with decorators ON_PHANTOM_OOP_REF or ON_WEAK_OOP_REF.
+  void load_heap_oop(Register dest, const Address &a,
+                     Register tmp1, Register tmp2,
+                     DecoratorSet decorators = 0, Label *is_null = NULL);
+  void store_heap_oop(Register Roop, const Address &a,
+                      Register tmp1, Register tmp2, Register tmp3,
+                      DecoratorSet decorators = 0);
+
   void oop_encoder(Register Rdst, Register Rsrc, bool maybeNULL,
                    Register Rbase = Z_R1, int pow2_offset = -1, bool only32bitValid = false);
   void oop_decoder(Register Rdst, Register Rsrc, bool maybeNULL,

src/hotspot/cpu/s390/methodHandles_s390.cpp

@@ -197,15 +197,18 @@ void MethodHandles::jump_to_lambda_form(MacroAssembler* _masm,
   __ verify_oop(recv);
   __ load_heap_oop(method_temp,
                    Address(recv,
-                             NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes())));
+                           NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes())),
+                   noreg, noreg, OOP_NOT_NULL);
   __ verify_oop(method_temp);
   __ load_heap_oop(method_temp,
                    Address(method_temp,
-                             NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes())));
+                           NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes())),
+                   noreg, noreg, OOP_NOT_NULL);
   __ verify_oop(method_temp);
   __ load_heap_oop(method_temp,
                    Address(method_temp,
-                  NONZERO(java_lang_invoke_MemberName::method_offset_in_bytes())));
+                           NONZERO(java_lang_invoke_MemberName::method_offset_in_bytes())),
+                   noreg, noreg, OOP_NOT_NULL);
   __ verify_oop(method_temp);
   __ z_lg(method_temp,
           Address(method_temp,
@@ -405,7 +408,8 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
       NearLabel L_ok;
       Register temp2_defc = temp2;
 
-      __ load_heap_oop(temp2_defc, member_clazz);
+      __ load_heap_oop(temp2_defc, member_clazz,
+                       noreg, noreg, OOP_NOT_NULL);
       load_klass_from_Class(_masm, temp2_defc, temp3, temp4);
       __ verify_klass_ptr(temp2_defc);
       __ check_klass_subtype(temp1_recv_klass, temp2_defc, temp3, temp4, L_ok);
@@ -431,7 +435,8 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
       if (VerifyMethodHandles) {
         verify_ref_kind(_masm, JVM_REF_invokeSpecial, member_reg, temp3);
       }
-      __ load_heap_oop(Z_method, member_vmtarget);
+      __ load_heap_oop(Z_method, member_vmtarget,
+                       noreg, noreg, OOP_NOT_NULL);
       __ z_lg(Z_method, vmtarget_method);
       method_is_live = true;
       break;
@@ -440,7 +445,8 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
       if (VerifyMethodHandles) {
         verify_ref_kind(_masm, JVM_REF_invokeStatic, member_reg, temp3);
       }
-      __ load_heap_oop(Z_method, member_vmtarget);
+      __ load_heap_oop(Z_method, member_vmtarget,
+                       noreg, noreg, OOP_NOT_NULL);
       __ z_lg(Z_method, vmtarget_method);
       method_is_live = true;
       break;
@@ -481,7 +487,8 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
 
       Register temp3_intf = temp3;
 
-      __ load_heap_oop(temp3_intf, member_clazz);
+      __ load_heap_oop(temp3_intf, member_clazz,
+                       noreg, noreg, OOP_NOT_NULL);
       load_klass_from_Class(_masm, temp3_intf, temp2, temp4);
 
       Register Z_index = Z_method;

src/hotspot/cpu/s390/s390.ad

@@ -2239,8 +2239,8 @@ encode %{
       // Go through the vtable. Get receiver klass. Receiver already
       // checked for non-null. If we'll go thru a C2I adapter, the
       // interpreter expects method in Z_method.
-      // Use Z_method to temporarily hold the klass oop. Z_R1_scratch is destroyed
+      // Use Z_method to temporarily hold the klass oop.
-      // by load_heap_oop_not_null.
+      // Z_R1_scratch is destroyed.
       __ load_klass(Z_method, Z_R2);
 
       int entry_offset = in_bytes(Klass::vtable_start_offset()) + vtable_index * vtableEntry::size_in_bytes();

src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp

@@ -512,9 +512,7 @@ address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
   __ z_bre(slow_path);
 
   //  Load the value of the referent field.
- BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  __ load_heap_oop(pre_val, Address(pre_val, referent_offset), scratch1, scratch2, ON_WEAK_OOP_REF);
- bs->load_at(_masm, IN_HEAP | ON_WEAK_OOP_REF, T_OBJECT,
-                   Address(pre_val, referent_offset), pre_val, scratch1, scratch2);
 
   // Restore caller sp for c2i case.
   __ resize_frame_absolute(Z_R10, Z_R0, true); // Cut the stack back to where the caller started.

src/hotspot/cpu/s390/templateTable_s390.cpp

@@ -200,8 +200,7 @@ static void do_oop_store(InterpreterMacroAssembler* _masm,
                          Register           tmp3,
                          DecoratorSet       decorators) {
   assert_different_registers(tmp1, tmp2, tmp3, val, addr.base());
-  BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  __ store_heap_oop(val, addr, tmp1, tmp2, tmp3, decorators);
-  bs->store_at(_masm, decorators, T_OBJECT, addr, val, tmp1, tmp2, tmp3);
 }
 
 static void do_oop_load(InterpreterMacroAssembler* _masm,
@@ -212,8 +211,7 @@ static void do_oop_load(InterpreterMacroAssembler* _masm,
                         DecoratorSet decorators) {
   assert_different_registers(addr.base(), tmp1, tmp2);
   assert_different_registers(dst, tmp1, tmp2);
-  BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  __ load_heap_oop(dst, addr, tmp1, tmp2, decorators);
-  bs->load_at(_masm, decorators, T_OBJECT, addr, dst, tmp1, tmp2);
 }
 
 Address TemplateTable::at_bcp(int offset) {