8058255: Native jbyte Atomic::cmpxchg for supported x86 platforms

Use the native cmpxchgb instruction on x86.

Reviewed-by: dholmes, kbarrett, phh

hotspot/src/cpu/sparc/vm/stubGenerator_sparc.cpp

@@ -4813,6 +4813,7 @@ class StubGenerator: public StubCodeGenerator {
     StubRoutines::_atomic_add_entry          = generate_atomic_add();
     StubRoutines::_atomic_xchg_ptr_entry     = StubRoutines::_atomic_xchg_entry;
     StubRoutines::_atomic_cmpxchg_ptr_entry  = StubRoutines::_atomic_cmpxchg_entry;
+    StubRoutines::_atomic_cmpxchg_byte_entry = ShouldNotCallThisStub();
     StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
     StubRoutines::_atomic_add_ptr_entry      = StubRoutines::_atomic_add_entry;
 #endif  // COMPILER2 !=> _LP64

hotspot/src/cpu/x86/vm/assembler_x86.cpp

@@ -1297,6 +1297,17 @@ void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
   emit_operand(reg, adr);
 }
 
+// The 8-bit cmpxchg compares the value at adr with the contents of rax,
+// and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
+// The ZF is set if the compared values were equal, and cleared otherwise.
+void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
+  InstructionMark im(this);
+  prefix(adr, reg, true);
+  emit_int8(0x0F);
+  emit_int8((unsigned char)0xB0);
+  emit_operand(reg, adr);
+}
+
 void Assembler::comisd(XMMRegister dst, Address src) {
   // NOTE: dbx seems to decode this as comiss even though the
   // 0x66 is there. Strangly ucomisd comes out correct

hotspot/src/cpu/x86/vm/assembler_x86.hpp

@@ -1006,6 +1006,7 @@ private:
 
   void cmpxchg8 (Address adr);
 
+  void cmpxchgb(Register reg, Address adr);
   void cmpxchgl(Register reg, Address adr);
 
   void cmpxchgq(Register reg, Address adr);

hotspot/src/cpu/x86/vm/stubGenerator_x86_64.cpp

@@ -594,7 +594,33 @@ class StubGenerator: public StubCodeGenerator {
     return start;
   }
 
-  // Support for jint atomic::atomic_cmpxchg_long(jlong exchange_value,
+  // Support for jbyte atomic::atomic_cmpxchg(jbyte exchange_value, volatile jbyte* dest,
+  //                                          jbyte compare_value)
+  //
+  // Arguments :
+  //    c_rarg0: exchange_value
+  //    c_rarg1: dest
+  //    c_rarg2: compare_value
+  //
+  // Result:
+  //    if ( compare_value == *dest ) {
+  //       *dest = exchange_value
+  //       return compare_value;
+  //    else
+  //       return *dest;
+  address generate_atomic_cmpxchg_byte() {
+    StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_byte");
+    address start = __ pc();
+
+    __ movsbq(rax, c_rarg2);
+   if ( os::is_MP() ) __ lock();
+    __ cmpxchgb(c_rarg0, Address(c_rarg1, 0));
+    __ ret(0);
+
+    return start;
+  }
+
+  // Support for jlong atomic::atomic_cmpxchg(jlong exchange_value,
   //                                          volatile jlong* dest,
   //                                          jlong compare_value)
   // Arguments :
@@ -3894,6 +3920,7 @@ class StubGenerator: public StubCodeGenerator {
     StubRoutines::_atomic_xchg_entry         = generate_atomic_xchg();
     StubRoutines::_atomic_xchg_ptr_entry     = generate_atomic_xchg_ptr();
     StubRoutines::_atomic_cmpxchg_entry      = generate_atomic_cmpxchg();
+    StubRoutines::_atomic_cmpxchg_byte_entry = generate_atomic_cmpxchg_byte();
     StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
     StubRoutines::_atomic_add_entry          = generate_atomic_add();
     StubRoutines::_atomic_add_ptr_entry      = generate_atomic_add_ptr();

hotspot/src/cpu/zero/vm/stubGenerator_zero.cpp

@@ -207,6 +207,7 @@ class StubGenerator: public StubCodeGenerator {
     StubRoutines::_atomic_xchg_ptr_entry     = ShouldNotCallThisStub();
     StubRoutines::_atomic_cmpxchg_entry      = ShouldNotCallThisStub();
     StubRoutines::_atomic_cmpxchg_ptr_entry  = ShouldNotCallThisStub();
+    StubRoutines::_atomic_cmpxchg_byte_entry = ShouldNotCallThisStub();
     StubRoutines::_atomic_cmpxchg_long_entry = ShouldNotCallThisStub();
     StubRoutines::_atomic_add_entry          = ShouldNotCallThisStub();
     StubRoutines::_atomic_add_ptr_entry      = ShouldNotCallThisStub();

hotspot/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp

@@ -88,6 +88,15 @@ inline void*    Atomic::xchg_ptr(void*    exchange_value, volatile void*     des
   return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
 }
 
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte    Atomic::cmpxchg    (jbyte    exchange_value, volatile jbyte*    dest, jbyte    compare_value) {
+  int mp = os::is_MP();
+  __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
+                    : "=a" (exchange_value)
+                    : "q" (exchange_value), "a" (compare_value), "r" (dest), "r" (mp)
+                    : "cc", "memory");
+  return exchange_value;
+}
 
 inline jint     Atomic::cmpxchg    (jint     exchange_value, volatile jint*     dest, jint     compare_value) {
   int mp = os::is_MP();

hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp

@@ -88,6 +88,15 @@ inline void*    Atomic::xchg_ptr(void*    exchange_value, volatile void*     des
   return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
 }
 
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte    Atomic::cmpxchg    (jbyte    exchange_value, volatile jbyte*    dest, jbyte    compare_value) {
+  int mp = os::is_MP();
+  __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
+                    : "=a" (exchange_value)
+                    : "q" (exchange_value), "a" (compare_value), "r" (dest), "r" (mp)
+                    : "cc", "memory");
+  return exchange_value;
+}
 
 inline jint     Atomic::cmpxchg    (jint     exchange_value, volatile jint*     dest, jint     compare_value) {
   int mp = os::is_MP();

hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp

@@ -68,6 +68,8 @@ inline void Atomic::dec_ptr(volatile void*     dest) { (void)add_ptr(-1, dest);
 extern "C" {
   jint _Atomic_add(jint add_value, volatile jint* dest IS_MP_DECL());
   jint _Atomic_xchg(jint exchange_value, volatile jint* dest);
+  jbyte _Atomic_cmpxchg_byte(jbyte exchange_value, volatile jbyte* dest,
+                       jbyte compare_value IS_MP_DECL());
   jint _Atomic_cmpxchg(jint exchange_value, volatile jint* dest,
                        jint compare_value IS_MP_DECL());
   jlong _Atomic_cmpxchg_long(jlong exchange_value, volatile jlong* dest,
@@ -82,6 +84,11 @@ inline jint     Atomic::xchg       (jint     exchange_value, volatile jint*
   return _Atomic_xchg(exchange_value, dest);
 }
 
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte    Atomic::cmpxchg    (jbyte    exchange_value, volatile jbyte*    dest, jbyte    compare_value) {
+  return _Atomic_cmpxchg_byte(exchange_value, dest, compare_value IS_MP_ARG());
+}
+
 inline jint     Atomic::cmpxchg    (jint     exchange_value, volatile jint*     dest, jint     compare_value) {
   return _Atomic_cmpxchg(exchange_value, dest, compare_value IS_MP_ARG());
 }
@@ -217,6 +224,15 @@ extern "C" {
     return exchange_value;
   }
 
+
+  inline jbyte _Atomic_cmpxchg_byte(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, int mp) {
+    __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
+                    : "=a" (exchange_value)
+                    : "q" (exchange_value), "a" (compare_value), "r" (dest), "r" (mp)
+                    : "cc", "memory");
+    return exchange_value;
+  }
+
   // This is the interface to the atomic instruction in solaris_i486.s.
   jlong _Atomic_cmpxchg_long_gcc(jlong exchange_value, volatile jlong* dest, jlong compare_value, int mp);
 

hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_32.il

@@ -76,6 +76,23 @@
       xchgl    (%ecx), %eax
       .end
 
+  // Support for jbyte Atomic::cmpxchg(jbyte exchange_value, 
+  //                                   volatile jbyte *dest, 
+  //                                   jbyte compare_value)
+  // An additional bool (os::is_MP()) is passed as the last argument.
+      .inline _Atomic_cmpxchg_byte,4
+      movb     8(%esp), %al   // compare_value
+      movb     0(%esp), %cl   // exchange_value
+      movl     4(%esp), %edx   // dest
+      cmp      $0, 12(%esp)    // MP test
+      jne      1f
+      cmpxchgb %cl, (%edx)
+      jmp      2f
+1:    lock
+      cmpxchgb %cl, (%edx)
+2:
+      .end
+
   // Support for jint Atomic::cmpxchg(jint exchange_value, 
   //                                  volatile jint *dest, 
   //                                  jint compare_value)

hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_64.il

@@ -77,6 +77,15 @@
       movq     %rdi, %rax
       .end
 
+  // Support for jbyte Atomic::cmpxchg(jbyte exchange_value, 
+  //                                   volatile jbyte *dest, 
+  //                                   jbyte compare_value)
+      .inline _Atomic_cmpxchg_byte,3
+      movb     %dl, %al      // compare_value
+      lock
+      cmpxchgb %dil, (%rsi)
+      .end
+
   // Support for jint Atomic::cmpxchg(jint exchange_value, 
   //                                  volatile jint *dest, 
   //                                  jint compare_value)

hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp

@@ -123,6 +123,11 @@ inline jint     Atomic::cmpxchg    (jint     exchange_value, volatile jint*
   return (*os::atomic_cmpxchg_func)(exchange_value, dest, compare_value);
 }
 
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte    Atomic::cmpxchg    (jbyte    exchange_value, volatile jbyte*    dest, jbyte    compare_value) {
+    return (*os::atomic_cmpxchg_byte_func)(exchange_value, dest, compare_value);
+}
+
 inline jlong    Atomic::cmpxchg    (jlong    exchange_value, volatile jlong*    dest, jlong    compare_value) {
   return (*os::atomic_cmpxchg_long_func)(exchange_value, dest, compare_value);
 }
@@ -212,6 +217,19 @@ inline void*    Atomic::xchg_ptr(void*    exchange_value, volatile void*     des
   return (void*)xchg((jint)exchange_value, (volatile jint*)dest);
 }
 
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte    Atomic::cmpxchg    (jbyte    exchange_value, volatile jbyte*    dest, jbyte    compare_value) {
+  // alternative for InterlockedCompareExchange
+  int mp = os::is_MP();
+  __asm {
+    mov edx, dest
+    mov cl, exchange_value
+    mov al, compare_value
+    LOCK_IF_MP(mp)
+    cmpxchg byte ptr [edx], cl
+  }
+}
+
 inline jint     Atomic::cmpxchg    (jint     exchange_value, volatile jint*     dest, jint     compare_value) {
   // alternative for InterlockedCompareExchange
   int mp = os::is_MP();

hotspot/src/os_cpu/windows_x86/vm/os_windows_x86.cpp

@@ -220,6 +220,7 @@ void os::initialize_thread(Thread* thr) {
 typedef jint      xchg_func_t            (jint,     volatile jint*);
 typedef intptr_t  xchg_ptr_func_t        (intptr_t, volatile intptr_t*);
 typedef jint      cmpxchg_func_t         (jint,     volatile jint*,  jint);
+typedef jbyte     cmpxchg_byte_func_t    (jbyte,    volatile jbyte*, jbyte);
 typedef jlong     cmpxchg_long_func_t    (jlong,    volatile jlong*, jlong);
 typedef jint      add_func_t             (jint,     volatile jint*);
 typedef intptr_t  add_ptr_func_t         (intptr_t, volatile intptr_t*);
@@ -272,6 +273,23 @@ jint os::atomic_cmpxchg_bootstrap(jint exchange_value, volatile jint* dest, jint
     *dest = exchange_value;
   return old_value;
 }
+
+jbyte os::atomic_cmpxchg_byte_bootstrap(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+  // try to use the stub:
+  cmpxchg_byte_func_t* func = CAST_TO_FN_PTR(cmpxchg_byte_func_t*, StubRoutines::atomic_cmpxchg_byte_entry());
+
+  if (func != NULL) {
+    os::atomic_cmpxchg_byte_func = func;
+    return (*func)(exchange_value, dest, compare_value);
+  }
+  assert(Threads::number_of_threads() == 0, "for bootstrap only");
+
+  jbyte old_value = *dest;
+  if (old_value == compare_value)
+    *dest = exchange_value;
+  return old_value;
+}
+
 #endif // AMD64
 
 jlong os::atomic_cmpxchg_long_bootstrap(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
@@ -321,6 +339,7 @@ intptr_t os::atomic_add_ptr_bootstrap(intptr_t add_value, volatile intptr_t* des
 xchg_func_t*         os::atomic_xchg_func         = os::atomic_xchg_bootstrap;
 xchg_ptr_func_t*     os::atomic_xchg_ptr_func     = os::atomic_xchg_ptr_bootstrap;
 cmpxchg_func_t*      os::atomic_cmpxchg_func      = os::atomic_cmpxchg_bootstrap;
+cmpxchg_byte_func_t* os::atomic_cmpxchg_byte_func = os::atomic_cmpxchg_byte_bootstrap;
 add_func_t*          os::atomic_add_func          = os::atomic_add_bootstrap;
 add_ptr_func_t*      os::atomic_add_ptr_func      = os::atomic_add_ptr_bootstrap;
 

hotspot/src/os_cpu/windows_x86/vm/os_windows_x86.hpp

@@ -33,6 +33,7 @@
   static intptr_t  (*atomic_xchg_ptr_func)      (intptr_t,  volatile intptr_t*);
 
   static jint      (*atomic_cmpxchg_func)       (jint,      volatile jint*,  jint);
+  static jbyte     (*atomic_cmpxchg_byte_func)  (jbyte,     volatile jbyte*, jbyte);
   static jlong     (*atomic_cmpxchg_long_func)  (jlong,     volatile jlong*, jlong);
 
   static jint      (*atomic_add_func)           (jint,      volatile jint*);
@@ -42,6 +43,7 @@
   static intptr_t  atomic_xchg_ptr_bootstrap    (intptr_t,  volatile intptr_t*);
 
   static jint      atomic_cmpxchg_bootstrap     (jint,      volatile jint*,  jint);
+  static jbyte     atomic_cmpxchg_byte_bootstrap(jbyte,     volatile jbyte*, jbyte);
 #else
 
   static jlong (*atomic_cmpxchg_long_func)  (jlong, volatile jlong*, jlong);

hotspot/src/share/vm/runtime/atomic.cpp

@@ -25,7 +25,13 @@
 #include "precompiled.hpp"
 #include "runtime/atomic.inline.hpp"
 
-jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+/*
+ * This is the default implementation of byte-sized cmpxchg. It emulates jbyte-sized cmpxchg
+ * in terms of jint-sized cmpxchg. Platforms may override this by defining their own inline definition
+ * as well as defining VM_HAS_SPECIALIZED_CMPXCHG_BYTE. This will cause the platform specific
+ * implementation to be used instead.
+ */
+jbyte Atomic::cmpxchg_general(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
   assert(sizeof(jbyte) == 1, "assumption.");
   uintptr_t dest_addr = (uintptr_t)dest;
   uintptr_t offset = dest_addr % sizeof(jint);

hotspot/src/share/vm/runtime/atomic.hpp

@@ -28,6 +28,9 @@
 #include "memory/allocation.hpp"
 
 class Atomic : AllStatic {
+ private:
+  static jbyte cmpxchg_general(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value);
+
  public:
   // Atomic operations on jlong types are not available on all 32-bit
   // platforms. If atomic ops on jlongs are defined here they must only
@@ -104,7 +107,7 @@ class Atomic : AllStatic {
   // *dest with exchange_value if the comparison succeeded. Returns prior
   // value of *dest. cmpxchg*() provide:
   // <fence> compare-and-exchange <membar StoreLoad|StoreStore>
-  static jbyte           cmpxchg    (jbyte        exchange_value, volatile jbyte*        dest, jbyte        compare_value);
+  inline static jbyte    cmpxchg    (jbyte        exchange_value, volatile jbyte*        dest, jbyte        compare_value);
   inline static jint     cmpxchg    (jint         exchange_value, volatile jint*         dest, jint         compare_value);
   // See comment above about using jlong atomics on 32-bit platforms
   inline static jlong    cmpxchg    (jlong        exchange_value, volatile jlong*        dest, jlong        compare_value);

hotspot/src/share/vm/runtime/atomic.inline.hpp

@@ -87,4 +87,12 @@ inline void Atomic::dec(volatile size_t* dest) {
   dec_ptr((volatile intptr_t*) dest);
 }
 
+#ifndef VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+// See comment in atomic.cpp how to override.
+inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte *dest, jbyte comparand)
+{
+  return cmpxchg_general(exchange_value, dest, comparand);
+}
+#endif // VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+
 #endif // SHARE_VM_RUNTIME_ATOMIC_INLINE_HPP

hotspot/src/share/vm/runtime/stubRoutines.cpp

@@ -62,6 +62,7 @@ address StubRoutines::_atomic_store_entry                       = NULL;
 address StubRoutines::_atomic_store_ptr_entry                   = NULL;
 address StubRoutines::_atomic_cmpxchg_entry                     = NULL;
 address StubRoutines::_atomic_cmpxchg_ptr_entry                 = NULL;
+address StubRoutines::_atomic_cmpxchg_byte_entry                = NULL;
 address StubRoutines::_atomic_cmpxchg_long_entry                = NULL;
 address StubRoutines::_atomic_add_entry                         = NULL;
 address StubRoutines::_atomic_add_ptr_entry                     = NULL;

hotspot/src/share/vm/runtime/stubRoutines.hpp

@@ -126,6 +126,7 @@ class StubRoutines: AllStatic {
   static address _atomic_store_ptr_entry;
   static address _atomic_cmpxchg_entry;
   static address _atomic_cmpxchg_ptr_entry;
+  static address _atomic_cmpxchg_byte_entry;
   static address _atomic_cmpxchg_long_entry;
   static address _atomic_add_entry;
   static address _atomic_add_ptr_entry;
@@ -282,6 +283,7 @@ class StubRoutines: AllStatic {
   static address atomic_store_ptr_entry()                  { return _atomic_store_ptr_entry; }
   static address atomic_cmpxchg_entry()                    { return _atomic_cmpxchg_entry; }
   static address atomic_cmpxchg_ptr_entry()                { return _atomic_cmpxchg_ptr_entry; }
+  static address atomic_cmpxchg_byte_entry()               { return _atomic_cmpxchg_byte_entry; }
   static address atomic_cmpxchg_long_entry()               { return _atomic_cmpxchg_long_entry; }
   static address atomic_add_entry()                        { return _atomic_add_entry; }
   static address atomic_add_ptr_entry()                    { return _atomic_add_ptr_entry; }