8241673: Shenandoah: refactor anti-false-sharing padding

Reviewed-by: rkennke

src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.hpp

@@ -29,6 +29,7 @@
 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
 #include "gc/shenandoah/shenandoahLock.hpp"
 #include "gc/shenandoah/shenandoahNMethod.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
 #include "memory/iterator.hpp"
 #include "utilities/globalDefinitions.hpp"
@@ -40,10 +41,10 @@ class ShenandoahParallelCodeHeapIterator {
   friend class CodeCache;
 private:
   CodeHeap*     _heap;
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int));
+  shenandoah_padding(0);
   volatile int  _claimed_idx;
   volatile bool _finished;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 public:
   ShenandoahParallelCodeHeapIterator(CodeHeap* heap);
   void parallel_blobs_do(CodeBlobClosure* f);

src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp

@@ -28,6 +28,7 @@
 #include "memory/allocation.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 
 class ShenandoahCollectionSet : public CHeapObj<mtGC> {
   friend class ShenandoahHeap;
@@ -46,9 +47,9 @@ private:
   size_t                _used;
   size_t                _region_count;
 
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
+  shenandoah_padding(0);
   volatile jint         _current_index;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
 public:
   ShenandoahCollectionSet(ShenandoahHeap* heap, char* heap_base, size_t size);

src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp

@@ -28,6 +28,7 @@
 #include "gc/shared/gcCause.hpp"
 #include "gc/shared/concurrentGCThread.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
 #include "runtime/task.hpp"
 #include "utilities/ostream.hpp"
@@ -85,9 +86,9 @@ private:
   GCCause::Cause       _requested_gc_cause;
   ShenandoahHeap::ShenandoahDegenPoint _degen_point;
 
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
+  shenandoah_padding(0);
   volatile size_t _allocs_seen;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
   bool check_cancellation_or_degen(ShenandoahHeap::ShenandoahDegenPoint point);
   void service_concurrent_normal_cycle(GCCause::Cause cause);

src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_HPP
 #define SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_HPP
 
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/globalDefinitions.hpp"
 
@@ -81,9 +82,9 @@ class ShenandoahEvacOOMHandler {
 private:
   static const jint OOM_MARKER_MASK;
 
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile jint));
+  shenandoah_padding(0);
   volatile jint _threads_in_evac;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
   void wait_for_no_evac_threads();
 

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -54,6 +54,7 @@
 #include "gc/shenandoah/shenandoahNormalMode.hpp"
 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
 #include "gc/shenandoah/shenandoahPassiveMode.hpp"
 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
@@ -1321,9 +1322,9 @@ private:
   ShenandoahHeap* const _heap;
   ShenandoahHeapRegionClosure* const _blk;
 
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
+  shenandoah_padding(0);
   volatile size_t _index;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
 public:
   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -32,6 +32,7 @@
 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
 #include "gc/shenandoah/shenandoahLock.hpp"
 #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
 #include "gc/shenandoah/shenandoahUnload.hpp"
 #include "services/memoryManager.hpp"
@@ -66,9 +67,9 @@ class ShenandoahRegionIterator : public StackObj {
 private:
   ShenandoahHeap* _heap;
 
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
+  shenandoah_padding(0);
   volatile size_t _index;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
   // No implicit copying: iterators should be passed by reference to capture the state
   NONCOPYABLE(ShenandoahRegionIterator);
@@ -161,11 +162,11 @@ public:
 private:
            size_t _initial_size;
            size_t _minimum_size;
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
+  shenandoah_padding(0);
   volatile size_t _used;
   volatile size_t _committed;
   volatile size_t _bytes_allocated_since_gc_start;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
 public:
   void increase_used(size_t bytes);

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -30,6 +30,7 @@
 #include "gc/shenandoah/shenandoahAsserts.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahPacer.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "utilities/sizes.hpp"
 
 class VMStructs;
@@ -224,9 +225,9 @@ private:
   // Global allocation counter, increased for each allocation under Shenandoah heap lock.
   // Padded to avoid false sharing with the read-only fields above.
   struct PaddedAllocSeqNum {
-    DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(uint64_t));
+    shenandoah_padding(0);
     uint64_t value;
-    DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+    shenandoah_padding(1);
 
     PaddedAllocSeqNum() {
       // start with 1, reserve 0 for uninitialized value
@@ -265,7 +266,7 @@ private:
   HeapWord* _update_watermark;
 
   // Claim some space at the end to protect next region
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(0);
 
 public:
   ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start, size_t size_words, size_t index, bool committed);

src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.hpp

@@ -28,6 +28,7 @@
 #include "memory/allocation.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 class ShenandoahHeapRegionSet;
@@ -37,9 +38,9 @@ private:
   const ShenandoahHeapRegionSet* _set;
   ShenandoahHeap* const _heap;
 
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile jint));
+  shenandoah_padding(0);
   volatile jint _current_index;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
   // No implicit copying: iterators should be passed by reference to capture the state
   NONCOPYABLE(ShenandoahHeapRegionSetIterator);

src/hotspot/share/gc/shenandoah/shenandoahLock.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHLOCK_HPP
 #define SHARE_GC_SHENANDOAH_SHENANDOAHLOCK_HPP
 
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
 #include "runtime/safepoint.hpp"
 #include "runtime/thread.hpp"
@@ -33,11 +34,11 @@ class ShenandoahLock  {
 private:
   enum LockState { unlocked = 0, locked = 1 };
 
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int));
+  shenandoah_padding(0);
   volatile int _state;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile Thread*));
+  shenandoah_padding(1);
   volatile Thread* _owner;
-  DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(2);
 
 public:
   ShenandoahLock() : _state(unlocked), _owner(NULL) {};

src/hotspot/share/gc/shenandoah/shenandoahNMethod.hpp

@@ -28,6 +28,7 @@
 #include "code/nmethod.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahLock.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/growableArray.hpp"
 
@@ -91,9 +92,9 @@ private:
   ShenandoahNMethod** const   _array;
   const int                   _length;
 
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
+  shenandoah_padding(0);
   volatile size_t       _claimed;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
 public:
   ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table);

src/hotspot/share/gc/shenandoah/shenandoahPacer.hpp

@@ -26,6 +26,7 @@
 #define SHARE_GC_SHENANDOAH_SHENANDOAHPACER_HPP
 
 #include "gc/shenandoah/shenandoahNumberSeq.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
 
 class ShenandoahHeap;
@@ -50,14 +51,14 @@ private:
   volatile double _tax_rate;
 
   // Heavily updated, protect from accidental false sharing
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile intptr_t));
+  shenandoah_padding(0);
   volatile intptr_t _budget;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
   // Heavily updated, protect from accidental false sharing
-  DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile intptr_t));
+  shenandoah_padding(2);
   volatile intptr_t _progress;
-  DEFINE_PAD_MINUS_SIZE(3, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(3);
 
 public:
   ShenandoahPacer(ShenandoahHeap* heap) :

src/hotspot/share/gc/shenandoah/shenandoahPadding.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2020, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHPADDING_INLINE_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHPADDING_INLINE_HPP
+
+// 64 bytes is enough to cover all existing architectures. If we have some
+// other platforms, we would need to provide the architecture-specific
+// versions here. Shared code provides DEFAULT_CACHE_LINE_SIZE, which is
+// inconveniently large by default.
+
+#define shenandoah_padding(id) \
+  DEFINE_PAD_MINUS_SIZE(id, 64, 0)
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHPADDING_INLINE_HPP

src/hotspot/share/gc/shenandoah/shenandoahSharedVariables.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSHAREDVARIABLES_HPP
 #define SHARE_GC_SHENANDOAH_SHENANDOAHSHAREDVARIABLES_HPP
 
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
 #include "runtime/atomic.hpp"
 
@@ -39,9 +40,9 @@ typedef struct ShenandoahSharedFlag {
     SET = 1
   };
 
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile ShenandoahSharedValue));
+  shenandoah_padding(0);
   volatile ShenandoahSharedValue value;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
   ShenandoahSharedFlag() {
     unset();
@@ -107,9 +108,9 @@ private:
 } ShenandoahSharedFlag;
 
 typedef struct ShenandoahSharedBitmap {
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile ShenandoahSharedValue));
+  shenandoah_padding(0);
   volatile ShenandoahSharedValue value;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
   ShenandoahSharedBitmap() {
     clear();
@@ -201,9 +202,9 @@ private:
 
 template<class T>
 struct ShenandoahSharedEnumFlag {
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile ShenandoahSharedValue));
+  shenandoah_padding(0);
   volatile ShenandoahSharedValue value;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
   ShenandoahSharedEnumFlag() {
     value = 0;

src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.hpp

@@ -27,6 +27,7 @@
 
 #include "gc/shared/taskTerminator.hpp"
 #include "gc/shared/taskqueue.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/mutex.hpp"
@@ -272,9 +273,9 @@ typedef Padded<ShenandoahBufferedOverflowTaskQueue> ShenandoahObjToScanQueue;
 template <class T, MEMFLAGS F>
 class ParallelClaimableQueueSet: public GenericTaskQueueSet<T, F> {
 private:
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile jint));
+  shenandoah_padding(0);
   volatile jint     _claimed_index;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+  shenandoah_padding(1);
 
   debug_only(uint   _reserved;  )