mirror of
https://github.com/openjdk/jdk.git
synced 2025-08-27 06:45:07 +02:00
8192943: Optimize atomic accumulators using VarHandle getAndSet
Reviewed-by: martin, psandoz, chegar
This commit is contained in:
Doug Lea
parent
0b3b384a27
commit
00d1900dc9
5 changed files with 113 additions and 117 deletions
|
|
@ -105,20 +105,20 @@ public class DoubleAccumulator extends Striped64 implements Serializable {
|
|||
* @param x the value
|
||||
*/
|
||||
public void accumulate(double x) {
|
||||
Cell[] as; long b, v, r; int m; Cell a;
|
||||
if ((as = cells) != null
|
||||
Cell[] cs; long b, v, r; int m; Cell c;
|
||||
if ((cs = cells) != null
|
||||
|| ((r = doubleToRawLongBits
|
||||
(function.applyAsDouble(longBitsToDouble(b = base), x))) != b
|
||||
&& !casBase(b, r))) {
|
||||
boolean uncontended = true;
|
||||
if (as == null
|
||||
|| (m = as.length - 1) < 0
|
||||
|| (a = as[getProbe() & m]) == null
|
||||
if (cs == null
|
||||
|| (m = cs.length - 1) < 0
|
||||
|| (c = cs[getProbe() & m]) == null
|
||||
|| !(uncontended =
|
||||
((r = doubleToRawLongBits
|
||||
(function.applyAsDouble
|
||||
(longBitsToDouble(v = a.value), x))) == v)
|
||||
|| a.cas(v, r)))
|
||||
(longBitsToDouble(v = c.value), x))) == v)
|
||||
|| c.cas(v, r)))
|
||||
doubleAccumulate(x, function, uncontended);
|
||||
}
|
||||
}
|
||||
|
|
@ -133,13 +133,13 @@ public class DoubleAccumulator extends Striped64 implements Serializable {
|
|||
* @return the current value
|
||||
*/
|
||||
public double get() {
|
||||
Cell[] as = cells;
|
||||
Cell[] cs = cells;
|
||||
double result = longBitsToDouble(base);
|
||||
if (as != null) {
|
||||
for (Cell a : as)
|
||||
if (a != null)
|
||||
if (cs != null) {
|
||||
for (Cell c : cs)
|
||||
if (c != null)
|
||||
result = function.applyAsDouble
|
||||
(result, longBitsToDouble(a.value));
|
||||
(result, longBitsToDouble(c.value));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
@ -153,12 +153,12 @@ public class DoubleAccumulator extends Striped64 implements Serializable {
|
|||
* updating.
|
||||
*/
|
||||
public void reset() {
|
||||
Cell[] as = cells;
|
||||
Cell[] cs = cells;
|
||||
base = identity;
|
||||
if (as != null) {
|
||||
for (Cell a : as)
|
||||
if (a != null)
|
||||
a.reset(identity);
|
||||
if (cs != null) {
|
||||
for (Cell c : cs)
|
||||
if (c != null)
|
||||
c.reset(identity);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -173,14 +173,12 @@ public class DoubleAccumulator extends Striped64 implements Serializable {
|
|||
* @return the value before reset
|
||||
*/
|
||||
public double getThenReset() {
|
||||
Cell[] as = cells;
|
||||
double result = longBitsToDouble(base);
|
||||
base = identity;
|
||||
if (as != null) {
|
||||
for (Cell a : as) {
|
||||
if (a != null) {
|
||||
double v = longBitsToDouble(a.value);
|
||||
a.reset(identity);
|
||||
Cell[] cs = cells;
|
||||
double result = longBitsToDouble(getAndSetBase(identity));
|
||||
if (cs != null) {
|
||||
for (Cell c : cs) {
|
||||
if (c != null) {
|
||||
double v = longBitsToDouble(c.getAndSet(identity));
|
||||
result = function.applyAsDouble(result, v);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,15 +87,15 @@ public class DoubleAdder extends Striped64 implements Serializable {
|
|||
* @param x the value to add
|
||||
*/
|
||||
public void add(double x) {
|
||||
Cell[] as; long b, v; int m; Cell a;
|
||||
if ((as = cells) != null ||
|
||||
Cell[] cs; long b, v; int m; Cell c;
|
||||
if ((cs = cells) != null ||
|
||||
!casBase(b = base,
|
||||
Double.doubleToRawLongBits
|
||||
(Double.longBitsToDouble(b) + x))) {
|
||||
boolean uncontended = true;
|
||||
if (as == null || (m = as.length - 1) < 0 ||
|
||||
(a = as[getProbe() & m]) == null ||
|
||||
!(uncontended = a.cas(v = a.value,
|
||||
if (cs == null || (m = cs.length - 1) < 0 ||
|
||||
(c = cs[getProbe() & m]) == null ||
|
||||
!(uncontended = c.cas(v = c.value,
|
||||
Double.doubleToRawLongBits
|
||||
(Double.longBitsToDouble(v) + x))))
|
||||
doubleAccumulate(x, null, uncontended);
|
||||
|
|
@ -115,12 +115,12 @@ public class DoubleAdder extends Striped64 implements Serializable {
|
|||
* @return the sum
|
||||
*/
|
||||
public double sum() {
|
||||
Cell[] as = cells;
|
||||
Cell[] cs = cells;
|
||||
double sum = Double.longBitsToDouble(base);
|
||||
if (as != null) {
|
||||
for (Cell a : as)
|
||||
if (a != null)
|
||||
sum += Double.longBitsToDouble(a.value);
|
||||
if (cs != null) {
|
||||
for (Cell c : cs)
|
||||
if (c != null)
|
||||
sum += Double.longBitsToDouble(c.value);
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
|
@ -133,12 +133,12 @@ public class DoubleAdder extends Striped64 implements Serializable {
|
|||
* known that no threads are concurrently updating.
|
||||
*/
|
||||
public void reset() {
|
||||
Cell[] as = cells;
|
||||
Cell[] cs = cells;
|
||||
base = 0L; // relies on fact that double 0 must have same rep as long
|
||||
if (as != null) {
|
||||
for (Cell a : as)
|
||||
if (a != null)
|
||||
a.reset();
|
||||
if (cs != null) {
|
||||
for (Cell c : cs)
|
||||
if (c != null)
|
||||
c.reset();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -153,16 +153,12 @@ public class DoubleAdder extends Striped64 implements Serializable {
|
|||
* @return the sum
|
||||
*/
|
||||
public double sumThenReset() {
|
||||
Cell[] as = cells;
|
||||
double sum = Double.longBitsToDouble(base);
|
||||
base = 0L;
|
||||
if (as != null) {
|
||||
for (Cell a : as) {
|
||||
if (a != null) {
|
||||
long v = a.value;
|
||||
a.reset();
|
||||
sum += Double.longBitsToDouble(v);
|
||||
}
|
||||
Cell[] cs = cells;
|
||||
double sum = Double.longBitsToDouble(getAndSetBase(0L));
|
||||
if (cs != null) {
|
||||
for (Cell c : cs) {
|
||||
if (c != null)
|
||||
sum += Double.longBitsToDouble(c.getAndSet(0L));
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
|
|
|
|||
|
|
@ -103,17 +103,17 @@ public class LongAccumulator extends Striped64 implements Serializable {
|
|||
* @param x the value
|
||||
*/
|
||||
public void accumulate(long x) {
|
||||
Cell[] as; long b, v, r; int m; Cell a;
|
||||
if ((as = cells) != null
|
||||
Cell[] cs; long b, v, r; int m; Cell c;
|
||||
if ((cs = cells) != null
|
||||
|| ((r = function.applyAsLong(b = base, x)) != b
|
||||
&& !casBase(b, r))) {
|
||||
boolean uncontended = true;
|
||||
if (as == null
|
||||
|| (m = as.length - 1) < 0
|
||||
|| (a = as[getProbe() & m]) == null
|
||||
if (cs == null
|
||||
|| (m = cs.length - 1) < 0
|
||||
|| (c = cs[getProbe() & m]) == null
|
||||
|| !(uncontended =
|
||||
(r = function.applyAsLong(v = a.value, x)) == v
|
||||
|| a.cas(v, r)))
|
||||
(r = function.applyAsLong(v = c.value, x)) == v
|
||||
|| c.cas(v, r)))
|
||||
longAccumulate(x, function, uncontended);
|
||||
}
|
||||
}
|
||||
|
|
@ -128,12 +128,12 @@ public class LongAccumulator extends Striped64 implements Serializable {
|
|||
* @return the current value
|
||||
*/
|
||||
public long get() {
|
||||
Cell[] as = cells;
|
||||
Cell[] cs = cells;
|
||||
long result = base;
|
||||
if (as != null) {
|
||||
for (Cell a : as)
|
||||
if (a != null)
|
||||
result = function.applyAsLong(result, a.value);
|
||||
if (cs != null) {
|
||||
for (Cell c : cs)
|
||||
if (c != null)
|
||||
result = function.applyAsLong(result, c.value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
@ -147,12 +147,12 @@ public class LongAccumulator extends Striped64 implements Serializable {
|
|||
* updating.
|
||||
*/
|
||||
public void reset() {
|
||||
Cell[] as = cells;
|
||||
Cell[] cs = cells;
|
||||
base = identity;
|
||||
if (as != null) {
|
||||
for (Cell a : as)
|
||||
if (a != null)
|
||||
a.reset(identity);
|
||||
if (cs != null) {
|
||||
for (Cell c : cs)
|
||||
if (c != null)
|
||||
c.reset(identity);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -167,14 +167,12 @@ public class LongAccumulator extends Striped64 implements Serializable {
|
|||
* @return the value before reset
|
||||
*/
|
||||
public long getThenReset() {
|
||||
Cell[] as = cells;
|
||||
long result = base;
|
||||
base = identity;
|
||||
if (as != null) {
|
||||
for (Cell a : as) {
|
||||
if (a != null) {
|
||||
long v = a.value;
|
||||
a.reset(identity);
|
||||
Cell[] cs = cells;
|
||||
long result = getAndSetBase(identity);
|
||||
if (cs != null) {
|
||||
for (Cell c : cs) {
|
||||
if (c != null) {
|
||||
long v = c.getAndSet(identity);
|
||||
result = function.applyAsLong(result, v);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,12 +83,12 @@ public class LongAdder extends Striped64 implements Serializable {
|
|||
* @param x the value to add
|
||||
*/
|
||||
public void add(long x) {
|
||||
Cell[] as; long b, v; int m; Cell a;
|
||||
if ((as = cells) != null || !casBase(b = base, b + x)) {
|
||||
Cell[] cs; long b, v; int m; Cell c;
|
||||
if ((cs = cells) != null || !casBase(b = base, b + x)) {
|
||||
boolean uncontended = true;
|
||||
if (as == null || (m = as.length - 1) < 0 ||
|
||||
(a = as[getProbe() & m]) == null ||
|
||||
!(uncontended = a.cas(v = a.value, v + x)))
|
||||
if (cs == null || (m = cs.length - 1) < 0 ||
|
||||
(c = cs[getProbe() & m]) == null ||
|
||||
!(uncontended = c.cas(v = c.value, v + x)))
|
||||
longAccumulate(x, null, uncontended);
|
||||
}
|
||||
}
|
||||
|
|
@ -117,12 +117,12 @@ public class LongAdder extends Striped64 implements Serializable {
|
|||
* @return the sum
|
||||
*/
|
||||
public long sum() {
|
||||
Cell[] as = cells;
|
||||
Cell[] cs = cells;
|
||||
long sum = base;
|
||||
if (as != null) {
|
||||
for (Cell a : as)
|
||||
if (a != null)
|
||||
sum += a.value;
|
||||
if (cs != null) {
|
||||
for (Cell c : cs)
|
||||
if (c != null)
|
||||
sum += c.value;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
|
@ -135,12 +135,12 @@ public class LongAdder extends Striped64 implements Serializable {
|
|||
* known that no threads are concurrently updating.
|
||||
*/
|
||||
public void reset() {
|
||||
Cell[] as = cells;
|
||||
Cell[] cs = cells;
|
||||
base = 0L;
|
||||
if (as != null) {
|
||||
for (Cell a : as)
|
||||
if (a != null)
|
||||
a.reset();
|
||||
if (cs != null) {
|
||||
for (Cell c : cs)
|
||||
if (c != null)
|
||||
c.reset();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -155,15 +155,12 @@ public class LongAdder extends Striped64 implements Serializable {
|
|||
* @return the sum
|
||||
*/
|
||||
public long sumThenReset() {
|
||||
Cell[] as = cells;
|
||||
long sum = base;
|
||||
base = 0L;
|
||||
if (as != null) {
|
||||
for (Cell a : as) {
|
||||
if (a != null) {
|
||||
sum += a.value;
|
||||
a.reset();
|
||||
}
|
||||
Cell[] cs = cells;
|
||||
long sum = getAndSetBase(0L);
|
||||
if (cs != null) {
|
||||
for (Cell c : cs) {
|
||||
if (c != null)
|
||||
sum += c.getAndSet(0L);
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
|
|
|
|||
|
|
@ -133,6 +133,9 @@ abstract class Striped64 extends Number {
|
|||
final void reset(long identity) {
|
||||
VALUE.setVolatile(this, identity);
|
||||
}
|
||||
final long getAndSet(long val) {
|
||||
return (long)VALUE.getAndSet(this, val);
|
||||
}
|
||||
|
||||
// VarHandle mechanics
|
||||
private static final VarHandle VALUE;
|
||||
|
|
@ -178,6 +181,10 @@ abstract class Striped64 extends Number {
|
|||
return BASE.compareAndSet(this, cmp, val);
|
||||
}
|
||||
|
||||
final long getAndSetBase(long val) {
|
||||
return (long)BASE.getAndSet(this, val);
|
||||
}
|
||||
|
||||
/**
|
||||
* CASes the cellsBusy field from 0 to 1 to acquire lock.
|
||||
*/
|
||||
|
|
@ -228,9 +235,9 @@ abstract class Striped64 extends Number {
|
|||
}
|
||||
boolean collide = false; // True if last slot nonempty
|
||||
done: for (;;) {
|
||||
Cell[] as; Cell a; int n; long v;
|
||||
if ((as = cells) != null && (n = as.length) > 0) {
|
||||
if ((a = as[(n - 1) & h]) == null) {
|
||||
Cell[] cs; Cell c; int n; long v;
|
||||
if ((cs = cells) != null && (n = cs.length) > 0) {
|
||||
if ((c = cs[(n - 1) & h]) == null) {
|
||||
if (cellsBusy == 0) { // Try to attach new Cell
|
||||
Cell r = new Cell(x); // Optimistically create
|
||||
if (cellsBusy == 0 && casCellsBusy()) {
|
||||
|
|
@ -252,17 +259,17 @@ abstract class Striped64 extends Number {
|
|||
}
|
||||
else if (!wasUncontended) // CAS already known to fail
|
||||
wasUncontended = true; // Continue after rehash
|
||||
else if (a.cas(v = a.value,
|
||||
else if (c.cas(v = c.value,
|
||||
(fn == null) ? v + x : fn.applyAsLong(v, x)))
|
||||
break;
|
||||
else if (n >= NCPU || cells != as)
|
||||
else if (n >= NCPU || cells != cs)
|
||||
collide = false; // At max size or stale
|
||||
else if (!collide)
|
||||
collide = true;
|
||||
else if (cellsBusy == 0 && casCellsBusy()) {
|
||||
try {
|
||||
if (cells == as) // Expand table unless stale
|
||||
cells = Arrays.copyOf(as, n << 1);
|
||||
if (cells == cs) // Expand table unless stale
|
||||
cells = Arrays.copyOf(cs, n << 1);
|
||||
} finally {
|
||||
cellsBusy = 0;
|
||||
}
|
||||
|
|
@ -271,9 +278,9 @@ abstract class Striped64 extends Number {
|
|||
}
|
||||
h = advanceProbe(h);
|
||||
}
|
||||
else if (cellsBusy == 0 && cells == as && casCellsBusy()) {
|
||||
else if (cellsBusy == 0 && cells == cs && casCellsBusy()) {
|
||||
try { // Initialize table
|
||||
if (cells == as) {
|
||||
if (cells == cs) {
|
||||
Cell[] rs = new Cell[2];
|
||||
rs[h & 1] = new Cell(x);
|
||||
cells = rs;
|
||||
|
|
@ -312,9 +319,9 @@ abstract class Striped64 extends Number {
|
|||
}
|
||||
boolean collide = false; // True if last slot nonempty
|
||||
done: for (;;) {
|
||||
Cell[] as; Cell a; int n; long v;
|
||||
if ((as = cells) != null && (n = as.length) > 0) {
|
||||
if ((a = as[(n - 1) & h]) == null) {
|
||||
Cell[] cs; Cell c; int n; long v;
|
||||
if ((cs = cells) != null && (n = cs.length) > 0) {
|
||||
if ((c = cs[(n - 1) & h]) == null) {
|
||||
if (cellsBusy == 0) { // Try to attach new Cell
|
||||
Cell r = new Cell(Double.doubleToRawLongBits(x));
|
||||
if (cellsBusy == 0 && casCellsBusy()) {
|
||||
|
|
@ -336,16 +343,16 @@ abstract class Striped64 extends Number {
|
|||
}
|
||||
else if (!wasUncontended) // CAS already known to fail
|
||||
wasUncontended = true; // Continue after rehash
|
||||
else if (a.cas(v = a.value, apply(fn, v, x)))
|
||||
else if (c.cas(v = c.value, apply(fn, v, x)))
|
||||
break;
|
||||
else if (n >= NCPU || cells != as)
|
||||
else if (n >= NCPU || cells != cs)
|
||||
collide = false; // At max size or stale
|
||||
else if (!collide)
|
||||
collide = true;
|
||||
else if (cellsBusy == 0 && casCellsBusy()) {
|
||||
try {
|
||||
if (cells == as) // Expand table unless stale
|
||||
cells = Arrays.copyOf(as, n << 1);
|
||||
if (cells == cs) // Expand table unless stale
|
||||
cells = Arrays.copyOf(cs, n << 1);
|
||||
} finally {
|
||||
cellsBusy = 0;
|
||||
}
|
||||
|
|
@ -354,9 +361,9 @@ abstract class Striped64 extends Number {
|
|||
}
|
||||
h = advanceProbe(h);
|
||||
}
|
||||
else if (cellsBusy == 0 && cells == as && casCellsBusy()) {
|
||||
else if (cellsBusy == 0 && cells == cs && casCellsBusy()) {
|
||||
try { // Initialize table
|
||||
if (cells == as) {
|
||||
if (cells == cs) {
|
||||
Cell[] rs = new Cell[2];
|
||||
rs[h & 1] = new Cell(Double.doubleToRawLongBits(x));
|
||||
cells = rs;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue