archive/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2025-08-28 15:24:43 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

8284960: Integration of JEP 426: Vector API (Fourth Incubator)

Browse source 
Co-authored-by: Jatin Bhateja <jbhateja@openjdk.org>
Co-authored-by: Paul Sandoz <psandoz@openjdk.org>
Co-authored-by: Sandhya Viswanathan <sviswanathan@openjdk.org>
Co-authored-by: Smita Kamath <svkamath@openjdk.org>
Co-authored-by: Joshua Zhu <jzhu@openjdk.org>
Co-authored-by: Xiaohong Gong <xgong@openjdk.org>
Co-authored-by: John R Rose <jrose@openjdk.org>
Co-authored-by: Eric Liu <eliu@openjdk.org>
Co-authored-by: Ningsheng Jian <njian@openjdk.org>
Reviewed-by: ngasson, vlivanov, mcimadamore, jlahoda, kvn
This commit is contained in:
Jatin Bhateja 2022-05-31 16:02:09 +00:00
parent 171a7cdd5d
commit 6f6486e977
227 changed files with 20949 additions and 21221 deletions
Download patch file Download diff file Expand all files Collapse all files

125
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java
View file

@ -28,6 +28,10 @@ import java.util.Objects;
import jdk.internal.vm.annotation.ForceInline;
import jdk.internal.misc.Unsafe;
import jdk.internal.vm.vector.VectorSupport;
import static jdk.incubator.vector.VectorOperators.*;
abstract class AbstractMask<E> extends VectorMask<E> {
@ -77,7 +81,15 @@ abstract class AbstractMask<E> extends VectorMask<E> {
@Override
public void intoArray(boolean[] bits, int i) {
System.arraycopy(getBits(), 0, bits, i, length());
AbstractSpecies<E> vsp = (AbstractSpecies<E>) vectorSpecies();
int laneCount = vsp.laneCount();
i = VectorIntrinsics.checkFromIndexSize(i, laneCount, bits.length);
VectorSupport.store(
vsp.maskType(), vsp.elementType(), laneCount,
bits, (long) i + Unsafe.ARRAY_BOOLEAN_BASE_OFFSET,
this, bits, i,
(c, idx, s) -> System.arraycopy(s.getBits(), 0, c, (int) idx, s.length()));
}
@Override
@ -192,6 +204,15 @@ abstract class AbstractMask<E> extends VectorMask<E> {
return this.andNot(badMask);
}
@Override
@ForceInline
public VectorMask<E> indexInRange(long offset, long limit) {
int vlength = length();
Vector<E> iota = vectorSpecies().zero().addIndex(1);
VectorMask<E> badMask = checkIndex0(offset, limit, iota, vlength);
return this.andNot(badMask);
}
/*package-private*/
@ForceInline
AbstractVector<E>
@ -215,7 +236,7 @@ abstract class AbstractMask<E> extends VectorMask<E> {
*/
/*package-private*/
@ForceInline
void checkIndexByLane(int offset, int alength,
void checkIndexByLane(int offset, int length,
Vector<E> iota,
int esize) {
if (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK == 0) {
@ -229,15 +250,15 @@ abstract class AbstractMask<E> extends VectorMask<E> {
int vlength = length();
VectorMask<E> badMask;
if (esize == 1) {
badMask = checkIndex0(offset, alength, iota, vlength);
badMask = checkIndex0(offset, length, iota, vlength);
} else if (offset >= 0) {
// Masked access to multi-byte lanes in byte array.
// It could be aligned anywhere.
int elemCount = Math.min(vlength, (alength - offset) / esize);
int elemCount = Math.min(vlength, (length - offset) / esize);
badMask = checkIndex0(0, elemCount, iota, vlength);
} else {
int clipOffset = Math.max(offset, -(vlength * esize));
badMask = checkIndex0(clipOffset, alength,
badMask = checkIndex0(clipOffset, length,
iota.lanewise(VectorOperators.MUL, esize),
vlength * esize);
}
@ -245,20 +266,20 @@ abstract class AbstractMask<E> extends VectorMask<E> {
if (badMask.anyTrue()) {
int badLane = badMask.firstTrue();
throw ((AbstractMask<E>)badMask)
.checkIndexFailed(offset, badLane, alength, esize);
.checkIndexFailed(offset, badLane, length, esize);
}
}
private
@ForceInline
VectorMask<E> checkIndex0(int offset, int alength,
VectorMask<E> checkIndex0(int offset, int length,
Vector<E> iota, int vlength) {
// An active lane is bad if its number is greater than
// alength-offset, since when added to offset it will step off
// length-offset, since when added to offset it will step off
// of the end of the array. To avoid overflow when
// converting, clip the comparison value to [0..vlength]
// inclusive.
int indexLimit = Math.max(0, Math.min(alength - offset, vlength));
int indexLimit = Math.max(0, Math.min(length - offset, vlength));
VectorMask<E> badMask =
iota.compare(GE, iota.broadcast(indexLimit));
if (offset < 0) {
@ -280,14 +301,90 @@ abstract class AbstractMask<E> extends VectorMask<E> {
return badMask;
}
private IndexOutOfBoundsException checkIndexFailed(int offset, int lane,
int alength, int esize) {
/**
* Test if a masked memory access at a given offset into an array
* of the given length will stay within the array.
* The per-lane offsets are iota*esize.
*/
/*package-private*/
@ForceInline
void checkIndexByLane(long offset, long length,
Vector<E> iota,
int esize) {
if (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK == 0) {
return;
}
// Although the specification is simple, the implementation is
// tricky, because the value iota*esize might possibly
// overflow. So we calculate our test values as scalars,
// clipping to the range [-1..VLENGTH], and test them against
// the unscaled iota vector, whose values are in [0..VLENGTH-1].
int vlength = length();
VectorMask<E> badMask;
if (esize == 1) {
badMask = checkIndex0(offset, length, iota, vlength);
} else if (offset >= 0) {
// Masked access to multi-byte lanes in byte array.
// It could be aligned anywhere.
// 0 <= elemCount <= vlength
int elemCount = (int) Math.min(vlength, (length - offset) / esize);
badMask = checkIndex0(0, elemCount, iota, vlength);
} else {
// -vlength * esize <= clipOffset <= 0
int clipOffset = (int) Math.max(offset, -(vlength * esize));
badMask = checkIndex0(clipOffset, length,
iota.lanewise(VectorOperators.MUL, esize),
vlength * esize);
}
badMask = badMask.and(this);
if (badMask.anyTrue()) {
int badLane = badMask.firstTrue();
throw ((AbstractMask<E>)badMask)
.checkIndexFailed(offset, badLane, length, esize);
}
}
private
@ForceInline
VectorMask<E> checkIndex0(long offset, long length,
Vector<E> iota, int vlength) {
// An active lane is bad if its number is greater than
// length-offset, since when added to offset it will step off
// of the end of the array. To avoid overflow when
// converting, clip the comparison value to [0..vlength]
// inclusive.
// 0 <= indexLimit <= vlength
int indexLimit = (int) Math.max(0, Math.min(length - offset, vlength));
VectorMask<E> badMask =
iota.compare(GE, iota.broadcast(indexLimit));
if (offset < 0) {
// An active lane is bad if its number is less than
// -offset, because when added to offset it will then
// address an array element at a negative index. To avoid
// overflow when converting, clip the comparison value at
// vlength. This specific expression works correctly even
// when offset is Integer.MIN_VALUE.
// 0 <= firstGoodIndex <= vlength
int firstGoodIndex = (int) -Math.max(offset, -vlength);
VectorMask<E> badMask2 =
iota.compare(LT, iota.broadcast(firstGoodIndex));
if (indexLimit >= vlength) {
badMask = badMask2; // 1st badMask is all true
} else {
badMask = badMask.or(badMask2);
}
}
return badMask;
}
private IndexOutOfBoundsException checkIndexFailed(long offset, int lane,
long length, int esize) {
String msg = String.format("Masked range check failed: "+
"vector mask %s out of bounds at "+
"index %d+%d in array of length %d",
this, offset, lane * esize, alength);
"index %d+%d for length %d",
this, offset, lane * esize, length);
if (esize != 1) {
msg += String.format(" (each lane spans %d array elements)", esize);
msg += String.format(" (each lane spans %d elements)", esize);
}
throw new IndexOutOfBoundsException(msg);
}

21
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -24,10 +24,11 @@
*/
package jdk.incubator.vector;
import java.lang.foreign.MemorySegment;
import jdk.internal.vm.annotation.ForceInline;
import jdk.internal.vm.annotation.Stable;
import java.nio.ByteOrder;
import java.lang.reflect.Array;
import java.nio.ByteOrder;
import java.util.Arrays;
import java.util.function.Function;
import java.util.function.IntUnaryOperator;
@ -203,12 +204,24 @@ abstract class AbstractSpecies<E> extends jdk.internal.vm.vector.VectorSupport.V
return VectorIntrinsics.roundDown(length, laneCount);
}
@Override
@ForceInline
public final long loopBound(long length) {
return VectorIntrinsics.roundDown(length, laneCount);
}
@Override
@ForceInline
public final VectorMask<E> indexInRange(int offset, int limit) {
return maskAll(true).indexInRange(offset, limit);
}
@Override
@ForceInline
public final VectorMask<E> indexInRange(long offset, long limit) {
return maskAll(true).indexInRange(offset, limit);
}
@Override
@ForceInline
public final <F> VectorSpecies<F> withLanes(Class<F> newType) {
@ -349,9 +362,9 @@ abstract class AbstractSpecies<E> extends jdk.internal.vm.vector.VectorSupport.V
@ForceInline
@Override
public final Vector<E> fromByteArray(byte[] a, int offset, ByteOrder bo) {
public final Vector<E> fromMemorySegment(MemorySegment ms, long offset, ByteOrder bo) {
return dummyVector()
.fromByteArray0(a, offset)
.fromMemorySegment0(ms, offset)
.maybeSwap(bo);
}

31
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -24,10 +24,10 @@
*/
package jdk.incubator.vector;
import java.lang.foreign.MemorySegment;
import jdk.internal.vm.annotation.ForceInline;
import jdk.internal.vm.vector.VectorSupport;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.function.IntUnaryOperator;
@ -194,7 +194,7 @@ abstract class AbstractVector<E> extends Vector<E> {
abstract AbstractShuffle<E> shuffleFromOp(IntUnaryOperator fn);
/*package-private*/
abstract AbstractVector<E> fromByteArray0(byte[] a, int offset);
abstract AbstractVector<E> fromMemorySegment0(MemorySegment ms, long offset);
/*package-private*/
abstract AbstractVector<E> maybeSwap(ByteOrder bo);
@ -504,23 +504,23 @@ abstract class AbstractVector<E> extends Vector<E> {
AbstractVector<F> defaultReinterpret(AbstractSpecies<F> rsp) {
int blen = Math.max(this.bitSize(), rsp.vectorBitSize()) / Byte.SIZE;
ByteOrder bo = ByteOrder.nativeOrder();
ByteBuffer bb = ByteBuffer.allocate(blen);
this.intoByteBuffer(bb, 0, bo);
MemorySegment ms = MemorySegment.ofArray(new byte[blen]);
this.intoMemorySegment(ms, 0, bo);
VectorMask<F> m = rsp.maskAll(true);
// enum-switches don't optimize properly JDK-8161245
switch (rsp.laneType.switchKey) {
case LaneType.SK_BYTE:
return ByteVector.fromByteBuffer(rsp.check(byte.class), bb, 0, bo, m.check(byte.class)).check0(rsp);
return ByteVector.fromMemorySegment(rsp.check(byte.class), ms, 0, bo, m.check(byte.class)).check0(rsp);
case LaneType.SK_SHORT:
return ShortVector.fromByteBuffer(rsp.check(short.class), bb, 0, bo, m.check(short.class)).check0(rsp);
return ShortVector.fromMemorySegment(rsp.check(short.class), ms, 0, bo, m.check(short.class)).check0(rsp);
case LaneType.SK_INT:
return IntVector.fromByteBuffer(rsp.check(int.class), bb, 0, bo, m.check(int.class)).check0(rsp);
return IntVector.fromMemorySegment(rsp.check(int.class), ms, 0, bo, m.check(int.class)).check0(rsp);
case LaneType.SK_LONG:
return LongVector.fromByteBuffer(rsp.check(long.class), bb, 0, bo, m.check(long.class)).check0(rsp);
return LongVector.fromMemorySegment(rsp.check(long.class), ms, 0, bo, m.check(long.class)).check0(rsp);
case LaneType.SK_FLOAT:
return FloatVector.fromByteBuffer(rsp.check(float.class), bb, 0, bo, m.check(float.class)).check0(rsp);
return FloatVector.fromMemorySegment(rsp.check(float.class), ms, 0, bo, m.check(float.class)).check0(rsp);
case LaneType.SK_DOUBLE:
return DoubleVector.fromByteBuffer(rsp.check(double.class), bb, 0, bo, m.check(double.class)).check0(rsp);
return DoubleVector.fromMemorySegment(rsp.check(double.class), ms, 0, bo, m.check(double.class)).check0(rsp);
default:
throw new AssertionError(rsp.toString());
}
@ -730,15 +730,6 @@ abstract class AbstractVector<E> extends Vector<E> {
throw new AssertionError();
}
// Byte buffer wrappers.
static ByteBuffer wrapper(ByteBuffer bb, ByteOrder bo) {
return bb.duplicate().order(bo);
}
static ByteBuffer wrapper(byte[] a, ByteOrder bo) {
return ByteBuffer.wrap(a).order(bo);
}
static {
// Recode uses of VectorSupport.reinterpret if this assertion fails:
assert(REGISTER_ENDIAN == ByteOrder.LITTLE_ENDIAN);

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Byte128Vector extends ByteVector {
(Byte128Vector) v); // specialize
}
@Override
@ForceInline
public Byte128Vector compress(VectorMask<Byte> m) {
return (Byte128Vector)
super.compressTemplate(Byte128Mask.class,
(Byte128Mask) m); // specialize
}
@Override
@ForceInline
public Byte128Vector expand(VectorMask<Byte> m) {
return (Byte128Vector)
super.expandTemplate(Byte128Mask.class,
(Byte128Mask) m); // specialize
}
@Override
@ForceInline
public Byte128Vector selectFrom(Vector<Byte> v) {
@ -677,6 +693,15 @@ final class Byte128Vector extends ByteVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Byte128Mask compress() {
return (Byte128Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Byte128Vector.class, Byte128Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -876,29 +901,15 @@ final class Byte128Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromByteArray0Template(Byte128Mask.class, a, offset, (Byte128Mask) m); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
return super.fromByteBuffer0Template(Byte128Mask.class, bb, offset, (Byte128Mask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(Byte128Mask.class, ms, offset, (Byte128Mask) m); // specialize
}
@ForceInline
@ -926,22 +937,8 @@ final class Byte128Vector extends ByteVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
super.intoByteArray0Template(Byte128Mask.class, a, offset, (Byte128Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
super.intoByteBuffer0Template(Byte128Mask.class, bb, offset, (Byte128Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
super.intoMemorySegment0Template(Byte128Mask.class, ms, offset, (Byte128Mask) m);
}
@ -950,3 +947,4 @@ final class Byte128Vector extends ByteVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Byte256Vector extends ByteVector {
(Byte256Vector) v); // specialize
}
@Override
@ForceInline
public Byte256Vector compress(VectorMask<Byte> m) {
return (Byte256Vector)
super.compressTemplate(Byte256Mask.class,
(Byte256Mask) m); // specialize
}
@Override
@ForceInline
public Byte256Vector expand(VectorMask<Byte> m) {
return (Byte256Vector)
super.expandTemplate(Byte256Mask.class,
(Byte256Mask) m); // specialize
}
@Override
@ForceInline
public Byte256Vector selectFrom(Vector<Byte> v) {
@ -709,6 +725,15 @@ final class Byte256Vector extends ByteVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Byte256Mask compress() {
return (Byte256Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Byte256Vector.class, Byte256Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -908,29 +933,15 @@ final class Byte256Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromByteArray0Template(Byte256Mask.class, a, offset, (Byte256Mask) m); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
return super.fromByteBuffer0Template(Byte256Mask.class, bb, offset, (Byte256Mask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(Byte256Mask.class, ms, offset, (Byte256Mask) m); // specialize
}
@ForceInline
@ -958,22 +969,8 @@ final class Byte256Vector extends ByteVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
super.intoByteArray0Template(Byte256Mask.class, a, offset, (Byte256Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
super.intoByteBuffer0Template(Byte256Mask.class, bb, offset, (Byte256Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
super.intoMemorySegment0Template(Byte256Mask.class, ms, offset, (Byte256Mask) m);
}
@ -982,3 +979,4 @@ final class Byte256Vector extends ByteVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Byte512Vector extends ByteVector {
(Byte512Vector) v); // specialize
}
@Override
@ForceInline
public Byte512Vector compress(VectorMask<Byte> m) {
return (Byte512Vector)
super.compressTemplate(Byte512Mask.class,
(Byte512Mask) m); // specialize
}
@Override
@ForceInline
public Byte512Vector expand(VectorMask<Byte> m) {
return (Byte512Vector)
super.expandTemplate(Byte512Mask.class,
(Byte512Mask) m); // specialize
}
@Override
@ForceInline
public Byte512Vector selectFrom(Vector<Byte> v) {
@ -773,6 +789,15 @@ final class Byte512Vector extends ByteVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Byte512Mask compress() {
return (Byte512Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Byte512Vector.class, Byte512Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -972,29 +997,15 @@ final class Byte512Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromByteArray0Template(Byte512Mask.class, a, offset, (Byte512Mask) m); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
return super.fromByteBuffer0Template(Byte512Mask.class, bb, offset, (Byte512Mask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(Byte512Mask.class, ms, offset, (Byte512Mask) m); // specialize
}
@ForceInline
@ -1022,22 +1033,8 @@ final class Byte512Vector extends ByteVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
super.intoByteArray0Template(Byte512Mask.class, a, offset, (Byte512Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
super.intoByteBuffer0Template(Byte512Mask.class, bb, offset, (Byte512Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
super.intoMemorySegment0Template(Byte512Mask.class, ms, offset, (Byte512Mask) m);
}
@ -1046,3 +1043,4 @@ final class Byte512Vector extends ByteVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Byte64Vector extends ByteVector {
(Byte64Vector) v); // specialize
}
@Override
@ForceInline
public Byte64Vector compress(VectorMask<Byte> m) {
return (Byte64Vector)
super.compressTemplate(Byte64Mask.class,
(Byte64Mask) m); // specialize
}
@Override
@ForceInline
public Byte64Vector expand(VectorMask<Byte> m) {
return (Byte64Vector)
super.expandTemplate(Byte64Mask.class,
(Byte64Mask) m); // specialize
}
@Override
@ForceInline
public Byte64Vector selectFrom(Vector<Byte> v) {
@ -661,6 +677,15 @@ final class Byte64Vector extends ByteVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Byte64Mask compress() {
return (Byte64Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Byte64Vector.class, Byte64Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -860,29 +885,15 @@ final class Byte64Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromByteArray0Template(Byte64Mask.class, a, offset, (Byte64Mask) m); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
return super.fromByteBuffer0Template(Byte64Mask.class, bb, offset, (Byte64Mask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(Byte64Mask.class, ms, offset, (Byte64Mask) m); // specialize
}
@ForceInline
@ -910,22 +921,8 @@ final class Byte64Vector extends ByteVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
super.intoByteArray0Template(Byte64Mask.class, a, offset, (Byte64Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
super.intoByteBuffer0Template(Byte64Mask.class, bb, offset, (Byte64Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
super.intoMemorySegment0Template(Byte64Mask.class, ms, offset, (Byte64Mask) m);
}
@ -934,3 +931,4 @@ final class Byte64Vector extends ByteVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class ByteMaxVector extends ByteVector {
(ByteMaxVector) v); // specialize
}
@Override
@ForceInline
public ByteMaxVector compress(VectorMask<Byte> m) {
return (ByteMaxVector)
super.compressTemplate(ByteMaxMask.class,
(ByteMaxMask) m); // specialize
}
@Override
@ForceInline
public ByteMaxVector expand(VectorMask<Byte> m) {
return (ByteMaxVector)
super.expandTemplate(ByteMaxMask.class,
(ByteMaxMask) m); // specialize
}
@Override
@ForceInline
public ByteMaxVector selectFrom(Vector<Byte> v) {
@ -647,6 +663,15 @@ final class ByteMaxVector extends ByteVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public ByteMaxMask compress() {
return (ByteMaxMask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
ByteMaxVector.class, ByteMaxMask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -846,29 +871,15 @@ final class ByteMaxVector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromByteArray0Template(ByteMaxMask.class, a, offset, (ByteMaxMask) m); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ByteVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
return super.fromByteBuffer0Template(ByteMaxMask.class, bb, offset, (ByteMaxMask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(ByteMaxMask.class, ms, offset, (ByteMaxMask) m); // specialize
}
@ForceInline
@ -896,22 +907,8 @@ final class ByteMaxVector extends ByteVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Byte> m) {
super.intoByteArray0Template(ByteMaxMask.class, a, offset, (ByteMaxMask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m) {
super.intoByteBuffer0Template(ByteMaxMask.class, bb, offset, (ByteMaxMask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
super.intoMemorySegment0Template(ByteMaxMask.class, ms, offset, (ByteMaxMask) m);
}
@ -920,3 +917,4 @@ final class ByteMaxVector extends ByteVector {
// ================================================
}

584
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java
View file

@ -24,14 +24,14 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import jdk.internal.foreign.AbstractMemorySegmentImpl;
import jdk.internal.misc.ScopedMemoryAccess;
import jdk.internal.misc.Unsafe;
import jdk.internal.vm.annotation.ForceInline;
@ -57,6 +57,8 @@ public abstract class ByteVector extends AbstractVector<Byte> {
static final int FORBID_OPCODE_KIND = VO_ONLYFP;
static final ValueLayout.OfByte ELEMENT_LAYOUT = ValueLayout.JAVA_BYTE.withBitAlignment(8);
@ForceInline
static int opCode(Operator op) {
return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
@ -351,6 +353,45 @@ public abstract class ByteVector extends AbstractVector<Byte> {
return vectorFactory(res);
}
/*package-private*/
interface FLdLongOp {
byte apply(MemorySegment memory, long offset, int i);
}
/*package-private*/
@ForceInline
final
ByteVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
//dummy; no vec = vec();
byte[] res = new byte[length()];
for (int i = 0; i < res.length; i++) {
res[i] = f.apply(memory, offset, i);
}
return vectorFactory(res);
}
/*package-private*/
@ForceInline
final
ByteVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Byte> m,
FLdLongOp f) {
//byte[] vec = vec();
byte[] res = new byte[length()];
boolean[] mbits = ((AbstractMask<Byte>)m).getBits();
for (int i = 0; i < res.length; i++) {
if (mbits[i]) {
res[i] = f.apply(memory, offset, i);
}
}
return vectorFactory(res);
}
static byte memorySegmentGet(MemorySegment ms, long o, int i) {
return ms.get(ELEMENT_LAYOUT, o + i * 1L);
}
interface FStOp<M> {
void apply(M memory, int offset, int i, byte a);
}
@ -381,6 +422,40 @@ public abstract class ByteVector extends AbstractVector<Byte> {
}
}
interface FStLongOp {
void apply(MemorySegment memory, long offset, int i, byte a);
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
FStLongOp f) {
byte[] vec = vec();
for (int i = 0; i < vec.length; i++) {
f.apply(memory, offset, i, vec[i]);
}
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
VectorMask<Byte> m,
FStLongOp f) {
byte[] vec = vec();
boolean[] mbits = ((AbstractMask<Byte>)m).getBits();
for (int i = 0; i < vec.length; i++) {
if (mbits[i]) {
f.apply(memory, offset, i, vec[i]);
}
}
}
static void memorySegmentSet(MemorySegment ms, long o, int i, byte e) {
ms.set(ELEMENT_LAYOUT, o + i * 1L, e);
}
// Binary test
/*package-private*/
@ -431,6 +506,36 @@ public abstract class ByteVector extends AbstractVector<Byte> {
return ((byte)bits);
}
static ByteVector expandHelper(Vector<Byte> v, VectorMask<Byte> m) {
VectorSpecies<Byte> vsp = m.vectorSpecies();
ByteVector r = (ByteVector) vsp.zero();
ByteVector vi = (ByteVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(i, vi.lane(j++));
}
}
return r;
}
static ByteVector compressHelper(Vector<Byte> v, VectorMask<Byte> m) {
VectorSpecies<Byte> vsp = m.vectorSpecies();
ByteVector r = (ByteVector) vsp.zero();
ByteVector vi = (ByteVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(j++, vi.lane(i));
}
}
return r;
}
// Static factories (other than memory operations)
// Note: A surprising behavior in javadoc
@ -620,6 +725,16 @@ public abstract class ByteVector extends AbstractVector<Byte> {
v0.uOp(m, (i, a) -> (byte) -a);
case VECTOR_OP_ABS: return (v0, m) ->
v0.uOp(m, (i, a) -> (byte) Math.abs(a));
case VECTOR_OP_BIT_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (byte) bitCount(a));
case VECTOR_OP_TZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (byte) numberOfTrailingZeros(a));
case VECTOR_OP_LZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (byte) numberOfLeadingZeros(a));
case VECTOR_OP_REVERSE: return (v0, m) ->
v0.uOp(m, (i, a) -> reverse(a));
case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
v0.uOp(m, (i, a) -> a);
default: return null;
}
}
@ -1746,6 +1861,25 @@ public abstract class ByteVector extends AbstractVector<Byte> {
return lanewise(ABS);
}
static int bitCount(byte a) {
return Integer.bitCount((int)a & 0xFF);
}
static int numberOfTrailingZeros(byte a) {
return a != 0 ? Integer.numberOfTrailingZeros(a) : 8;
}
static int numberOfLeadingZeros(byte a) {
return a >= 0 ? Integer.numberOfLeadingZeros(a) - 24 : 0;
}
static byte reverse(byte a) {
if (a == 0 || a == -1) return a;
byte b = rotateLeft(a, 4);
b = (byte) (((b & 0x55) << 1) | ((b & 0xAA) >>> 1));
b = (byte) (((b & 0x33) << 2) | ((b & 0xCC) >>> 2));
return b;
}
// not (~)
/**
* Computes the bitwise logical complement ({@code ~})
@ -2372,6 +2506,45 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
ByteVector compress(VectorMask<Byte> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Byte>>
ByteVector compressTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (ByteVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_COMPRESS, getClass(), masktype,
byte.class, length(), this, m,
(v1, m1) -> compressHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
ByteVector expand(VectorMask<Byte> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Byte>>
ByteVector expandTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (ByteVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_EXPAND, getClass(), masktype,
byte.class, length(), this, m,
(v1, m1) -> expandHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
*/
@ -2784,90 +2957,6 @@ public abstract class ByteVector extends AbstractVector<Byte> {
return res;
}
/**
* Loads a vector from a byte array starting at an offset.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
*/
@ForceInline
public static
ByteVector fromByteArray(VectorSpecies<Byte> species,
byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
ByteSpecies vsp = (ByteSpecies) species;
return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
}
/**
* Loads a vector from a byte array starting at an offset
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code byte} (zero).
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
* where the mask is set
*/
@ForceInline
public static
ByteVector fromByteArray(VectorSpecies<Byte> species,
byte[] a, int offset,
ByteOrder bo,
VectorMask<Byte> m) {
ByteSpecies vsp = (ByteSpecies) species;
if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
ByteBuffer wb = wrapper(a, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Byte>)m,
(wb_, o, i) -> wb_.get(o + i * 1));
}
/**
* Loads a vector from an array of type {@code byte[]}
* starting at an offset.
@ -3174,44 +3263,49 @@ public abstract class ByteVector extends AbstractVector<Byte> {
}
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer.
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* {@link #fromMemorySegment(VectorSpecies,MemorySegment,long,ByteOrder,VectorMask)
* fromMemorySegment()} as follows:
* <pre>{@code
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* return fromMemorySegment(species, ms, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*1 < 0}
* or {@code offset+N*1 >= bb.limit()}
* or {@code offset+N*1 >= ms.byteSize()}
* for any lane {@code N} in the vector
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
ByteVector fromByteBuffer(VectorSpecies<Byte> species,
ByteBuffer bb, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
ByteVector fromMemorySegment(VectorSpecies<Byte> species,
MemorySegment ms, long offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), ms.byteSize());
ByteSpecies vsp = (ByteSpecies) species;
return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset).maybeSwap(bo);
}
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code byte} (zero).
@ -3222,12 +3316,11 @@ public abstract class ByteVector extends AbstractVector<Byte> {
* <p>
* The following pseudocode illustrates the behavior:
* <pre>{@code
* ByteBuffer eb = bb.duplicate()
* .position(offset);
* var slice = ms.asSlice(offset);
* byte[] ar = new byte[species.length()];
* for (int n = 0; n < ar.length; n++) {
* if (m.laneIsSet(n)) {
* ar[n] = eb.get(n);
* ar[n] = slice.getAtIndex(ValuaLayout.JAVA_BYTE.withBitAlignment(8), n);
* }
* }
* ByteVector r = ByteVector.fromArray(species, ar, 0);
@ -3236,33 +3329,36 @@ public abstract class ByteVector extends AbstractVector<Byte> {
* The byte order argument is ignored.
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*1 < 0}
* or {@code offset+N*1 >= bb.limit()}
* or {@code offset+N*1 >= ms.byteSize()}
* for any lane {@code N} in the vector
* where the mask is set
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
ByteVector fromByteBuffer(VectorSpecies<Byte> species,
ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Byte> m) {
ByteVector fromMemorySegment(VectorSpecies<Byte> species,
MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Byte> m) {
ByteSpecies vsp = (ByteSpecies) species;
if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
if (offset >= 0 && offset <= (ms.byteSize() - species.vectorByteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, bb.limit());
ByteBuffer wb = wrapper(bb, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Byte>)m,
(wb_, o, i) -> wb_.get(o + i * 1));
checkMaskFromIndexSize(offset, vsp, m, 1, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, ByteVector::memorySegmentGet);
}
// Memory store operations
@ -3292,7 +3388,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
this,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3443,7 +3539,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
normalized,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
}
@ -3581,67 +3677,40 @@ public abstract class ByteVector extends AbstractVector<Byte> {
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, byteSize(), a.length);
maybeSwap(bo).intoByteArray0(a, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo,
VectorMask<Byte> m) {
if (m.allTrue()) {
intoByteArray(a, offset, bo);
} else {
ByteSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
maybeSwap(bo).intoByteArray0(a, offset, m);
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo) {
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
offset = checkFromIndexSize(offset, byteSize(), ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo) {
if (ScopedMemoryAccess.isReadOnly(bb)) {
throw new ReadOnlyBufferException();
}
offset = checkFromIndexSize(offset, byteSize(), bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Byte> m) {
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Byte> m) {
if (m.allTrue()) {
intoByteBuffer(bb, offset, bo);
intoMemorySegment(ms, offset, bo);
} else {
if (bb.isReadOnly()) {
throw new ReadOnlyBufferException();
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
ByteSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 1, bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset, m);
checkMaskFromIndexSize(offset, vsp, m, 1, ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset, m);
}
}
@ -3675,7 +3744,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3692,7 +3761,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3709,7 +3778,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
}
@ -3726,78 +3795,37 @@ public abstract class ByteVector extends AbstractVector<Byte> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
}
@Override
abstract
ByteVector fromByteArray0(byte[] a, int offset);
ByteVector fromMemorySegment0(MemorySegment bb, long offset);
@ForceInline
final
ByteVector fromByteArray0Template(byte[] a, int offset) {
ByteVector fromMemorySegment0Template(MemorySegment ms, long offset) {
ByteSpecies vsp = vspecies();
return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.get(o + i * 1));
});
}
abstract
ByteVector fromByteArray0(byte[] a, int offset, VectorMask<Byte> m);
@ForceInline
final
<M extends VectorMask<Byte>>
ByteVector fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
ByteSpecies vsp = vspecies();
m.check(vsp);
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.get(o + i * 1));
});
}
abstract
ByteVector fromByteBuffer0(ByteBuffer bb, int offset);
@ForceInline
final
ByteVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
ByteSpecies vsp = vspecies();
return ScopedMemoryAccess.loadFromByteBuffer(
return ScopedMemoryAccess.loadFromMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
bb, offset, vsp,
(buf, off, s) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.get(o + i * 1));
(AbstractMemorySegmentImpl) ms, offset, vsp,
(msp, off, s) -> {
return s.ldLongOp((MemorySegment) msp, off, ByteVector::memorySegmentGet);
});
}
abstract
ByteVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m);
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m);
@ForceInline
final
<M extends VectorMask<Byte>>
ByteVector fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
ByteVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
ByteSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromByteBufferMasked(
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
bb, offset, m, vsp,
(buf, off, s, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.get(o + i * 1));
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, ByteVector::memorySegmentGet);
});
}
@ -3816,7 +3844,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
a, arrayAddress(a, offset),
this, a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_+i] = e));
}
@ -3833,7 +3861,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
a, arrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3852,75 +3880,37 @@ public abstract class ByteVector extends AbstractVector<Byte> {
a, booleanArrayAddress(a, offset),
normalized, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
}
abstract
void intoByteArray0(byte[] a, int offset);
@ForceInline
final
void intoByteArray0Template(byte[] a, int offset) {
void intoMemorySegment0(MemorySegment ms, long offset) {
ByteSpecies vsp = vspecies();
VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, a, offset,
(arr, off, v) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off,
(tb_, o, i, e) -> tb_.put(o + i * 1, e));
});
}
abstract
void intoByteArray0(byte[] a, int offset, VectorMask<Byte> m);
@ForceInline
final
<M extends VectorMask<Byte>>
void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
ByteSpecies vsp = vspecies();
m.check(vsp);
VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(tb_, o, i, e) -> tb_.put(o + i * 1, e));
});
}
@ForceInline
final
void intoByteBuffer0(ByteBuffer bb, int offset) {
ByteSpecies vsp = vspecies();
ScopedMemoryAccess.storeIntoByteBuffer(
ScopedMemoryAccess.storeIntoMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
this, bb, offset,
(buf, off, v) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off,
(wb_, o, i, e) -> wb_.put(o + i * 1, e));
this,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v) -> {
v.stLongOp((MemorySegment) msp, off, ByteVector::memorySegmentSet);
});
}
abstract
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Byte> m);
void intoMemorySegment0(MemorySegment bb, long offset, VectorMask<Byte> m);
@ForceInline
final
<M extends VectorMask<Byte>>
void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
void intoMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
ByteSpecies vsp = vspecies();
m.check(vsp);
ScopedMemoryAccess.storeIntoByteBufferMasked(
ScopedMemoryAccess.storeIntoMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
this, m, bb, offset,
(buf, off, v, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(wb_, o, i, e) -> wb_.put(o + i * 1, e));
this, m,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v, vm) -> {
v.stLongOp((MemorySegment) msp, off, vm, ByteVector::memorySegmentSet);
});
}
@ -3937,6 +3927,16 @@ public abstract class ByteVector extends AbstractVector<Byte> {
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
private static
void checkMaskFromIndexSize(long offset,
ByteSpecies vsp,
VectorMask<Byte> m,
int scale,
long limit) {
((AbstractMask<Byte>)m)
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
@ForceInline
private void conditionalStoreNYI(int offset,
ByteSpecies vsp,
@ -4256,6 +4256,21 @@ public abstract class ByteVector extends AbstractVector<Byte> {
return dummyVector().ldOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
ByteVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
ByteVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Byte> m,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
<M> void stOp(M memory, int offset, FStOp<M> f) {
@ -4270,6 +4285,20 @@ public abstract class ByteVector extends AbstractVector<Byte> {
dummyVector().stOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset, FStLongOp f) {
dummyVector().stLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset,
AbstractMask<Byte> m,
FStLongOp f) {
dummyVector().stLongOp(memory, offset, m, f);
}
// N.B. Make sure these constant vectors and
// masks load up correctly into registers.
//
@ -4383,3 +4412,4 @@ public abstract class ByteVector extends AbstractVector<Byte> {
public static final VectorSpecies<Byte> SPECIES_PREFERRED
= (ByteSpecies) VectorSpecies.ofPreferred(byte.class);
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class Double128Vector extends DoubleVector {
(Double128Vector) v); // specialize
}
@Override
@ForceInline
public Double128Vector compress(VectorMask<Double> m) {
return (Double128Vector)
super.compressTemplate(Double128Mask.class,
(Double128Mask) m); // specialize
}
@Override
@ForceInline
public Double128Vector expand(VectorMask<Double> m) {
return (Double128Vector)
super.expandTemplate(Double128Mask.class,
(Double128Mask) m); // specialize
}
@Override
@ForceInline
public Double128Vector selectFrom(Vector<Double> v) {
@ -638,6 +654,15 @@ final class Double128Vector extends DoubleVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Double128Mask compress() {
return (Double128Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Double128Vector.class, Double128Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -830,29 +855,15 @@ final class Double128Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset, VectorMask<Double> m) {
return super.fromByteArray0Template(Double128Mask.class, a, offset, (Double128Mask) m); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
return super.fromByteBuffer0Template(Double128Mask.class, bb, offset, (Double128Mask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(Double128Mask.class, ms, offset, (Double128Mask) m); // specialize
}
@ForceInline
@ -880,22 +891,8 @@ final class Double128Vector extends DoubleVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Double> m) {
super.intoByteArray0Template(Double128Mask.class, a, offset, (Double128Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
super.intoByteBuffer0Template(Double128Mask.class, bb, offset, (Double128Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
super.intoMemorySegment0Template(Double128Mask.class, ms, offset, (Double128Mask) m);
}
@ -904,3 +901,4 @@ final class Double128Vector extends DoubleVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class Double256Vector extends DoubleVector {
(Double256Vector) v); // specialize
}
@Override
@ForceInline
public Double256Vector compress(VectorMask<Double> m) {
return (Double256Vector)
super.compressTemplate(Double256Mask.class,
(Double256Mask) m); // specialize
}
@Override
@ForceInline
public Double256Vector expand(VectorMask<Double> m) {
return (Double256Vector)
super.expandTemplate(Double256Mask.class,
(Double256Mask) m); // specialize
}
@Override
@ForceInline
public Double256Vector selectFrom(Vector<Double> v) {
@ -642,6 +658,15 @@ final class Double256Vector extends DoubleVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Double256Mask compress() {
return (Double256Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Double256Vector.class, Double256Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -834,29 +859,15 @@ final class Double256Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset, VectorMask<Double> m) {
return super.fromByteArray0Template(Double256Mask.class, a, offset, (Double256Mask) m); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
return super.fromByteBuffer0Template(Double256Mask.class, bb, offset, (Double256Mask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(Double256Mask.class, ms, offset, (Double256Mask) m); // specialize
}
@ForceInline
@ -884,22 +895,8 @@ final class Double256Vector extends DoubleVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Double> m) {
super.intoByteArray0Template(Double256Mask.class, a, offset, (Double256Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
super.intoByteBuffer0Template(Double256Mask.class, bb, offset, (Double256Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
super.intoMemorySegment0Template(Double256Mask.class, ms, offset, (Double256Mask) m);
}
@ -908,3 +905,4 @@ final class Double256Vector extends DoubleVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class Double512Vector extends DoubleVector {
(Double512Vector) v); // specialize
}
@Override
@ForceInline
public Double512Vector compress(VectorMask<Double> m) {
return (Double512Vector)
super.compressTemplate(Double512Mask.class,
(Double512Mask) m); // specialize
}
@Override
@ForceInline
public Double512Vector expand(VectorMask<Double> m) {
return (Double512Vector)
super.expandTemplate(Double512Mask.class,
(Double512Mask) m); // specialize
}
@Override
@ForceInline
public Double512Vector selectFrom(Vector<Double> v) {
@ -650,6 +666,15 @@ final class Double512Vector extends DoubleVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Double512Mask compress() {
return (Double512Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Double512Vector.class, Double512Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -842,29 +867,15 @@ final class Double512Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset, VectorMask<Double> m) {
return super.fromByteArray0Template(Double512Mask.class, a, offset, (Double512Mask) m); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
return super.fromByteBuffer0Template(Double512Mask.class, bb, offset, (Double512Mask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(Double512Mask.class, ms, offset, (Double512Mask) m); // specialize
}
@ForceInline
@ -892,22 +903,8 @@ final class Double512Vector extends DoubleVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Double> m) {
super.intoByteArray0Template(Double512Mask.class, a, offset, (Double512Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
super.intoByteBuffer0Template(Double512Mask.class, bb, offset, (Double512Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
super.intoMemorySegment0Template(Double512Mask.class, ms, offset, (Double512Mask) m);
}
@ -916,3 +913,4 @@ final class Double512Vector extends DoubleVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class Double64Vector extends DoubleVector {
(Double64Vector) v); // specialize
}
@Override
@ForceInline
public Double64Vector compress(VectorMask<Double> m) {
return (Double64Vector)
super.compressTemplate(Double64Mask.class,
(Double64Mask) m); // specialize
}
@Override
@ForceInline
public Double64Vector expand(VectorMask<Double> m) {
return (Double64Vector)
super.expandTemplate(Double64Mask.class,
(Double64Mask) m); // specialize
}
@Override
@ForceInline
public Double64Vector selectFrom(Vector<Double> v) {
@ -636,6 +652,15 @@ final class Double64Vector extends DoubleVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Double64Mask compress() {
return (Double64Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Double64Vector.class, Double64Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -828,29 +853,15 @@ final class Double64Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset, VectorMask<Double> m) {
return super.fromByteArray0Template(Double64Mask.class, a, offset, (Double64Mask) m); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
return super.fromByteBuffer0Template(Double64Mask.class, bb, offset, (Double64Mask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(Double64Mask.class, ms, offset, (Double64Mask) m); // specialize
}
@ForceInline
@ -878,22 +889,8 @@ final class Double64Vector extends DoubleVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Double> m) {
super.intoByteArray0Template(Double64Mask.class, a, offset, (Double64Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
super.intoByteBuffer0Template(Double64Mask.class, bb, offset, (Double64Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
super.intoMemorySegment0Template(Double64Mask.class, ms, offset, (Double64Mask) m);
}
@ -902,3 +899,4 @@ final class Double64Vector extends DoubleVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class DoubleMaxVector extends DoubleVector {
(DoubleMaxVector) v); // specialize
}
@Override
@ForceInline
public DoubleMaxVector compress(VectorMask<Double> m) {
return (DoubleMaxVector)
super.compressTemplate(DoubleMaxMask.class,
(DoubleMaxMask) m); // specialize
}
@Override
@ForceInline
public DoubleMaxVector expand(VectorMask<Double> m) {
return (DoubleMaxVector)
super.expandTemplate(DoubleMaxMask.class,
(DoubleMaxMask) m); // specialize
}
@Override
@ForceInline
public DoubleMaxVector selectFrom(Vector<Double> v) {
@ -635,6 +651,15 @@ final class DoubleMaxVector extends DoubleVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public DoubleMaxMask compress() {
return (DoubleMaxMask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
DoubleMaxVector.class, DoubleMaxMask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -827,29 +852,15 @@ final class DoubleMaxVector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteArray0(byte[] a, int offset, VectorMask<Double> m) {
return super.fromByteArray0Template(DoubleMaxMask.class, a, offset, (DoubleMaxMask) m); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
return super.fromByteBuffer0Template(DoubleMaxMask.class, bb, offset, (DoubleMaxMask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(DoubleMaxMask.class, ms, offset, (DoubleMaxMask) m); // specialize
}
@ForceInline
@ -877,22 +888,8 @@ final class DoubleMaxVector extends DoubleVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Double> m) {
super.intoByteArray0Template(DoubleMaxMask.class, a, offset, (DoubleMaxMask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m) {
super.intoByteBuffer0Template(DoubleMaxMask.class, bb, offset, (DoubleMaxMask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
super.intoMemorySegment0Template(DoubleMaxMask.class, ms, offset, (DoubleMaxMask) m);
}
@ -901,3 +898,4 @@ final class DoubleMaxVector extends DoubleVector {
// ================================================
}

549
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java
View file

@ -24,14 +24,14 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import jdk.internal.foreign.AbstractMemorySegmentImpl;
import jdk.internal.misc.ScopedMemoryAccess;
import jdk.internal.misc.Unsafe;
import jdk.internal.vm.annotation.ForceInline;
@ -57,6 +57,8 @@ public abstract class DoubleVector extends AbstractVector<Double> {
static final int FORBID_OPCODE_KIND = VO_NOFP;
static final ValueLayout.OfDouble ELEMENT_LAYOUT = ValueLayout.JAVA_DOUBLE.withBitAlignment(8);
@ForceInline
static int opCode(Operator op) {
return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
@ -351,6 +353,45 @@ public abstract class DoubleVector extends AbstractVector<Double> {
return vectorFactory(res);
}
/*package-private*/
interface FLdLongOp {
double apply(MemorySegment memory, long offset, int i);
}
/*package-private*/
@ForceInline
final
DoubleVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
//dummy; no vec = vec();
double[] res = new double[length()];
for (int i = 0; i < res.length; i++) {
res[i] = f.apply(memory, offset, i);
}
return vectorFactory(res);
}
/*package-private*/
@ForceInline
final
DoubleVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Double> m,
FLdLongOp f) {
//double[] vec = vec();
double[] res = new double[length()];
boolean[] mbits = ((AbstractMask<Double>)m).getBits();
for (int i = 0; i < res.length; i++) {
if (mbits[i]) {
res[i] = f.apply(memory, offset, i);
}
}
return vectorFactory(res);
}
static double memorySegmentGet(MemorySegment ms, long o, int i) {
return ms.get(ELEMENT_LAYOUT, o + i * 8L);
}
interface FStOp<M> {
void apply(M memory, int offset, int i, double a);
}
@ -381,6 +422,40 @@ public abstract class DoubleVector extends AbstractVector<Double> {
}
}
interface FStLongOp {
void apply(MemorySegment memory, long offset, int i, double a);
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
FStLongOp f) {
double[] vec = vec();
for (int i = 0; i < vec.length; i++) {
f.apply(memory, offset, i, vec[i]);
}
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
VectorMask<Double> m,
FStLongOp f) {
double[] vec = vec();
boolean[] mbits = ((AbstractMask<Double>)m).getBits();
for (int i = 0; i < vec.length; i++) {
if (mbits[i]) {
f.apply(memory, offset, i, vec[i]);
}
}
}
static void memorySegmentSet(MemorySegment ms, long o, int i, double e) {
ms.set(ELEMENT_LAYOUT, o + i * 8L, e);
}
// Binary test
/*package-private*/
@ -420,6 +495,36 @@ public abstract class DoubleVector extends AbstractVector<Double> {
return Double.longBitsToDouble((long)bits);
}
static DoubleVector expandHelper(Vector<Double> v, VectorMask<Double> m) {
VectorSpecies<Double> vsp = m.vectorSpecies();
DoubleVector r = (DoubleVector) vsp.zero();
DoubleVector vi = (DoubleVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(i, vi.lane(j++));
}
}
return r;
}
static DoubleVector compressHelper(Vector<Double> v, VectorMask<Double> m) {
VectorSpecies<Double> vsp = m.vectorSpecies();
DoubleVector r = (DoubleVector) vsp.zero();
DoubleVector vi = (DoubleVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(j++, vi.lane(i));
}
}
return r;
}
// Static factories (other than memory operations)
// Note: A surprising behavior in javadoc
@ -1594,6 +1699,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
}
// sqrt
/**
* Computes the square root of this vector.
@ -2241,6 +2347,45 @@ public abstract class DoubleVector extends AbstractVector<Double> {
DoubleVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
DoubleVector compress(VectorMask<Double> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Double>>
DoubleVector compressTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (DoubleVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_COMPRESS, getClass(), masktype,
double.class, length(), this, m,
(v1, m1) -> compressHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
DoubleVector expand(VectorMask<Double> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Double>>
DoubleVector expandTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (DoubleVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_EXPAND, getClass(), masktype,
double.class, length(), this, m,
(v1, m1) -> expandHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
*/
@ -2609,90 +2754,6 @@ public abstract class DoubleVector extends AbstractVector<Double> {
return toArray();
}
/**
* Loads a vector from a byte array starting at an offset.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
*/
@ForceInline
public static
DoubleVector fromByteArray(VectorSpecies<Double> species,
byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
DoubleSpecies vsp = (DoubleSpecies) species;
return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
}
/**
* Loads a vector from a byte array starting at an offset
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code double} (positive zero).
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
* where the mask is set
*/
@ForceInline
public static
DoubleVector fromByteArray(VectorSpecies<Double> species,
byte[] a, int offset,
ByteOrder bo,
VectorMask<Double> m) {
DoubleSpecies vsp = (DoubleSpecies) species;
if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 8, a.length);
ByteBuffer wb = wrapper(a, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Double>)m,
(wb_, o, i) -> wb_.getDouble(o + i * 8));
}
/**
* Loads a vector from an array of type {@code double[]}
* starting at an offset.
@ -2883,44 +2944,49 @@ public abstract class DoubleVector extends AbstractVector<Double> {
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer.
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* {@link #fromMemorySegment(VectorSpecies,MemorySegment,long,ByteOrder,VectorMask)
* fromMemorySegment()} as follows:
* <pre>{@code
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* return fromMemorySegment(species, ms, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*8 < 0}
* or {@code offset+N*8 >= bb.limit()}
* or {@code offset+N*8 >= ms.byteSize()}
* for any lane {@code N} in the vector
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
DoubleVector fromByteBuffer(VectorSpecies<Double> species,
ByteBuffer bb, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
DoubleVector fromMemorySegment(VectorSpecies<Double> species,
MemorySegment ms, long offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), ms.byteSize());
DoubleSpecies vsp = (DoubleSpecies) species;
return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset).maybeSwap(bo);
}
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code double} (positive zero).
@ -2931,13 +2997,11 @@ public abstract class DoubleVector extends AbstractVector<Double> {
* <p>
* The following pseudocode illustrates the behavior:
* <pre>{@code
* DoubleBuffer eb = bb.duplicate()
* .position(offset)
* .order(bo).asDoubleBuffer();
* var slice = ms.asSlice(offset);
* double[] ar = new double[species.length()];
* for (int n = 0; n < ar.length; n++) {
* if (m.laneIsSet(n)) {
* ar[n] = eb.get(n);
* ar[n] = slice.getAtIndex(ValuaLayout.JAVA_DOUBLE.withBitAlignment(8), n);
* }
* }
* DoubleVector r = DoubleVector.fromArray(species, ar, 0);
@ -2951,33 +3015,36 @@ public abstract class DoubleVector extends AbstractVector<Double> {
* the bytes of lane values.
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*8 < 0}
* or {@code offset+N*8 >= bb.limit()}
* or {@code offset+N*8 >= ms.byteSize()}
* for any lane {@code N} in the vector
* where the mask is set
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
DoubleVector fromByteBuffer(VectorSpecies<Double> species,
ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Double> m) {
DoubleVector fromMemorySegment(VectorSpecies<Double> species,
MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Double> m) {
DoubleSpecies vsp = (DoubleSpecies) species;
if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
if (offset >= 0 && offset <= (ms.byteSize() - species.vectorByteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 8, bb.limit());
ByteBuffer wb = wrapper(bb, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Double>)m,
(wb_, o, i) -> wb_.getDouble(o + i * 8));
checkMaskFromIndexSize(offset, vsp, m, 8, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, DoubleVector::memorySegmentGet);
}
// Memory store operations
@ -3007,7 +3074,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
this,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3167,67 +3234,40 @@ public abstract class DoubleVector extends AbstractVector<Double> {
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, byteSize(), a.length);
maybeSwap(bo).intoByteArray0(a, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo,
VectorMask<Double> m) {
if (m.allTrue()) {
intoByteArray(a, offset, bo);
} else {
DoubleSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 8, a.length);
maybeSwap(bo).intoByteArray0(a, offset, m);
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo) {
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
offset = checkFromIndexSize(offset, byteSize(), ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo) {
if (ScopedMemoryAccess.isReadOnly(bb)) {
throw new ReadOnlyBufferException();
}
offset = checkFromIndexSize(offset, byteSize(), bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Double> m) {
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Double> m) {
if (m.allTrue()) {
intoByteBuffer(bb, offset, bo);
intoMemorySegment(ms, offset, bo);
} else {
if (bb.isReadOnly()) {
throw new ReadOnlyBufferException();
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
DoubleSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 8, bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset, m);
checkMaskFromIndexSize(offset, vsp, m, 8, ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset, m);
}
}
@ -3261,7 +3301,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3278,7 +3318,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3336,74 +3376,33 @@ public abstract class DoubleVector extends AbstractVector<Double> {
@Override
abstract
DoubleVector fromByteArray0(byte[] a, int offset);
DoubleVector fromMemorySegment0(MemorySegment bb, long offset);
@ForceInline
final
DoubleVector fromByteArray0Template(byte[] a, int offset) {
DoubleVector fromMemorySegment0Template(MemorySegment ms, long offset) {
DoubleSpecies vsp = vspecies();
return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getDouble(o + i * 8));
});
}
abstract
DoubleVector fromByteArray0(byte[] a, int offset, VectorMask<Double> m);
@ForceInline
final
<M extends VectorMask<Double>>
DoubleVector fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
DoubleSpecies vsp = vspecies();
m.check(vsp);
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getDouble(o + i * 8));
});
}
abstract
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset);
@ForceInline
final
DoubleVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
DoubleSpecies vsp = vspecies();
return ScopedMemoryAccess.loadFromByteBuffer(
return ScopedMemoryAccess.loadFromMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
bb, offset, vsp,
(buf, off, s) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getDouble(o + i * 8));
(AbstractMemorySegmentImpl) ms, offset, vsp,
(msp, off, s) -> {
return s.ldLongOp((MemorySegment) msp, off, DoubleVector::memorySegmentGet);
});
}
abstract
DoubleVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m);
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m);
@ForceInline
final
<M extends VectorMask<Double>>
DoubleVector fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
DoubleVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
DoubleSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromByteBufferMasked(
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
bb, offset, m, vsp,
(buf, off, s, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getDouble(o + i * 8));
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, DoubleVector::memorySegmentGet);
});
}
@ -3422,7 +3421,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
a, arrayAddress(a, offset),
this, a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_+i] = e));
}
@ -3439,7 +3438,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
a, arrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3497,71 +3496,33 @@ public abstract class DoubleVector extends AbstractVector<Double> {
}
abstract
void intoByteArray0(byte[] a, int offset);
@ForceInline
final
void intoByteArray0Template(byte[] a, int offset) {
void intoMemorySegment0(MemorySegment ms, long offset) {
DoubleSpecies vsp = vspecies();
VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, a, offset,
(arr, off, v) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off,
(tb_, o, i, e) -> tb_.putDouble(o + i * 8, e));
});
}
abstract
void intoByteArray0(byte[] a, int offset, VectorMask<Double> m);
@ForceInline
final
<M extends VectorMask<Double>>
void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
DoubleSpecies vsp = vspecies();
m.check(vsp);
VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(tb_, o, i, e) -> tb_.putDouble(o + i * 8, e));
});
}
@ForceInline
final
void intoByteBuffer0(ByteBuffer bb, int offset) {
DoubleSpecies vsp = vspecies();
ScopedMemoryAccess.storeIntoByteBuffer(
ScopedMemoryAccess.storeIntoMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
this, bb, offset,
(buf, off, v) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off,
(wb_, o, i, e) -> wb_.putDouble(o + i * 8, e));
this,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v) -> {
v.stLongOp((MemorySegment) msp, off, DoubleVector::memorySegmentSet);
});
}
abstract
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Double> m);
void intoMemorySegment0(MemorySegment bb, long offset, VectorMask<Double> m);
@ForceInline
final
<M extends VectorMask<Double>>
void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
void intoMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
DoubleSpecies vsp = vspecies();
m.check(vsp);
ScopedMemoryAccess.storeIntoByteBufferMasked(
ScopedMemoryAccess.storeIntoMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
this, m, bb, offset,
(buf, off, v, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(wb_, o, i, e) -> wb_.putDouble(o + i * 8, e));
this, m,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v, vm) -> {
v.stLongOp((MemorySegment) msp, off, vm, DoubleVector::memorySegmentSet);
});
}
@ -3578,6 +3539,16 @@ public abstract class DoubleVector extends AbstractVector<Double> {
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
private static
void checkMaskFromIndexSize(long offset,
DoubleSpecies vsp,
VectorMask<Double> m,
int scale,
long limit) {
((AbstractMask<Double>)m)
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
@ForceInline
private void conditionalStoreNYI(int offset,
DoubleSpecies vsp,
@ -3888,6 +3859,21 @@ public abstract class DoubleVector extends AbstractVector<Double> {
return dummyVector().ldOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
DoubleVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
DoubleVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Double> m,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
<M> void stOp(M memory, int offset, FStOp<M> f) {
@ -3902,6 +3888,20 @@ public abstract class DoubleVector extends AbstractVector<Double> {
dummyVector().stOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset, FStLongOp f) {
dummyVector().stLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset,
AbstractMask<Double> m,
FStLongOp f) {
dummyVector().stLongOp(memory, offset, m, f);
}
// N.B. Make sure these constant vectors and
// masks load up correctly into registers.
//
@ -4015,3 +4015,4 @@ public abstract class DoubleVector extends AbstractVector<Double> {
public static final VectorSpecies<Double> SPECIES_PREFERRED
= (DoubleSpecies) VectorSpecies.ofPreferred(double.class);
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class Float128Vector extends FloatVector {
(Float128Vector) v); // specialize
}
@Override
@ForceInline
public Float128Vector compress(VectorMask<Float> m) {
return (Float128Vector)
super.compressTemplate(Float128Mask.class,
(Float128Mask) m); // specialize
}
@Override
@ForceInline
public Float128Vector expand(VectorMask<Float> m) {
return (Float128Vector)
super.expandTemplate(Float128Mask.class,
(Float128Mask) m); // specialize
}
@Override
@ForceInline
public Float128Vector selectFrom(Vector<Float> v) {
@ -642,6 +658,15 @@ final class Float128Vector extends FloatVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Float128Mask compress() {
return (Float128Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Float128Vector.class, Float128Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -834,29 +859,15 @@ final class Float128Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset, VectorMask<Float> m) {
return super.fromByteArray0Template(Float128Mask.class, a, offset, (Float128Mask) m); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
return super.fromByteBuffer0Template(Float128Mask.class, bb, offset, (Float128Mask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(Float128Mask.class, ms, offset, (Float128Mask) m); // specialize
}
@ForceInline
@ -884,22 +895,8 @@ final class Float128Vector extends FloatVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Float> m) {
super.intoByteArray0Template(Float128Mask.class, a, offset, (Float128Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
super.intoByteBuffer0Template(Float128Mask.class, bb, offset, (Float128Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
super.intoMemorySegment0Template(Float128Mask.class, ms, offset, (Float128Mask) m);
}
@ -908,3 +905,4 @@ final class Float128Vector extends FloatVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class Float256Vector extends FloatVector {
(Float256Vector) v); // specialize
}
@Override
@ForceInline
public Float256Vector compress(VectorMask<Float> m) {
return (Float256Vector)
super.compressTemplate(Float256Mask.class,
(Float256Mask) m); // specialize
}
@Override
@ForceInline
public Float256Vector expand(VectorMask<Float> m) {
return (Float256Vector)
super.expandTemplate(Float256Mask.class,
(Float256Mask) m); // specialize
}
@Override
@ForceInline
public Float256Vector selectFrom(Vector<Float> v) {
@ -650,6 +666,15 @@ final class Float256Vector extends FloatVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Float256Mask compress() {
return (Float256Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Float256Vector.class, Float256Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -842,29 +867,15 @@ final class Float256Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset, VectorMask<Float> m) {
return super.fromByteArray0Template(Float256Mask.class, a, offset, (Float256Mask) m); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
return super.fromByteBuffer0Template(Float256Mask.class, bb, offset, (Float256Mask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(Float256Mask.class, ms, offset, (Float256Mask) m); // specialize
}
@ForceInline
@ -892,22 +903,8 @@ final class Float256Vector extends FloatVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Float> m) {
super.intoByteArray0Template(Float256Mask.class, a, offset, (Float256Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
super.intoByteBuffer0Template(Float256Mask.class, bb, offset, (Float256Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
super.intoMemorySegment0Template(Float256Mask.class, ms, offset, (Float256Mask) m);
}
@ -916,3 +913,4 @@ final class Float256Vector extends FloatVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class Float512Vector extends FloatVector {
(Float512Vector) v); // specialize
}
@Override
@ForceInline
public Float512Vector compress(VectorMask<Float> m) {
return (Float512Vector)
super.compressTemplate(Float512Mask.class,
(Float512Mask) m); // specialize
}
@Override
@ForceInline
public Float512Vector expand(VectorMask<Float> m) {
return (Float512Vector)
super.expandTemplate(Float512Mask.class,
(Float512Mask) m); // specialize
}
@Override
@ForceInline
public Float512Vector selectFrom(Vector<Float> v) {
@ -666,6 +682,15 @@ final class Float512Vector extends FloatVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Float512Mask compress() {
return (Float512Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Float512Vector.class, Float512Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -858,29 +883,15 @@ final class Float512Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset, VectorMask<Float> m) {
return super.fromByteArray0Template(Float512Mask.class, a, offset, (Float512Mask) m); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
return super.fromByteBuffer0Template(Float512Mask.class, bb, offset, (Float512Mask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(Float512Mask.class, ms, offset, (Float512Mask) m); // specialize
}
@ForceInline
@ -908,22 +919,8 @@ final class Float512Vector extends FloatVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Float> m) {
super.intoByteArray0Template(Float512Mask.class, a, offset, (Float512Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
super.intoByteBuffer0Template(Float512Mask.class, bb, offset, (Float512Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
super.intoMemorySegment0Template(Float512Mask.class, ms, offset, (Float512Mask) m);
}
@ -932,3 +929,4 @@ final class Float512Vector extends FloatVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class Float64Vector extends FloatVector {
(Float64Vector) v); // specialize
}
@Override
@ForceInline
public Float64Vector compress(VectorMask<Float> m) {
return (Float64Vector)
super.compressTemplate(Float64Mask.class,
(Float64Mask) m); // specialize
}
@Override
@ForceInline
public Float64Vector expand(VectorMask<Float> m) {
return (Float64Vector)
super.expandTemplate(Float64Mask.class,
(Float64Mask) m); // specialize
}
@Override
@ForceInline
public Float64Vector selectFrom(Vector<Float> v) {
@ -638,6 +654,15 @@ final class Float64Vector extends FloatVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Float64Mask compress() {
return (Float64Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Float64Vector.class, Float64Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -830,29 +855,15 @@ final class Float64Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset, VectorMask<Float> m) {
return super.fromByteArray0Template(Float64Mask.class, a, offset, (Float64Mask) m); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
return super.fromByteBuffer0Template(Float64Mask.class, bb, offset, (Float64Mask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(Float64Mask.class, ms, offset, (Float64Mask) m); // specialize
}
@ForceInline
@ -880,22 +891,8 @@ final class Float64Vector extends FloatVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Float> m) {
super.intoByteArray0Template(Float64Mask.class, a, offset, (Float64Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
super.intoByteBuffer0Template(Float64Mask.class, bb, offset, (Float64Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
super.intoMemorySegment0Template(Float64Mask.class, ms, offset, (Float64Mask) m);
}
@ -904,3 +901,4 @@ final class Float64Vector extends FloatVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -461,6 +461,22 @@ final class FloatMaxVector extends FloatVector {
(FloatMaxVector) v); // specialize
}
@Override
@ForceInline
public FloatMaxVector compress(VectorMask<Float> m) {
return (FloatMaxVector)
super.compressTemplate(FloatMaxMask.class,
(FloatMaxMask) m); // specialize
}
@Override
@ForceInline
public FloatMaxVector expand(VectorMask<Float> m) {
return (FloatMaxVector)
super.expandTemplate(FloatMaxMask.class,
(FloatMaxMask) m); // specialize
}
@Override
@ForceInline
public FloatMaxVector selectFrom(Vector<Float> v) {
@ -635,6 +651,15 @@ final class FloatMaxVector extends FloatVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public FloatMaxMask compress() {
return (FloatMaxMask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
FloatMaxVector.class, FloatMaxMask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -827,29 +852,15 @@ final class FloatMaxVector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteArray0(byte[] a, int offset, VectorMask<Float> m) {
return super.fromByteArray0Template(FloatMaxMask.class, a, offset, (FloatMaxMask) m); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
FloatVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
return super.fromByteBuffer0Template(FloatMaxMask.class, bb, offset, (FloatMaxMask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(FloatMaxMask.class, ms, offset, (FloatMaxMask) m); // specialize
}
@ForceInline
@ -877,22 +888,8 @@ final class FloatMaxVector extends FloatVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Float> m) {
super.intoByteArray0Template(FloatMaxMask.class, a, offset, (FloatMaxMask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m) {
super.intoByteBuffer0Template(FloatMaxMask.class, bb, offset, (FloatMaxMask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
super.intoMemorySegment0Template(FloatMaxMask.class, ms, offset, (FloatMaxMask) m);
}
@ -901,3 +898,4 @@ final class FloatMaxVector extends FloatVector {
// ================================================
}

549
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java
View file

@ -24,14 +24,14 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import jdk.internal.foreign.AbstractMemorySegmentImpl;
import jdk.internal.misc.ScopedMemoryAccess;
import jdk.internal.misc.Unsafe;
import jdk.internal.vm.annotation.ForceInline;
@ -57,6 +57,8 @@ public abstract class FloatVector extends AbstractVector<Float> {
static final int FORBID_OPCODE_KIND = VO_NOFP;
static final ValueLayout.OfFloat ELEMENT_LAYOUT = ValueLayout.JAVA_FLOAT.withBitAlignment(8);
@ForceInline
static int opCode(Operator op) {
return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
@ -351,6 +353,45 @@ public abstract class FloatVector extends AbstractVector<Float> {
return vectorFactory(res);
}
/*package-private*/
interface FLdLongOp {
float apply(MemorySegment memory, long offset, int i);
}
/*package-private*/
@ForceInline
final
FloatVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
//dummy; no vec = vec();
float[] res = new float[length()];
for (int i = 0; i < res.length; i++) {
res[i] = f.apply(memory, offset, i);
}
return vectorFactory(res);
}
/*package-private*/
@ForceInline
final
FloatVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Float> m,
FLdLongOp f) {
//float[] vec = vec();
float[] res = new float[length()];
boolean[] mbits = ((AbstractMask<Float>)m).getBits();
for (int i = 0; i < res.length; i++) {
if (mbits[i]) {
res[i] = f.apply(memory, offset, i);
}
}
return vectorFactory(res);
}
static float memorySegmentGet(MemorySegment ms, long o, int i) {
return ms.get(ELEMENT_LAYOUT, o + i * 4L);
}
interface FStOp<M> {
void apply(M memory, int offset, int i, float a);
}
@ -381,6 +422,40 @@ public abstract class FloatVector extends AbstractVector<Float> {
}
}
interface FStLongOp {
void apply(MemorySegment memory, long offset, int i, float a);
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
FStLongOp f) {
float[] vec = vec();
for (int i = 0; i < vec.length; i++) {
f.apply(memory, offset, i, vec[i]);
}
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
VectorMask<Float> m,
FStLongOp f) {
float[] vec = vec();
boolean[] mbits = ((AbstractMask<Float>)m).getBits();
for (int i = 0; i < vec.length; i++) {
if (mbits[i]) {
f.apply(memory, offset, i, vec[i]);
}
}
}
static void memorySegmentSet(MemorySegment ms, long o, int i, float e) {
ms.set(ELEMENT_LAYOUT, o + i * 4L, e);
}
// Binary test
/*package-private*/
@ -420,6 +495,36 @@ public abstract class FloatVector extends AbstractVector<Float> {
return Float.intBitsToFloat((int)bits);
}
static FloatVector expandHelper(Vector<Float> v, VectorMask<Float> m) {
VectorSpecies<Float> vsp = m.vectorSpecies();
FloatVector r = (FloatVector) vsp.zero();
FloatVector vi = (FloatVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(i, vi.lane(j++));
}
}
return r;
}
static FloatVector compressHelper(Vector<Float> v, VectorMask<Float> m) {
VectorSpecies<Float> vsp = m.vectorSpecies();
FloatVector r = (FloatVector) vsp.zero();
FloatVector vi = (FloatVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(j++, vi.lane(i));
}
}
return r;
}
// Static factories (other than memory operations)
// Note: A surprising behavior in javadoc
@ -1602,6 +1707,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
}
// sqrt
/**
* Computes the square root of this vector.
@ -2253,6 +2359,45 @@ public abstract class FloatVector extends AbstractVector<Float> {
FloatVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
FloatVector compress(VectorMask<Float> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Float>>
FloatVector compressTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (FloatVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_COMPRESS, getClass(), masktype,
float.class, length(), this, m,
(v1, m1) -> compressHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
FloatVector expand(VectorMask<Float> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Float>>
FloatVector expandTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (FloatVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_EXPAND, getClass(), masktype,
float.class, length(), this, m,
(v1, m1) -> expandHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
*/
@ -2633,90 +2778,6 @@ public abstract class FloatVector extends AbstractVector<Float> {
return res;
}
/**
* Loads a vector from a byte array starting at an offset.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
*/
@ForceInline
public static
FloatVector fromByteArray(VectorSpecies<Float> species,
byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
FloatSpecies vsp = (FloatSpecies) species;
return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
}
/**
* Loads a vector from a byte array starting at an offset
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code float} (positive zero).
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
* where the mask is set
*/
@ForceInline
public static
FloatVector fromByteArray(VectorSpecies<Float> species,
byte[] a, int offset,
ByteOrder bo,
VectorMask<Float> m) {
FloatSpecies vsp = (FloatSpecies) species;
if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 4, a.length);
ByteBuffer wb = wrapper(a, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Float>)m,
(wb_, o, i) -> wb_.getFloat(o + i * 4));
}
/**
* Loads a vector from an array of type {@code float[]}
* starting at an offset.
@ -2889,44 +2950,49 @@ public abstract class FloatVector extends AbstractVector<Float> {
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer.
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* {@link #fromMemorySegment(VectorSpecies,MemorySegment,long,ByteOrder,VectorMask)
* fromMemorySegment()} as follows:
* <pre>{@code
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* return fromMemorySegment(species, ms, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*4 < 0}
* or {@code offset+N*4 >= bb.limit()}
* or {@code offset+N*4 >= ms.byteSize()}
* for any lane {@code N} in the vector
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
FloatVector fromByteBuffer(VectorSpecies<Float> species,
ByteBuffer bb, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
FloatVector fromMemorySegment(VectorSpecies<Float> species,
MemorySegment ms, long offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), ms.byteSize());
FloatSpecies vsp = (FloatSpecies) species;
return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset).maybeSwap(bo);
}
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code float} (positive zero).
@ -2937,13 +3003,11 @@ public abstract class FloatVector extends AbstractVector<Float> {
* <p>
* The following pseudocode illustrates the behavior:
* <pre>{@code
* FloatBuffer eb = bb.duplicate()
* .position(offset)
* .order(bo).asFloatBuffer();
* var slice = ms.asSlice(offset);
* float[] ar = new float[species.length()];
* for (int n = 0; n < ar.length; n++) {
* if (m.laneIsSet(n)) {
* ar[n] = eb.get(n);
* ar[n] = slice.getAtIndex(ValuaLayout.JAVA_FLOAT.withBitAlignment(8), n);
* }
* }
* FloatVector r = FloatVector.fromArray(species, ar, 0);
@ -2957,33 +3021,36 @@ public abstract class FloatVector extends AbstractVector<Float> {
* the bytes of lane values.
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*4 < 0}
* or {@code offset+N*4 >= bb.limit()}
* or {@code offset+N*4 >= ms.byteSize()}
* for any lane {@code N} in the vector
* where the mask is set
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
FloatVector fromByteBuffer(VectorSpecies<Float> species,
ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Float> m) {
FloatVector fromMemorySegment(VectorSpecies<Float> species,
MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Float> m) {
FloatSpecies vsp = (FloatSpecies) species;
if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
if (offset >= 0 && offset <= (ms.byteSize() - species.vectorByteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 4, bb.limit());
ByteBuffer wb = wrapper(bb, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Float>)m,
(wb_, o, i) -> wb_.getFloat(o + i * 4));
checkMaskFromIndexSize(offset, vsp, m, 4, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, FloatVector::memorySegmentGet);
}
// Memory store operations
@ -3013,7 +3080,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
this,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3154,67 +3221,40 @@ public abstract class FloatVector extends AbstractVector<Float> {
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, byteSize(), a.length);
maybeSwap(bo).intoByteArray0(a, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo,
VectorMask<Float> m) {
if (m.allTrue()) {
intoByteArray(a, offset, bo);
} else {
FloatSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 4, a.length);
maybeSwap(bo).intoByteArray0(a, offset, m);
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo) {
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
offset = checkFromIndexSize(offset, byteSize(), ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo) {
if (ScopedMemoryAccess.isReadOnly(bb)) {
throw new ReadOnlyBufferException();
}
offset = checkFromIndexSize(offset, byteSize(), bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Float> m) {
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Float> m) {
if (m.allTrue()) {
intoByteBuffer(bb, offset, bo);
intoMemorySegment(ms, offset, bo);
} else {
if (bb.isReadOnly()) {
throw new ReadOnlyBufferException();
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
FloatSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 4, bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset, m);
checkMaskFromIndexSize(offset, vsp, m, 4, ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset, m);
}
}
@ -3248,7 +3288,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3265,7 +3305,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3305,74 +3345,33 @@ public abstract class FloatVector extends AbstractVector<Float> {
@Override
abstract
FloatVector fromByteArray0(byte[] a, int offset);
FloatVector fromMemorySegment0(MemorySegment bb, long offset);
@ForceInline
final
FloatVector fromByteArray0Template(byte[] a, int offset) {
FloatVector fromMemorySegment0Template(MemorySegment ms, long offset) {
FloatSpecies vsp = vspecies();
return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getFloat(o + i * 4));
});
}
abstract
FloatVector fromByteArray0(byte[] a, int offset, VectorMask<Float> m);
@ForceInline
final
<M extends VectorMask<Float>>
FloatVector fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
FloatSpecies vsp = vspecies();
m.check(vsp);
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getFloat(o + i * 4));
});
}
abstract
FloatVector fromByteBuffer0(ByteBuffer bb, int offset);
@ForceInline
final
FloatVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
FloatSpecies vsp = vspecies();
return ScopedMemoryAccess.loadFromByteBuffer(
return ScopedMemoryAccess.loadFromMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
bb, offset, vsp,
(buf, off, s) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getFloat(o + i * 4));
(AbstractMemorySegmentImpl) ms, offset, vsp,
(msp, off, s) -> {
return s.ldLongOp((MemorySegment) msp, off, FloatVector::memorySegmentGet);
});
}
abstract
FloatVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m);
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m);
@ForceInline
final
<M extends VectorMask<Float>>
FloatVector fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
FloatVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
FloatSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromByteBufferMasked(
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
bb, offset, m, vsp,
(buf, off, s, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getFloat(o + i * 4));
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, FloatVector::memorySegmentGet);
});
}
@ -3391,7 +3390,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
a, arrayAddress(a, offset),
this, a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_+i] = e));
}
@ -3408,7 +3407,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
a, arrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3447,71 +3446,33 @@ public abstract class FloatVector extends AbstractVector<Float> {
}
abstract
void intoByteArray0(byte[] a, int offset);
@ForceInline
final
void intoByteArray0Template(byte[] a, int offset) {
void intoMemorySegment0(MemorySegment ms, long offset) {
FloatSpecies vsp = vspecies();
VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, a, offset,
(arr, off, v) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off,
(tb_, o, i, e) -> tb_.putFloat(o + i * 4, e));
});
}
abstract
void intoByteArray0(byte[] a, int offset, VectorMask<Float> m);
@ForceInline
final
<M extends VectorMask<Float>>
void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
FloatSpecies vsp = vspecies();
m.check(vsp);
VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(tb_, o, i, e) -> tb_.putFloat(o + i * 4, e));
});
}
@ForceInline
final
void intoByteBuffer0(ByteBuffer bb, int offset) {
FloatSpecies vsp = vspecies();
ScopedMemoryAccess.storeIntoByteBuffer(
ScopedMemoryAccess.storeIntoMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
this, bb, offset,
(buf, off, v) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off,
(wb_, o, i, e) -> wb_.putFloat(o + i * 4, e));
this,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v) -> {
v.stLongOp((MemorySegment) msp, off, FloatVector::memorySegmentSet);
});
}
abstract
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Float> m);
void intoMemorySegment0(MemorySegment bb, long offset, VectorMask<Float> m);
@ForceInline
final
<M extends VectorMask<Float>>
void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
void intoMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
FloatSpecies vsp = vspecies();
m.check(vsp);
ScopedMemoryAccess.storeIntoByteBufferMasked(
ScopedMemoryAccess.storeIntoMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
this, m, bb, offset,
(buf, off, v, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(wb_, o, i, e) -> wb_.putFloat(o + i * 4, e));
this, m,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v, vm) -> {
v.stLongOp((MemorySegment) msp, off, vm, FloatVector::memorySegmentSet);
});
}
@ -3528,6 +3489,16 @@ public abstract class FloatVector extends AbstractVector<Float> {
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
private static
void checkMaskFromIndexSize(long offset,
FloatSpecies vsp,
VectorMask<Float> m,
int scale,
long limit) {
((AbstractMask<Float>)m)
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
@ForceInline
private void conditionalStoreNYI(int offset,
FloatSpecies vsp,
@ -3838,6 +3809,21 @@ public abstract class FloatVector extends AbstractVector<Float> {
return dummyVector().ldOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
FloatVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
FloatVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Float> m,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
<M> void stOp(M memory, int offset, FStOp<M> f) {
@ -3852,6 +3838,20 @@ public abstract class FloatVector extends AbstractVector<Float> {
dummyVector().stOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset, FStLongOp f) {
dummyVector().stLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset,
AbstractMask<Float> m,
FStLongOp f) {
dummyVector().stLongOp(memory, offset, m, f);
}
// N.B. Make sure these constant vectors and
// masks load up correctly into registers.
//
@ -3965,3 +3965,4 @@ public abstract class FloatVector extends AbstractVector<Float> {
public static final VectorSpecies<Float> SPECIES_PREFERRED
= (FloatSpecies) VectorSpecies.ofPreferred(float.class);
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Int128Vector extends IntVector {
(Int128Vector) v); // specialize
}
@Override
@ForceInline
public Int128Vector compress(VectorMask<Integer> m) {
return (Int128Vector)
super.compressTemplate(Int128Mask.class,
(Int128Mask) m); // specialize
}
@Override
@ForceInline
public Int128Vector expand(VectorMask<Integer> m) {
return (Int128Vector)
super.expandTemplate(Int128Mask.class,
(Int128Mask) m); // specialize
}
@Override
@ForceInline
public Int128Vector selectFrom(Vector<Integer> v) {
@ -653,6 +669,15 @@ final class Int128Vector extends IntVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Int128Mask compress() {
return (Int128Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Int128Vector.class, Int128Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -845,29 +870,15 @@ final class Int128Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
return super.fromByteArray0Template(Int128Mask.class, a, offset, (Int128Mask) m); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
return super.fromByteBuffer0Template(Int128Mask.class, bb, offset, (Int128Mask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(Int128Mask.class, ms, offset, (Int128Mask) m); // specialize
}
@ForceInline
@ -895,22 +906,8 @@ final class Int128Vector extends IntVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
super.intoByteArray0Template(Int128Mask.class, a, offset, (Int128Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
super.intoByteBuffer0Template(Int128Mask.class, bb, offset, (Int128Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
super.intoMemorySegment0Template(Int128Mask.class, ms, offset, (Int128Mask) m);
}
@ -919,3 +916,4 @@ final class Int128Vector extends IntVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Int256Vector extends IntVector {
(Int256Vector) v); // specialize
}
@Override
@ForceInline
public Int256Vector compress(VectorMask<Integer> m) {
return (Int256Vector)
super.compressTemplate(Int256Mask.class,
(Int256Mask) m); // specialize
}
@Override
@ForceInline
public Int256Vector expand(VectorMask<Integer> m) {
return (Int256Vector)
super.expandTemplate(Int256Mask.class,
(Int256Mask) m); // specialize
}
@Override
@ForceInline
public Int256Vector selectFrom(Vector<Integer> v) {
@ -661,6 +677,15 @@ final class Int256Vector extends IntVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Int256Mask compress() {
return (Int256Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Int256Vector.class, Int256Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -853,29 +878,15 @@ final class Int256Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
return super.fromByteArray0Template(Int256Mask.class, a, offset, (Int256Mask) m); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
return super.fromByteBuffer0Template(Int256Mask.class, bb, offset, (Int256Mask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(Int256Mask.class, ms, offset, (Int256Mask) m); // specialize
}
@ForceInline
@ -903,22 +914,8 @@ final class Int256Vector extends IntVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
super.intoByteArray0Template(Int256Mask.class, a, offset, (Int256Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
super.intoByteBuffer0Template(Int256Mask.class, bb, offset, (Int256Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
super.intoMemorySegment0Template(Int256Mask.class, ms, offset, (Int256Mask) m);
}
@ -927,3 +924,4 @@ final class Int256Vector extends IntVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Int512Vector extends IntVector {
(Int512Vector) v); // specialize
}
@Override
@ForceInline
public Int512Vector compress(VectorMask<Integer> m) {
return (Int512Vector)
super.compressTemplate(Int512Mask.class,
(Int512Mask) m); // specialize
}
@Override
@ForceInline
public Int512Vector expand(VectorMask<Integer> m) {
return (Int512Vector)
super.expandTemplate(Int512Mask.class,
(Int512Mask) m); // specialize
}
@Override
@ForceInline
public Int512Vector selectFrom(Vector<Integer> v) {
@ -677,6 +693,15 @@ final class Int512Vector extends IntVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Int512Mask compress() {
return (Int512Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Int512Vector.class, Int512Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -869,29 +894,15 @@ final class Int512Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
return super.fromByteArray0Template(Int512Mask.class, a, offset, (Int512Mask) m); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
return super.fromByteBuffer0Template(Int512Mask.class, bb, offset, (Int512Mask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(Int512Mask.class, ms, offset, (Int512Mask) m); // specialize
}
@ForceInline
@ -919,22 +930,8 @@ final class Int512Vector extends IntVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
super.intoByteArray0Template(Int512Mask.class, a, offset, (Int512Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
super.intoByteBuffer0Template(Int512Mask.class, bb, offset, (Int512Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
super.intoMemorySegment0Template(Int512Mask.class, ms, offset, (Int512Mask) m);
}
@ -943,3 +940,4 @@ final class Int512Vector extends IntVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Int64Vector extends IntVector {
(Int64Vector) v); // specialize
}
@Override
@ForceInline
public Int64Vector compress(VectorMask<Integer> m) {
return (Int64Vector)
super.compressTemplate(Int64Mask.class,
(Int64Mask) m); // specialize
}
@Override
@ForceInline
public Int64Vector expand(VectorMask<Integer> m) {
return (Int64Vector)
super.expandTemplate(Int64Mask.class,
(Int64Mask) m); // specialize
}
@Override
@ForceInline
public Int64Vector selectFrom(Vector<Integer> v) {
@ -649,6 +665,15 @@ final class Int64Vector extends IntVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Int64Mask compress() {
return (Int64Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Int64Vector.class, Int64Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -841,29 +866,15 @@ final class Int64Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
return super.fromByteArray0Template(Int64Mask.class, a, offset, (Int64Mask) m); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
return super.fromByteBuffer0Template(Int64Mask.class, bb, offset, (Int64Mask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(Int64Mask.class, ms, offset, (Int64Mask) m); // specialize
}
@ForceInline
@ -891,22 +902,8 @@ final class Int64Vector extends IntVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
super.intoByteArray0Template(Int64Mask.class, a, offset, (Int64Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
super.intoByteBuffer0Template(Int64Mask.class, bb, offset, (Int64Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
super.intoMemorySegment0Template(Int64Mask.class, ms, offset, (Int64Mask) m);
}
@ -915,3 +912,4 @@ final class Int64Vector extends IntVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class IntMaxVector extends IntVector {
(IntMaxVector) v); // specialize
}
@Override
@ForceInline
public IntMaxVector compress(VectorMask<Integer> m) {
return (IntMaxVector)
super.compressTemplate(IntMaxMask.class,
(IntMaxMask) m); // specialize
}
@Override
@ForceInline
public IntMaxVector expand(VectorMask<Integer> m) {
return (IntMaxVector)
super.expandTemplate(IntMaxMask.class,
(IntMaxMask) m); // specialize
}
@Override
@ForceInline
public IntMaxVector selectFrom(Vector<Integer> v) {
@ -647,6 +663,15 @@ final class IntMaxVector extends IntVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public IntMaxMask compress() {
return (IntMaxMask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
IntMaxVector.class, IntMaxMask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -850,29 +875,15 @@ final class IntMaxVector extends IntVector {
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
return super.fromByteArray0Template(IntMaxMask.class, a, offset, (IntMaxMask) m); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
IntVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
return super.fromByteBuffer0Template(IntMaxMask.class, bb, offset, (IntMaxMask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(IntMaxMask.class, ms, offset, (IntMaxMask) m); // specialize
}
@ForceInline
@ -900,22 +911,8 @@ final class IntMaxVector extends IntVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Integer> m) {
super.intoByteArray0Template(IntMaxMask.class, a, offset, (IntMaxMask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m) {
super.intoByteBuffer0Template(IntMaxMask.class, bb, offset, (IntMaxMask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
super.intoMemorySegment0Template(IntMaxMask.class, ms, offset, (IntMaxMask) m);
}
@ -924,3 +921,4 @@ final class IntMaxVector extends IntVector {
// ================================================
}

563
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java
View file

@ -24,14 +24,14 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import jdk.internal.foreign.AbstractMemorySegmentImpl;
import jdk.internal.misc.ScopedMemoryAccess;
import jdk.internal.misc.Unsafe;
import jdk.internal.vm.annotation.ForceInline;
@ -57,6 +57,8 @@ public abstract class IntVector extends AbstractVector<Integer> {
static final int FORBID_OPCODE_KIND = VO_ONLYFP;
static final ValueLayout.OfInt ELEMENT_LAYOUT = ValueLayout.JAVA_INT.withBitAlignment(8);
@ForceInline
static int opCode(Operator op) {
return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
@ -351,6 +353,45 @@ public abstract class IntVector extends AbstractVector<Integer> {
return vectorFactory(res);
}
/*package-private*/
interface FLdLongOp {
int apply(MemorySegment memory, long offset, int i);
}
/*package-private*/
@ForceInline
final
IntVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
//dummy; no vec = vec();
int[] res = new int[length()];
for (int i = 0; i < res.length; i++) {
res[i] = f.apply(memory, offset, i);
}
return vectorFactory(res);
}
/*package-private*/
@ForceInline
final
IntVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Integer> m,
FLdLongOp f) {
//int[] vec = vec();
int[] res = new int[length()];
boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
for (int i = 0; i < res.length; i++) {
if (mbits[i]) {
res[i] = f.apply(memory, offset, i);
}
}
return vectorFactory(res);
}
static int memorySegmentGet(MemorySegment ms, long o, int i) {
return ms.get(ELEMENT_LAYOUT, o + i * 4L);
}
interface FStOp<M> {
void apply(M memory, int offset, int i, int a);
}
@ -381,6 +422,40 @@ public abstract class IntVector extends AbstractVector<Integer> {
}
}
interface FStLongOp {
void apply(MemorySegment memory, long offset, int i, int a);
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
FStLongOp f) {
int[] vec = vec();
for (int i = 0; i < vec.length; i++) {
f.apply(memory, offset, i, vec[i]);
}
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
VectorMask<Integer> m,
FStLongOp f) {
int[] vec = vec();
boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
for (int i = 0; i < vec.length; i++) {
if (mbits[i]) {
f.apply(memory, offset, i, vec[i]);
}
}
}
static void memorySegmentSet(MemorySegment ms, long o, int i, int e) {
ms.set(ELEMENT_LAYOUT, o + i * 4L, e);
}
// Binary test
/*package-private*/
@ -431,6 +506,36 @@ public abstract class IntVector extends AbstractVector<Integer> {
return ((int)bits);
}
static IntVector expandHelper(Vector<Integer> v, VectorMask<Integer> m) {
VectorSpecies<Integer> vsp = m.vectorSpecies();
IntVector r = (IntVector) vsp.zero();
IntVector vi = (IntVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(i, vi.lane(j++));
}
}
return r;
}
static IntVector compressHelper(Vector<Integer> v, VectorMask<Integer> m) {
VectorSpecies<Integer> vsp = m.vectorSpecies();
IntVector r = (IntVector) vsp.zero();
IntVector vi = (IntVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(j++, vi.lane(i));
}
}
return r;
}
// Static factories (other than memory operations)
// Note: A surprising behavior in javadoc
@ -620,6 +725,16 @@ public abstract class IntVector extends AbstractVector<Integer> {
v0.uOp(m, (i, a) -> (int) -a);
case VECTOR_OP_ABS: return (v0, m) ->
v0.uOp(m, (i, a) -> (int) Math.abs(a));
case VECTOR_OP_BIT_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (int) Integer.bitCount(a));
case VECTOR_OP_TZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (int) Integer.numberOfTrailingZeros(a));
case VECTOR_OP_LZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (int) Integer.numberOfLeadingZeros(a));
case VECTOR_OP_REVERSE: return (v0, m) ->
v0.uOp(m, (i, a) -> (int) Integer.reverse(a));
case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
v0.uOp(m, (i, a) -> (int) Integer.reverseBytes(a));
default: return null;
}
}
@ -760,6 +875,10 @@ public abstract class IntVector extends AbstractVector<Integer> {
v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
case VECTOR_OP_COMPRESS_BITS: return (v0, v1, vm) ->
v0.bOp(v1, vm, (i, a, n) -> Integer.compress(a, n));
case VECTOR_OP_EXPAND_BITS: return (v0, v1, vm) ->
v0.bOp(v1, vm, (i, a, n) -> Integer.expand(a, n));
default: return null;
}
}
@ -1745,6 +1864,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
return lanewise(ABS);
}
// not (~)
/**
* Computes the bitwise logical complement ({@code ~})
@ -2371,6 +2491,45 @@ public abstract class IntVector extends AbstractVector<Integer> {
IntVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
IntVector compress(VectorMask<Integer> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Integer>>
IntVector compressTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (IntVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_COMPRESS, getClass(), masktype,
int.class, length(), this, m,
(v1, m1) -> compressHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
IntVector expand(VectorMask<Integer> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Integer>>
IntVector expandTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (IntVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_EXPAND, getClass(), masktype,
int.class, length(), this, m,
(v1, m1) -> expandHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
*/
@ -2776,90 +2935,6 @@ public abstract class IntVector extends AbstractVector<Integer> {
return res;
}
/**
* Loads a vector from a byte array starting at an offset.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
*/
@ForceInline
public static
IntVector fromByteArray(VectorSpecies<Integer> species,
byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
IntSpecies vsp = (IntSpecies) species;
return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
}
/**
* Loads a vector from a byte array starting at an offset
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code int} (zero).
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
* where the mask is set
*/
@ForceInline
public static
IntVector fromByteArray(VectorSpecies<Integer> species,
byte[] a, int offset,
ByteOrder bo,
VectorMask<Integer> m) {
IntSpecies vsp = (IntSpecies) species;
if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 4, a.length);
ByteBuffer wb = wrapper(a, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Integer>)m,
(wb_, o, i) -> wb_.getInt(o + i * 4));
}
/**
* Loads a vector from an array of type {@code int[]}
* starting at an offset.
@ -3032,44 +3107,49 @@ public abstract class IntVector extends AbstractVector<Integer> {
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer.
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* {@link #fromMemorySegment(VectorSpecies,MemorySegment,long,ByteOrder,VectorMask)
* fromMemorySegment()} as follows:
* <pre>{@code
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* return fromMemorySegment(species, ms, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*4 < 0}
* or {@code offset+N*4 >= bb.limit()}
* or {@code offset+N*4 >= ms.byteSize()}
* for any lane {@code N} in the vector
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
IntVector fromByteBuffer(VectorSpecies<Integer> species,
ByteBuffer bb, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
IntVector fromMemorySegment(VectorSpecies<Integer> species,
MemorySegment ms, long offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), ms.byteSize());
IntSpecies vsp = (IntSpecies) species;
return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset).maybeSwap(bo);
}
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code int} (zero).
@ -3080,13 +3160,11 @@ public abstract class IntVector extends AbstractVector<Integer> {
* <p>
* The following pseudocode illustrates the behavior:
* <pre>{@code
* IntBuffer eb = bb.duplicate()
* .position(offset)
* .order(bo).asIntBuffer();
* var slice = ms.asSlice(offset);
* int[] ar = new int[species.length()];
* for (int n = 0; n < ar.length; n++) {
* if (m.laneIsSet(n)) {
* ar[n] = eb.get(n);
* ar[n] = slice.getAtIndex(ValuaLayout.JAVA_INT.withBitAlignment(8), n);
* }
* }
* IntVector r = IntVector.fromArray(species, ar, 0);
@ -3100,33 +3178,36 @@ public abstract class IntVector extends AbstractVector<Integer> {
* the bytes of lane values.
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*4 < 0}
* or {@code offset+N*4 >= bb.limit()}
* or {@code offset+N*4 >= ms.byteSize()}
* for any lane {@code N} in the vector
* where the mask is set
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
IntVector fromByteBuffer(VectorSpecies<Integer> species,
ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Integer> m) {
IntVector fromMemorySegment(VectorSpecies<Integer> species,
MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Integer> m) {
IntSpecies vsp = (IntSpecies) species;
if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
if (offset >= 0 && offset <= (ms.byteSize() - species.vectorByteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 4, bb.limit());
ByteBuffer wb = wrapper(bb, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Integer>)m,
(wb_, o, i) -> wb_.getInt(o + i * 4));
checkMaskFromIndexSize(offset, vsp, m, 4, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, IntVector::memorySegmentGet);
}
// Memory store operations
@ -3156,7 +3237,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
this,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3297,67 +3378,40 @@ public abstract class IntVector extends AbstractVector<Integer> {
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, byteSize(), a.length);
maybeSwap(bo).intoByteArray0(a, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo,
VectorMask<Integer> m) {
if (m.allTrue()) {
intoByteArray(a, offset, bo);
} else {
IntSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 4, a.length);
maybeSwap(bo).intoByteArray0(a, offset, m);
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo) {
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
offset = checkFromIndexSize(offset, byteSize(), ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo) {
if (ScopedMemoryAccess.isReadOnly(bb)) {
throw new ReadOnlyBufferException();
}
offset = checkFromIndexSize(offset, byteSize(), bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Integer> m) {
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Integer> m) {
if (m.allTrue()) {
intoByteBuffer(bb, offset, bo);
intoMemorySegment(ms, offset, bo);
} else {
if (bb.isReadOnly()) {
throw new ReadOnlyBufferException();
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
IntSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 4, bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset, m);
checkMaskFromIndexSize(offset, vsp, m, 4, ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset, m);
}
}
@ -3391,7 +3445,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3408,7 +3462,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3448,74 +3502,33 @@ public abstract class IntVector extends AbstractVector<Integer> {
@Override
abstract
IntVector fromByteArray0(byte[] a, int offset);
IntVector fromMemorySegment0(MemorySegment bb, long offset);
@ForceInline
final
IntVector fromByteArray0Template(byte[] a, int offset) {
IntVector fromMemorySegment0Template(MemorySegment ms, long offset) {
IntSpecies vsp = vspecies();
return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getInt(o + i * 4));
});
}
abstract
IntVector fromByteArray0(byte[] a, int offset, VectorMask<Integer> m);
@ForceInline
final
<M extends VectorMask<Integer>>
IntVector fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
IntSpecies vsp = vspecies();
m.check(vsp);
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getInt(o + i * 4));
});
}
abstract
IntVector fromByteBuffer0(ByteBuffer bb, int offset);
@ForceInline
final
IntVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
IntSpecies vsp = vspecies();
return ScopedMemoryAccess.loadFromByteBuffer(
return ScopedMemoryAccess.loadFromMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
bb, offset, vsp,
(buf, off, s) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getInt(o + i * 4));
(AbstractMemorySegmentImpl) ms, offset, vsp,
(msp, off, s) -> {
return s.ldLongOp((MemorySegment) msp, off, IntVector::memorySegmentGet);
});
}
abstract
IntVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m);
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m);
@ForceInline
final
<M extends VectorMask<Integer>>
IntVector fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
IntVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
IntSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromByteBufferMasked(
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
bb, offset, m, vsp,
(buf, off, s, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getInt(o + i * 4));
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, IntVector::memorySegmentGet);
});
}
@ -3534,7 +3547,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
a, arrayAddress(a, offset),
this, a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_+i] = e));
}
@ -3551,7 +3564,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
a, arrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3590,71 +3603,33 @@ public abstract class IntVector extends AbstractVector<Integer> {
}
abstract
void intoByteArray0(byte[] a, int offset);
@ForceInline
final
void intoByteArray0Template(byte[] a, int offset) {
void intoMemorySegment0(MemorySegment ms, long offset) {
IntSpecies vsp = vspecies();
VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, a, offset,
(arr, off, v) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off,
(tb_, o, i, e) -> tb_.putInt(o + i * 4, e));
});
}
abstract
void intoByteArray0(byte[] a, int offset, VectorMask<Integer> m);
@ForceInline
final
<M extends VectorMask<Integer>>
void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
IntSpecies vsp = vspecies();
m.check(vsp);
VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(tb_, o, i, e) -> tb_.putInt(o + i * 4, e));
});
}
@ForceInline
final
void intoByteBuffer0(ByteBuffer bb, int offset) {
IntSpecies vsp = vspecies();
ScopedMemoryAccess.storeIntoByteBuffer(
ScopedMemoryAccess.storeIntoMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
this, bb, offset,
(buf, off, v) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off,
(wb_, o, i, e) -> wb_.putInt(o + i * 4, e));
this,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v) -> {
v.stLongOp((MemorySegment) msp, off, IntVector::memorySegmentSet);
});
}
abstract
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m);
void intoMemorySegment0(MemorySegment bb, long offset, VectorMask<Integer> m);
@ForceInline
final
<M extends VectorMask<Integer>>
void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
void intoMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
IntSpecies vsp = vspecies();
m.check(vsp);
ScopedMemoryAccess.storeIntoByteBufferMasked(
ScopedMemoryAccess.storeIntoMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
this, m, bb, offset,
(buf, off, v, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(wb_, o, i, e) -> wb_.putInt(o + i * 4, e));
this, m,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v, vm) -> {
v.stLongOp((MemorySegment) msp, off, vm, IntVector::memorySegmentSet);
});
}
@ -3671,6 +3646,16 @@ public abstract class IntVector extends AbstractVector<Integer> {
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
private static
void checkMaskFromIndexSize(long offset,
IntSpecies vsp,
VectorMask<Integer> m,
int scale,
long limit) {
((AbstractMask<Integer>)m)
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
@ForceInline
private void conditionalStoreNYI(int offset,
IntSpecies vsp,
@ -3981,6 +3966,21 @@ public abstract class IntVector extends AbstractVector<Integer> {
return dummyVector().ldOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
IntVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
IntVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Integer> m,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
<M> void stOp(M memory, int offset, FStOp<M> f) {
@ -3995,6 +3995,20 @@ public abstract class IntVector extends AbstractVector<Integer> {
dummyVector().stOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset, FStLongOp f) {
dummyVector().stLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset,
AbstractMask<Integer> m,
FStLongOp f) {
dummyVector().stLongOp(memory, offset, m, f);
}
// N.B. Make sure these constant vectors and
// masks load up correctly into registers.
//
@ -4108,3 +4122,4 @@ public abstract class IntVector extends AbstractVector<Integer> {
public static final VectorSpecies<Integer> SPECIES_PREFERRED
= (IntSpecies) VectorSpecies.ofPreferred(int.class);
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -464,6 +464,22 @@ final class Long128Vector extends LongVector {
(Long128Vector) v); // specialize
}
@Override
@ForceInline
public Long128Vector compress(VectorMask<Long> m) {
return (Long128Vector)
super.compressTemplate(Long128Mask.class,
(Long128Mask) m); // specialize
}
@Override
@ForceInline
public Long128Vector expand(VectorMask<Long> m) {
return (Long128Vector)
super.expandTemplate(Long128Mask.class,
(Long128Mask) m); // specialize
}
@Override
@ForceInline
public Long128Vector selectFrom(Vector<Long> v) {
@ -639,6 +655,15 @@ final class Long128Vector extends LongVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Long128Mask compress() {
return (Long128Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Long128Vector.class, Long128Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -831,29 +856,15 @@ final class Long128Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset, VectorMask<Long> m) {
return super.fromByteArray0Template(Long128Mask.class, a, offset, (Long128Mask) m); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
return super.fromByteBuffer0Template(Long128Mask.class, bb, offset, (Long128Mask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(Long128Mask.class, ms, offset, (Long128Mask) m); // specialize
}
@ForceInline
@ -881,22 +892,8 @@ final class Long128Vector extends LongVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Long> m) {
super.intoByteArray0Template(Long128Mask.class, a, offset, (Long128Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
super.intoByteBuffer0Template(Long128Mask.class, bb, offset, (Long128Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
super.intoMemorySegment0Template(Long128Mask.class, ms, offset, (Long128Mask) m);
}
@ -905,3 +902,4 @@ final class Long128Vector extends LongVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -464,6 +464,22 @@ final class Long256Vector extends LongVector {
(Long256Vector) v); // specialize
}
@Override
@ForceInline
public Long256Vector compress(VectorMask<Long> m) {
return (Long256Vector)
super.compressTemplate(Long256Mask.class,
(Long256Mask) m); // specialize
}
@Override
@ForceInline
public Long256Vector expand(VectorMask<Long> m) {
return (Long256Vector)
super.expandTemplate(Long256Mask.class,
(Long256Mask) m); // specialize
}
@Override
@ForceInline
public Long256Vector selectFrom(Vector<Long> v) {
@ -643,6 +659,15 @@ final class Long256Vector extends LongVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Long256Mask compress() {
return (Long256Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Long256Vector.class, Long256Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -835,29 +860,15 @@ final class Long256Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset, VectorMask<Long> m) {
return super.fromByteArray0Template(Long256Mask.class, a, offset, (Long256Mask) m); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
return super.fromByteBuffer0Template(Long256Mask.class, bb, offset, (Long256Mask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(Long256Mask.class, ms, offset, (Long256Mask) m); // specialize
}
@ForceInline
@ -885,22 +896,8 @@ final class Long256Vector extends LongVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Long> m) {
super.intoByteArray0Template(Long256Mask.class, a, offset, (Long256Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
super.intoByteBuffer0Template(Long256Mask.class, bb, offset, (Long256Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
super.intoMemorySegment0Template(Long256Mask.class, ms, offset, (Long256Mask) m);
}
@ -909,3 +906,4 @@ final class Long256Vector extends LongVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -464,6 +464,22 @@ final class Long512Vector extends LongVector {
(Long512Vector) v); // specialize
}
@Override
@ForceInline
public Long512Vector compress(VectorMask<Long> m) {
return (Long512Vector)
super.compressTemplate(Long512Mask.class,
(Long512Mask) m); // specialize
}
@Override
@ForceInline
public Long512Vector expand(VectorMask<Long> m) {
return (Long512Vector)
super.expandTemplate(Long512Mask.class,
(Long512Mask) m); // specialize
}
@Override
@ForceInline
public Long512Vector selectFrom(Vector<Long> v) {
@ -651,6 +667,15 @@ final class Long512Vector extends LongVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Long512Mask compress() {
return (Long512Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Long512Vector.class, Long512Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -843,29 +868,15 @@ final class Long512Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset, VectorMask<Long> m) {
return super.fromByteArray0Template(Long512Mask.class, a, offset, (Long512Mask) m); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
return super.fromByteBuffer0Template(Long512Mask.class, bb, offset, (Long512Mask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(Long512Mask.class, ms, offset, (Long512Mask) m); // specialize
}
@ForceInline
@ -893,22 +904,8 @@ final class Long512Vector extends LongVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Long> m) {
super.intoByteArray0Template(Long512Mask.class, a, offset, (Long512Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
super.intoByteBuffer0Template(Long512Mask.class, bb, offset, (Long512Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
super.intoMemorySegment0Template(Long512Mask.class, ms, offset, (Long512Mask) m);
}
@ -917,3 +914,4 @@ final class Long512Vector extends LongVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -464,6 +464,22 @@ final class Long64Vector extends LongVector {
(Long64Vector) v); // specialize
}
@Override
@ForceInline
public Long64Vector compress(VectorMask<Long> m) {
return (Long64Vector)
super.compressTemplate(Long64Mask.class,
(Long64Mask) m); // specialize
}
@Override
@ForceInline
public Long64Vector expand(VectorMask<Long> m) {
return (Long64Vector)
super.expandTemplate(Long64Mask.class,
(Long64Mask) m); // specialize
}
@Override
@ForceInline
public Long64Vector selectFrom(Vector<Long> v) {
@ -637,6 +653,15 @@ final class Long64Vector extends LongVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Long64Mask compress() {
return (Long64Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Long64Vector.class, Long64Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -829,29 +854,15 @@ final class Long64Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset, VectorMask<Long> m) {
return super.fromByteArray0Template(Long64Mask.class, a, offset, (Long64Mask) m); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
return super.fromByteBuffer0Template(Long64Mask.class, bb, offset, (Long64Mask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(Long64Mask.class, ms, offset, (Long64Mask) m); // specialize
}
@ForceInline
@ -879,22 +890,8 @@ final class Long64Vector extends LongVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Long> m) {
super.intoByteArray0Template(Long64Mask.class, a, offset, (Long64Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
super.intoByteBuffer0Template(Long64Mask.class, bb, offset, (Long64Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
super.intoMemorySegment0Template(Long64Mask.class, ms, offset, (Long64Mask) m);
}
@ -903,3 +900,4 @@ final class Long64Vector extends LongVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -464,6 +464,22 @@ final class LongMaxVector extends LongVector {
(LongMaxVector) v); // specialize
}
@Override
@ForceInline
public LongMaxVector compress(VectorMask<Long> m) {
return (LongMaxVector)
super.compressTemplate(LongMaxMask.class,
(LongMaxMask) m); // specialize
}
@Override
@ForceInline
public LongMaxVector expand(VectorMask<Long> m) {
return (LongMaxVector)
super.expandTemplate(LongMaxMask.class,
(LongMaxMask) m); // specialize
}
@Override
@ForceInline
public LongMaxVector selectFrom(Vector<Long> v) {
@ -637,6 +653,15 @@ final class LongMaxVector extends LongVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public LongMaxMask compress() {
return (LongMaxMask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
LongMaxVector.class, LongMaxMask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -829,29 +854,15 @@ final class LongMaxVector extends LongVector {
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteArray0(byte[] a, int offset, VectorMask<Long> m) {
return super.fromByteArray0Template(LongMaxMask.class, a, offset, (LongMaxMask) m); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
LongVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
return super.fromByteBuffer0Template(LongMaxMask.class, bb, offset, (LongMaxMask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(LongMaxMask.class, ms, offset, (LongMaxMask) m); // specialize
}
@ForceInline
@ -879,22 +890,8 @@ final class LongMaxVector extends LongVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Long> m) {
super.intoByteArray0Template(LongMaxMask.class, a, offset, (LongMaxMask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m) {
super.intoByteBuffer0Template(LongMaxMask.class, bb, offset, (LongMaxMask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
super.intoMemorySegment0Template(LongMaxMask.class, ms, offset, (LongMaxMask) m);
}
@ -903,3 +900,4 @@ final class LongMaxVector extends LongVector {
// ================================================
}

563
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java
View file

@ -24,14 +24,14 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import jdk.internal.foreign.AbstractMemorySegmentImpl;
import jdk.internal.misc.ScopedMemoryAccess;
import jdk.internal.misc.Unsafe;
import jdk.internal.vm.annotation.ForceInline;
@ -57,6 +57,8 @@ public abstract class LongVector extends AbstractVector<Long> {
static final int FORBID_OPCODE_KIND = VO_ONLYFP;
static final ValueLayout.OfLong ELEMENT_LAYOUT = ValueLayout.JAVA_LONG.withBitAlignment(8);
@ForceInline
static int opCode(Operator op) {
return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
@ -351,6 +353,45 @@ public abstract class LongVector extends AbstractVector<Long> {
return vectorFactory(res);
}
/*package-private*/
interface FLdLongOp {
long apply(MemorySegment memory, long offset, int i);
}
/*package-private*/
@ForceInline
final
LongVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
//dummy; no vec = vec();
long[] res = new long[length()];
for (int i = 0; i < res.length; i++) {
res[i] = f.apply(memory, offset, i);
}
return vectorFactory(res);
}
/*package-private*/
@ForceInline
final
LongVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Long> m,
FLdLongOp f) {
//long[] vec = vec();
long[] res = new long[length()];
boolean[] mbits = ((AbstractMask<Long>)m).getBits();
for (int i = 0; i < res.length; i++) {
if (mbits[i]) {
res[i] = f.apply(memory, offset, i);
}
}
return vectorFactory(res);
}
static long memorySegmentGet(MemorySegment ms, long o, int i) {
return ms.get(ELEMENT_LAYOUT, o + i * 8L);
}
interface FStOp<M> {
void apply(M memory, int offset, int i, long a);
}
@ -381,6 +422,40 @@ public abstract class LongVector extends AbstractVector<Long> {
}
}
interface FStLongOp {
void apply(MemorySegment memory, long offset, int i, long a);
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
FStLongOp f) {
long[] vec = vec();
for (int i = 0; i < vec.length; i++) {
f.apply(memory, offset, i, vec[i]);
}
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
VectorMask<Long> m,
FStLongOp f) {
long[] vec = vec();
boolean[] mbits = ((AbstractMask<Long>)m).getBits();
for (int i = 0; i < vec.length; i++) {
if (mbits[i]) {
f.apply(memory, offset, i, vec[i]);
}
}
}
static void memorySegmentSet(MemorySegment ms, long o, int i, long e) {
ms.set(ELEMENT_LAYOUT, o + i * 8L, e);
}
// Binary test
/*package-private*/
@ -431,6 +506,36 @@ public abstract class LongVector extends AbstractVector<Long> {
return ((long)bits);
}
static LongVector expandHelper(Vector<Long> v, VectorMask<Long> m) {
VectorSpecies<Long> vsp = m.vectorSpecies();
LongVector r = (LongVector) vsp.zero();
LongVector vi = (LongVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(i, vi.lane(j++));
}
}
return r;
}
static LongVector compressHelper(Vector<Long> v, VectorMask<Long> m) {
VectorSpecies<Long> vsp = m.vectorSpecies();
LongVector r = (LongVector) vsp.zero();
LongVector vi = (LongVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(j++, vi.lane(i));
}
}
return r;
}
// Static factories (other than memory operations)
// Note: A surprising behavior in javadoc
@ -578,6 +683,16 @@ public abstract class LongVector extends AbstractVector<Long> {
v0.uOp(m, (i, a) -> (long) -a);
case VECTOR_OP_ABS: return (v0, m) ->
v0.uOp(m, (i, a) -> (long) Math.abs(a));
case VECTOR_OP_BIT_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (long) Long.bitCount(a));
case VECTOR_OP_TZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (long) Long.numberOfTrailingZeros(a));
case VECTOR_OP_LZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (long) Long.numberOfLeadingZeros(a));
case VECTOR_OP_REVERSE: return (v0, m) ->
v0.uOp(m, (i, a) -> (long) Long.reverse(a));
case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
v0.uOp(m, (i, a) -> (long) Long.reverseBytes(a));
default: return null;
}
}
@ -718,6 +833,10 @@ public abstract class LongVector extends AbstractVector<Long> {
v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
case VECTOR_OP_COMPRESS_BITS: return (v0, v1, vm) ->
v0.bOp(v1, vm, (i, a, n) -> Long.compress(a, n));
case VECTOR_OP_EXPAND_BITS: return (v0, v1, vm) ->
v0.bOp(v1, vm, (i, a, n) -> Long.expand(a, n));
default: return null;
}
}
@ -1658,6 +1777,7 @@ public abstract class LongVector extends AbstractVector<Long> {
return lanewise(ABS);
}
// not (~)
/**
* Computes the bitwise logical complement ({@code ~})
@ -2237,6 +2357,45 @@ public abstract class LongVector extends AbstractVector<Long> {
LongVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
LongVector compress(VectorMask<Long> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Long>>
LongVector compressTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (LongVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_COMPRESS, getClass(), masktype,
long.class, length(), this, m,
(v1, m1) -> compressHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
LongVector expand(VectorMask<Long> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Long>>
LongVector expandTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (LongVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_EXPAND, getClass(), masktype,
long.class, length(), this, m,
(v1, m1) -> expandHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
*/
@ -2637,90 +2796,6 @@ public abstract class LongVector extends AbstractVector<Long> {
return res;
}
/**
* Loads a vector from a byte array starting at an offset.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
*/
@ForceInline
public static
LongVector fromByteArray(VectorSpecies<Long> species,
byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
LongSpecies vsp = (LongSpecies) species;
return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
}
/**
* Loads a vector from a byte array starting at an offset
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code long} (zero).
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
* where the mask is set
*/
@ForceInline
public static
LongVector fromByteArray(VectorSpecies<Long> species,
byte[] a, int offset,
ByteOrder bo,
VectorMask<Long> m) {
LongSpecies vsp = (LongSpecies) species;
if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 8, a.length);
ByteBuffer wb = wrapper(a, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Long>)m,
(wb_, o, i) -> wb_.getLong(o + i * 8));
}
/**
* Loads a vector from an array of type {@code long[]}
* starting at an offset.
@ -2911,44 +2986,49 @@ public abstract class LongVector extends AbstractVector<Long> {
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer.
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* {@link #fromMemorySegment(VectorSpecies,MemorySegment,long,ByteOrder,VectorMask)
* fromMemorySegment()} as follows:
* <pre>{@code
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* return fromMemorySegment(species, ms, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*8 < 0}
* or {@code offset+N*8 >= bb.limit()}
* or {@code offset+N*8 >= ms.byteSize()}
* for any lane {@code N} in the vector
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
LongVector fromByteBuffer(VectorSpecies<Long> species,
ByteBuffer bb, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
LongVector fromMemorySegment(VectorSpecies<Long> species,
MemorySegment ms, long offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), ms.byteSize());
LongSpecies vsp = (LongSpecies) species;
return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset).maybeSwap(bo);
}
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code long} (zero).
@ -2959,13 +3039,11 @@ public abstract class LongVector extends AbstractVector<Long> {
* <p>
* The following pseudocode illustrates the behavior:
* <pre>{@code
* LongBuffer eb = bb.duplicate()
* .position(offset)
* .order(bo).asLongBuffer();
* var slice = ms.asSlice(offset);
* long[] ar = new long[species.length()];
* for (int n = 0; n < ar.length; n++) {
* if (m.laneIsSet(n)) {
* ar[n] = eb.get(n);
* ar[n] = slice.getAtIndex(ValuaLayout.JAVA_LONG.withBitAlignment(8), n);
* }
* }
* LongVector r = LongVector.fromArray(species, ar, 0);
@ -2979,33 +3057,36 @@ public abstract class LongVector extends AbstractVector<Long> {
* the bytes of lane values.
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*8 < 0}
* or {@code offset+N*8 >= bb.limit()}
* or {@code offset+N*8 >= ms.byteSize()}
* for any lane {@code N} in the vector
* where the mask is set
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
LongVector fromByteBuffer(VectorSpecies<Long> species,
ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Long> m) {
LongVector fromMemorySegment(VectorSpecies<Long> species,
MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Long> m) {
LongSpecies vsp = (LongSpecies) species;
if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
if (offset >= 0 && offset <= (ms.byteSize() - species.vectorByteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 8, bb.limit());
ByteBuffer wb = wrapper(bb, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Long>)m,
(wb_, o, i) -> wb_.getLong(o + i * 8));
checkMaskFromIndexSize(offset, vsp, m, 8, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, LongVector::memorySegmentGet);
}
// Memory store operations
@ -3035,7 +3116,7 @@ public abstract class LongVector extends AbstractVector<Long> {
this,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3195,67 +3276,40 @@ public abstract class LongVector extends AbstractVector<Long> {
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, byteSize(), a.length);
maybeSwap(bo).intoByteArray0(a, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo,
VectorMask<Long> m) {
if (m.allTrue()) {
intoByteArray(a, offset, bo);
} else {
LongSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 8, a.length);
maybeSwap(bo).intoByteArray0(a, offset, m);
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo) {
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
offset = checkFromIndexSize(offset, byteSize(), ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo) {
if (ScopedMemoryAccess.isReadOnly(bb)) {
throw new ReadOnlyBufferException();
}
offset = checkFromIndexSize(offset, byteSize(), bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Long> m) {
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Long> m) {
if (m.allTrue()) {
intoByteBuffer(bb, offset, bo);
intoMemorySegment(ms, offset, bo);
} else {
if (bb.isReadOnly()) {
throw new ReadOnlyBufferException();
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
LongSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 8, bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset, m);
checkMaskFromIndexSize(offset, vsp, m, 8, ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset, m);
}
}
@ -3289,7 +3343,7 @@ public abstract class LongVector extends AbstractVector<Long> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3306,7 +3360,7 @@ public abstract class LongVector extends AbstractVector<Long> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3364,74 +3418,33 @@ public abstract class LongVector extends AbstractVector<Long> {
@Override
abstract
LongVector fromByteArray0(byte[] a, int offset);
LongVector fromMemorySegment0(MemorySegment bb, long offset);
@ForceInline
final
LongVector fromByteArray0Template(byte[] a, int offset) {
LongVector fromMemorySegment0Template(MemorySegment ms, long offset) {
LongSpecies vsp = vspecies();
return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getLong(o + i * 8));
});
}
abstract
LongVector fromByteArray0(byte[] a, int offset, VectorMask<Long> m);
@ForceInline
final
<M extends VectorMask<Long>>
LongVector fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
LongSpecies vsp = vspecies();
m.check(vsp);
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getLong(o + i * 8));
});
}
abstract
LongVector fromByteBuffer0(ByteBuffer bb, int offset);
@ForceInline
final
LongVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
LongSpecies vsp = vspecies();
return ScopedMemoryAccess.loadFromByteBuffer(
return ScopedMemoryAccess.loadFromMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
bb, offset, vsp,
(buf, off, s) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getLong(o + i * 8));
(AbstractMemorySegmentImpl) ms, offset, vsp,
(msp, off, s) -> {
return s.ldLongOp((MemorySegment) msp, off, LongVector::memorySegmentGet);
});
}
abstract
LongVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m);
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m);
@ForceInline
final
<M extends VectorMask<Long>>
LongVector fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
LongVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
LongSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromByteBufferMasked(
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
bb, offset, m, vsp,
(buf, off, s, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getLong(o + i * 8));
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, LongVector::memorySegmentGet);
});
}
@ -3450,7 +3463,7 @@ public abstract class LongVector extends AbstractVector<Long> {
a, arrayAddress(a, offset),
this, a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_+i] = e));
}
@ -3467,7 +3480,7 @@ public abstract class LongVector extends AbstractVector<Long> {
a, arrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3525,71 +3538,33 @@ public abstract class LongVector extends AbstractVector<Long> {
}
abstract
void intoByteArray0(byte[] a, int offset);
@ForceInline
final
void intoByteArray0Template(byte[] a, int offset) {
void intoMemorySegment0(MemorySegment ms, long offset) {
LongSpecies vsp = vspecies();
VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, a, offset,
(arr, off, v) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off,
(tb_, o, i, e) -> tb_.putLong(o + i * 8, e));
});
}
abstract
void intoByteArray0(byte[] a, int offset, VectorMask<Long> m);
@ForceInline
final
<M extends VectorMask<Long>>
void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
LongSpecies vsp = vspecies();
m.check(vsp);
VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(tb_, o, i, e) -> tb_.putLong(o + i * 8, e));
});
}
@ForceInline
final
void intoByteBuffer0(ByteBuffer bb, int offset) {
LongSpecies vsp = vspecies();
ScopedMemoryAccess.storeIntoByteBuffer(
ScopedMemoryAccess.storeIntoMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
this, bb, offset,
(buf, off, v) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off,
(wb_, o, i, e) -> wb_.putLong(o + i * 8, e));
this,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v) -> {
v.stLongOp((MemorySegment) msp, off, LongVector::memorySegmentSet);
});
}
abstract
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m);
void intoMemorySegment0(MemorySegment bb, long offset, VectorMask<Long> m);
@ForceInline
final
<M extends VectorMask<Long>>
void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
void intoMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
LongSpecies vsp = vspecies();
m.check(vsp);
ScopedMemoryAccess.storeIntoByteBufferMasked(
ScopedMemoryAccess.storeIntoMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
this, m, bb, offset,
(buf, off, v, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(wb_, o, i, e) -> wb_.putLong(o + i * 8, e));
this, m,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v, vm) -> {
v.stLongOp((MemorySegment) msp, off, vm, LongVector::memorySegmentSet);
});
}
@ -3606,6 +3581,16 @@ public abstract class LongVector extends AbstractVector<Long> {
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
private static
void checkMaskFromIndexSize(long offset,
LongSpecies vsp,
VectorMask<Long> m,
int scale,
long limit) {
((AbstractMask<Long>)m)
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
@ForceInline
private void conditionalStoreNYI(int offset,
LongSpecies vsp,
@ -3907,6 +3892,21 @@ public abstract class LongVector extends AbstractVector<Long> {
return dummyVector().ldOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
LongVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
LongVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Long> m,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
<M> void stOp(M memory, int offset, FStOp<M> f) {
@ -3921,6 +3921,20 @@ public abstract class LongVector extends AbstractVector<Long> {
dummyVector().stOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset, FStLongOp f) {
dummyVector().stLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset,
AbstractMask<Long> m,
FStLongOp f) {
dummyVector().stLongOp(memory, offset, m, f);
}
// N.B. Make sure these constant vectors and
// masks load up correctly into registers.
//
@ -4034,3 +4048,4 @@ public abstract class LongVector extends AbstractVector<Long> {
public static final VectorSpecies<Long> SPECIES_PREFERRED
= (LongSpecies) VectorSpecies.ofPreferred(long.class);
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Short128Vector extends ShortVector {
(Short128Vector) v); // specialize
}
@Override
@ForceInline
public Short128Vector compress(VectorMask<Short> m) {
return (Short128Vector)
super.compressTemplate(Short128Mask.class,
(Short128Mask) m); // specialize
}
@Override
@ForceInline
public Short128Vector expand(VectorMask<Short> m) {
return (Short128Vector)
super.expandTemplate(Short128Mask.class,
(Short128Mask) m); // specialize
}
@Override
@ForceInline
public Short128Vector selectFrom(Vector<Short> v) {
@ -661,6 +677,15 @@ final class Short128Vector extends ShortVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Short128Mask compress() {
return (Short128Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Short128Vector.class, Short128Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -860,29 +885,15 @@ final class Short128Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset, VectorMask<Short> m) {
return super.fromByteArray0Template(Short128Mask.class, a, offset, (Short128Mask) m); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
return super.fromByteBuffer0Template(Short128Mask.class, bb, offset, (Short128Mask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(Short128Mask.class, ms, offset, (Short128Mask) m); // specialize
}
@ForceInline
@ -904,22 +915,8 @@ final class Short128Vector extends ShortVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Short> m) {
super.intoByteArray0Template(Short128Mask.class, a, offset, (Short128Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
super.intoByteBuffer0Template(Short128Mask.class, bb, offset, (Short128Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
super.intoMemorySegment0Template(Short128Mask.class, ms, offset, (Short128Mask) m);
}
@ForceInline
@ -934,3 +931,4 @@ final class Short128Vector extends ShortVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Short256Vector extends ShortVector {
(Short256Vector) v); // specialize
}
@Override
@ForceInline
public Short256Vector compress(VectorMask<Short> m) {
return (Short256Vector)
super.compressTemplate(Short256Mask.class,
(Short256Mask) m); // specialize
}
@Override
@ForceInline
public Short256Vector expand(VectorMask<Short> m) {
return (Short256Vector)
super.expandTemplate(Short256Mask.class,
(Short256Mask) m); // specialize
}
@Override
@ForceInline
public Short256Vector selectFrom(Vector<Short> v) {
@ -677,6 +693,15 @@ final class Short256Vector extends ShortVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Short256Mask compress() {
return (Short256Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Short256Vector.class, Short256Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -876,29 +901,15 @@ final class Short256Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset, VectorMask<Short> m) {
return super.fromByteArray0Template(Short256Mask.class, a, offset, (Short256Mask) m); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
return super.fromByteBuffer0Template(Short256Mask.class, bb, offset, (Short256Mask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(Short256Mask.class, ms, offset, (Short256Mask) m); // specialize
}
@ForceInline
@ -920,22 +931,8 @@ final class Short256Vector extends ShortVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Short> m) {
super.intoByteArray0Template(Short256Mask.class, a, offset, (Short256Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
super.intoByteBuffer0Template(Short256Mask.class, bb, offset, (Short256Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
super.intoMemorySegment0Template(Short256Mask.class, ms, offset, (Short256Mask) m);
}
@ForceInline
@ -950,3 +947,4 @@ final class Short256Vector extends ShortVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Short512Vector extends ShortVector {
(Short512Vector) v); // specialize
}
@Override
@ForceInline
public Short512Vector compress(VectorMask<Short> m) {
return (Short512Vector)
super.compressTemplate(Short512Mask.class,
(Short512Mask) m); // specialize
}
@Override
@ForceInline
public Short512Vector expand(VectorMask<Short> m) {
return (Short512Vector)
super.expandTemplate(Short512Mask.class,
(Short512Mask) m); // specialize
}
@Override
@ForceInline
public Short512Vector selectFrom(Vector<Short> v) {
@ -709,6 +725,15 @@ final class Short512Vector extends ShortVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Short512Mask compress() {
return (Short512Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Short512Vector.class, Short512Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -908,29 +933,15 @@ final class Short512Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset, VectorMask<Short> m) {
return super.fromByteArray0Template(Short512Mask.class, a, offset, (Short512Mask) m); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
return super.fromByteBuffer0Template(Short512Mask.class, bb, offset, (Short512Mask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(Short512Mask.class, ms, offset, (Short512Mask) m); // specialize
}
@ForceInline
@ -952,22 +963,8 @@ final class Short512Vector extends ShortVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Short> m) {
super.intoByteArray0Template(Short512Mask.class, a, offset, (Short512Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
super.intoByteBuffer0Template(Short512Mask.class, bb, offset, (Short512Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
super.intoMemorySegment0Template(Short512Mask.class, ms, offset, (Short512Mask) m);
}
@ForceInline
@ -982,3 +979,4 @@ final class Short512Vector extends ShortVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class Short64Vector extends ShortVector {
(Short64Vector) v); // specialize
}
@Override
@ForceInline
public Short64Vector compress(VectorMask<Short> m) {
return (Short64Vector)
super.compressTemplate(Short64Mask.class,
(Short64Mask) m); // specialize
}
@Override
@ForceInline
public Short64Vector expand(VectorMask<Short> m) {
return (Short64Vector)
super.expandTemplate(Short64Mask.class,
(Short64Mask) m); // specialize
}
@Override
@ForceInline
public Short64Vector selectFrom(Vector<Short> v) {
@ -653,6 +669,15 @@ final class Short64Vector extends ShortVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public Short64Mask compress() {
return (Short64Mask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
Short64Vector.class, Short64Mask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -852,29 +877,15 @@ final class Short64Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset, VectorMask<Short> m) {
return super.fromByteArray0Template(Short64Mask.class, a, offset, (Short64Mask) m); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
return super.fromByteBuffer0Template(Short64Mask.class, bb, offset, (Short64Mask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(Short64Mask.class, ms, offset, (Short64Mask) m); // specialize
}
@ForceInline
@ -896,22 +907,8 @@ final class Short64Vector extends ShortVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Short> m) {
super.intoByteArray0Template(Short64Mask.class, a, offset, (Short64Mask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
super.intoByteBuffer0Template(Short64Mask.class, bb, offset, (Short64Mask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
super.intoMemorySegment0Template(Short64Mask.class, ms, offset, (Short64Mask) m);
}
@ForceInline
@ -926,3 +923,4 @@ final class Short64Vector extends ShortVector {
// ================================================
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -474,6 +474,22 @@ final class ShortMaxVector extends ShortVector {
(ShortMaxVector) v); // specialize
}
@Override
@ForceInline
public ShortMaxVector compress(VectorMask<Short> m) {
return (ShortMaxVector)
super.compressTemplate(ShortMaxMask.class,
(ShortMaxMask) m); // specialize
}
@Override
@ForceInline
public ShortMaxVector expand(VectorMask<Short> m) {
return (ShortMaxVector)
super.expandTemplate(ShortMaxMask.class,
(ShortMaxMask) m); // specialize
}
@Override
@ForceInline
public ShortMaxVector selectFrom(Vector<Short> v) {
@ -647,6 +663,15 @@ final class ShortMaxVector extends ShortVector {
return xor(maskAll(true));
}
@Override
@ForceInline
public ShortMaxMask compress() {
return (ShortMaxMask)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
ShortMaxVector.class, ShortMaxMask.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -846,29 +871,15 @@ final class ShortMaxVector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteArray0(byte[] a, int offset, VectorMask<Short> m) {
return super.fromByteArray0Template(ShortMaxMask.class, a, offset, (ShortMaxMask) m); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
ShortVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
return super.fromByteBuffer0Template(ShortMaxMask.class, bb, offset, (ShortMaxMask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(ShortMaxMask.class, ms, offset, (ShortMaxMask) m); // specialize
}
@ForceInline
@ -890,22 +901,8 @@ final class ShortMaxVector extends ShortVector {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<Short> m) {
super.intoByteArray0Template(ShortMaxMask.class, a, offset, (ShortMaxMask) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m) {
super.intoByteBuffer0Template(ShortMaxMask.class, bb, offset, (ShortMaxMask) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
super.intoMemorySegment0Template(ShortMaxMask.class, ms, offset, (ShortMaxMask) m);
}
@ForceInline
@ -920,3 +917,4 @@ final class ShortMaxVector extends ShortVector {
// ================================================
}

586
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java
View file

@ -24,14 +24,14 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import jdk.internal.foreign.AbstractMemorySegmentImpl;
import jdk.internal.misc.ScopedMemoryAccess;
import jdk.internal.misc.Unsafe;
import jdk.internal.vm.annotation.ForceInline;
@ -57,6 +57,8 @@ public abstract class ShortVector extends AbstractVector<Short> {
static final int FORBID_OPCODE_KIND = VO_ONLYFP;
static final ValueLayout.OfShort ELEMENT_LAYOUT = ValueLayout.JAVA_SHORT.withBitAlignment(8);
@ForceInline
static int opCode(Operator op) {
return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
@ -351,6 +353,45 @@ public abstract class ShortVector extends AbstractVector<Short> {
return vectorFactory(res);
}
/*package-private*/
interface FLdLongOp {
short apply(MemorySegment memory, long offset, int i);
}
/*package-private*/
@ForceInline
final
ShortVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
//dummy; no vec = vec();
short[] res = new short[length()];
for (int i = 0; i < res.length; i++) {
res[i] = f.apply(memory, offset, i);
}
return vectorFactory(res);
}
/*package-private*/
@ForceInline
final
ShortVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Short> m,
FLdLongOp f) {
//short[] vec = vec();
short[] res = new short[length()];
boolean[] mbits = ((AbstractMask<Short>)m).getBits();
for (int i = 0; i < res.length; i++) {
if (mbits[i]) {
res[i] = f.apply(memory, offset, i);
}
}
return vectorFactory(res);
}
static short memorySegmentGet(MemorySegment ms, long o, int i) {
return ms.get(ELEMENT_LAYOUT, o + i * 2L);
}
interface FStOp<M> {
void apply(M memory, int offset, int i, short a);
}
@ -381,6 +422,40 @@ public abstract class ShortVector extends AbstractVector<Short> {
}
}
interface FStLongOp {
void apply(MemorySegment memory, long offset, int i, short a);
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
FStLongOp f) {
short[] vec = vec();
for (int i = 0; i < vec.length; i++) {
f.apply(memory, offset, i, vec[i]);
}
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
VectorMask<Short> m,
FStLongOp f) {
short[] vec = vec();
boolean[] mbits = ((AbstractMask<Short>)m).getBits();
for (int i = 0; i < vec.length; i++) {
if (mbits[i]) {
f.apply(memory, offset, i, vec[i]);
}
}
}
static void memorySegmentSet(MemorySegment ms, long o, int i, short e) {
ms.set(ELEMENT_LAYOUT, o + i * 2L, e);
}
// Binary test
/*package-private*/
@ -431,6 +506,36 @@ public abstract class ShortVector extends AbstractVector<Short> {
return ((short)bits);
}
static ShortVector expandHelper(Vector<Short> v, VectorMask<Short> m) {
VectorSpecies<Short> vsp = m.vectorSpecies();
ShortVector r = (ShortVector) vsp.zero();
ShortVector vi = (ShortVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(i, vi.lane(j++));
}
}
return r;
}
static ShortVector compressHelper(Vector<Short> v, VectorMask<Short> m) {
VectorSpecies<Short> vsp = m.vectorSpecies();
ShortVector r = (ShortVector) vsp.zero();
ShortVector vi = (ShortVector) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(j++, vi.lane(i));
}
}
return r;
}
// Static factories (other than memory operations)
// Note: A surprising behavior in javadoc
@ -620,6 +725,16 @@ public abstract class ShortVector extends AbstractVector<Short> {
v0.uOp(m, (i, a) -> (short) -a);
case VECTOR_OP_ABS: return (v0, m) ->
v0.uOp(m, (i, a) -> (short) Math.abs(a));
case VECTOR_OP_BIT_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (short) bitCount(a));
case VECTOR_OP_TZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (short) numberOfTrailingZeros(a));
case VECTOR_OP_LZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> (short) numberOfLeadingZeros(a));
case VECTOR_OP_REVERSE: return (v0, m) ->
v0.uOp(m, (i, a) -> reverse(a));
case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
v0.uOp(m, (i, a) -> (short) Short.reverseBytes(a));
default: return null;
}
}
@ -1746,6 +1861,26 @@ public abstract class ShortVector extends AbstractVector<Short> {
return lanewise(ABS);
}
static int bitCount(short a) {
return Integer.bitCount((int)a & 0xFFFF);
}
static int numberOfTrailingZeros(short a) {
return a != 0 ? Integer.numberOfTrailingZeros(a) : 16;
}
static int numberOfLeadingZeros(short a) {
return a >= 0 ? Integer.numberOfLeadingZeros(a) - 16 : 0;
}
static short reverse(short a) {
if (a == 0 || a == -1) return a;
short b = rotateLeft(a, 8);
b = (short) (((b & 0x5555) << 1) | ((b & 0xAAAA) >>> 1));
b = (short) (((b & 0x3333) << 2) | ((b & 0xCCCC) >>> 2));
b = (short) (((b & 0x0F0F) << 4) | ((b & 0xF0F0) >>> 4));
return b;
}
// not (~)
/**
* Computes the bitwise logical complement ({@code ~})
@ -2372,6 +2507,45 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
ShortVector compress(VectorMask<Short> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Short>>
ShortVector compressTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (ShortVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_COMPRESS, getClass(), masktype,
short.class, length(), this, m,
(v1, m1) -> compressHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
ShortVector expand(VectorMask<Short> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<Short>>
ShortVector expandTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return (ShortVector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_EXPAND, getClass(), masktype,
short.class, length(), this, m,
(v1, m1) -> expandHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
*/
@ -2784,90 +2958,6 @@ public abstract class ShortVector extends AbstractVector<Short> {
return res;
}
/**
* Loads a vector from a byte array starting at an offset.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
*/
@ForceInline
public static
ShortVector fromByteArray(VectorSpecies<Short> species,
byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
ShortSpecies vsp = (ShortSpecies) species;
return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
}
/**
* Loads a vector from a byte array starting at an offset
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code short} (zero).
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
* where the mask is set
*/
@ForceInline
public static
ShortVector fromByteArray(VectorSpecies<Short> species,
byte[] a, int offset,
ByteOrder bo,
VectorMask<Short> m) {
ShortSpecies vsp = (ShortSpecies) species;
if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 2, a.length);
ByteBuffer wb = wrapper(a, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Short>)m,
(wb_, o, i) -> wb_.getShort(o + i * 2));
}
/**
* Loads a vector from an array of type {@code short[]}
* starting at an offset.
@ -3167,44 +3257,49 @@ public abstract class ShortVector extends AbstractVector<Short> {
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer.
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* {@link #fromMemorySegment(VectorSpecies,MemorySegment,long,ByteOrder,VectorMask)
* fromMemorySegment()} as follows:
* <pre>{@code
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* return fromMemorySegment(species, ms, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*2 < 0}
* or {@code offset+N*2 >= bb.limit()}
* or {@code offset+N*2 >= ms.byteSize()}
* for any lane {@code N} in the vector
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
ShortVector fromByteBuffer(VectorSpecies<Short> species,
ByteBuffer bb, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
ShortVector fromMemorySegment(VectorSpecies<Short> species,
MemorySegment ms, long offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), ms.byteSize());
ShortSpecies vsp = (ShortSpecies) species;
return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset).maybeSwap(bo);
}
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code short} (zero).
@ -3215,13 +3310,11 @@ public abstract class ShortVector extends AbstractVector<Short> {
* <p>
* The following pseudocode illustrates the behavior:
* <pre>{@code
* ShortBuffer eb = bb.duplicate()
* .position(offset)
* .order(bo).asShortBuffer();
* var slice = ms.asSlice(offset);
* short[] ar = new short[species.length()];
* for (int n = 0; n < ar.length; n++) {
* if (m.laneIsSet(n)) {
* ar[n] = eb.get(n);
* ar[n] = slice.getAtIndex(ValuaLayout.JAVA_SHORT.withBitAlignment(8), n);
* }
* }
* ShortVector r = ShortVector.fromArray(species, ar, 0);
@ -3235,33 +3328,36 @@ public abstract class ShortVector extends AbstractVector<Short> {
* the bytes of lane values.
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*2 < 0}
* or {@code offset+N*2 >= bb.limit()}
* or {@code offset+N*2 >= ms.byteSize()}
* for any lane {@code N} in the vector
* where the mask is set
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
ShortVector fromByteBuffer(VectorSpecies<Short> species,
ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Short> m) {
ShortVector fromMemorySegment(VectorSpecies<Short> species,
MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Short> m) {
ShortSpecies vsp = (ShortSpecies) species;
if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
if (offset >= 0 && offset <= (ms.byteSize() - species.vectorByteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 2, bb.limit());
ByteBuffer wb = wrapper(bb, bo);
return vsp.ldOp(wb, offset, (AbstractMask<Short>)m,
(wb_, o, i) -> wb_.getShort(o + i * 2));
checkMaskFromIndexSize(offset, vsp, m, 2, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, ShortVector::memorySegmentGet);
}
// Memory store operations
@ -3291,7 +3387,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
this,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -3437,7 +3533,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
this,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
}
@ -3567,67 +3663,40 @@ public abstract class ShortVector extends AbstractVector<Short> {
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, byteSize(), a.length);
maybeSwap(bo).intoByteArray0(a, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo,
VectorMask<Short> m) {
if (m.allTrue()) {
intoByteArray(a, offset, bo);
} else {
ShortSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 2, a.length);
maybeSwap(bo).intoByteArray0(a, offset, m);
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo) {
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
offset = checkFromIndexSize(offset, byteSize(), ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo) {
if (ScopedMemoryAccess.isReadOnly(bb)) {
throw new ReadOnlyBufferException();
}
offset = checkFromIndexSize(offset, byteSize(), bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<Short> m) {
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<Short> m) {
if (m.allTrue()) {
intoByteBuffer(bb, offset, bo);
intoMemorySegment(ms, offset, bo);
} else {
if (bb.isReadOnly()) {
throw new ReadOnlyBufferException();
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
ShortSpecies vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, 2, bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset, m);
checkMaskFromIndexSize(offset, vsp, m, 2, ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset, m);
}
}
@ -3661,7 +3730,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3678,7 +3747,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -3694,7 +3763,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> (short) arr_[off_ + i]));
}
@ -3711,79 +3780,38 @@ public abstract class ShortVector extends AbstractVector<Short> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (short) arr_[off_ + i]));
}
@Override
abstract
ShortVector fromByteArray0(byte[] a, int offset);
ShortVector fromMemorySegment0(MemorySegment bb, long offset);
@ForceInline
final
ShortVector fromByteArray0Template(byte[] a, int offset) {
ShortVector fromMemorySegment0Template(MemorySegment ms, long offset) {
ShortSpecies vsp = vspecies();
return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getShort(o + i * 2));
});
}
abstract
ShortVector fromByteArray0(byte[] a, int offset, VectorMask<Short> m);
@ForceInline
final
<M extends VectorMask<Short>>
ShortVector fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
ShortSpecies vsp = vspecies();
m.check(vsp);
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getShort(o + i * 2));
});
}
abstract
ShortVector fromByteBuffer0(ByteBuffer bb, int offset);
@ForceInline
final
ShortVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
ShortSpecies vsp = vspecies();
return ScopedMemoryAccess.loadFromByteBuffer(
return ScopedMemoryAccess.loadFromMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
bb, offset, vsp,
(buf, off, s) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.getShort(o + i * 2));
(AbstractMemorySegmentImpl) ms, offset, vsp,
(msp, off, s) -> {
return s.ldLongOp((MemorySegment) msp, off, ShortVector::memorySegmentGet);
});
}
abstract
ShortVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m);
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m);
@ForceInline
final
<M extends VectorMask<Short>>
ShortVector fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
ShortVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
ShortSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromByteBufferMasked(
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
bb, offset, m, vsp,
(buf, off, s, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.getShort(o + i * 2));
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, ShortVector::memorySegmentGet);
});
}
@ -3802,7 +3830,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
a, arrayAddress(a, offset),
this, a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_+i] = e));
}
@ -3819,77 +3847,39 @@ public abstract class ShortVector extends AbstractVector<Short> {
a, arrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
abstract
void intoByteArray0(byte[] a, int offset);
@ForceInline
final
void intoByteArray0Template(byte[] a, int offset) {
void intoMemorySegment0(MemorySegment ms, long offset) {
ShortSpecies vsp = vspecies();
VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, a, offset,
(arr, off, v) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off,
(tb_, o, i, e) -> tb_.putShort(o + i * 2, e));
});
}
abstract
void intoByteArray0(byte[] a, int offset, VectorMask<Short> m);
@ForceInline
final
<M extends VectorMask<Short>>
void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
ShortSpecies vsp = vspecies();
m.check(vsp);
VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(tb_, o, i, e) -> tb_.putShort(o + i * 2, e));
});
}
@ForceInline
final
void intoByteBuffer0(ByteBuffer bb, int offset) {
ShortSpecies vsp = vspecies();
ScopedMemoryAccess.storeIntoByteBuffer(
ScopedMemoryAccess.storeIntoMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
this, bb, offset,
(buf, off, v) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off,
(wb_, o, i, e) -> wb_.putShort(o + i * 2, e));
this,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v) -> {
v.stLongOp((MemorySegment) msp, off, ShortVector::memorySegmentSet);
});
}
abstract
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Short> m);
void intoMemorySegment0(MemorySegment bb, long offset, VectorMask<Short> m);
@ForceInline
final
<M extends VectorMask<Short>>
void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
void intoMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
ShortSpecies vsp = vspecies();
m.check(vsp);
ScopedMemoryAccess.storeIntoByteBufferMasked(
ScopedMemoryAccess.storeIntoMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
this, m, bb, offset,
(buf, off, v, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(wb_, o, i, e) -> wb_.putShort(o + i * 2, e));
this, m,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v, vm) -> {
v.stLongOp((MemorySegment) msp, off, vm, ShortVector::memorySegmentSet);
});
}
@ -3907,7 +3897,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
a, charArrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
}
@ -3923,6 +3913,16 @@ public abstract class ShortVector extends AbstractVector<Short> {
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
private static
void checkMaskFromIndexSize(long offset,
ShortSpecies vsp,
VectorMask<Short> m,
int scale,
long limit) {
((AbstractMask<Short>)m)
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
@ForceInline
private void conditionalStoreNYI(int offset,
ShortSpecies vsp,
@ -4250,6 +4250,21 @@ public abstract class ShortVector extends AbstractVector<Short> {
return dummyVector().ldOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
ShortVector ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
ShortVector ldLongOp(MemorySegment memory, long offset,
VectorMask<Short> m,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
<M> void stOp(M memory, int offset, FStOp<M> f) {
@ -4264,6 +4279,20 @@ public abstract class ShortVector extends AbstractVector<Short> {
dummyVector().stOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset, FStLongOp f) {
dummyVector().stLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset,
AbstractMask<Short> m,
FStLongOp f) {
dummyVector().stLongOp(memory, offset, m, f);
}
// N.B. Make sure these constant vectors and
// masks load up correctly into registers.
//
@ -4377,3 +4406,4 @@ public abstract class ShortVector extends AbstractVector<Short> {
public static final VectorSpecies<Short> SPECIES_PREFERRED
= (ShortSpecies) VectorSpecies.ofPreferred(short.class);
}

198
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Vector.java
View file

@ -24,7 +24,8 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.nio.ByteOrder;
import java.util.Arrays;
@ -763,11 +764,11 @@ import java.util.Arrays;
* first vector lane value occupies the first position in memory, and so on,
* up to the length of the vector. Further, the memory order of stored
* vector lanes corresponds to increasing index values in a Java array or
* in a {@link java.nio.ByteBuffer}.
* in a {@link java.lang.foreign.MemorySegment}.
*
* <p> Byte order for lane storage is chosen such that the stored
* vector values can be read or written as single primitive values,
* within the array or buffer that holds the vector, producing the
* within the array or segment that holds the vector, producing the
* same values as the lane-wise values within the vector.
* This fact is independent of the convenient fiction that lane values
* inside of vectors are stored in little-endian order.
@ -1039,6 +1040,12 @@ import java.util.Arrays;
* can encode a mathematical permutation as well as many other
* patterns of data movement.
*
* <li>The {@link #compress(VectorMask)} and {@link #expand(VectorMask)}
* methods, which select up to {@code VLENGTH} lanes from an
* input vector, and assemble them in lane order. The selection of lanes
* is controlled by a {@code VectorMask}, with set lane elements mapping, by
* compression or expansion in lane order, source lanes to destination lanes.
*
* </ul>
* <p> Some vector operations are not lane-wise, but rather move data
* across lane boundaries. Such operations are typically rare in SIMD
@ -2689,6 +2696,46 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
*/
public abstract Vector<E> rearrange(VectorShuffle<E> s, Vector<E> v);
/**
* Compresses the lane elements of this vector selecting lanes
* under the control of a specific mask.
*
* This is a cross-lane operation that compresses the lane
* elements of this vector as selected by the specified mask.
*
* For each lane {@code N} of the mask, if the mask at
* lane {@code N} is set, the element at lane {@code N}
* of input vector is selected and stored into the output
* vector contiguously starting from the lane {@code 0}.
* All the upper remaining lanes, if any, of the output
* vector are set to zero.
*
* @param m the mask controlling the compression
* @return the compressed lane elements of this vector
* @since 19
*/
public abstract Vector<E> compress(VectorMask<E> m);
/**
* Expands the lane elements of this vector
* under the control of a specific mask.
*
* This is a cross-lane operation that expands the contiguous lane
* elements of this vector into lanes of an output vector
* as selected by the specified mask.
*
* For each lane {@code N} of the mask, if the mask at
* lane {@code N} is set, the next contiguous element of input vector
* starting from lane {@code 0} is selected and stored into the output
* vector at lane {@code N}.
* All the remaining lanes, if any, of the output vector are set to zero.
*
* @param m the mask controlling the compression
* @return the expanded lane elements of this vector
* @since 19
*/
public abstract Vector<E> expand(VectorMask<E> m);
/**
* Using index values stored in the lanes of this vector,
* assemble values stored in second vector {@code v}.
@ -2854,9 +2901,8 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
* implementation costs.
*
* <p> The method behaves as if this vector is stored into a byte
* buffer or array using little-endian byte ordering and then the
* desired vector is loaded from the same byte buffer or array
* using the same ordering.
* array using little-endian byte ordering and then the desired vector is loaded from the same byte
* array using the same ordering.
*
* <p> The following pseudocode illustrates the behavior:
* <pre>{@code
@ -2865,15 +2911,15 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
* int M = (domSize > ranSize ? domSize / ranSize : ranSize / domSize);
* assert Math.abs(part) < M;
* assert (part == 0) || (part > 0) == (domSize > ranSize);
* byte[] ra = new byte[Math.max(domSize, ranSize)];
* MemorySegment ms = MemorySegment.ofArray(new byte[Math.max(domSize, ranSize)]);
* if (domSize > ranSize) { // expansion
* this.intoByteArray(ra, 0, ByteOrder.native());
* this.intoMemorySegment(ms, 0, ByteOrder.native());
* int origin = part * ranSize;
* return species.fromByteArray(ra, origin, ByteOrder.native());
* return species.fromMemorySegment(ms, origin, ByteOrder.native());
* } else { // contraction or size-invariant
* int origin = (-part) * domSize;
* this.intoByteArray(ra, origin, ByteOrder.native());
* return species.fromByteArray(ra, 0, ByteOrder.native());
* this.intoMemorySegment(ms, origin, ByteOrder.native());
* return species.fromMemorySegment(ms, 0, ByteOrder.native());
* }
* }</pre>
*
@ -2910,8 +2956,8 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
*
* @return a {@code ByteVector} with the same shape and information content
* @see Vector#reinterpretShape(VectorSpecies,int)
* @see IntVector#intoByteArray(byte[], int, ByteOrder)
* @see FloatVector#intoByteArray(byte[], int, ByteOrder)
* @see IntVector#intoMemorySegment(java.lang.foreign.MemorySegment, long, java.nio.ByteOrder)
* @see FloatVector#intoMemorySegment(java.lang.foreign.MemorySegment, long, java.nio.ByteOrder)
* @see VectorSpecies#withLanes(Class)
*/
public abstract ByteVector reinterpretAsBytes();
@ -3319,8 +3365,8 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
//Array stores
/**
* Stores this vector into a byte array starting at an offset
* using explicit byte order.
* Stores this vector into a {@linkplain MemorySegment memory segment}
* starting at an offset using explicit byte order.
* <p>
* Bytes are extracted from primitive lane elements according
* to the specified byte ordering.
@ -3328,88 +3374,33 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it calls
* {@link #intoByteBuffer(ByteBuffer,int,ByteOrder,VectorMask)
* intoByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* var m = maskAll(true);
* intoByteBuffer(bb, offset, bo, m);
* }</pre>
*
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
*/
public abstract void intoByteArray(byte[] a, int offset,
ByteOrder bo);
/**
* Stores this vector into a byte array starting at an offset
* using explicit byte order and a mask.
* <p>
* Bytes are extracted from primitive lane elements according
* to the specified byte ordering.
* The lanes are stored according to their
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it calls
* {@link #intoByteBuffer(ByteBuffer,int,ByteOrder,VectorMask)
* intoByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* intoByteBuffer(bb, offset, bo, m);
* }</pre>
*
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
* where the mask is set
*/
public abstract void intoByteArray(byte[] a, int offset,
ByteOrder bo,
VectorMask<E> m);
/**
* Stores this vector into a byte buffer starting at an offset
* using explicit byte order.
* <p>
* Bytes are extracted from primitive lane elements according
* to the specified byte ordering.
* The lanes are stored according to their
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it calls
* {@link #intoByteBuffer(ByteBuffer,int,ByteOrder,VectorMask)
* intoByteBuffer()} as follows:
* {@link #intoMemorySegment(MemorySegment,long,ByteOrder,VectorMask)
* intoMemorySegment()} as follows:
* <pre>{@code
* var m = maskAll(true);
* intoByteBuffer(bb, offset, bo, m);
* intoMemorySegment(ms, offset, bo, m);
* }</pre>
*
* @param bb the byte buffer
* @param offset the offset into the array
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > bb.limit()}
* or {@code offset+(N+1)*ESIZE > ms.byteSize()}
* for any lane {@code N} in the vector
* @throws java.nio.ReadOnlyBufferException
* if the byte buffer is read-only
* @throws UnsupportedOperationException
* if the memory segment is read-only
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
public abstract void intoByteBuffer(ByteBuffer bb, int offset, ByteOrder bo);
public abstract void intoMemorySegment(MemorySegment ms, long offset, ByteOrder bo);
/**
* Stores this vector into a byte buffer starting at an offset
* using explicit byte order and a mask.
* Stores this vector into a {@linkplain MemorySegment memory segment}
* starting at an offset using explicit byte order and a mask.
* <p>
* Bytes are extracted from primitive lane elements according
* to the specified byte ordering.
@ -3417,28 +3408,18 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* The following pseudocode illustrates the behavior, where
* the primitive element type is not of {@code byte},
* {@code EBuffer} is the primitive buffer type, {@code ETYPE} is the
* {@code JAVA_E} is the layout of the primitive element type, {@code ETYPE} is the
* primitive element type, and {@code EVector} is the primitive
* vector type for this vector:
* <pre>{@code
* EBuffer eb = bb.duplicate()
* .position(offset)
* .order(bo).asEBuffer();
* ETYPE[] a = this.toArray();
* var slice = ms.asSlice(offset)
* for (int n = 0; n < a.length; n++) {
* if (m.laneIsSet(n)) {
* eb.put(n, a[n]);
* slice.setAtIndex(ValueLayout.JAVA_E.withBitAlignment(8), n);
* }
* }
* }</pre>
* When the primitive element type is of {@code byte} the primitive
* byte buffer is obtained as follows, where operation on the buffer
* remains the same as in the prior pseudocode:
* <pre>{@code
* ByteBuffer eb = bb.duplicate()
* .position(offset);
* }</pre>
*
* @implNote
* This operation is likely to be more efficient if
@ -3451,20 +3432,25 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
* {@code byte}, the byte order argument is
* ignored.
*
* @param bb the byte buffer
* @param offset the offset into the array
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > bb.limit()}
* or {@code offset+(N+1)*ESIZE > ms.byteSize()}
* for any lane {@code N} in the vector
* where the mask is set
* @throws java.nio.ReadOnlyBufferException
* if the byte buffer is read-only
* @throws UnsupportedOperationException
* if the memory segment is read-only
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
public abstract void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo, VectorMask<E> m);
public abstract void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo, VectorMask<E> m);
/**
* Returns a packed array containing all the lane values.

33
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorIntrinsics.java
View file

@ -54,6 +54,16 @@ import java.util.Objects;
}
}
@ForceInline
static long checkFromIndexSize(long ix, long vlen, long length) {
switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
case 0: return ix; // no range check
case 1: return Objects.checkFromIndexSize(ix, vlen, length);
case 2: return Objects.checkIndex(ix, length - (vlen - 1));
default: throw new InternalError();
}
}
@ForceInline
static IntVector checkIndex(IntVector vix, int length) {
switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
@ -92,9 +102,30 @@ import java.util.Objects;
if (index >= 0) {
return index - (index % size);
} else {
return index - Math.floorMod(index, Math.abs(size));
return index - Math.floorMod(index, size);
}
}
// If the index is not already a multiple of size,
// round it down to the next smaller multiple of size.
// It is an error if size is less than zero.
@ForceInline
static long roundDown(long index, int size) {
if ((size & (size - 1)) == 0) {
// Size is zero or a power of two, so we got this.
return index & ~(size - 1);
} else {
return roundDownNPOT(index, size);
}
}
private static long roundDownNPOT(long index, int size) {
if (index >= 0) {
return index - (index % size);
} else {
return index - Math.floorMod(index, size);
}
}
@ForceInline
static int wrapToRange(int index, int size) {
if ((size & (size - 1)) == 0) {

47
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMask.java
View file

@ -210,7 +210,7 @@ public abstract class VectorMask<E> extends jdk.internal.vm.vector.VectorSupport
bits, (long) offset + Unsafe.ARRAY_BOOLEAN_BASE_OFFSET,
bits, offset, vsp,
(c, idx, s)
-> s.opm(n -> c[idx + n]));
-> s.opm(n -> c[((int )idx) + n]));
}
/**
@ -471,6 +471,39 @@ public abstract class VectorMask<E> extends jdk.internal.vm.vector.VectorSupport
*/
public abstract VectorMask<E> indexInRange(int offset, int limit);
/**
* Removes lanes numbered {@code N} from this mask where the
* adjusted index {@code N+offset}, is not in the range
* {@code [0..limit-1]}.
*
* <p> In all cases the series of set and unset lanes is assigned
* as if by using infinite precision or {@code VLENGTH-}saturating
* additions or subtractions, without overflow or wrap-around.
*
* @apiNote
*
* This method performs a SIMD emulation of the check performed by
* {@link Objects#checkIndex(long,long)}, on the index numbers in
* the range {@code [offset..offset+VLENGTH-1]}. If an exception
* is desired, the resulting mask can be compared with the
* original mask; if they are not equal, then at least one lane
* was out of range, and exception processing can be performed.
*
* <p> A mask which is a series of {@code N} set lanes followed by
* a series of unset lanes can be obtained by calling
* {@code allTrue.indexInRange(0, N)}, where {@code allTrue} is a
* mask of all true bits. A mask of {@code N1} unset lanes
* followed by {@code N2} set lanes can be obtained by calling
* {@code allTrue.indexInRange(-N1, N2)}.
*
* @param offset the starting index
* @param limit the upper-bound (exclusive) of index range
* @return the original mask, with out-of-range lanes unset
* @see VectorSpecies#indexInRange(long, long)
* @since 19
*/
public abstract VectorMask<E> indexInRange(long offset, long limit);
/**
* Returns a vector representation of this mask, the
* lane bits of which are set or unset in correspondence
@ -621,6 +654,18 @@ public abstract class VectorMask<E> extends jdk.internal.vm.vector.VectorSupport
return Objects.hash(vectorSpecies(), Arrays.hashCode(toArray()));
}
/**
* Compresses set lanes from this mask.
*
* Returns a mask which is a series of {@code N} set lanes
* followed by a series of unset lanes, where {@code N} is
* the true count of this mask.
*
* @return the compressed mask of this mask
* @since 19
*/
public abstract VectorMask<E> compress();
// ==== JROSE NAME CHANGES ====
// TYPE CHANGED

28
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java
View file

@ -452,6 +452,26 @@ public abstract class VectorOperators {
public static final Unary ABS = unary("ABS", "abs", VectorSupport.VECTOR_OP_ABS, VO_ALL);
/** Produce {@code -a}. */
public static final Unary NEG = unary("NEG", "-a", VectorSupport.VECTOR_OP_NEG, VO_ALL|VO_SPECIAL);
/** Produce {@code bitCount(a)}
* @since 19
*/
public static final Unary BIT_COUNT = unary("BIT_COUNT", "bitCount", VectorSupport.VECTOR_OP_BIT_COUNT, VO_NOFP);
/** Produce {@code numberOfTrailingZeros(a)}
* @since 19
*/
public static final Unary TRAILING_ZEROS_COUNT = unary("TRAILING_ZEROS_COUNT", "numberOfTrailingZeros", VectorSupport.VECTOR_OP_TZ_COUNT, VO_NOFP);
/** Produce {@code numberOfLeadingZeros(a)}
* @since 19
*/
public static final Unary LEADING_ZEROS_COUNT = unary("LEADING_ZEROS_COUNT", "numberOfLeadingZeros", VectorSupport.VECTOR_OP_LZ_COUNT, VO_NOFP);
/** Produce {@code reverse(a)}
* @since 19
*/
public static final Unary REVERSE = unary("REVERSE", "reverse", VectorSupport.VECTOR_OP_REVERSE, VO_NOFP);
/** Produce {@code reverseBytes(a)}
* @since 19
*/
public static final Unary REVERSE_BYTES = unary("REVERSE_BYTES", "reverseBytes", VectorSupport.VECTOR_OP_REVERSE_BYTES, VO_NOFP);
/** Produce {@code sin(a)}. Floating only.
* Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
@ -556,6 +576,14 @@ public abstract class VectorOperators {
public static final /*bitwise*/ Binary ROL = binary("ROL", "rotateLeft", VectorSupport.VECTOR_OP_LROTATE, VO_SHIFT);
/** Produce {@code rotateRight(a,n)}. Integral only. */
public static final /*bitwise*/ Binary ROR = binary("ROR", "rotateRight", VectorSupport.VECTOR_OP_RROTATE, VO_SHIFT);
/** Produce {@code compress(a,n)}. Integral, {@code int} and {@code long}, only.
* @since 19
*/
public static final /*bitwise*/ Binary COMPRESS_BITS = binary("COMPRESS_BITS", "compressBits", VectorSupport.VECTOR_OP_COMPRESS_BITS, VO_NOFP);
/** Produce {@code expand(a,n)}. Integral, {@code int} and {@code long}, only.
* @since 19
*/
public static final /*bitwise*/ Binary EXPAND_BITS = binary("EXPAND_BITS", "expandBits", VectorSupport.VECTOR_OP_EXPAND_BITS, VO_NOFP);
/** Produce {@code atan2(a,b)}. See Floating only.
* Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java
View file

@ -24,6 +24,8 @@
*/
package jdk.incubator.vector;
import java.lang.foreign.MemorySegment;
import java.nio.ByteOrder;
import java.util.function.IntUnaryOperator;
@ -149,11 +151,37 @@ public interface VectorSpecies<E> {
* @return the largest multiple of the vector length not greater
* than the given length
* @throws IllegalArgumentException if the {@code length} is
negative and the result would overflow to a positive value
* negative and the result would overflow to a positive value
* @see Math#floorMod(int, int)
*/
int loopBound(int length);
/**
* Loop control function which returns the largest multiple of
* {@code VLENGTH} that is less than or equal to the given
* {@code length} value.
* Here, {@code VLENGTH} is the result of {@code this.length()},
* and {@code length} is interpreted as a number of lanes.
* The resulting value {@code R} satisfies this inequality:
* <pre>{@code R <= length < R+VLENGTH}
* </pre>
* <p> Specifically, this method computes
* {@code length - floorMod(length, VLENGTH)}, where
* {@link Math#floorMod(long,int) floorMod} computes a remainder
* value by rounding its quotient toward negative infinity.
* As long as {@code VLENGTH} is a power of two, then the result
* is also equal to {@code length & ~(VLENGTH - 1)}.
*
* @param length the input length
* @return the largest multiple of the vector length not greater
* than the given length
* @throws IllegalArgumentException if the {@code length} is
* negative and the result would overflow to a positive value
* @see Math#floorMod(long, int)
* @since 19
*/
long loopBound(long length);
/**
* Returns a mask of this species where only
* the lanes at index N such that the adjusted index
@ -171,6 +199,24 @@ public interface VectorSpecies<E> {
*/
VectorMask<E> indexInRange(int offset, int limit);
/**
* Returns a mask of this species where only
* the lanes at index N such that the adjusted index
* {@code N+offset} is in the range {@code [0..limit-1]}
* are set.
*
* <p>
* This method returns the value of the expression
* {@code maskAll(true).indexInRange(offset, limit)}
*
* @param offset the starting index
* @param limit the upper-bound (exclusive) of index range
* @return a mask with out-of-range lanes unset
* @see VectorMask#indexInRange(long, long)
* @since 19
*/
VectorMask<E> indexInRange(long offset, long limit);
/**
* Checks that this species has the given element type,
* and returns this species unchanged.
@ -433,31 +479,31 @@ public interface VectorSpecies<E> {
// Defined when ETYPE is known.
/**
* Loads a vector of this species from a byte array starting
* at an offset.
* Loads a vector of this species from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* Equivalent to
* {@code IntVector.fromByteArray(this,a,offset,bo)}
* or an equivalent {@code fromByteArray} method,
* {@code IntVector.fromMemorySegment(this,ms,offset,bo)},
* on the vector type corresponding to
* this species.
*
* @param a a byte array
* @param offset the index of the first byte to load
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @return a vector of the given species filled from the byte array
* @return a vector of the given species filled from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
* @see IntVector#fromByteArray(VectorSpecies,byte[],int,ByteOrder)
* @see FloatVector#fromByteArray(VectorSpecies,byte[],int,ByteOrder)
* @see IntVector#fromMemorySegment(VectorSpecies, java.lang.foreign.MemorySegment, long, java.nio.ByteOrder)
* @see FloatVector#fromMemorySegment(VectorSpecies, java.lang.foreign.MemorySegment, long, java.nio.ByteOrder)
* @since 19
*/
Vector<E> fromByteArray(byte[] a, int offset, ByteOrder bo);
Vector<E> fromMemorySegment(MemorySegment ms, long offset, ByteOrder bo);
/**
* Returns a mask of this species

652
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template
View file

@ -24,14 +24,14 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import jdk.internal.foreign.AbstractMemorySegmentImpl;
import jdk.internal.misc.ScopedMemoryAccess;
import jdk.internal.misc.Unsafe;
import jdk.internal.vm.annotation.ForceInline;
@ -61,6 +61,8 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
static final int FORBID_OPCODE_KIND = VO_ONLYFP;
#end[FP]
static final ValueLayout.Of$Type$ ELEMENT_LAYOUT = ValueLayout.JAVA_$TYPE$.withBitAlignment(8);
@ForceInline
static int opCode(Operator op) {
return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
@ -355,6 +357,45 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
return vectorFactory(res);
}
/*package-private*/
interface FLdLongOp {
$type$ apply(MemorySegment memory, long offset, int i);
}
/*package-private*/
@ForceInline
final
$abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
//dummy; no vec = vec();
$type$[] res = new $type$[length()];
for (int i = 0; i < res.length; i++) {
res[i] = f.apply(memory, offset, i);
}
return vectorFactory(res);
}
/*package-private*/
@ForceInline
final
$abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
VectorMask<$Boxtype$> m,
FLdLongOp f) {
//$type$[] vec = vec();
$type$[] res = new $type$[length()];
boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
for (int i = 0; i < res.length; i++) {
if (mbits[i]) {
res[i] = f.apply(memory, offset, i);
}
}
return vectorFactory(res);
}
static $type$ memorySegmentGet(MemorySegment ms, long o, int i) {
return ms.get(ELEMENT_LAYOUT, o + i * $sizeInBytes$L);
}
interface FStOp<M> {
void apply(M memory, int offset, int i, $type$ a);
}
@ -385,6 +426,40 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
}
}
interface FStLongOp {
void apply(MemorySegment memory, long offset, int i, $type$ a);
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
FStLongOp f) {
$type$[] vec = vec();
for (int i = 0; i < vec.length; i++) {
f.apply(memory, offset, i, vec[i]);
}
}
/*package-private*/
@ForceInline
final
void stLongOp(MemorySegment memory, long offset,
VectorMask<$Boxtype$> m,
FStLongOp f) {
$type$[] vec = vec();
boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
for (int i = 0; i < vec.length; i++) {
if (mbits[i]) {
f.apply(memory, offset, i, vec[i]);
}
}
}
static void memorySegmentSet(MemorySegment ms, long o, int i, $type$ e) {
ms.set(ELEMENT_LAYOUT, o + i * $sizeInBytes$L, e);
}
// Binary test
/*package-private*/
@ -445,6 +520,36 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
return {#if[FP]?$Type$.$bitstype$BitsTo$Type$}(($bitstype$)bits);
}
static $abstractvectortype$ expandHelper(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
VectorSpecies<$Boxtype$> vsp = m.vectorSpecies();
$abstractvectortype$ r = ($abstractvectortype$) vsp.zero();
$abstractvectortype$ vi = ($abstractvectortype$) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(i, vi.lane(j++));
}
}
return r;
}
static $abstractvectortype$ compressHelper(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
VectorSpecies<$Boxtype$> vsp = m.vectorSpecies();
$abstractvectortype$ r = ($abstractvectortype$) vsp.zero();
$abstractvectortype$ vi = ($abstractvectortype$) v;
if (m.allTrue()) {
return vi;
}
for (int i = 0, j = 0; i < vsp.length(); i++) {
if (m.laneIsSet(i)) {
r = r.withLane(j++, vi.lane(i));
}
}
return r;
}
// Static factories (other than memory operations)
// Note: A surprising behavior in javadoc
@ -646,6 +751,36 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
v0.uOp(m, (i, a) -> ($type$) -a);
case VECTOR_OP_ABS: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) Math.abs(a));
#if[!FP]
#if[intOrLong]
case VECTOR_OP_BIT_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) $Boxtype$.bitCount(a));
case VECTOR_OP_TZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) $Boxtype$.numberOfTrailingZeros(a));
case VECTOR_OP_LZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) $Boxtype$.numberOfLeadingZeros(a));
case VECTOR_OP_REVERSE: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) $Boxtype$.reverse(a));
#else[intOrLong]
case VECTOR_OP_BIT_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) bitCount(a));
case VECTOR_OP_TZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) numberOfTrailingZeros(a));
case VECTOR_OP_LZ_COUNT: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) numberOfLeadingZeros(a));
case VECTOR_OP_REVERSE: return (v0, m) ->
v0.uOp(m, (i, a) -> reverse(a));
#end[intOrLong]
#if[BITWISE]
#if[byte]
case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
v0.uOp(m, (i, a) -> a);
#else[byte]
case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) $Boxtype$.reverseBytes(a));
#end[byte]
#end[BITWISE]
#end[!FP]
#if[FP]
case VECTOR_OP_SIN: return (v0, m) ->
v0.uOp(m, (i, a) -> ($type$) Math.sin(a));
@ -839,6 +974,12 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
#if[intOrLong]
case VECTOR_OP_COMPRESS_BITS: return (v0, v1, vm) ->
v0.bOp(v1, vm, (i, a, n) -> $Boxtype$.compress(a, n));
case VECTOR_OP_EXPAND_BITS: return (v0, v1, vm) ->
v0.bOp(v1, vm, (i, a, n) -> $Boxtype$.expand(a, n));
#end[intOrLong]
#end[BITWISE]
#if[FP]
case VECTOR_OP_OR: return (v0, v1, vm) ->
@ -1987,6 +2128,56 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
return lanewise(ABS);
}
#if[!FP]
#if[!intOrLong]
static int bitCount($type$ a) {
#if[short]
return Integer.bitCount((int)a & 0xFFFF);
#else[short]
return Integer.bitCount((int)a & 0xFF);
#end[short]
}
#end[!intOrLong]
#end[!FP]
#if[!FP]
#if[!intOrLong]
static int numberOfTrailingZeros($type$ a) {
#if[short]
return a != 0 ? Integer.numberOfTrailingZeros(a) : 16;
#else[short]
return a != 0 ? Integer.numberOfTrailingZeros(a) : 8;
#end[short]
}
#end[!intOrLong]
#end[!FP]
#if[!FP]
#if[!intOrLong]
static int numberOfLeadingZeros($type$ a) {
#if[short]
return a >= 0 ? Integer.numberOfLeadingZeros(a) - 16 : 0;
#else[short]
return a >= 0 ? Integer.numberOfLeadingZeros(a) - 24 : 0;
#end[short]
}
static $type$ reverse($type$ a) {
if (a == 0 || a == -1) return a;
#if[short]
$type$ b = rotateLeft(a, 8);
b = ($type$) (((b & 0x5555) << 1) | ((b & 0xAAAA) >>> 1));
b = ($type$) (((b & 0x3333) << 2) | ((b & 0xCCCC) >>> 2));
b = ($type$) (((b & 0x0F0F) << 4) | ((b & 0xF0F0) >>> 4));
#else[short]
$type$ b = rotateLeft(a, 4);
b = ($type$) (((b & 0x55) << 1) | ((b & 0xAA) >>> 1));
b = ($type$) (((b & 0x33) << 2) | ((b & 0xCC) >>> 2));
#end[short]
return b;
}
#end[!intOrLong]
#end[!FP]
#if[BITWISE]
// not (~)
/**
@ -2695,6 +2886,45 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Vector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
$Type$Vector compress(VectorMask<$Boxtype$> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<$Boxtype$>>
$Type$Vector compressTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return ($Type$Vector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_COMPRESS, getClass(), masktype,
$type$.class, length(), this, m,
(v1, m1) -> compressHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
public abstract
$Type$Vector expand(VectorMask<$Boxtype$> m);
/*package-private*/
@ForceInline
final
<M extends AbstractMask<$Boxtype$>>
$Type$Vector expandTemplate(Class<M> masktype, M m) {
m.check(masktype, this);
return ($Type$Vector) VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_EXPAND, getClass(), masktype,
$type$.class, length(), this, m,
(v1, m1) -> expandHelper(v1, m1));
}
/**
* {@inheritDoc} <!--workaround-->
*/
@ -3302,90 +3532,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
}
#end[double]
/**
* Loads a vector from a byte array starting at an offset.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
*/
@ForceInline
public static
$abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
$Type$Species vsp = ($Type$Species) species;
return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
}
/**
* Loads a vector from a byte array starting at an offset
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code $type$} ({#if[FP]?positive }zero).
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* <pre>{@code
* var bb = ByteBuffer.wrap(a);
* return fromByteBuffer(species, bb, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param a the byte array
* @param offset the offset into the array
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte array
* @throws IndexOutOfBoundsException
* if {@code offset+N*ESIZE < 0}
* or {@code offset+(N+1)*ESIZE > a.length}
* for any lane {@code N} in the vector
* where the mask is set
*/
@ForceInline
public static
$abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
byte[] a, int offset,
ByteOrder bo,
VectorMask<$Boxtype$> m) {
$Type$Species vsp = ($Type$Species) species;
if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
ByteBuffer wb = wrapper(a, bo);
return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
(wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
}
/**
* Loads a vector from an array of type {@code $type$[]}
* starting at an offset.
@ -3917,44 +4063,49 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
#end[byte]
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer.
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment.
* Bytes are composed into primitive lane elements according
* to the specified byte order.
* The vector is arranged into lanes according to
* <a href="Vector.html#lane-order">memory ordering</a>.
* <p>
* This method behaves as if it returns the result of calling
* {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
* fromByteBuffer()} as follows:
* {@link #fromMemorySegment(VectorSpecies,MemorySegment,long,ByteOrder,VectorMask)
* fromMemorySegment()} as follows:
* <pre>{@code
* var m = species.maskAll(true);
* return fromByteBuffer(species, bb, offset, bo, m);
* return fromMemorySegment(species, ms, offset, bo, m);
* }</pre>
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*$sizeInBytes$ < 0}
* or {@code offset+N*$sizeInBytes$ >= bb.limit()}
* or {@code offset+N*$sizeInBytes$ >= ms.byteSize()}
* for any lane {@code N} in the vector
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
$abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
ByteBuffer bb, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
$abstractvectortype$ fromMemorySegment(VectorSpecies<$Boxtype$> species,
MemorySegment ms, long offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, species.vectorByteSize(), ms.byteSize());
$Type$Species vsp = ($Type$Species) species;
return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset).maybeSwap(bo);
}
/**
* Loads a vector from a {@linkplain ByteBuffer byte buffer}
* starting at an offset into the byte buffer
* Loads a vector from a {@linkplain MemorySegment memory segment}
* starting at an offset into the memory segment
* and using a mask.
* Lanes where the mask is unset are filled with the default
* value of {@code $type$} ({#if[FP]?positive }zero).
@ -3965,15 +4116,11 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
* <p>
* The following pseudocode illustrates the behavior:
* <pre>{@code
* $Type$Buffer eb = bb.duplicate()
* .position(offset){#if[byte]?;}
#if[!byte]
* .order(bo).as$Type$Buffer();
#end[!byte]
* var slice = ms.asSlice(offset);
* $type$[] ar = new $type$[species.length()];
* for (int n = 0; n < ar.length; n++) {
* if (m.laneIsSet(n)) {
* ar[n] = eb.get(n);
* ar[n] = slice.getAtIndex(ValuaLayout.JAVA_$TYPE$.withBitAlignment(8), n);
* }
* }
* $abstractvectortype$ r = $abstractvectortype$.fromArray(species, ar, 0);
@ -3991,33 +4138,36 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
#end[!byte]
*
* @param species species of desired vector
* @param bb the byte buffer
* @param offset the offset into the byte buffer
* @param ms the memory segment
* @param offset the offset into the memory segment
* @param bo the intended byte order
* @param m the mask controlling lane selection
* @return a vector loaded from a byte buffer
* @return a vector loaded from the memory segment
* @throws IndexOutOfBoundsException
* if {@code offset+N*$sizeInBytes$ < 0}
* or {@code offset+N*$sizeInBytes$ >= bb.limit()}
* or {@code offset+N*$sizeInBytes$ >= ms.byteSize()}
* for any lane {@code N} in the vector
* where the mask is set
* @throws IllegalArgumentException if the memory segment is a heap segment that is
* not backed by a {@code byte[]} array.
* @throws IllegalStateException if the memory segment's session is not alive,
* or if access occurs from a thread other than the thread owning the session.
* @since 19
*/
@ForceInline
public static
$abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<$Boxtype$> m) {
$abstractvectortype$ fromMemorySegment(VectorSpecies<$Boxtype$> species,
MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<$Boxtype$> m) {
$Type$Species vsp = ($Type$Species) species;
if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
if (offset >= 0 && offset <= (ms.byteSize() - species.vectorByteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
ByteBuffer wb = wrapper(bb, bo);
return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
(wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, $abstractvectortype$::memorySegmentGet);
}
// Memory store operations
@ -4047,7 +4197,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
this,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -4264,7 +4414,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
this,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
}
@ -4423,7 +4573,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
normalized,
a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
}
@ -4562,67 +4712,40 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo) {
offset = checkFromIndexSize(offset, byteSize(), a.length);
maybeSwap(bo).intoByteArray0(a, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteArray(byte[] a, int offset,
ByteOrder bo,
VectorMask<$Boxtype$> m) {
if (m.allTrue()) {
intoByteArray(a, offset, bo);
} else {
$Type$Species vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
maybeSwap(bo).intoByteArray0(a, offset, m);
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo) {
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
offset = checkFromIndexSize(offset, byteSize(), ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo) {
if (ScopedMemoryAccess.isReadOnly(bb)) {
throw new ReadOnlyBufferException();
}
offset = checkFromIndexSize(offset, byteSize(), bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset);
}
/**
* {@inheritDoc} <!--workaround-->
*/
@Override
@ForceInline
public final
void intoByteBuffer(ByteBuffer bb, int offset,
ByteOrder bo,
VectorMask<$Boxtype$> m) {
void intoMemorySegment(MemorySegment ms, long offset,
ByteOrder bo,
VectorMask<$Boxtype$> m) {
if (m.allTrue()) {
intoByteBuffer(bb, offset, bo);
intoMemorySegment(ms, offset, bo);
} else {
if (bb.isReadOnly()) {
throw new ReadOnlyBufferException();
if (ms.isReadOnly()) {
throw new UnsupportedOperationException("Attempt to write a read-only segment");
}
$Type$Species vsp = vspecies();
checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
maybeSwap(bo).intoByteBuffer0(bb, offset, m);
checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, ms.byteSize());
maybeSwap(bo).intoMemorySegment0(ms, offset, m);
}
}
@ -4656,7 +4779,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -4673,7 +4796,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
}
@ -4750,7 +4873,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> (short) arr_[off_ + i]));
}
@ -4767,7 +4890,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (short) arr_[off_ + i]));
}
#end[short]
@ -4784,7 +4907,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, off,
(arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
}
@ -4801,79 +4924,38 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, off, vm,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
}
#end[byte]
@Override
abstract
$abstractvectortype$ fromByteArray0(byte[] a, int offset);
$abstractvectortype$ fromMemorySegment0(MemorySegment bb, long offset);
@ForceInline
final
$abstractvectortype$ fromByteArray0Template(byte[] a, int offset) {
$abstractvectortype$ fromMemorySegment0Template(MemorySegment ms, long offset) {
$Type$Species vsp = vspecies();
return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
a, offset, vsp,
(arr, off, s) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
});
}
abstract
$abstractvectortype$ fromByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
@ForceInline
final
<M extends VectorMask<$Boxtype$>>
$abstractvectortype$ fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
$Type$Species vsp = vspecies();
m.check(vsp);
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset), m,
a, offset, vsp,
(arr, off, s, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
});
}
abstract
$abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset);
@ForceInline
final
$abstractvectortype$ fromByteBuffer0Template(ByteBuffer bb, int offset) {
$Type$Species vsp = vspecies();
return ScopedMemoryAccess.loadFromByteBuffer(
return ScopedMemoryAccess.loadFromMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
bb, offset, vsp,
(buf, off, s) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off,
(wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
(AbstractMemorySegmentImpl) ms, offset, vsp,
(msp, off, s) -> {
return s.ldLongOp((MemorySegment) msp, off, $abstractvectortype$::memorySegmentGet);
});
}
abstract
$abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
$abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m);
@ForceInline
final
<M extends VectorMask<$Boxtype$>>
$abstractvectortype$ fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
$abstractvectortype$ fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
$Type$Species vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromByteBufferMasked(
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
bb, offset, m, vsp,
(buf, off, s, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
return s.ldOp(wb, off, vm,
(wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, $abstractvectortype$::memorySegmentGet);
});
}
@ -4892,7 +4974,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
a, arrayAddress(a, offset),
this, a, offset,
(arr, off, v)
-> v.stOp(arr, off,
-> v.stOp(arr, (int) off,
(arr_, off_, i, e) -> arr_[off_+i] = e));
}
@ -4909,7 +4991,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
a, arrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = e));
}
@ -4990,76 +5072,38 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
a, booleanArrayAddress(a, offset),
normalized, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
}
#end[byte]
abstract
void intoByteArray0(byte[] a, int offset);
@ForceInline
final
void intoByteArray0Template(byte[] a, int offset) {
void intoMemorySegment0(MemorySegment ms, long offset) {
$Type$Species vsp = vspecies();
VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, a, offset,
(arr, off, v) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off,
(tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
});
}
abstract
void intoByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
@ForceInline
final
<M extends VectorMask<$Boxtype$>>
void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
$Type$Species vsp = vspecies();
m.check(vsp);
VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, byteArrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm) -> {
ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
});
}
@ForceInline
final
void intoByteBuffer0(ByteBuffer bb, int offset) {
$Type$Species vsp = vspecies();
ScopedMemoryAccess.storeIntoByteBuffer(
ScopedMemoryAccess.storeIntoMemorySegment(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
this, bb, offset,
(buf, off, v) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off,
(wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
this,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v) -> {
v.stLongOp((MemorySegment) msp, off, $abstractvectortype$::memorySegmentSet);
});
}
abstract
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
void intoMemorySegment0(MemorySegment bb, long offset, VectorMask<$Boxtype$> m);
@ForceInline
final
<M extends VectorMask<$Boxtype$>>
void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
void intoMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
$Type$Species vsp = vspecies();
m.check(vsp);
ScopedMemoryAccess.storeIntoByteBufferMasked(
ScopedMemoryAccess.storeIntoMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
this, m, bb, offset,
(buf, off, v, vm) -> {
ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
v.stOp(wb, off, vm,
(wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
this, m,
(AbstractMemorySegmentImpl) ms, offset,
(msp, off, v, vm) -> {
v.stLongOp((MemorySegment) msp, off, vm, $abstractvectortype$::memorySegmentSet);
});
}
@ -5078,7 +5122,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
a, charArrayAddress(a, offset),
this, m, a, offset,
(arr, off, v, vm)
-> v.stOp(arr, off, vm,
-> v.stOp(arr, (int) off, vm,
(arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
}
#end[short]
@ -5095,6 +5139,16 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
private static
void checkMaskFromIndexSize(long offset,
$Type$Species vsp,
VectorMask<$Boxtype$> m,
int scale,
long limit) {
((AbstractMask<$Boxtype$>)m)
.checkIndexByLane(offset, limit, vsp.iota(), scale);
}
@ForceInline
private void conditionalStoreNYI(int offset,
$Type$Species vsp,
@ -5463,6 +5517,21 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
return dummyVector().ldOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
$abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
$abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
VectorMask<$Boxtype$> m,
FLdLongOp f) {
return dummyVector().ldLongOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
<M> void stOp(M memory, int offset, FStOp<M> f) {
@ -5477,6 +5546,20 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
dummyVector().stOp(memory, offset, m, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset, FStLongOp f) {
dummyVector().stLongOp(memory, offset, f);
}
/*package-private*/
@ForceInline
void stLongOp(MemorySegment memory, long offset,
AbstractMask<$Boxtype$> m,
FStLongOp f) {
dummyVector().stLongOp(memory, offset, m, f);
}
// N.B. Make sure these constant vectors and
// masks load up correctly into registers.
//
@ -5590,3 +5673,4 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
public static final VectorSpecies<$Boxtype$> SPECIES_PREFERRED
= ($Type$Species) VectorSpecies.ofPreferred($type$.class);
}

68
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template
View file

@ -24,7 +24,7 @@
*/
package jdk.incubator.vector;
import java.nio.ByteBuffer;
import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;
@ -480,6 +480,22 @@ final class $vectortype$ extends $abstractvectortype$ {
($vectortype$) v); // specialize
}
@Override
@ForceInline
public $vectortype$ compress(VectorMask<$Boxtype$> m) {
return ($vectortype$)
super.compressTemplate($masktype$.class,
($masktype$) m); // specialize
}
@Override
@ForceInline
public $vectortype$ expand(VectorMask<$Boxtype$> m) {
return ($vectortype$)
super.expandTemplate($masktype$.class,
($masktype$) m); // specialize
}
@Override
@ForceInline
public $vectortype$ selectFrom(Vector<$Boxtype$> v) {
@ -920,6 +936,15 @@ final class $vectortype$ extends $abstractvectortype$ {
return xor(maskAll(true));
}
@Override
@ForceInline
public $masktype$ compress() {
return ($masktype$)VectorSupport.compressExpandOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
$vectortype$.class, $masktype$.class, ETYPE, VLENGTH, null, this,
(v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
}
// Binary operations
@Override
@ -1159,29 +1184,15 @@ final class $vectortype$ extends $abstractvectortype$ {
@ForceInline
@Override
final
$abstractvectortype$ fromByteArray0(byte[] a, int offset) {
return super.fromByteArray0Template(a, offset); // specialize
$abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset) {
return super.fromMemorySegment0Template(ms, offset); // specialize
}
@ForceInline
@Override
final
$abstractvectortype$ fromByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m) {
return super.fromByteArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
}
@ForceInline
@Override
final
$abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset) {
return super.fromByteBuffer0Template(bb, offset); // specialize
}
@ForceInline
@Override
final
$abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m) {
return super.fromByteBuffer0Template($masktype$.class, bb, offset, ($masktype$) m); // specialize
$abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m) {
return super.fromMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m); // specialize
}
@ForceInline
@ -1219,22 +1230,8 @@ final class $vectortype$ extends $abstractvectortype$ {
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset) {
super.intoByteArray0Template(a, offset); // specialize
}
@ForceInline
@Override
final
void intoByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m) {
super.intoByteArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
}
@ForceInline
@Override
final
void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m) {
super.intoByteBuffer0Template($masktype$.class, bb, offset, ($masktype$) m);
void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m) {
super.intoMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m);
}
#if[short]
@ -1251,3 +1248,4 @@ final class $vectortype$ extends $abstractvectortype$ {
// ================================================
}