archive/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2025-08-28 07:14:30 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

8319111: Mismatched MemorySegment heap access is not consistently intrinsified

Browse source 
Reviewed-by: sviswanathan, psandoz, thartmann
This commit is contained in:
Jatin Bhateja 2023-12-06 09:58:06 +00:00
parent a0920aa436
commit 2678e4cd94
14 changed files with 205 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/hotspot/share/classfile/vmIntrinsics.hpp
View file

@ -997,6 +997,7 @@ class methodHandle;
"I" \ "I" \
"Ljava/lang/Object;" \ "Ljava/lang/Object;" \
"J" \ "J" \
"Z" \
"Ljava/lang/Object;" \ "Ljava/lang/Object;" \
"J" \ "J" \
"Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \ "Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \
@ -1011,6 +1012,7 @@ class methodHandle;
"I" \ "I" \
"Ljava/lang/Object;" \ "Ljava/lang/Object;" \
"J" \ "J" \
"Z" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \ "Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
"I" \ "I" \
"Ljava/lang/Object;" \ "Ljava/lang/Object;" \
@ -1026,6 +1028,7 @@ class methodHandle;
"I" \ "I" \
"Ljava/lang/Object;" \ "Ljava/lang/Object;" \
"J" \ "J" \
"Z" \
"Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \ "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \
"Ljava/lang/Object;" \ "Ljava/lang/Object;" \
"J" \ "J" \
@ -1040,6 +1043,7 @@ class methodHandle;
"I" \ "I" \
"Ljava/lang/Object;" \ "Ljava/lang/Object;" \
"J" \ "J" \
"Z" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \ "Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \ "Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
"Ljava/lang/Object;" \ "Ljava/lang/Object;" \

155
src/hotspot/share/opto/vectorIntrinsics.cpp
View file

@ -928,7 +928,7 @@ bool LibraryCallKit::inline_vector_frombits_coerced() {
return true; return true;
} }
static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) { static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
assert(arr_type != nullptr, "unexpected"); assert(arr_type != nullptr, "unexpected");
BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type(); BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
if (elem_bt == arr_elem_bt) { if (elem_bt == arr_elem_bt) {
@ -940,41 +940,44 @@ static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_ty
// Load/store of byte vector from/to boolean[] is supported // Load/store of byte vector from/to boolean[] is supported
return true; return true;
} else { } else {
return false; return mismatched_ms;
} }
} }
// public static // public static
// <C, // <C,
// VM, // VM extends VectorPayload,
// E, // E,
// S extends VectorSpecies<E>> // S extends VectorSpecies<E>>
// VM load(Class<? extends VM> vmClass, Class<E> elementType, int length, // VM load(Class<? extends VM> vmClass, Class<E> eClass,
// int length,
// Object base, long offset, // Unsafe addressing // Object base, long offset, // Unsafe addressing
// boolean fromSegment,
// C container, long index, S s, // Arguments for default implementation // C container, long index, S s, // Arguments for default implementation
// LoadOperation<C, VM, E, S> defaultImpl) // LoadOperation<C, VM, S> defaultImpl) {
//
// public static // public static
// <C, // <C,
// V extends Vector<?>> // V extends VectorPayload>
// void store(Class<?> vectorClass, Class<?> elementType, int length, // void store(Class<?> vClass, Class<?> eClass,
// int length,
// Object base, long offset, // Unsafe addressing // Object base, long offset, // Unsafe addressing
// V v, // boolean fromSegment,
// C container, long index, // Arguments for default implementation // V v, C container, long index, // Arguments for default implementation
// StoreVectorOperation<C, V> defaultImpl) // StoreVectorOperation<C, V> defaultImpl) {
bool LibraryCallKit::inline_vector_mem_operation(bool is_store) { bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr(); const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr(); const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(2))->isa_int(); const TypeInt* vlen = gvn().type(argument(2))->isa_int();
const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) { vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s", tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
NodeClassNames[argument(0)->Opcode()], NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()], NodeClassNames[argument(1)->Opcode()],
NodeClassNames[argument(2)->Opcode()]); NodeClassNames[argument(2)->Opcode()],
NodeClassNames[argument(6)->Opcode()]);
} }
return false; // not enough info for intrinsification return false; // not enough info for intrinsification
} }
@ -1033,11 +1036,10 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
const bool needs_cpu_membar = is_mixed_access || is_mismatched_access; const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
// Now handle special case where load/store happens from/to byte array but element type is not byte. bool mismatched_ms = from_ms->get_con() && !is_mask && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
bool using_byte_array = arr_type != nullptr && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE; BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
// Handle loading masks. int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
// If there is no consistency between array and vector element types, it must be special byte array case or loading masks if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
if (arr_type != nullptr && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no", tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
is_store, is_store ? "store" : "load", is_store, is_store ? "store" : "load",
@ -1047,20 +1049,34 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
set_sp(old_sp); set_sp(old_sp);
return false; return false;
} }
// Since we are using byte array, we need to double check that the byte operations are supported by backend.
if (using_byte_array) { // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
int byte_num_elem = num_elem * type2aelembytes(elem_bt); if (mismatched_ms) {
if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed) if (is_store) {
|| !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) { if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
|| !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no", tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
is_store, is_store ? "store" : "load", is_store, "store",
byte_num_elem, type2name(elem_bt)); num_elem, type2name(elem_bt));
} }
set_map(old_map); set_map(old_map);
set_sp(old_sp); set_sp(old_sp);
return false; // not supported return false; // not supported
} }
} else {
if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
|| !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
is_store, "load",
mem_num_elem, type2name(mem_elem_bt));
}
set_map(old_map);
set_sp(old_sp);
return false; // not supported
}
}
} }
if (is_mask) { if (is_mask) {
if (!is_store) { if (!is_store) {
@ -1085,7 +1101,7 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
} }
if (is_store) { if (is_store) {
Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
if (val == nullptr) { if (val == nullptr) {
set_map(old_map); set_map(old_map);
set_sp(old_sp); set_sp(old_sp);
@ -1095,9 +1111,9 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
// In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector. // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
int store_num_elem = num_elem; int store_num_elem = num_elem;
if (using_byte_array) { if (mismatched_ms) {
store_num_elem = num_elem * type2aelembytes(elem_bt); store_num_elem = mem_num_elem;
const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem); const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type)); val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
} }
if (is_mask) { if (is_mask) {
@ -1108,9 +1124,8 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
} else { } else {
// When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load. // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
Node* vload = nullptr; Node* vload = nullptr;
if (using_byte_array) { if (mismatched_ms) {
int load_num_elem = num_elem * type2aelembytes(elem_bt); vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem); const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type)); vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
} else { } else {
@ -1142,37 +1157,41 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
// E, // E,
// S extends VectorSpecies<E>, // S extends VectorSpecies<E>,
// M extends VectorMask<E>> // M extends VectorMask<E>>
// V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
// int length, Object base, long offset, M m, int offsetInRange, // int length, Object base, long offset, // Unsafe addressing
// boolean fromSegment,
// M m, int offsetInRange,
// C container, long index, S s, // Arguments for default implementation // C container, long index, S s, // Arguments for default implementation
// LoadVectorMaskedOperation<C, V, S, M> defaultImpl) { // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
//
// public static // public static
// <C, // <C,
// V extends Vector<E>, // V extends Vector<E>,
// M extends VectorMask<E>, // M extends VectorMask<E>,
// E> // E>
// void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
// int length, Object base, long offset, // int length,
// V v, M m, // Object base, long offset, // Unsafe addressing
// C container, long index, // Arguments for default implementation // boolean fromSegment,
// StoreVectorMaskedOperation<C, V, M, E> defaultImpl) { // V v, M m, C container, long index, // Arguments for default implementation
// // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) { bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr(); const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr(); const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr(); const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(3))->isa_int(); const TypeInt* vlen = gvn().type(argument(3))->isa_int();
const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr || if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
elem_klass->const_oop() == nullptr || !vlen->is_con()) { elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s", tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
NodeClassNames[argument(0)->Opcode()], NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()], NodeClassNames[argument(1)->Opcode()],
NodeClassNames[argument(2)->Opcode()], NodeClassNames[argument(2)->Opcode()],
NodeClassNames[argument(3)->Opcode()]); NodeClassNames[argument(3)->Opcode()],
NodeClassNames[argument(7)->Opcode()]);
} }
return false; // not enough info for intrinsification return false; // not enough info for intrinsification
} }
@ -1212,10 +1231,10 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
const TypePtr *addr_type = gvn().type(addr)->isa_ptr(); const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
const TypeAryPtr* arr_type = addr_type->isa_aryptr(); const TypeAryPtr* arr_type = addr_type->isa_aryptr();
// Now handle special case where load/store happens from/to byte array but element type is not byte. bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
bool using_byte_array = arr_type != nullptr && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE; BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
// If there is no consistency between array and vector element types, it must be special byte array case // If there is no consistency between array and vector element types, it must be special byte array case
if (arr_type != nullptr && !using_byte_array && !elem_consistent_with_arr(elem_bt, arr_type)) { if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s", tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
is_store, is_store ? "storeMasked" : "loadMasked", is_store, is_store ? "storeMasked" : "loadMasked",
@ -1226,8 +1245,8 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
return false; return false;
} }
int mem_num_elem = using_byte_array ? num_elem * type2aelembytes(elem_bt) : num_elem; int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
BasicType mem_elem_bt = using_byte_array ? T_BYTE : elem_bt; BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked, bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
mem_num_elem, mem_elem_bt, VecMaskUseLoad); mem_num_elem, mem_elem_bt, VecMaskUseLoad);
@ -1240,7 +1259,7 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
needs_predicate = true; needs_predicate = true;
} else { } else {
// Masked vector load with IOOBE always uses the predicated load. // Masked vector load with IOOBE always uses the predicated load.
const TypeInt* offset_in_range = gvn().type(argument(8))->isa_int(); const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
if (!offset_in_range->is_con()) { if (!offset_in_range->is_con()) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** missing constant: offsetInRange=%s", tty->print_cr(" ** missing constant: offsetInRange=%s",
@ -1255,9 +1274,9 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
if (needs_predicate) { if (needs_predicate) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: op=%s vlen=%d etype=%s using_byte_array=%d", tty->print_cr(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
is_store ? "storeMasked" : "loadMasked", is_store ? "storeMasked" : "loadMasked",
num_elem, type2name(elem_bt), using_byte_array ? 1 : 0); num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
} }
set_map(old_map); set_map(old_map);
set_sp(old_sp); set_sp(old_sp);
@ -1270,8 +1289,8 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) || if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) { !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: op=loadMasked vlen=%d etype=%s using_byte_array=%d", tty->print_cr(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
num_elem, type2name(elem_bt), using_byte_array ? 1 : 0); num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
} }
set_map(old_map); set_map(old_map);
set_sp(old_sp); set_sp(old_sp);
@ -1280,10 +1299,10 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
// Since we are using byte array, we need to double check that the vector reinterpret operation // Since we are using byte array, we need to double check that the vector reinterpret operation
// with byte type is supported by backend. // with byte type is supported by backend.
if (using_byte_array) { if (mismatched_ms) {
if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) { if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s using_byte_array=1", tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
is_store, is_store ? "storeMasked" : "loadMasked", is_store, is_store ? "storeMasked" : "loadMasked",
num_elem, type2name(elem_bt)); num_elem, type2name(elem_bt));
} }
@ -1318,12 +1337,12 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass); const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
Node* mask = unbox_vector(is_store ? argument(8) : argument(7), mbox_type, elem_bt, num_elem); Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
if (mask == nullptr) { if (mask == nullptr) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** unbox failed mask=%s", tty->print_cr(" ** unbox failed mask=%s",
is_store ? NodeClassNames[argument(8)->Opcode()] is_store ? NodeClassNames[argument(9)->Opcode()]
: NodeClassNames[argument(7)->Opcode()]); : NodeClassNames[argument(8)->Opcode()]);
} }
set_map(old_map); set_map(old_map);
set_sp(old_sp); set_sp(old_sp);
@ -1331,11 +1350,11 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
} }
if (is_store) { if (is_store) {
Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem); Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
if (val == nullptr) { if (val == nullptr) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** unbox failed vector=%s", tty->print_cr(" ** unbox failed vector=%s",
NodeClassNames[argument(7)->Opcode()]); NodeClassNames[argument(8)->Opcode()]);
} }
set_map(old_map); set_map(old_map);
set_sp(old_sp); set_sp(old_sp);
@ -1343,7 +1362,7 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
} }
set_all_memory(reset_memory()); set_all_memory(reset_memory());
if (using_byte_array) { if (mismatched_ms) {
// Reinterpret the incoming vector to byte vector. // Reinterpret the incoming vector to byte vector.
const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem); const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type)); val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
@ -1357,7 +1376,7 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
} else { } else {
Node* vload = nullptr; Node* vload = nullptr;
if (using_byte_array) { if (mismatched_ms) {
// Reinterpret the vector mask to byte type. // Reinterpret the vector mask to byte type.
const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem); const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem); const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
@ -1376,7 +1395,7 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
vload = gvn().transform(new VectorBlendNode(zero, vload, mask)); vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
} }
if (using_byte_array) { if (mismatched_ms) {
const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem); const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type)); vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
} }
@ -1525,7 +1544,7 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
const TypeAryPtr* arr_type = addr_type->isa_aryptr(); const TypeAryPtr* arr_type = addr_type->isa_aryptr();
// The array must be consistent with vector type // The array must be consistent with vector type
if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type))) { if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
if (C->print_intrinsics()) { if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no", tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
is_scatter, is_scatter ? "scatter" : "gather", is_scatter, is_scatter ? "scatter" : "gather",

8
src/java.base/share/classes/jdk/internal/misc/X-ScopedMemoryAccess.java.template
View file

@ -355,7 +355,7 @@ public class ScopedMemoryAccess {
session.checkValidStateRaw(); session.checkValidStateRaw();
return VectorSupport.load(vmClass, e, length, return VectorSupport.load(vmClass, e, length,
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, true,
msp, offset, s, msp, offset, s,
defaultImpl); defaultImpl);
} finally { } finally {
@ -397,7 +397,7 @@ public class ScopedMemoryAccess {
session.checkValidStateRaw(); session.checkValidStateRaw();
return VectorSupport.loadMasked(vmClass, maskClass, e, length, return VectorSupport.loadMasked(vmClass, maskClass, e, length,
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, m, offsetInRange, msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, true, m, offsetInRange,
msp, offset, s, msp, offset, s,
defaultImpl); defaultImpl);
} finally { } finally {
@ -438,7 +438,7 @@ public class ScopedMemoryAccess {
session.checkValidStateRaw(); session.checkValidStateRaw();
VectorSupport.store(vmClass, e, length, VectorSupport.store(vmClass, e, length,
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, true,
v, v,
msp, offset, msp, offset,
defaultImpl); defaultImpl);
@ -480,7 +480,7 @@ public class ScopedMemoryAccess {
session.checkValidStateRaw(); session.checkValidStateRaw();
VectorSupport.storeMasked(vmClass, maskClass, e, length, VectorSupport.storeMasked(vmClass, maskClass, e, length,
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, true,
v, m, v, m,
msp, offset, msp, offset,
defaultImpl); defaultImpl);

8
src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java
View file

@ -421,7 +421,7 @@ public class VectorSupport {
S extends VectorSpecies<E>> S extends VectorSpecies<E>>
VM load(Class<? extends VM> vmClass, Class<E> eClass, VM load(Class<? extends VM> vmClass, Class<E> eClass,
int length, int length,
Object base, long offset, Object base, long offset, boolean fromSegment,
C container, long index, S s, C container, long index, S s,
LoadOperation<C, VM, S> defaultImpl) { LoadOperation<C, VM, S> defaultImpl) {
assert isNonCapturingLambda(defaultImpl) : defaultImpl; assert isNonCapturingLambda(defaultImpl) : defaultImpl;
@ -445,7 +445,7 @@ public class VectorSupport {
S extends VectorSpecies<E>, S extends VectorSpecies<E>,
M extends VectorMask<E>> M extends VectorMask<E>>
V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
int length, Object base, long offset, int length, Object base, long offset, boolean fromSegment,
M m, int offsetInRange, M m, int offsetInRange,
C container, long index, S s, C container, long index, S s,
LoadVectorMaskedOperation<C, V, S, M> defaultImpl) { LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
@ -494,7 +494,7 @@ public class VectorSupport {
V extends VectorPayload> V extends VectorPayload>
void store(Class<?> vClass, Class<?> eClass, void store(Class<?> vClass, Class<?> eClass,
int length, int length,
Object base, long offset, Object base, long offset, boolean fromSegment,
V v, C container, long index, V v, C container, long index,
StoreVectorOperation<C, V> defaultImpl) { StoreVectorOperation<C, V> defaultImpl) {
assert isNonCapturingLambda(defaultImpl) : defaultImpl; assert isNonCapturingLambda(defaultImpl) : defaultImpl;
@ -515,7 +515,7 @@ public class VectorSupport {
E> E>
void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
int length, int length,
Object base, long offset, Object base, long offset, boolean fromSegment,
V v, M m, C container, long index, V v, M m, C container, long index,
StoreVectorMaskedOperation<C, V, M> defaultImpl) { StoreVectorMaskedOperation<C, V, M> defaultImpl) {
assert isNonCapturingLambda(defaultImpl) : defaultImpl; assert isNonCapturingLambda(defaultImpl) : defaultImpl;

2
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java
View file

@ -74,7 +74,7 @@ abstract class AbstractMask<E> extends VectorMask<E> {
i = VectorIntrinsics.checkFromIndexSize(i, laneCount, bits.length); i = VectorIntrinsics.checkFromIndexSize(i, laneCount, bits.length);
VectorSupport.store( VectorSupport.store(
vsp.maskType(), vsp.elementType(), laneCount, vsp.maskType(), vsp.elementType(), laneCount,
bits, (long) i + Unsafe.ARRAY_BOOLEAN_BASE_OFFSET, bits, (long) i + Unsafe.ARRAY_BOOLEAN_BASE_OFFSET, false,
this, bits, i, this, bits, i,
(c, idx, s) -> System.arraycopy(s.getBits(), 0, c, (int) idx, s.length())); (c, idx, s) -> System.arraycopy(s.getBits(), 0, c, (int) idx, s.length()));

18
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java
View file

@ -3373,7 +3373,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteSpecies vsp = vspecies(); ByteSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, this,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -3526,7 +3526,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteVector normalized = this.and((byte) 1); ByteVector normalized = this.and((byte) 1);
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), a, booleanArrayAddress(a, offset), false,
normalized, normalized,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -3737,7 +3737,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteSpecies vsp = vspecies(); ByteSpecies vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3754,7 +3754,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteSpecies vsp = vspecies(); ByteSpecies vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m, offsetInRange, a, arrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3771,7 +3771,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteSpecies vsp = vspecies(); ByteSpecies vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), a, booleanArrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0))); (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
@ -3788,7 +3788,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteSpecies vsp = vspecies(); ByteSpecies vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), m, offsetInRange, a, booleanArrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0))); (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
@ -3836,7 +3836,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteSpecies vsp = vspecies(); ByteSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, a, offset, this, a, offset,
(arr, off, v) (arr, off, v)
-> v.stOp(arr, (int) off, -> v.stOp(arr, (int) off,
@ -3853,7 +3853,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteSpecies vsp = vspecies(); ByteSpecies vsp = vspecies();
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, m, a, offset, this, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,
@ -3872,7 +3872,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteVector normalized = this.and((byte) 1); ByteVector normalized = this.and((byte) 1);
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), a, booleanArrayAddress(a, offset), false,
normalized, m, a, offset, normalized, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,

10
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java
View file

@ -3061,7 +3061,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
DoubleSpecies vsp = vspecies(); DoubleSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, this,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -3294,7 +3294,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
DoubleSpecies vsp = vspecies(); DoubleSpecies vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3311,7 +3311,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
DoubleSpecies vsp = vspecies(); DoubleSpecies vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m, offsetInRange, a, arrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3413,7 +3413,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
DoubleSpecies vsp = vspecies(); DoubleSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, a, offset, this, a, offset,
(arr, off, v) (arr, off, v)
-> v.stOp(arr, (int) off, -> v.stOp(arr, (int) off,
@ -3430,7 +3430,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
DoubleSpecies vsp = vspecies(); DoubleSpecies vsp = vspecies();
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, m, a, offset, this, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,

10
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java
View file

@ -3067,7 +3067,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
FloatSpecies vsp = vspecies(); FloatSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, this,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -3281,7 +3281,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
FloatSpecies vsp = vspecies(); FloatSpecies vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3298,7 +3298,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
FloatSpecies vsp = vspecies(); FloatSpecies vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m, offsetInRange, a, arrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3382,7 +3382,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
FloatSpecies vsp = vspecies(); FloatSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, a, offset, this, a, offset,
(arr, off, v) (arr, off, v)
-> v.stOp(arr, (int) off, -> v.stOp(arr, (int) off,
@ -3399,7 +3399,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
FloatSpecies vsp = vspecies(); FloatSpecies vsp = vspecies();
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, m, a, offset, this, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,

10
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java
View file

@ -3223,7 +3223,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
IntSpecies vsp = vspecies(); IntSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, this,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -3437,7 +3437,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
IntSpecies vsp = vspecies(); IntSpecies vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3454,7 +3454,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
IntSpecies vsp = vspecies(); IntSpecies vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m, offsetInRange, a, arrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3538,7 +3538,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
IntSpecies vsp = vspecies(); IntSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, a, offset, this, a, offset,
(arr, off, v) (arr, off, v)
-> v.stOp(arr, (int) off, -> v.stOp(arr, (int) off,
@ -3555,7 +3555,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
IntSpecies vsp = vspecies(); IntSpecies vsp = vspecies();
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, m, a, offset, this, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,

10
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java
View file

@ -3102,7 +3102,7 @@ public abstract class LongVector extends AbstractVector<Long> {
LongSpecies vsp = vspecies(); LongSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, this,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -3335,7 +3335,7 @@ public abstract class LongVector extends AbstractVector<Long> {
LongSpecies vsp = vspecies(); LongSpecies vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3352,7 +3352,7 @@ public abstract class LongVector extends AbstractVector<Long> {
LongSpecies vsp = vspecies(); LongSpecies vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m, offsetInRange, a, arrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3454,7 +3454,7 @@ public abstract class LongVector extends AbstractVector<Long> {
LongSpecies vsp = vspecies(); LongSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, a, offset, this, a, offset,
(arr, off, v) (arr, off, v)
-> v.stOp(arr, (int) off, -> v.stOp(arr, (int) off,
@ -3471,7 +3471,7 @@ public abstract class LongVector extends AbstractVector<Long> {
LongSpecies vsp = vspecies(); LongSpecies vsp = vspecies();
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, m, a, offset, this, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,

18
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java
View file

@ -3372,7 +3372,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortSpecies vsp = vspecies(); ShortSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, this,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -3520,7 +3520,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortSpecies vsp = vspecies(); ShortSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), a, charArrayAddress(a, offset), false,
this, this,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -3723,7 +3723,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortSpecies vsp = vspecies(); ShortSpecies vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3740,7 +3740,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortSpecies vsp = vspecies(); ShortSpecies vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m, offsetInRange, a, arrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -3756,7 +3756,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortSpecies vsp = vspecies(); ShortSpecies vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), a, charArrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> (short) arr_[off_ + i])); (arr_, off_, i) -> (short) arr_[off_ + i]));
@ -3773,7 +3773,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortSpecies vsp = vspecies(); ShortSpecies vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), m, offsetInRange, a, charArrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (short) arr_[off_ + i])); (arr_, off_, i) -> (short) arr_[off_ + i]));
@ -3822,7 +3822,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortSpecies vsp = vspecies(); ShortSpecies vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, a, offset, this, a, offset,
(arr, off, v) (arr, off, v)
-> v.stOp(arr, (int) off, -> v.stOp(arr, (int) off,
@ -3839,7 +3839,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortSpecies vsp = vspecies(); ShortSpecies vsp = vspecies();
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, m, a, offset, this, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,
@ -3889,7 +3889,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
ShortSpecies vsp = vspecies(); ShortSpecies vsp = vspecies();
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), a, charArrayAddress(a, offset), false,
this, m, a, offset, this, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,

2
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMask.java
View file

@ -207,7 +207,7 @@ public abstract class VectorMask<E> extends jdk.internal.vm.vector.VectorSupport
offset = VectorIntrinsics.checkFromIndexSize(offset, laneCount, bits.length); offset = VectorIntrinsics.checkFromIndexSize(offset, laneCount, bits.length);
return VectorSupport.load( return VectorSupport.load(
vsp.maskType(), vsp.elementType(), laneCount, vsp.maskType(), vsp.elementType(), laneCount,
bits, (long) offset + Unsafe.ARRAY_BOOLEAN_BASE_OFFSET, bits, (long) offset + Unsafe.ARRAY_BOOLEAN_BASE_OFFSET, false,
bits, offset, vsp, bits, offset, vsp,
(c, idx, s) (c, idx, s)
-> s.opm(n -> c[((int )idx) + n])); -> s.opm(n -> c[((int )idx) + n]));

26
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template
View file

@ -4178,7 +4178,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, this,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -4397,7 +4397,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), a, charArrayAddress(a, offset), false,
this, this,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -4558,7 +4558,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
ByteVector normalized = this.and((byte) 1); ByteVector normalized = this.and((byte) 1);
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), a, booleanArrayAddress(a, offset), false,
normalized, normalized,
a, offset, a, offset,
(arr, off, v) (arr, off, v)
@ -4770,7 +4770,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -4787,7 +4787,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m, offsetInRange, a, arrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i])); (arr_, off_, i) -> arr_[off_ + i]));
@ -4864,7 +4864,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), a, charArrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> (short) arr_[off_ + i])); (arr_, off_, i) -> (short) arr_[off_ + i]));
@ -4881,7 +4881,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), m, offsetInRange, a, charArrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (short) arr_[off_ + i])); (arr_, off_, i) -> (short) arr_[off_ + i]));
@ -4898,7 +4898,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
return VectorSupport.load( return VectorSupport.load(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), a, booleanArrayAddress(a, offset), false,
a, offset, vsp, a, offset, vsp,
(arr, off, s) -> s.ldOp(arr, (int) off, (arr, off, s) -> s.ldOp(arr, (int) off,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0))); (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
@ -4915,7 +4915,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
return VectorSupport.loadMasked( return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), m, offsetInRange, a, booleanArrayAddress(a, offset), false, m, offsetInRange,
a, offset, vsp, a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm, (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0))); (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
@ -4964,7 +4964,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
VectorSupport.store( VectorSupport.store(
vsp.vectorType(), vsp.elementType(), vsp.laneCount(), vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, a, offset, this, a, offset,
(arr, off, v) (arr, off, v)
-> v.stOp(arr, (int) off, -> v.stOp(arr, (int) off,
@ -4981,7 +4981,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), a, arrayAddress(a, offset), false,
this, m, a, offset, this, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,
@ -5062,7 +5062,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
ByteVector normalized = this.and((byte) 1); ByteVector normalized = this.and((byte) 1);
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), a, booleanArrayAddress(a, offset), false,
normalized, m, a, offset, normalized, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,
@ -5112,7 +5112,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = vspecies(); $Type$Species vsp = vspecies();
VectorSupport.storeMasked( VectorSupport.storeMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(), vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), a, charArrayAddress(a, offset), false,
this, m, a, offset, this, m, a, offset,
(arr, off, v, vm) (arr, off, v, vm)
-> v.stOp(arr, (int) off, vm, -> v.stOp(arr, (int) off, vm,

26
test/micro/org/openjdk/bench/jdk/incubator/vector/TestLoadSegmentVarious.java
View file

@ -55,6 +55,9 @@ public class TestLoadSegmentVarious {
private static final VectorSpecies<Integer> INTEGER_SPECIES = VectorSpecies.ofLargestShape(int.class); private static final VectorSpecies<Integer> INTEGER_SPECIES = VectorSpecies.ofLargestShape(int.class);
private static final VectorSpecies<Double> DOUBLE_SPECIES = VectorSpecies.ofLargestShape(double.class); private static final VectorSpecies<Double> DOUBLE_SPECIES = VectorSpecies.ofLargestShape(double.class);
private static final VectorMask<Integer> INTEGER_MASK = VectorMask.fromLong(INTEGER_SPECIES, (1 << (INTEGER_SPECIES.length() / 2)) - 1);
private static final VectorMask<Double> DOUBLE_MASK = VectorMask.fromLong(DOUBLE_SPECIES, (1 << (DOUBLE_SPECIES.length() / 2)) - 1);
// Must be evenly dividable by Double.BYTES // Must be evenly dividable by Double.BYTES
@Param("1024") @Param("1024")
private int size; private int size;
@ -253,4 +256,27 @@ public class TestLoadSegmentVarious {
} }
} }
@Benchmark
public void intVectorFromIntBackedSegmentMasked(Blackhole bh) {
for (int i = 0; i < INTEGER_SPECIES.loopBound(intSrcArray.length); i += INTEGER_SPECIES.vectorByteSize()) {
var v = IntVector.fromMemorySegment(INTEGER_SPECIES, doubleSegment, i, ByteOrder.nativeOrder(), INTEGER_MASK);
bh.consume(v);
}
}
@Benchmark
public void intVectorFromDoubleBackedSegmentMasked(Blackhole bh) {
for (int i = 0; i < INTEGER_SPECIES.loopBound(intSrcArray.length); i += INTEGER_SPECIES.vectorByteSize()) {
var v = IntVector.fromMemorySegment(INTEGER_SPECIES, doubleSegment, i, ByteOrder.nativeOrder(), INTEGER_MASK);
bh.consume(v);
}
}
@Benchmark
public void doubleVectorFromIntBackedSegmentMasked(Blackhole bh) {
for (int i = 0; i < DOUBLE_SPECIES.loopBound(doubleSrcArray.length); i += DOUBLE_SPECIES.vectorByteSize()) {
var v = DoubleVector.fromMemorySegment(DOUBLE_SPECIES, intSegment, i, ByteOrder.nativeOrder(), DOUBLE_MASK);
bh.consume(v);
}
}
} }