From 50210bd5cc50b35cfbacae95ae7cf70affd37ed2 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Mon, 18 Nov 2024 18:45:52 -0800 Subject: [PATCH 01/25] Remove vector math intrinsics --- src/hotspot/cpu/aarch64/aarch64.ad | 4 +- .../cpu/aarch64/stubGenerator_aarch64.cpp | 75 ---------- src/hotspot/cpu/riscv/riscv.ad | 2 +- src/hotspot/cpu/riscv/stubGenerator_riscv.cpp | 54 ------- src/hotspot/cpu/x86/stubGenerator_x86_64.cpp | 64 --------- src/hotspot/cpu/x86/x86_64.ad | 7 +- src/hotspot/share/classfile/modules.cpp | 4 - src/hotspot/share/jvmci/jvmci_globals.cpp | 3 +- src/hotspot/share/jvmci/jvmci_globals.hpp | 4 +- src/hotspot/share/opto/c2_globals.hpp | 3 - src/hotspot/share/opto/vectorIntrinsics.cpp | 132 ++---------------- src/hotspot/share/prims/vectorSupport.cpp | 42 ------ src/hotspot/share/prims/vectorSupport.hpp | 26 ---- src/hotspot/share/runtime/arguments.cpp | 5 - src/hotspot/share/runtime/stubRoutines.cpp | 2 - src/hotspot/share/runtime/stubRoutines.hpp | 4 - .../jdk/incubator/vector/DoubleVector.java | 16 +++ .../jdk/incubator/vector/FloatVector.java | 16 +++ .../jdk/incubator/vector/VectorOperators.java | 38 ++--- 19 files changed, 74 insertions(+), 427 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 5bfe697d12c0b..142e4d3b0be54 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -2306,11 +2306,11 @@ const RegMask* Matcher::predicate_reg_mask(void) { } bool Matcher::supports_vector_calling_convention(void) { - return EnableVectorSupport && UseVectorStubs; + return EnableVectorSupport; } OptoRegPair Matcher::vector_return_value(uint ideal_reg) { - assert(EnableVectorSupport && UseVectorStubs, "sanity"); + assert(EnableVectorSupport, "sanity"); int lo = V0_num; int hi = V0_H_num; if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) { diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index f0f145e3d7612..9688c6ea88829 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -9769,79 +9769,6 @@ class StubGenerator: public StubCodeGenerator { // } }; - void generate_vector_math_stubs() { - // Get native vector math stub routine addresses - void* libsleef = nullptr; - char ebuf[1024]; - char dll_name[JVM_MAXPATHLEN]; - if (os::dll_locate_lib(dll_name, sizeof(dll_name), Arguments::get_dll_dir(), "sleef")) { - libsleef = os::dll_load(dll_name, ebuf, sizeof ebuf); - } - if (libsleef == nullptr) { - log_info(library)("Failed to load native vector math library, %s!", ebuf); - return; - } - // Method naming convention - // All the methods are named as _ - // Where: - // is the operation name, e.g. sin - // is optional to indicate float/double - // "f/d" for vector float/double operation - // is the number of elements in the vector - // "2/4" for neon, and "x" for sve - // is the precision level - // "u10/u05" represents 1.0/0.5 ULP error bounds - // We use "u10" for all operations by default - // But for those functions do not have u10 support, we use "u05" instead - // indicates neon/sve - // "sve/advsimd" for sve/neon implementations - // e.g. sinfx_u10sve is the method for computing vector float sin using SVE instructions - // cosd2_u10advsimd is the method for computing 2 elements vector double cos using NEON instructions - // - log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "sleef" JNI_LIB_SUFFIX, p2i(libsleef)); - - // Math vector stubs implemented with SVE for scalable vector size. - if (UseSVE > 0) { - for (int op = 0; op < VectorSupport::NUM_VECTOR_OP_MATH; op++) { - int vop = VectorSupport::VECTOR_OP_MATH_START + op; - // Skip "tanh" because there is performance regression - if (vop == VectorSupport::VECTOR_OP_TANH) { - continue; - } - - // The native library does not support u10 level of "hypot". - const char* ulf = (vop == VectorSupport::VECTOR_OP_HYPOT) ? "u05" : "u10"; - - snprintf(ebuf, sizeof(ebuf), "%sfx_%ssve", VectorSupport::mathname[op], ulf); - StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op] = (address)os::dll_lookup(libsleef, ebuf); - - snprintf(ebuf, sizeof(ebuf), "%sdx_%ssve", VectorSupport::mathname[op], ulf); - StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op] = (address)os::dll_lookup(libsleef, ebuf); - } - } - - // Math vector stubs implemented with NEON for 64/128 bits vector size. - for (int op = 0; op < VectorSupport::NUM_VECTOR_OP_MATH; op++) { - int vop = VectorSupport::VECTOR_OP_MATH_START + op; - // Skip "tanh" because there is performance regression - if (vop == VectorSupport::VECTOR_OP_TANH) { - continue; - } - - // The native library does not support u10 level of "hypot". - const char* ulf = (vop == VectorSupport::VECTOR_OP_HYPOT) ? "u05" : "u10"; - - snprintf(ebuf, sizeof(ebuf), "%sf4_%sadvsimd", VectorSupport::mathname[op], ulf); - StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libsleef, ebuf); - - snprintf(ebuf, sizeof(ebuf), "%sf4_%sadvsimd", VectorSupport::mathname[op], ulf); - StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libsleef, ebuf); - - snprintf(ebuf, sizeof(ebuf), "%sd2_%sadvsimd", VectorSupport::mathname[op], ulf); - StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libsleef, ebuf); - } - } - // Initialization void generate_initial_stubs() { // Generate initial stubs and initializes the entry points @@ -9995,8 +9922,6 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_montgomerySquare = g.generate_multiply(); } - generate_vector_math_stubs(); - #endif // COMPILER2 if (UseChaCha20Intrinsics) { diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 59171d84c9b80..596dfb29628a6 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1944,7 +1944,7 @@ const RegMask* Matcher::predicate_reg_mask(void) { // Vector calling convention not yet implemented. bool Matcher::supports_vector_calling_convention(void) { - return EnableVectorSupport && UseVectorStubs; + return EnableVectorSupport; } OptoRegPair Matcher::vector_return_value(uint ideal_reg) { diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index 4527a32926f52..fb4539267ae98 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -6458,58 +6458,6 @@ static const int64_t right_3_bits = right_n_bits(3); return start; } - void generate_vector_math_stubs() { - if (!UseRVV) { - log_info(library)("vector is not supported, skip loading vector math (sleef) library!"); - return; - } - - // Get native vector math stub routine addresses - void* libsleef = nullptr; - char ebuf[1024]; - char dll_name[JVM_MAXPATHLEN]; - if (os::dll_locate_lib(dll_name, sizeof(dll_name), Arguments::get_dll_dir(), "sleef")) { - libsleef = os::dll_load(dll_name, ebuf, sizeof ebuf); - } - if (libsleef == nullptr) { - log_info(library)("Failed to load native vector math (sleef) library, %s!", ebuf); - return; - } - - // Method naming convention - // All the methods are named as _ - // - // Where: - // is the operation name, e.g. sin, cos - // is to indicate float/double - // "fx/dx" for vector float/double operation - // is the precision level - // "u10/u05" represents 1.0/0.5 ULP error bounds - // We use "u10" for all operations by default - // But for those functions do not have u10 support, we use "u05" instead - // rvv, indicates riscv vector extension - // - // e.g. sinfx_u10rvv is the method for computing vector float sin using rvv instructions - // - log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "sleef" JNI_LIB_SUFFIX, p2i(libsleef)); - - for (int op = 0; op < VectorSupport::NUM_VECTOR_OP_MATH; op++) { - int vop = VectorSupport::VECTOR_OP_MATH_START + op; - if (vop == VectorSupport::VECTOR_OP_TANH) { // skip tanh because of performance regression - continue; - } - - // The native library does not support u10 level of "hypot". - const char* ulf = (vop == VectorSupport::VECTOR_OP_HYPOT) ? "u05" : "u10"; - - snprintf(ebuf, sizeof(ebuf), "%sfx_%srvv", VectorSupport::mathname[op], ulf); - StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op] = (address)os::dll_lookup(libsleef, ebuf); - - snprintf(ebuf, sizeof(ebuf), "%sdx_%srvv", VectorSupport::mathname[op], ulf); - StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op] = (address)os::dll_lookup(libsleef, ebuf); - } - } - #endif // COMPILER2 /** @@ -6741,8 +6689,6 @@ static const int64_t right_3_bits = right_n_bits(3); generate_string_indexof_stubs(); - generate_vector_math_stubs(); - #endif // COMPILER2 } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index 7e63e6fb49b62..2af6bcaa53d0c 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -4331,70 +4331,6 @@ void StubGenerator::generate_compiler_stubs() { } } - // Get svml stub routine addresses - void *libjsvml = nullptr; - char ebuf[1024]; - char dll_name[JVM_MAXPATHLEN]; - if (os::dll_locate_lib(dll_name, sizeof(dll_name), Arguments::get_dll_dir(), "jsvml")) { - libjsvml = os::dll_load(dll_name, ebuf, sizeof ebuf); - } - if (libjsvml != nullptr) { - // SVML method naming convention - // All the methods are named as __jsvml_op_ha_ - // Where: - // ha stands for high accuracy - // is optional to indicate float/double - // Set to f for vector float operation - // Omitted for vector double operation - // is the number of elements in the vector - // 1, 2, 4, 8, 16 - // e.g. 128 bit float vector has 4 float elements - // indicates the avx/sse level: - // z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2 - // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns - // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns - - log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "jsvml" JNI_LIB_SUFFIX, p2i(libjsvml)); - if (UseAVX > 2) { - for (int op = 0; op < VectorSupport::NUM_VECTOR_OP_MATH; op++) { - int vop = VectorSupport::VECTOR_OP_MATH_START + op; - if ((!VM_Version::supports_avx512dq()) && - (vop == VectorSupport::VECTOR_OP_LOG || vop == VectorSupport::VECTOR_OP_LOG10 || vop == VectorSupport::VECTOR_OP_POW)) { - continue; - } - snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf16_ha_z0", VectorSupport::mathname[op]); - StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf); - - snprintf(ebuf, sizeof(ebuf), "__jsvml_%s8_ha_z0", VectorSupport::mathname[op]); - StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf); - } - } - const char* avx_sse_str = (UseAVX >= 2) ? "l9" : ((UseAVX == 1) ? "e9" : "ex"); - for (int op = 0; op < VectorSupport::NUM_VECTOR_OP_MATH; op++) { - int vop = VectorSupport::VECTOR_OP_MATH_START + op; - if (vop == VectorSupport::VECTOR_OP_POW) { - continue; - } - snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::mathname[op], avx_sse_str); - StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf); - - snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::mathname[op], avx_sse_str); - StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf); - - snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf8_ha_%s", VectorSupport::mathname[op], avx_sse_str); - StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf); - - snprintf(ebuf, sizeof(ebuf), "__jsvml_%s1_ha_%s", VectorSupport::mathname[op], avx_sse_str); - StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf); - - snprintf(ebuf, sizeof(ebuf), "__jsvml_%s2_ha_%s", VectorSupport::mathname[op], avx_sse_str); - StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf); - - snprintf(ebuf, sizeof(ebuf), "__jsvml_%s4_ha_%s", VectorSupport::mathname[op], avx_sse_str); - StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf); - } - } - #endif // COMPILER2 #endif // COMPILER2_OR_JVMCI } diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index b94ff7dbd9e5a..9ad4ca088ecc1 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -1584,14 +1584,11 @@ uint MachUEPNode::size(PhaseRegAlloc* ra_) const //============================================================================= bool Matcher::supports_vector_calling_convention(void) { - if (EnableVectorSupport && UseVectorStubs) { - return true; - } - return false; + return EnableVectorSupport; } OptoRegPair Matcher::vector_return_value(uint ideal_reg) { - assert(EnableVectorSupport && UseVectorStubs, "sanity"); + assert(EnableVectorSupport, "sanity"); int lo = XMM0_num; int hi = XMM0b_num; if (ideal_reg == Op_VecX) hi = XMM0d_num; diff --git a/src/hotspot/share/classfile/modules.cpp b/src/hotspot/share/classfile/modules.cpp index 3f2ff90ccab31..07c382b208deb 100644 --- a/src/hotspot/share/classfile/modules.cpp +++ b/src/hotspot/share/classfile/modules.cpp @@ -466,13 +466,9 @@ void Modules::define_module(Handle module, jboolean is_open, jstring version, if (EnableVectorSupport && EnableVectorReboxing && FLAG_IS_DEFAULT(EnableVectorAggressiveReboxing)) { FLAG_SET_DEFAULT(EnableVectorAggressiveReboxing, true); } - if (EnableVectorSupport && FLAG_IS_DEFAULT(UseVectorStubs)) { - FLAG_SET_DEFAULT(UseVectorStubs, true); - } log_info(compilation)("EnableVectorSupport=%s", (EnableVectorSupport ? "true" : "false")); log_info(compilation)("EnableVectorReboxing=%s", (EnableVectorReboxing ? "true" : "false")); log_info(compilation)("EnableVectorAggressiveReboxing=%s", (EnableVectorAggressiveReboxing ? "true" : "false")); - log_info(compilation)("UseVectorStubs=%s", (UseVectorStubs ? "true" : "false")); } #endif // COMPILER2_OR_JVMCI } diff --git a/src/hotspot/share/jvmci/jvmci_globals.cpp b/src/hotspot/share/jvmci/jvmci_globals.cpp index dfc7e6b597046..a41351218f8f4 100644 --- a/src/hotspot/share/jvmci/jvmci_globals.cpp +++ b/src/hotspot/share/jvmci/jvmci_globals.cpp @@ -144,8 +144,9 @@ bool JVMCIGlobals::check_jvmci_flags_are_consistent() { JVMCI_FLAG_CHECKED(UseMulAddIntrinsic) JVMCI_FLAG_CHECKED(UseMontgomeryMultiplyIntrinsic) JVMCI_FLAG_CHECKED(UseMontgomerySquareIntrinsic) - JVMCI_FLAG_CHECKED(UseVectorStubs) #endif // !COMPILER2 + // + JVMCI_FLAG_CHECKED(UseVectorStubs) #ifndef PRODUCT #define JVMCI_CHECK4(type, name, value, ...) assert(name##checked, #name " flag not checked"); diff --git a/src/hotspot/share/jvmci/jvmci_globals.hpp b/src/hotspot/share/jvmci/jvmci_globals.hpp index 4da49b24e6ef9..5bcead9ff7b32 100644 --- a/src/hotspot/share/jvmci/jvmci_globals.hpp +++ b/src/hotspot/share/jvmci/jvmci_globals.hpp @@ -184,8 +184,8 @@ class fileStream; NOT_COMPILER2(product(bool, EnableVectorAggressiveReboxing, false, EXPERIMENTAL, \ "Enables aggressive reboxing of vectors")) \ \ - NOT_COMPILER2(product(bool, UseVectorStubs, false, EXPERIMENTAL, \ - "Use stubs for vector transcendental operations")) \ + product(bool, UseVectorStubs, false, EXPERIMENTAL, \ + "Use stubs for vector transcendental operations") \ // end of JVMCI_FLAGS diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp index ad88554fedd04..ab7180615b4f5 100644 --- a/src/hotspot/share/opto/c2_globals.hpp +++ b/src/hotspot/share/opto/c2_globals.hpp @@ -750,9 +750,6 @@ product(bool, EnableVectorAggressiveReboxing, false, EXPERIMENTAL, \ "Enables aggressive reboxing of vectors") \ \ - product(bool, UseVectorStubs, false, EXPERIMENTAL, \ - "Use stubs for vector transcendental operations") \ - \ product(bool, UseTypeSpeculation, true, \ "Speculatively propagate types from profiles") \ \ diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index e33d7b1968682..87a5f5fd85d52 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -366,17 +366,11 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { int num_elem = vlen->get_con(); int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt); int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc; - if ((opc != Op_CallLeafVector) && (sopc == 0)) { - log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt)); + if (sopc == 0 || num_elem == 1) { + log_if_needed(" ** operation not supported: arity=%d opc=%s[%d] vlen=%d etype=%s", + n, NodeClassNames[opc], opc, num_elem, type2name(elem_bt)); return false; // operation not supported } - if (num_elem == 1) { - if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) { - log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s", - n, opc, num_elem, type2name(elem_bt)); - return false; - } - } ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); @@ -384,22 +378,6 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { assert(!is_masked_op, "mask operations do not need mask to control"); } - if (opc == Op_CallLeafVector) { - if (!UseVectorStubs) { - log_if_needed(" ** vector stubs support is disabled"); - return false; - } - if (!Matcher::supports_vector_calling_convention()) { - log_if_needed(" ** no vector calling conventions supported"); - return false; - } - if (!Matcher::vector_size_supported(elem_bt, num_elem)) { - log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported", - num_elem, type2name(elem_bt)); - return false; - } - } - // When using mask, mask use type needs to be VecMaskUseLoad. VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed; @@ -464,30 +442,18 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { } Node* operation = nullptr; - if (opc == Op_CallLeafVector) { - assert(UseVectorStubs, "sanity"); - operation = gen_call_to_vector_math(opr->get_con(), elem_bt, num_elem, opd1, opd2); - if (operation == nullptr) { - log_if_needed(" ** Vector math call failed for %s_%s_%d", - (elem_bt == T_FLOAT) ? "float" : "double", - VectorSupport::mathname[opr->get_con() - VectorSupport::VECTOR_OP_MATH_START], - num_elem * type2aelembytes(elem_bt)); - return false; - } - } else { - const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass)); - switch (n) { - case 1: - case 2: { - operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned); - break; - } - case 3: { - operation = VectorNode::make(sopc, opd1, opd2, opd3, vt); - break; - } - default: fatal("unsupported arity: %d", n); + const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass)); + switch (n) { + case 1: + case 2: { + operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned); + break; + } + case 3: { + operation = VectorNode::make(sopc, opd1, opd2, opd3, vt); + break; } + default: fatal("unsupported arity: %d", n); } if (is_masked_op && mask != nullptr) { @@ -1844,50 +1810,6 @@ bool LibraryCallKit::inline_vector_rearrange() { return true; } -static address get_vector_math_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) { - address addr = nullptr; - assert(UseVectorStubs, "sanity"); - assert(name_ptr != nullptr, "unexpected"); - assert((vop >= VectorSupport::VECTOR_OP_MATH_START) && (vop <= VectorSupport::VECTOR_OP_MATH_END), "unexpected"); - int op = vop - VectorSupport::VECTOR_OP_MATH_START; - - switch(bits) { - case 64: //fallthough - case 128: //fallthough - case 256: //fallthough - case 512: - if (bt == T_FLOAT) { - snprintf(name_ptr, name_len, "vector_%s_float_%dbits_fixed", VectorSupport::mathname[op], bits); - addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op]; - } else { - assert(bt == T_DOUBLE, "must be FP type only"); - snprintf(name_ptr, name_len, "vector_%s_double_%dbits_fixed", VectorSupport::mathname[op], bits); - addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op]; - } - break; - default: - if (!Matcher::supports_scalable_vector() || !Matcher::vector_size_supported(bt, bits/type2aelembytes(bt)) ) { - snprintf(name_ptr, name_len, "invalid"); - addr = nullptr; - Unimplemented(); - } - break; - } - - if (addr == nullptr && Matcher::supports_scalable_vector()) { - if (bt == T_FLOAT) { - snprintf(name_ptr, name_len, "vector_%s_float_%dbits_scalable", VectorSupport::mathname[op], bits); - addr = StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op]; - } else { - assert(bt == T_DOUBLE, "must be FP type only"); - snprintf(name_ptr, name_len, "vector_%s_double_%dbits_scalable", VectorSupport::mathname[op], bits); - addr = StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op]; - } - } - - return addr; -} - // public static // , // M extends VectorMask, @@ -2044,32 +1966,6 @@ bool LibraryCallKit::inline_vector_select_from() { return true; } -Node* LibraryCallKit::gen_call_to_vector_math(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) { - assert(UseVectorStubs, "sanity"); - assert(vector_api_op_id >= VectorSupport::VECTOR_OP_MATH_START && vector_api_op_id <= VectorSupport::VECTOR_OP_MATH_END, "need valid op id"); - assert(opd1 != nullptr, "must not be null"); - const TypeVect* vt = TypeVect::make(bt, num_elem); - const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt); - char name[100] = ""; - - // Get address for vector math method. - address addr = get_vector_math_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100); - - if (addr == nullptr) { - return nullptr; - } - - assert(name[0] != '\0', "name must not be null"); - Node* operation = make_runtime_call(RC_VECTOR, - call_type, - addr, - name, - TypePtr::BOTTOM, - opd1, - opd2); - return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms)); -} - // public static // , // M extends VectorMask, diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index a00656f30ee1e..27e52966c8309 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -41,29 +41,6 @@ #include "opto/matcher.hpp" #endif // COMPILER2 -#ifdef COMPILER2 -const char* VectorSupport::mathname[VectorSupport::NUM_VECTOR_OP_MATH] = { - "tan", - "tanh", - "sin", - "sinh", - "cos", - "cosh", - "asin", - "acos", - "atan", - "atan2", - "cbrt", - "log", - "log10", - "log1p", - "pow", - "exp", - "expm1", - "hypot", -}; -#endif - bool VectorSupport::is_vector(Klass* klass) { return klass->is_subclass_of(vmClasses::vector_VectorPayload_klass()); } @@ -615,25 +592,6 @@ int VectorSupport::vop2ideal(jint id, BasicType bt) { break; } - case VECTOR_OP_TAN: - case VECTOR_OP_TANH: - case VECTOR_OP_SIN: - case VECTOR_OP_SINH: - case VECTOR_OP_COS: - case VECTOR_OP_COSH: - case VECTOR_OP_ASIN: - case VECTOR_OP_ACOS: - case VECTOR_OP_ATAN: - case VECTOR_OP_ATAN2: - case VECTOR_OP_CBRT: - case VECTOR_OP_LOG: - case VECTOR_OP_LOG10: - case VECTOR_OP_LOG1P: - case VECTOR_OP_POW: - case VECTOR_OP_EXP: - case VECTOR_OP_EXPM1: - case VECTOR_OP_HYPOT: - return Op_CallLeafVector; default: fatal("unknown op: %d", vop); } return 0; // Unimplemented diff --git a/src/hotspot/share/prims/vectorSupport.hpp b/src/hotspot/share/prims/vectorSupport.hpp index 688fb595099f6..5ba18cdfaa85e 100644 --- a/src/hotspot/share/prims/vectorSupport.hpp +++ b/src/hotspot/share/prims/vectorSupport.hpp @@ -101,36 +101,12 @@ class VectorSupport : AllStatic { VECTOR_OP_COMPRESS_BITS = 33, VECTOR_OP_EXPAND_BITS = 34, - // Vector Math Library - VECTOR_OP_TAN = 101, - VECTOR_OP_TANH = 102, - VECTOR_OP_SIN = 103, - VECTOR_OP_SINH = 104, - VECTOR_OP_COS = 105, - VECTOR_OP_COSH = 106, - VECTOR_OP_ASIN = 107, - VECTOR_OP_ACOS = 108, - VECTOR_OP_ATAN = 109, - VECTOR_OP_ATAN2 = 110, - VECTOR_OP_CBRT = 111, - VECTOR_OP_LOG = 112, - VECTOR_OP_LOG10 = 113, - VECTOR_OP_LOG1P = 114, - VECTOR_OP_POW = 115, - VECTOR_OP_EXP = 116, - VECTOR_OP_EXPM1 = 117, - VECTOR_OP_HYPOT = 118, - VECTOR_OP_SADD = 119, VECTOR_OP_SSUB = 120, VECTOR_OP_SUADD = 121, VECTOR_OP_SUSUB = 122, VECTOR_OP_UMIN = 123, VECTOR_OP_UMAX = 124, - - VECTOR_OP_MATH_START = VECTOR_OP_TAN, - VECTOR_OP_MATH_END = VECTOR_OP_HYPOT, - NUM_VECTOR_OP_MATH = VECTOR_OP_MATH_END - VECTOR_OP_MATH_START + 1 }; enum { @@ -147,8 +123,6 @@ class VectorSupport : AllStatic { MODE_BITS_COERCED_LONG_TO_MASK = 1 }; - static const char* mathname[VectorSupport::NUM_VECTOR_OP_MATH]; - static int vop2ideal(jint vop, BasicType bt); static bool has_scalar_op(jint id); static bool is_unsigned_op(jint id); diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index 0927e87c74853..3a2bdcaeb47d3 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -3789,11 +3789,6 @@ jint Arguments::apply_ergo() { } } FLAG_SET_DEFAULT(EnableVectorAggressiveReboxing, false); - - if (!FLAG_IS_DEFAULT(UseVectorStubs) && UseVectorStubs) { - warning("Disabling UseVectorStubs since EnableVectorSupport is turned off."); - } - FLAG_SET_DEFAULT(UseVectorStubs, false); } #endif // COMPILER2_OR_JVMCI diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index b1b1f1d60566a..358434938f29a 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -101,8 +101,6 @@ jint StubRoutines::_verify_oop_count = 0; address StubRoutines::_string_indexof_array[4] = { nullptr }; -address StubRoutines::_vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_VECTOR_OP_MATH] = {{nullptr}, {nullptr}}; -address StubRoutines::_vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_VECTOR_OP_MATH] = {{nullptr}, {nullptr}}; const char* StubRoutines::get_blob_name(StubGenBlobId id) { assert(0 <= id && id < StubGenBlobId::NUM_BLOBIDS, "invalid blob id"); diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index 3189415a6c5d6..7548a97ced899 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -305,10 +305,6 @@ class StubRoutines: AllStatic { /* special case: stub employs array of entries */ - // Vector Math Routines - static address _vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_VECTOR_OP_MATH]; - static address _vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_VECTOR_OP_MATH]; - static bool is_stub_code(address addr) { return contains(addr); } // generate code to implement method contains diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index 5fbf02f87bd93..c0aafcbb224ba 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -677,6 +677,11 @@ DoubleVector lanewiseTemplate(VectorOperators.Unary op) { if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); + } else { + int opc = opCode(op); + if (opc >= VECTOR_OP_TAN && opc <= VECTOR_OP_EXPM1) { + return UN_IMPL.find(op, opc, DoubleVector::unaryOperations).apply(this, null); + } } } int opc = opCode(op); @@ -702,6 +707,11 @@ DoubleVector lanewiseTemplate(VectorOperators.Unary op, if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); + } else { + int opc = opCode(op); + if (opc >= VECTOR_OP_TAN && opc <= VECTOR_OP_EXPM1) { + return UN_IMPL.find(op, opc, DoubleVector::unaryOperations).apply(this, m); + } } } int opc = opCode(op); @@ -780,6 +790,9 @@ DoubleVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = this.viewAsIntegralLanes().compare(EQ, (long) 0); return this.blend(that, mask.cast(vspecies())); + } else if (op == ATAN2 || op == POW || op == HYPOT) { + int opc = opCode(op); + return BIN_IMPL.find(op, opc, DoubleVector::binaryOperations).apply(this, that, null); } } @@ -814,6 +827,9 @@ DoubleVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = bits.compare(EQ, (long) 0, m.cast(bits.vspecies())); return this.blend(that, mask.cast(vspecies())); + } else if (op == ATAN2 || op == POW || op == HYPOT) { + int opc = opCode(op); + return BIN_IMPL.find(op, opc, DoubleVector::binaryOperations).apply(this, that, m); } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index 26fbe64742d6f..e46886b11a927 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -677,6 +677,11 @@ FloatVector lanewiseTemplate(VectorOperators.Unary op) { if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); + } else { + int opc = opCode(op); + if (opc >= VECTOR_OP_TAN && opc <= VECTOR_OP_EXPM1) { + return UN_IMPL.find(op, opc, FloatVector::unaryOperations).apply(this, null); + } } } int opc = opCode(op); @@ -702,6 +707,11 @@ FloatVector lanewiseTemplate(VectorOperators.Unary op, if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); + } else { + int opc = opCode(op); + if (opc >= VECTOR_OP_TAN && opc <= VECTOR_OP_EXPM1) { + return UN_IMPL.find(op, opc, FloatVector::unaryOperations).apply(this, m); + } } } int opc = opCode(op); @@ -780,6 +790,9 @@ FloatVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = this.viewAsIntegralLanes().compare(EQ, (int) 0); return this.blend(that, mask.cast(vspecies())); + } else if (op == ATAN2 || op == POW || op == HYPOT) { + int opc = opCode(op); + return BIN_IMPL.find(op, opc, FloatVector::binaryOperations).apply(this, that, null); } } @@ -814,6 +827,9 @@ FloatVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = bits.compare(EQ, (int) 0, m.cast(bits.vspecies())); return this.blend(that, mask.cast(vspecies())); + } else if (op == ATAN2 || op == POW || op == HYPOT) { + int opc = opCode(op); + return BIN_IMPL.find(op, opc, FloatVector::binaryOperations).apply(this, that, m); } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java index c5fe6dd444128..d0b2d6b4d035b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java @@ -476,67 +476,67 @@ static boolean opKind(Operator op, int bit) { /** Produce {@code sin(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary SIN = unary("SIN", "sin", VectorSupport.VECTOR_OP_SIN, VO_ONLYFP); + public static final /*float*/ Unary SIN = unary("SIN", "sin", VectorSupport.VECTOR_OP_SIN, VO_ONLYFP | VO_SPECIAL); /** Produce {@code cos(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary COS = unary("COS", "cos", VectorSupport.VECTOR_OP_COS, VO_ONLYFP); + public static final /*float*/ Unary COS = unary("COS", "cos", VectorSupport.VECTOR_OP_COS, VO_ONLYFP | VO_SPECIAL); /** Produce {@code tan(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary TAN = unary("TAN", "tan", VectorSupport.VECTOR_OP_TAN, VO_ONLYFP); + public static final /*float*/ Unary TAN = unary("TAN", "tan", VectorSupport.VECTOR_OP_TAN, VO_ONLYFP | VO_SPECIAL); /** Produce {@code asin(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary ASIN = unary("ASIN", "asin", VectorSupport.VECTOR_OP_ASIN, VO_ONLYFP); + public static final /*float*/ Unary ASIN = unary("ASIN", "asin", VectorSupport.VECTOR_OP_ASIN, VO_ONLYFP | VO_SPECIAL); /** Produce {@code acos(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary ACOS = unary("ACOS", "acos", VectorSupport.VECTOR_OP_ACOS, VO_ONLYFP); + public static final /*float*/ Unary ACOS = unary("ACOS", "acos", VectorSupport.VECTOR_OP_ACOS, VO_ONLYFP | VO_SPECIAL); /** Produce {@code atan(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary ATAN = unary("ATAN", "atan", VectorSupport.VECTOR_OP_ATAN, VO_ONLYFP); + public static final /*float*/ Unary ATAN = unary("ATAN", "atan", VectorSupport.VECTOR_OP_ATAN, VO_ONLYFP | VO_SPECIAL); /** Produce {@code exp(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary EXP = unary("EXP", "exp", VectorSupport.VECTOR_OP_EXP, VO_ONLYFP); + public static final /*float*/ Unary EXP = unary("EXP", "exp", VectorSupport.VECTOR_OP_EXP, VO_ONLYFP | VO_SPECIAL); /** Produce {@code log(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary LOG = unary("LOG", "log", VectorSupport.VECTOR_OP_LOG, VO_ONLYFP); + public static final /*float*/ Unary LOG = unary("LOG", "log", VectorSupport.VECTOR_OP_LOG, VO_ONLYFP | VO_SPECIAL); /** Produce {@code log10(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary LOG10 = unary("LOG10", "log10", VectorSupport.VECTOR_OP_LOG10, VO_ONLYFP); + public static final /*float*/ Unary LOG10 = unary("LOG10", "log10", VectorSupport.VECTOR_OP_LOG10, VO_ONLYFP | VO_SPECIAL); /** Produce {@code sqrt(a)}. Floating only. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary SQRT = unary("SQRT", "sqrt", VectorSupport.VECTOR_OP_SQRT, VO_ONLYFP); + public static final /*float*/ Unary SQRT = unary("SQRT", "sqrt", VectorSupport.VECTOR_OP_SQRT, VO_ONLYFP | VO_SPECIAL); /** Produce {@code cbrt(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary CBRT = unary("CBRT", "cbrt", VectorSupport.VECTOR_OP_CBRT, VO_ONLYFP); + public static final /*float*/ Unary CBRT = unary("CBRT", "cbrt", VectorSupport.VECTOR_OP_CBRT, VO_ONLYFP | VO_SPECIAL); /** Produce {@code sinh(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary SINH = unary("SINH", "sinh", VectorSupport.VECTOR_OP_SINH, VO_ONLYFP); + public static final /*float*/ Unary SINH = unary("SINH", "sinh", VectorSupport.VECTOR_OP_SINH, VO_ONLYFP | VO_SPECIAL); /** Produce {@code cosh(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary COSH = unary("COSH", "cosh", VectorSupport.VECTOR_OP_COSH, VO_ONLYFP); + public static final /*float*/ Unary COSH = unary("COSH", "cosh", VectorSupport.VECTOR_OP_COSH, VO_ONLYFP | VO_SPECIAL); /** Produce {@code tanh(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary TANH = unary("TANH", "tanh", VectorSupport.VECTOR_OP_TANH, VO_ONLYFP); + public static final /*float*/ Unary TANH = unary("TANH", "tanh", VectorSupport.VECTOR_OP_TANH, VO_ONLYFP | VO_SPECIAL); /** Produce {@code expm1(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary EXPM1 = unary("EXPM1", "expm1", VectorSupport.VECTOR_OP_EXPM1, VO_ONLYFP); + public static final /*float*/ Unary EXPM1 = unary("EXPM1", "expm1", VectorSupport.VECTOR_OP_EXPM1, VO_ONLYFP | VO_SPECIAL); /** Produce {@code log1p(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary LOG1P = unary("LOG1P", "log1p", VectorSupport.VECTOR_OP_LOG1P, VO_ONLYFP); + public static final /*float*/ Unary LOG1P = unary("LOG1P", "log1p", VectorSupport.VECTOR_OP_LOG1P, VO_ONLYFP | VO_SPECIAL); // Binary operators @@ -615,15 +615,15 @@ static boolean opKind(Operator op, int bit) { /** Produce {@code atan2(a,b)}. See Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Binary ATAN2 = binary("ATAN2", "atan2", VectorSupport.VECTOR_OP_ATAN2, VO_ONLYFP); + public static final /*float*/ Binary ATAN2 = binary("ATAN2", "atan2", VectorSupport.VECTOR_OP_ATAN2, VO_ONLYFP | VO_SPECIAL); /** Produce {@code pow(a,b)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Binary POW = binary("POW", "pow", VectorSupport.VECTOR_OP_POW, VO_ONLYFP); + public static final /*float*/ Binary POW = binary("POW", "pow", VectorSupport.VECTOR_OP_POW, VO_ONLYFP | VO_SPECIAL); /** Produce {@code hypot(a,b)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Binary HYPOT = binary("HYPOT", "hypot", VectorSupport.VECTOR_OP_HYPOT, VO_ONLYFP); + public static final /*float*/ Binary HYPOT = binary("HYPOT", "hypot", VectorSupport.VECTOR_OP_HYPOT, VO_ONLYFP | VO_SPECIAL); // Ternary operators From 57487c89fbed7ec9543a8dc95a6a0c813ba5ebd5 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Wed, 20 Nov 2024 13:47:59 -0800 Subject: [PATCH 02/25] Vector math library --- .../jdk/internal/vm/vector/VectorSupport.java | 46 +++++++ .../jdk/incubator/vector/DoubleVector.java | 53 ++++--- .../jdk/incubator/vector/FloatVector.java | 58 +++++--- .../incubator/vector/VectorMathLibrary.java | 130 ++++++++++++++++++ .../jdk/incubator/vector/VectorOperators.java | 39 +++--- 5 files changed, 273 insertions(+), 53 deletions(-) create mode 100644 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index 6e1c363f3d95d..e69258c6d807f 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -28,6 +28,7 @@ import jdk.internal.vm.annotation.IntrinsicCandidate; import jdk.internal.misc.Unsafe; +import java.lang.foreign.MemorySegment; import java.util.function.*; public class VectorSupport { @@ -114,6 +115,9 @@ public class VectorSupport { public static final int VECTOR_OP_EXPM1 = 117; public static final int VECTOR_OP_HYPOT = 118; + public static final int VECTOR_OP_MATHLIB_FIRST = VECTOR_OP_TAN; + public static final int VECTOR_OP_MATHLIB_LAST = VECTOR_OP_HYPOT; + public static final int VECTOR_OP_SADD = 119; public static final int VECTOR_OP_SSUB = 120; public static final int VECTOR_OP_SUADD = 121; @@ -334,6 +338,26 @@ V unaryOp(int oprId, /* ============================================================================ */ +// public interface LibraryUnaryOperation, +// M extends VectorMask> { +// V apply(MemorySegment entry, V v, M m); +// } + +// @IntrinsicCandidate + public static + , + M extends VectorMask, + E> + V libraryUnaryOp(long addr, + Class vClass, Class mClass, Class eClass, int length, + V v, M m, + UnaryOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(v, m); + } + + /* ============================================================================ */ + public interface BinaryOperation> { VM apply(VM v1, VM v2, M m); @@ -352,6 +376,28 @@ VM binaryOp(int oprId, assert isNonCapturingLambda(defaultImpl) : defaultImpl; return defaultImpl.apply(v1, v2, m); } + + /* ============================================================================ */ + +// public interface LibraryBinaryOperation> { +// V apply(MemorySegment entry, V v1, V v2, M m); +// } + +// @IntrinsicCandidate + public static + , + E> + V libraryBinaryOp(long addr, + Class vmClass, Class mClass, Class eClass, + int length, + V v1, V v2, M m, + BinaryOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(v1, v2, m); + } + /* ============================================================================ */ public interface SelectFromTwoVector> { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index c0aafcbb224ba..5e6c1632f0523 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -677,11 +677,8 @@ DoubleVector lanewiseTemplate(VectorOperators.Unary op) { if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); - } else { - int opc = opCode(op); - if (opc >= VECTOR_OP_TAN && opc <= VECTOR_OP_EXPM1) { - return UN_IMPL.find(op, opc, DoubleVector::unaryOperations).apply(this, null); - } + } else if (opKind(op, VO_MATHLIB)) { + return unaryMathOp(op, null, null); } } int opc = opCode(op); @@ -707,11 +704,8 @@ DoubleVector lanewiseTemplate(VectorOperators.Unary op, if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); - } else { - int opc = opCode(op); - if (opc >= VECTOR_OP_TAN && opc <= VECTOR_OP_EXPM1) { - return UN_IMPL.find(op, opc, DoubleVector::unaryOperations).apply(this, m); - } + } else if (opKind(op, VO_MATHLIB)) { + return unaryMathOp(op, maskClass, m); } } int opc = opCode(op); @@ -721,6 +715,21 @@ opc, getClass(), maskClass, double.class, length(), UN_IMPL.find(op, opc, DoubleVector::unaryOperations)); } + + @ForceInline + final + DoubleVector unaryMathOp(VectorOperators.Unary op, + Class> maskClass, + VectorMask m) { + int opc = opCode(op); + MemorySegment addr = VectorMathLibrary.lookup(op, opc, species()); + + return VectorSupport.libraryUnaryOp( + addr.address(), getClass(), maskClass, double.class, length(), // constants + this, m, + UN_IMPL.find(op, opc, DoubleVector::unaryOperations)); + } + private static final ImplCache>> UN_IMPL = new ImplCache<>(Unary.class, DoubleVector.class); @@ -790,9 +799,8 @@ DoubleVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = this.viewAsIntegralLanes().compare(EQ, (long) 0); return this.blend(that, mask.cast(vspecies())); - } else if (op == ATAN2 || op == POW || op == HYPOT) { - int opc = opCode(op); - return BIN_IMPL.find(op, opc, DoubleVector::binaryOperations).apply(this, that, null); + } else if (opKind(op, VO_MATHLIB)) { + return binaryMathOp(op, null, that, null); } } @@ -827,9 +835,8 @@ DoubleVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = bits.compare(EQ, (long) 0, m.cast(bits.vspecies())); return this.blend(that, mask.cast(vspecies())); - } else if (op == ATAN2 || op == POW || op == HYPOT) { - int opc = opCode(op); - return BIN_IMPL.find(op, opc, DoubleVector::binaryOperations).apply(this, that, m); + } else if (opKind(op, VO_MATHLIB)) { + return binaryMathOp(op, maskClass, that, m); } } @@ -840,6 +847,20 @@ opc, getClass(), maskClass, double.class, length(), BIN_IMPL.find(op, opc, DoubleVector::binaryOperations)); } + @ForceInline + final + DoubleVector binaryMathOp(VectorOperators.Binary op, + Class> maskClass, + DoubleVector that, VectorMask m) { + int opc = opCode(op); + MemorySegment addr = VectorMathLibrary.lookup(op, opc, species()); + + return VectorSupport.libraryBinaryOp( + addr.address(), getClass(), maskClass, double.class, length(), + this, that, m, + BIN_IMPL.find(op, opc, DoubleVector::binaryOperations)); + } + private static final ImplCache>> BIN_IMPL = new ImplCache<>(Binary.class, DoubleVector.class); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index e46886b11a927..f1a39b15b9ab7 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -677,11 +677,8 @@ FloatVector lanewiseTemplate(VectorOperators.Unary op) { if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); - } else { - int opc = opCode(op); - if (opc >= VECTOR_OP_TAN && opc <= VECTOR_OP_EXPM1) { - return UN_IMPL.find(op, opc, FloatVector::unaryOperations).apply(this, null); - } + } else if (opKind(op, VO_MATHLIB)) { + return unaryMathOp(op, null, null); } } int opc = opCode(op); @@ -707,11 +704,8 @@ FloatVector lanewiseTemplate(VectorOperators.Unary op, if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); - } else { - int opc = opCode(op); - if (opc >= VECTOR_OP_TAN && opc <= VECTOR_OP_EXPM1) { - return UN_IMPL.find(op, opc, FloatVector::unaryOperations).apply(this, m); - } + } else if (opKind(op, VO_MATHLIB)) { + return unaryMathOp(op, maskClass, m); } } int opc = opCode(op); @@ -721,6 +715,21 @@ opc, getClass(), maskClass, float.class, length(), UN_IMPL.find(op, opc, FloatVector::unaryOperations)); } + + @ForceInline + final + FloatVector unaryMathOp(VectorOperators.Unary op, + Class> maskClass, + VectorMask m) { + int opc = opCode(op); + MemorySegment addr = VectorMathLibrary.lookup(op, opc, species()); + + return VectorSupport.libraryUnaryOp( + addr.address(), getClass(), maskClass, float.class, length(), // constants + this, m, + UN_IMPL.find(op, opc, FloatVector::unaryOperations)); + } + private static final ImplCache>> UN_IMPL = new ImplCache<>(Unary.class, FloatVector.class); @@ -790,9 +799,8 @@ FloatVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = this.viewAsIntegralLanes().compare(EQ, (int) 0); return this.blend(that, mask.cast(vspecies())); - } else if (op == ATAN2 || op == POW || op == HYPOT) { - int opc = opCode(op); - return BIN_IMPL.find(op, opc, FloatVector::binaryOperations).apply(this, that, null); + } else if (opKind(op, VO_MATHLIB)) { + return binaryMathOp(op, null, that, null); } } @@ -815,8 +823,8 @@ FloatVector lanewise(VectorOperators.Binary op, @ForceInline final FloatVector lanewiseTemplate(VectorOperators.Binary op, - Class> maskClass, - Vector v, VectorMask m) { + Class> maskClass, + Vector v, VectorMask m) { FloatVector that = (FloatVector) v; that.check(this); m.check(maskClass, this); @@ -827,9 +835,8 @@ FloatVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = bits.compare(EQ, (int) 0, m.cast(bits.vspecies())); return this.blend(that, mask.cast(vspecies())); - } else if (op == ATAN2 || op == POW || op == HYPOT) { - int opc = opCode(op); - return BIN_IMPL.find(op, opc, FloatVector::binaryOperations).apply(this, that, m); + } else if (opKind(op, VO_MATHLIB)) { + return binaryMathOp(op, maskClass, that, m); } } @@ -840,6 +847,21 @@ opc, getClass(), maskClass, float.class, length(), BIN_IMPL.find(op, opc, FloatVector::binaryOperations)); } + + @ForceInline + final + FloatVector binaryMathOp(VectorOperators.Binary op, + Class> maskClass, + FloatVector that, VectorMask m) { + int opc = opCode(op); + MemorySegment addr = VectorMathLibrary.lookup(op, opc, species()); + + return VectorSupport.libraryBinaryOp( + addr.address(), getClass(), maskClass, float.class, length(), + this, that, m, + BIN_IMPL.find(op, opc, FloatVector::binaryOperations)); + } + private static final ImplCache>> BIN_IMPL = new ImplCache<>(Binary.class, FloatVector.class); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java new file mode 100644 index 0000000000000..1156ca080967b --- /dev/null +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -0,0 +1,130 @@ +package jdk.incubator.vector; + +import jdk.internal.vm.annotation.Stable; +import jdk.internal.vm.vector.VectorSupport; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; + +import static jdk.incubator.vector.VectorOperators.*; + +/*package-private*/ class VectorMathLibrary { + static final boolean DEBUG = Boolean.getBoolean("jdk.incubator.vector.VectorMathLibrary.DEBUG"); + + static final SymbolLookup LOOKUP = SymbolLookup.loaderLookup(); + + // SVML method naming convention + // All the methods are named as __jsvml__ha_ + // Where: + // ha stands for high accuracy + // is optional to indicate float/double + // Set to f for vector float operation + // Omitted for vector double operation + // is the number of elements in the vector + // 1, 2, 4, 8, 16 + // e.g. 128 bit float vector has 4 float elements + // indicates the avx/sse level: + // z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2 + // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns + // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns + static class SVML { + static { + loadNativeLibrary(); + } + + @SuppressWarnings({"removal", "restricted"}) + private static void loadNativeLibrary() { + System.loadLibrary("jsvml"); + } + + static int AVX = 0; + static String suffix(VectorShape vshape) { + String avx_sse_str = (AVX >= 2) ? "l9" : ((AVX == 1) ? "e9" : "ex"); + return switch (vshape) { + case S_64_BIT -> avx_sse_str; + case S_128_BIT -> avx_sse_str; + case S_256_BIT -> avx_sse_str; + case S_512_BIT -> "z0"; + case S_Max_BIT -> throw new InternalError("NYI"); + }; + } + + static String symbolName(Operator op, VectorSpecies vspecies) { + String suffix = suffix(vspecies.vectorShape()); + return String.format("__jsvml_%s%s%d_%s", op.operatorName(), vspecies.elementType(), vspecies.length(), suffix); + } + + // VectorSupport::VEC_SIZE_512: + // if ((!VM_Version::supports_avx512dq()) && + // (vop == VectorSupport::VECTOR_OP_LOG || vop == VectorSupport::VECTOR_OP_LOG10 || vop == VectorSupport::VECTOR_OP_POW)) { + // continue; + // } + // if (vop == VectorSupport::VECTOR_OP_POW) { + // continue; + // } + } + + static class SLEEF { + static { + loadNativeLibrary(); + } + + @SuppressWarnings({"removal", "restricted"}) + private static void loadNativeLibrary() { + System.loadLibrary("sleef"); + } + + static String suffix(VectorShape vshape) { + return "advsimd"; // FIXME + } + + static String precisionLevel(Operator op) { + return (op == HYPOT ? "u05" : "u10"); + } + + // Method naming convention + // All the methods are named as _ + // Where: + // is the operation name, e.g. sin + // is optional to indicate float/double + // "f/d" for vector float/double operation + // is the number of elements in the vector + // "2/4" for neon, and "x" for sve + // is the precision level + // "u10/u05" represents 1.0/0.5 ULP error bounds + // We use "u10" for all operations by default + // But for those functions do not have u10 support, we use "u05" instead + // indicates neon/sve + // "sve/advsimd" for sve/neon implementations + // e.g. sinfx_u10sve is the method for computing vector float sin using SVE instructions + // cosd2_u10advsimd is the method for computing 2 elements vector double cos using NEON instructions + static String symbolName(Operator op, VectorSpecies vspecies) { + return String.format("%s%s%d_%s%s", op.operatorName(), + (vspecies.elementType() == float.class ? "f" : "d"), + vspecies.length(), + precisionLevel(op), + suffix(vspecies.vectorShape())); + } + } + + static final int SIZE = VectorSupport.VECTOR_OP_MATHLIB_LAST - VectorSupport.VECTOR_OP_MATHLIB_FIRST + 1; + + static final @Stable MemorySegment[][][] LIBRARY = new MemorySegment[SIZE][LaneType.SK_LIMIT][VectorShape.SK_LIMIT]; // OP x SHAPE x TYPE + + static MemorySegment lookup(Operator op, int opc, VectorSpecies vspecies) { + int idx = opc - VectorSupport.VECTOR_OP_MATHLIB_FIRST; + int elem_idx = ((AbstractSpecies)vspecies).laneType.switchKey; + int shape_idx = vspecies.vectorShape().switchKey; + MemorySegment s = LIBRARY[idx][elem_idx][shape_idx]; + if (s == null) { + String symbol = SLEEF.symbolName(op, vspecies); // FIXME + s = LOOKUP.find(symbol).orElse(MemorySegment.NULL); // FIXME + if (DEBUG) { + System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, s.address()); + } + LIBRARY[idx][elem_idx][shape_idx] = s; + } + return s; + } + +} diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java index d0b2d6b4d035b..c8cb5e0a904cc 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java @@ -426,6 +426,7 @@ static boolean opKind(Operator op, int bit) { VO_SPECIAL = 0x080, // random special handling VO_NOFP = 0x100, VO_ONLYFP = 0x200, + VO_MATHLIB = 0x400, VO_OPCODE_VALID = 0x800, VO_OPCODE_SHIFT = 12, VO_OPCODE_LIMIT = 0x400, @@ -476,67 +477,67 @@ static boolean opKind(Operator op, int bit) { /** Produce {@code sin(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary SIN = unary("SIN", "sin", VectorSupport.VECTOR_OP_SIN, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary SIN = unary("SIN", "sin", VectorSupport.VECTOR_OP_SIN, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code cos(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary COS = unary("COS", "cos", VectorSupport.VECTOR_OP_COS, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary COS = unary("COS", "cos", VectorSupport.VECTOR_OP_COS, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code tan(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary TAN = unary("TAN", "tan", VectorSupport.VECTOR_OP_TAN, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary TAN = unary("TAN", "tan", VectorSupport.VECTOR_OP_TAN, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code asin(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary ASIN = unary("ASIN", "asin", VectorSupport.VECTOR_OP_ASIN, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary ASIN = unary("ASIN", "asin", VectorSupport.VECTOR_OP_ASIN, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code acos(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary ACOS = unary("ACOS", "acos", VectorSupport.VECTOR_OP_ACOS, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary ACOS = unary("ACOS", "acos", VectorSupport.VECTOR_OP_ACOS, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code atan(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary ATAN = unary("ATAN", "atan", VectorSupport.VECTOR_OP_ATAN, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary ATAN = unary("ATAN", "atan", VectorSupport.VECTOR_OP_ATAN, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code exp(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary EXP = unary("EXP", "exp", VectorSupport.VECTOR_OP_EXP, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary EXP = unary("EXP", "exp", VectorSupport.VECTOR_OP_EXP, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code log(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary LOG = unary("LOG", "log", VectorSupport.VECTOR_OP_LOG, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary LOG = unary("LOG", "log", VectorSupport.VECTOR_OP_LOG, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code log10(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary LOG10 = unary("LOG10", "log10", VectorSupport.VECTOR_OP_LOG10, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary LOG10 = unary("LOG10", "log10", VectorSupport.VECTOR_OP_LOG10, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code sqrt(a)}. Floating only. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary SQRT = unary("SQRT", "sqrt", VectorSupport.VECTOR_OP_SQRT, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary SQRT = unary("SQRT", "sqrt", VectorSupport.VECTOR_OP_SQRT, VO_ONLYFP); /** Produce {@code cbrt(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary CBRT = unary("CBRT", "cbrt", VectorSupport.VECTOR_OP_CBRT, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary CBRT = unary("CBRT", "cbrt", VectorSupport.VECTOR_OP_CBRT, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code sinh(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary SINH = unary("SINH", "sinh", VectorSupport.VECTOR_OP_SINH, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary SINH = unary("SINH", "sinh", VectorSupport.VECTOR_OP_SINH, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code cosh(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary COSH = unary("COSH", "cosh", VectorSupport.VECTOR_OP_COSH, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary COSH = unary("COSH", "cosh", VectorSupport.VECTOR_OP_COSH, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code tanh(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary TANH = unary("TANH", "tanh", VectorSupport.VECTOR_OP_TANH, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary TANH = unary("TANH", "tanh", VectorSupport.VECTOR_OP_TANH, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code expm1(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary EXPM1 = unary("EXPM1", "expm1", VectorSupport.VECTOR_OP_EXPM1, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary EXPM1 = unary("EXPM1", "expm1", VectorSupport.VECTOR_OP_EXPM1, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code log1p(a)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Unary LOG1P = unary("LOG1P", "log1p", VectorSupport.VECTOR_OP_LOG1P, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Unary LOG1P = unary("LOG1P", "log1p", VectorSupport.VECTOR_OP_LOG1P, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); // Binary operators @@ -615,15 +616,15 @@ static boolean opKind(Operator op, int bit) { /** Produce {@code atan2(a,b)}. See Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Binary ATAN2 = binary("ATAN2", "atan2", VectorSupport.VECTOR_OP_ATAN2, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Binary ATAN2 = binary("ATAN2", "atan2", VectorSupport.VECTOR_OP_ATAN2, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code pow(a,b)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Binary POW = binary("POW", "pow", VectorSupport.VECTOR_OP_POW, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Binary POW = binary("POW", "pow", VectorSupport.VECTOR_OP_POW, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); /** Produce {@code hypot(a,b)}. Floating only. * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above */ - public static final /*float*/ Binary HYPOT = binary("HYPOT", "hypot", VectorSupport.VECTOR_OP_HYPOT, VO_ONLYFP | VO_SPECIAL); + public static final /*float*/ Binary HYPOT = binary("HYPOT", "hypot", VectorSupport.VECTOR_OP_HYPOT, VO_ONLYFP | VO_SPECIAL | VO_MATHLIB); // Ternary operators From a92a2a88ff36787b4fde5b97519c09f3e726fb68 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Wed, 20 Nov 2024 17:20:12 -0800 Subject: [PATCH 03/25] VM intrinsics --- src/hotspot/share/ci/ciInstance.cpp | 6 ++ src/hotspot/share/ci/ciInstance.hpp | 1 + src/hotspot/share/classfile/vmIntrinsics.hpp | 26 ++++- src/hotspot/share/opto/c2compiler.cpp | 3 + src/hotspot/share/opto/library_call.cpp | 4 + src/hotspot/share/opto/library_call.hpp | 1 + src/hotspot/share/opto/vectorIntrinsics.cpp | 99 +++++++++++++++++++ .../jdk/internal/vm/vector/VectorSupport.java | 33 +++---- .../jdk/incubator/vector/DoubleVector.java | 34 +++---- .../jdk/incubator/vector/FloatVector.java | 33 +++---- .../incubator/vector/VectorMathLibrary.java | 19 ++-- 11 files changed, 196 insertions(+), 63 deletions(-) diff --git a/src/hotspot/share/ci/ciInstance.cpp b/src/hotspot/share/ci/ciInstance.cpp index ad456ba6726b8..9591298e3ab04 100644 --- a/src/hotspot/share/ci/ciInstance.cpp +++ b/src/hotspot/share/ci/ciInstance.cpp @@ -138,3 +138,9 @@ ciKlass* ciInstance::java_lang_Class_klass() { assert(java_lang_Class::as_Klass(get_oop()) != nullptr, "klass is null"); return CURRENT_ENV->get_metadata(java_lang_Class::as_Klass(get_oop()))->as_klass(); } + +char* ciInstance::java_lang_String_str(char* buf, size_t buflen) { + VM_ENTRY_MARK; + assert(get_oop()->is_a(vmClasses::String_klass()), "not a String"); + return java_lang_String::as_utf8_string(get_oop(), buf, buflen); +} diff --git a/src/hotspot/share/ci/ciInstance.hpp b/src/hotspot/share/ci/ciInstance.hpp index 3af07edcd9e89..1fb0998593088 100644 --- a/src/hotspot/share/ci/ciInstance.hpp +++ b/src/hotspot/share/ci/ciInstance.hpp @@ -67,6 +67,7 @@ class ciInstance : public ciObject { ciConstant field_value_by_offset(int field_offset); ciKlass* java_lang_Class_klass(); + char* java_lang_String_str(char* buf, size_t buflen); }; #endif // SHARE_CI_CIINSTANCE_HPP diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 93b67301b4bad..ee80a064a302f 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -980,7 +980,8 @@ class methodHandle; do_intrinsic(_VectorUnaryOp, jdk_internal_vm_vector_VectorSupport, vector_unary_op_name, vector_unary_op_sig, F_S) \ do_signature(vector_unary_op_sig, "(I" \ "Ljava/lang/Class;" \ - "Ljava/lang/Class;Ljava/lang/Class;" \ + "Ljava/lang/Class;" \ + "Ljava/lang/Class;" \ "I" \ "Ljdk/internal/vm/vector/VectorSupport$Vector;" \ "Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \ @@ -1001,6 +1002,29 @@ class methodHandle; "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;") \ do_name(vector_binary_op_name, "binaryOp") \ \ + do_intrinsic(_VectorUnaryLibOp, jdk_internal_vm_vector_VectorSupport, vector_unary_lib_op_name, vector_unary_lib_op_sig, F_S) \ + do_signature(vector_unary_lib_op_sig,"(J" \ + "Ljava/lang/Class;" \ + "Ljava/lang/Class;" \ + "I" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;" \ + "Ljdk/internal/vm/vector/VectorSupport$UnaryOperation;" \ + "Ljava/lang/String;)" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;") \ + do_name(vector_unary_lib_op_name, "libraryUnaryOp") \ + \ + do_intrinsic(_VectorBinaryLibOp, jdk_internal_vm_vector_VectorSupport, vector_binary_lib_op_name, vector_binary_lib_op_sig, F_S) \ + do_signature(vector_binary_lib_op_sig,"(J" \ + "Ljava/lang/Class;" \ + "Ljava/lang/Class;" \ + "I" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \ + "Ljdk/internal/vm/vector/VectorSupport$BinaryOperation;" \ + "Ljava/lang/String;)" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;") \ + do_name(vector_binary_lib_op_name, "libraryBinaryOp") \ + \ do_intrinsic(_VectorTernaryOp, jdk_internal_vm_vector_VectorSupport, vector_ternary_op_name, vector_ternary_op_sig, F_S) \ do_signature(vector_ternary_op_sig, "(I" \ "Ljava/lang/Class;" \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index 3effa8eee0498..b0dee4f24dcac 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -841,6 +841,9 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { case vmIntrinsics::_IndexVector: case vmIntrinsics::_IndexPartiallyInUpperRange: return EnableVectorSupport; + case vmIntrinsics::_VectorUnaryLibOp: + case vmIntrinsics::_VectorBinaryLibOp: + return EnableVectorSupport && Matcher::supports_vector_calling_convention(); case vmIntrinsics::_blackhole: #if INCLUDE_JVMTI case vmIntrinsics::_notifyJvmtiVThreadStart: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 3bb432ac6077b..97660a6104815 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -707,6 +707,10 @@ bool LibraryCallKit::try_to_inline(int predicate) { return inline_vector_nary_operation(1); case vmIntrinsics::_VectorBinaryOp: return inline_vector_nary_operation(2); + case vmIntrinsics::_VectorUnaryLibOp: + return inline_vector_call(1); + case vmIntrinsics::_VectorBinaryLibOp: + return inline_vector_call(2); case vmIntrinsics::_VectorTernaryOp: return inline_vector_nary_operation(3); case vmIntrinsics::_VectorFromBitsCoerced: diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 1f83e28932b70..a5ee0c521d309 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -362,6 +362,7 @@ class LibraryCallKit : public GraphKit { // Vector API support bool inline_vector_nary_operation(int n); + bool inline_vector_call(int arity); bool inline_vector_frombits_coerced(); bool inline_vector_mask_operation(); bool inline_vector_mem_operation(bool is_store); diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index 87a5f5fd85d52..9b37d97948073 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -476,6 +476,105 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { } // public static +// , E> +// V unaryOp(long address, Class vClass, Class elementType, int length, +// V v, +// UnaryOperation defaultImpl) +// +// public static +// +// V binaryOp(long address, Class vClass, Class elementType, int length, +// V v1, V v2, +// BinaryOperation defaultImpl) +bool LibraryCallKit::inline_vector_call(int arity) { + assert(Matcher::supports_vector_calling_convention(), "required"); + + const TypeLong* entry = gvn().type(argument(0))->isa_long(); + const TypeInstPtr* vector_klass = gvn().type(argument(2))->isa_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr(); + const TypeInt* vlen = gvn().type(argument(4))->isa_int(); + + if (entry == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || + !entry->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) { + log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()], + NodeClassNames[argument(4)->Opcode()]); + return false; // not enough info for intrinsification + } + + if (entry->get_con() == 0) { + log_if_needed(" ** missing entry point"); + return false; + } + + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type()); + return false; // should be primitive type + } + if (!is_klass_initialized(vector_klass)) { + log_if_needed(" ** klass argument not initialized"); + return false; + } + + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + if (!Matcher::vector_size_supported(elem_bt, num_elem)) { + log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported", + num_elem, type2name(elem_bt)); + return false; + } + + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); + if (opd1 == nullptr) { + log_if_needed(" ** unbox failed v1=%s", + NodeClassNames[argument(5)->Opcode()]); + return false; + } + + Node* opd2 = nullptr; + if (arity > 1) { + opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); + if (opd2 == nullptr) { + log_if_needed(" ** unbox failed v2=%s", + NodeClassNames[argument(6)->Opcode()]); + return false; + } + } + assert(arity == 1 || arity == 2, "not supported"); + const TypeVect* vt = TypeVect::make(elem_bt, num_elem); + const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(arity, vt, vt); + address entry_addr = (address)entry->get_con(); + + const char* debug_name = ""; + const TypeInstPtr* debug_name_oop = gvn().type(argument(8))->isa_instptr(); + if (debug_name_oop != nullptr && debug_name_oop->const_oop() && !debug_name_oop->const_oop()->is_null_object()) { + size_t buflen = 100; + char* buf = NEW_ARENA_ARRAY(C->comp_arena(), char, buflen); + debug_name = debug_name_oop->const_oop()->as_instance()->java_lang_String_str(buf, buflen); + } + Node* vcall = make_runtime_call(RC_VECTOR, + call_type, + entry_addr, + debug_name, + TypePtr::BOTTOM, + opd1, + opd2); + + vcall = gvn().transform(new ProjNode(gvn().transform(vcall), TypeFunc::Parms)); + + // Wrap it up in VectorBox to keep object type information. + Node* vbox = box_vector(vcall, vbox_type, elem_bt, num_elem); + set_result(vbox); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + // // long maskReductionCoerced(int oper, Class maskClass, Class elemClass, // int length, M m, VectorMaskOp defaultImpl) diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index e69258c6d807f..07d41c279da38 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -343,17 +343,15 @@ V unaryOp(int oprId, // V apply(MemorySegment entry, V v, M m); // } -// @IntrinsicCandidate + @IntrinsicCandidate public static - , - M extends VectorMask, - E> - V libraryUnaryOp(long addr, - Class vClass, Class mClass, Class eClass, int length, - V v, M m, - UnaryOperation defaultImpl) { + , E> + V libraryUnaryOp(long addr, Class vClass, Class eClass, int length, + V v, + UnaryOperation defaultImpl, + String debugName) { assert isNonCapturingLambda(defaultImpl) : defaultImpl; - return defaultImpl.apply(v, m); + return defaultImpl.apply(v, null); } /* ============================================================================ */ @@ -384,18 +382,15 @@ VM binaryOp(int oprId, // V apply(MemorySegment entry, V v1, V v2, M m); // } -// @IntrinsicCandidate + @IntrinsicCandidate public static - , - E> - V libraryBinaryOp(long addr, - Class vmClass, Class mClass, Class eClass, - int length, - V v1, V v2, M m, - BinaryOperation defaultImpl) { + + V libraryBinaryOp(long addr, Class vClass, Class eClass, int length, + V v1, V v2, + BinaryOperation defaultImpl, + String debugName) { assert isNonCapturingLambda(defaultImpl) : defaultImpl; - return defaultImpl.apply(v1, v2, m); + return defaultImpl.apply(v1, v2, null); } /* ============================================================================ */ diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index 5e6c1632f0523..71711a9623633 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -678,7 +678,7 @@ DoubleVector lanewiseTemplate(VectorOperators.Unary op) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); } else if (opKind(op, VO_MATHLIB)) { - return unaryMathOp(op, null, null); + return unaryMathOp(op); } } int opc = opCode(op); @@ -705,7 +705,7 @@ DoubleVector lanewiseTemplate(VectorOperators.Unary op, if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); } else if (opKind(op, VO_MATHLIB)) { - return unaryMathOp(op, maskClass, m); + return blend(unaryMathOp(op), m); } } int opc = opCode(op); @@ -718,16 +718,15 @@ opc, getClass(), maskClass, double.class, length(), @ForceInline final - DoubleVector unaryMathOp(VectorOperators.Unary op, - Class> maskClass, - VectorMask m) { + DoubleVector unaryMathOp(VectorOperators.Unary op) { int opc = opCode(op); - MemorySegment addr = VectorMathLibrary.lookup(op, opc, species()); + VectorMathLibrary.Entry entry = VectorMathLibrary.lookup(op, opc, species()); return VectorSupport.libraryUnaryOp( - addr.address(), getClass(), maskClass, double.class, length(), // constants - this, m, - UN_IMPL.find(op, opc, DoubleVector::unaryOperations)); + entry.entry().address(), getClass(), double.class, length(), // constants + this, + UN_IMPL.find(op, opc, DoubleVector::unaryOperations), + entry.name()); } private static final @@ -800,7 +799,7 @@ DoubleVector lanewiseTemplate(VectorOperators.Binary op, = this.viewAsIntegralLanes().compare(EQ, (long) 0); return this.blend(that, mask.cast(vspecies())); } else if (opKind(op, VO_MATHLIB)) { - return binaryMathOp(op, null, that, null); + return binaryMathOp(op, that); } } @@ -836,7 +835,7 @@ DoubleVector lanewiseTemplate(VectorOperators.Binary op, = bits.compare(EQ, (long) 0, m.cast(bits.vspecies())); return this.blend(that, mask.cast(vspecies())); } else if (opKind(op, VO_MATHLIB)) { - return binaryMathOp(op, maskClass, that, m); + return this.blend(binaryMathOp(op, that), m); } } @@ -849,16 +848,15 @@ opc, getClass(), maskClass, double.class, length(), @ForceInline final - DoubleVector binaryMathOp(VectorOperators.Binary op, - Class> maskClass, - DoubleVector that, VectorMask m) { + DoubleVector binaryMathOp(VectorOperators.Binary op, DoubleVector that) { int opc = opCode(op); - MemorySegment addr = VectorMathLibrary.lookup(op, opc, species()); + VectorMathLibrary.Entry entry = VectorMathLibrary.lookup(op, opc, species()); return VectorSupport.libraryBinaryOp( - addr.address(), getClass(), maskClass, double.class, length(), - this, that, m, - BIN_IMPL.find(op, opc, DoubleVector::binaryOperations)); + entry.entry().address(), getClass(), double.class, length(), + this, that, + BIN_IMPL.find(op, opc, DoubleVector::binaryOperations), + entry.name()); } private static final diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index f1a39b15b9ab7..1735f14db1529 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -678,7 +678,7 @@ FloatVector lanewiseTemplate(VectorOperators.Unary op) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); } else if (opKind(op, VO_MATHLIB)) { - return unaryMathOp(op, null, null); + return unaryMathOp(op); } } int opc = opCode(op); @@ -705,7 +705,7 @@ FloatVector lanewiseTemplate(VectorOperators.Unary op, if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); } else if (opKind(op, VO_MATHLIB)) { - return unaryMathOp(op, maskClass, m); + return blend(unaryMathOp(op), m); } } int opc = opCode(op); @@ -718,16 +718,15 @@ opc, getClass(), maskClass, float.class, length(), @ForceInline final - FloatVector unaryMathOp(VectorOperators.Unary op, - Class> maskClass, - VectorMask m) { + FloatVector unaryMathOp(VectorOperators.Unary op) { int opc = opCode(op); - MemorySegment addr = VectorMathLibrary.lookup(op, opc, species()); + VectorMathLibrary.Entry entry = VectorMathLibrary.lookup(op, opc, species()); return VectorSupport.libraryUnaryOp( - addr.address(), getClass(), maskClass, float.class, length(), // constants - this, m, - UN_IMPL.find(op, opc, FloatVector::unaryOperations)); + entry.entry().address(), getClass(), float.class, length(), // constants + this, + UN_IMPL.find(op, opc, FloatVector::unaryOperations), + entry.name()); } private static final @@ -800,7 +799,7 @@ FloatVector lanewiseTemplate(VectorOperators.Binary op, = this.viewAsIntegralLanes().compare(EQ, (int) 0); return this.blend(that, mask.cast(vspecies())); } else if (opKind(op, VO_MATHLIB)) { - return binaryMathOp(op, null, that, null); + return binaryMathOp(op, that); } } @@ -836,7 +835,7 @@ FloatVector lanewiseTemplate(VectorOperators.Binary op, = bits.compare(EQ, (int) 0, m.cast(bits.vspecies())); return this.blend(that, mask.cast(vspecies())); } else if (opKind(op, VO_MATHLIB)) { - return binaryMathOp(op, maskClass, that, m); + return this.blend(binaryMathOp(op, that), m); } } @@ -851,15 +850,15 @@ opc, getClass(), maskClass, float.class, length(), @ForceInline final FloatVector binaryMathOp(VectorOperators.Binary op, - Class> maskClass, - FloatVector that, VectorMask m) { + FloatVector that) { int opc = opCode(op); - MemorySegment addr = VectorMathLibrary.lookup(op, opc, species()); + VectorMathLibrary.Entry entry = VectorMathLibrary.lookup(op, opc, species()); return VectorSupport.libraryBinaryOp( - addr.address(), getClass(), maskClass, float.class, length(), - this, that, m, - BIN_IMPL.find(op, opc, FloatVector::binaryOperations)); + entry.entry().address(), getClass(), float.class, length(), + this, that, + BIN_IMPL.find(op, opc, FloatVector::binaryOperations), + entry.name()); } private static final diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index 1156ca080967b..e4647d1aef243 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -109,22 +109,25 @@ static String symbolName(Operator op, VectorSpecies vspecies) { static final int SIZE = VectorSupport.VECTOR_OP_MATHLIB_LAST - VectorSupport.VECTOR_OP_MATHLIB_FIRST + 1; - static final @Stable MemorySegment[][][] LIBRARY = new MemorySegment[SIZE][LaneType.SK_LIMIT][VectorShape.SK_LIMIT]; // OP x SHAPE x TYPE + record Entry (String name, MemorySegment entry) {} - static MemorySegment lookup(Operator op, int opc, VectorSpecies vspecies) { + static final @Stable Entry[][][] LIBRARY = new Entry[SIZE][LaneType.SK_LIMIT][VectorShape.SK_LIMIT]; // OP x SHAPE x TYPE + + static Entry lookup(Operator op, int opc, VectorSpecies vspecies) { int idx = opc - VectorSupport.VECTOR_OP_MATHLIB_FIRST; int elem_idx = ((AbstractSpecies)vspecies).laneType.switchKey; int shape_idx = vspecies.vectorShape().switchKey; - MemorySegment s = LIBRARY[idx][elem_idx][shape_idx]; - if (s == null) { + Entry entry = LIBRARY[idx][elem_idx][shape_idx]; + if (entry == null) { String symbol = SLEEF.symbolName(op, vspecies); // FIXME - s = LOOKUP.find(symbol).orElse(MemorySegment.NULL); // FIXME + MemorySegment addr = LOOKUP.find(symbol).orElse(MemorySegment.NULL); // FIXME if (DEBUG) { - System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, s.address()); + System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, addr.address()); } - LIBRARY[idx][elem_idx][shape_idx] = s; + entry = new Entry(symbol, addr); + LIBRARY[idx][elem_idx][shape_idx] = entry; } - return s; + return entry; } } From 714e4d04c1d7274fb22f04244da71a70d4426ecc Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Wed, 20 Nov 2024 19:17:45 -0800 Subject: [PATCH 04/25] VectorMathLib: Migrate to lambdas --- .../jdk/incubator/vector/DoubleVector.java | 20 +--- .../jdk/incubator/vector/FloatVector.java | 19 +--- .../incubator/vector/VectorMathLibrary.java | 96 +++++++++++++++---- 3 files changed, 87 insertions(+), 48 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index 71711a9623633..6eb3fe82299b1 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -719,14 +719,8 @@ opc, getClass(), maskClass, double.class, length(), @ForceInline final DoubleVector unaryMathOp(VectorOperators.Unary op) { - int opc = opCode(op); - VectorMathLibrary.Entry entry = VectorMathLibrary.lookup(op, opc, species()); - - return VectorSupport.libraryUnaryOp( - entry.entry().address(), getClass(), double.class, length(), // constants - this, - UN_IMPL.find(op, opc, DoubleVector::unaryOperations), - entry.name()); + return VectorMathLibrary.unaryMathOp(op, opCode(op), species(), DoubleVector::unaryOperations, + this); } private static final @@ -849,14 +843,8 @@ opc, getClass(), maskClass, double.class, length(), @ForceInline final DoubleVector binaryMathOp(VectorOperators.Binary op, DoubleVector that) { - int opc = opCode(op); - VectorMathLibrary.Entry entry = VectorMathLibrary.lookup(op, opc, species()); - - return VectorSupport.libraryBinaryOp( - entry.entry().address(), getClass(), double.class, length(), - this, that, - BIN_IMPL.find(op, opc, DoubleVector::binaryOperations), - entry.name()); + return VectorMathLibrary.binaryMathOp(op, opCode(op), species(), DoubleVector::binaryOperations, + this, that); } private static final diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index 1735f14db1529..733b08db55e67 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -719,14 +719,7 @@ opc, getClass(), maskClass, float.class, length(), @ForceInline final FloatVector unaryMathOp(VectorOperators.Unary op) { - int opc = opCode(op); - VectorMathLibrary.Entry entry = VectorMathLibrary.lookup(op, opc, species()); - - return VectorSupport.libraryUnaryOp( - entry.entry().address(), getClass(), float.class, length(), // constants - this, - UN_IMPL.find(op, opc, FloatVector::unaryOperations), - entry.name()); + return VectorMathLibrary.unaryMathOp(op, opCode(op), species(), FloatVector::unaryOperations, this); } private static final @@ -851,14 +844,8 @@ opc, getClass(), maskClass, float.class, length(), final FloatVector binaryMathOp(VectorOperators.Binary op, FloatVector that) { - int opc = opCode(op); - VectorMathLibrary.Entry entry = VectorMathLibrary.lookup(op, opc, species()); - - return VectorSupport.libraryBinaryOp( - entry.entry().address(), getClass(), float.class, length(), - this, that, - BIN_IMPL.find(op, opc, FloatVector::binaryOperations), - entry.name()); + return VectorMathLibrary.binaryMathOp(op, opCode(op), species(), FloatVector::binaryOperations, + this, that); } private static final diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index e4647d1aef243..878b88485c57a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -1,17 +1,20 @@ package jdk.incubator.vector; +import jdk.internal.vm.annotation.DontInline; +import jdk.internal.vm.annotation.ForceInline; import jdk.internal.vm.annotation.Stable; import jdk.internal.vm.vector.VectorSupport; import java.lang.foreign.MemorySegment; import java.lang.foreign.SymbolLookup; +import java.util.function.IntFunction; import static jdk.incubator.vector.VectorOperators.*; /*package-private*/ class VectorMathLibrary { - static final boolean DEBUG = Boolean.getBoolean("jdk.incubator.vector.VectorMathLibrary.DEBUG"); + private static final boolean DEBUG = Boolean.getBoolean("jdk.incubator.vector.VectorMathLibrary.DEBUG"); - static final SymbolLookup LOOKUP = SymbolLookup.loaderLookup(); + private static final SymbolLookup LOOKUP = SymbolLookup.loaderLookup(); // SVML method naming convention // All the methods are named as __jsvml__ha_ @@ -27,7 +30,7 @@ // z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2 // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns - static class SVML { + private static class SVML { static { loadNativeLibrary(); } @@ -64,7 +67,7 @@ static String symbolName(Operator op, VectorSpecies vspecies) { // } } - static class SLEEF { + private static class SLEEF { static { loadNativeLibrary(); } @@ -107,27 +110,88 @@ static String symbolName(Operator op, VectorSpecies vspecies) { } } - static final int SIZE = VectorSupport.VECTOR_OP_MATHLIB_LAST - VectorSupport.VECTOR_OP_MATHLIB_FIRST + 1; + private static final int SIZE = VectorSupport.VECTOR_OP_MATHLIB_LAST - VectorSupport.VECTOR_OP_MATHLIB_FIRST + 1; - record Entry (String name, MemorySegment entry) {} + private record Entry (String name, MemorySegment entry, T impl) {} - static final @Stable Entry[][][] LIBRARY = new Entry[SIZE][LaneType.SK_LIMIT][VectorShape.SK_LIMIT]; // OP x SHAPE x TYPE + private static final @Stable Entry[][][] LIBRARY = new Entry[SIZE][LaneType.SK_LIMIT][VectorShape.SK_LIMIT]; // OP x SHAPE x TYPE - static Entry lookup(Operator op, int opc, VectorSpecies vspecies) { + @ForceInline + private static Entry lookup(Operator op, int opc, VectorSpecies vspecies, IntFunction implSupplier) { int idx = opc - VectorSupport.VECTOR_OP_MATHLIB_FIRST; int elem_idx = ((AbstractSpecies)vspecies).laneType.switchKey; int shape_idx = vspecies.vectorShape().switchKey; - Entry entry = LIBRARY[idx][elem_idx][shape_idx]; + @SuppressWarnings({"unchecked"}) + Entry entry = (Entry)LIBRARY[idx][elem_idx][shape_idx]; if (entry == null) { - String symbol = SLEEF.symbolName(op, vspecies); // FIXME - MemorySegment addr = LOOKUP.find(symbol).orElse(MemorySegment.NULL); // FIXME - if (DEBUG) { - System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, addr.address()); - } - entry = new Entry(symbol, addr); - LIBRARY[idx][elem_idx][shape_idx] = entry; + entry = constructEntry(op, opc, vspecies, implSupplier); + LIBRARY[idx][elem_idx][shape_idx] = entry; // FIXME: CAS } return entry; } + @DontInline + private static + , T> + @SuppressWarnings({"unchecked"}) + Entry constructEntry(Operator op, int opc, VectorSpecies vspecies, IntFunction implSupplier) { + String symbol = SLEEF.symbolName(op, vspecies); // FIXME + MemorySegment addr = LOOKUP.find(symbol).orElse(MemorySegment.NULL); // FIXME + if (DEBUG) { + System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, addr.address()); + } + T impl; + if (addr != MemorySegment.NULL) { + @SuppressWarnings({"unchecked"}) + Class vt = (Class)vspecies.vectorType(); + if (op instanceof Unary) { + var defaultImpl = (VectorSupport.UnaryOperation) implSupplier.apply(opc); + + impl = (T)(VectorSupport.UnaryOperation) (v0, m) -> { + assert m == null; + return VectorSupport.libraryUnaryOp( + addr.address(), vt, vspecies.elementType(), vspecies.length(), + v0, + defaultImpl, // FIXME: should call the same native function + symbol); + }; + } else if (op instanceof Binary) { + var defaultImpl = (VectorSupport.BinaryOperation) implSupplier.apply(opc); + + impl = (T)(VectorSupport.BinaryOperation ) (v0, v1, m) -> { + assert m == null; + return VectorSupport.libraryBinaryOp( + addr.address(), vt, vspecies.elementType(), vspecies.length(), + v0, v1, + defaultImpl, // FIXME: should call the same native function + symbol); + }; + } else { + throw new InternalError("operation not supported: " + op); + } + } else { + impl = implSupplier.apply(opc); // use default implementation + } + return new Entry<>(symbol, addr, impl); + } + + @ForceInline + /*package-private*/ static + > + V unaryMathOp(VectorOperators.Unary op, int opc, VectorSpecies vspecies, + IntFunction> implSupplier, + V v1) { + var impl = lookup(op, opc, vspecies, implSupplier).impl; + return impl.apply(v1, null); + } + + @ForceInline + /*package-private*/ static + > + V binaryMathOp(VectorOperators.Binary op, int opc, VectorSpecies vspecies, + IntFunction> implSupplier, + V v1, V v2) { + var impl = lookup(op, opc, vspecies, implSupplier).impl; + return impl.apply(v1, v2, null); + } } From 02802d078039d9258236132894143dce76fa3d42 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Wed, 20 Nov 2024 19:41:39 -0800 Subject: [PATCH 05/25] cleanup --- .../incubator/vector/VectorMathLibrary.java | 68 ++++++++----------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index 878b88485c57a..dbf3d1a78bfd4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -133,45 +133,13 @@ private static Entry lookup(Operator op, int opc, VectorSpecies vspeci @DontInline private static , T> - @SuppressWarnings({"unchecked"}) Entry constructEntry(Operator op, int opc, VectorSpecies vspecies, IntFunction implSupplier) { String symbol = SLEEF.symbolName(op, vspecies); // FIXME MemorySegment addr = LOOKUP.find(symbol).orElse(MemorySegment.NULL); // FIXME if (DEBUG) { System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, addr.address()); } - T impl; - if (addr != MemorySegment.NULL) { - @SuppressWarnings({"unchecked"}) - Class vt = (Class)vspecies.vectorType(); - if (op instanceof Unary) { - var defaultImpl = (VectorSupport.UnaryOperation) implSupplier.apply(opc); - - impl = (T)(VectorSupport.UnaryOperation) (v0, m) -> { - assert m == null; - return VectorSupport.libraryUnaryOp( - addr.address(), vt, vspecies.elementType(), vspecies.length(), - v0, - defaultImpl, // FIXME: should call the same native function - symbol); - }; - } else if (op instanceof Binary) { - var defaultImpl = (VectorSupport.BinaryOperation) implSupplier.apply(opc); - - impl = (T)(VectorSupport.BinaryOperation ) (v0, v1, m) -> { - assert m == null; - return VectorSupport.libraryBinaryOp( - addr.address(), vt, vspecies.elementType(), vspecies.length(), - v0, v1, - defaultImpl, // FIXME: should call the same native function - symbol); - }; - } else { - throw new InternalError("operation not supported: " + op); - } - } else { - impl = implSupplier.apply(opc); // use default implementation - } + T impl = implSupplier.apply(opc); // FIXME: should call into the same native impl return new Entry<>(symbol, addr, impl); } @@ -180,9 +148,21 @@ Entry constructEntry(Operator op, int opc, VectorSpecies vspecies, IntFunc > V unaryMathOp(VectorOperators.Unary op, int opc, VectorSpecies vspecies, IntFunction> implSupplier, - V v1) { - var impl = lookup(op, opc, vspecies, implSupplier).impl; - return impl.apply(v1, null); + V v) { + var entry = lookup(op, opc, vspecies, implSupplier); + + long entryAddress = entry.entry.address(); + if (entryAddress != 0) { + @SuppressWarnings({"unchecked"}) + Class vt = (Class)vspecies.vectorType(); + return VectorSupport.libraryUnaryOp( + entry.entry.address(), vt, vspecies.elementType(), vspecies.length(), + v, + entry.impl, + entry.name); + } else { + return entry.impl.apply(v, null); + } } @ForceInline @@ -191,7 +171,19 @@ V unaryMathOp(VectorOperators.Unary op, int opc, VectorSpecies vspecies, V binaryMathOp(VectorOperators.Binary op, int opc, VectorSpecies vspecies, IntFunction> implSupplier, V v1, V v2) { - var impl = lookup(op, opc, vspecies, implSupplier).impl; - return impl.apply(v1, v2, null); + var entry = lookup(op, opc, vspecies, implSupplier); + + long entryAddress = entry.entry.address(); + if (entryAddress != 0) { + @SuppressWarnings({"unchecked"}) + Class vt = (Class)vspecies.vectorType(); + return VectorSupport.libraryBinaryOp( + entry.entry.address(), vt, vspecies.elementType(), vspecies.length(), + v1, v2, + entry.impl, + entry.name); + } else { + return entry.impl.apply(v1, v2, null); + } } } From 0734e25e84df66253146f4d245d73d4c3108187a Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Thu, 21 Nov 2024 13:04:54 -0800 Subject: [PATCH 06/25] SLEEF improvements --- src/java.base/share/classes/module-info.java | 1 + .../incubator/vector/VectorMathLibrary.java | 162 ++++++++++++++---- 2 files changed, 130 insertions(+), 33 deletions(-) diff --git a/src/java.base/share/classes/module-info.java b/src/java.base/share/classes/module-info.java index 22f40d9cead1d..6775dda269a38 100644 --- a/src/java.base/share/classes/module-info.java +++ b/src/java.base/share/classes/module-info.java @@ -272,6 +272,7 @@ java.security.jgss, java.smartcardio, jdk.charsets, + jdk.incubator.vector, jdk.internal.vm.ci, jdk.jlink, jdk.jpackage, diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index dbf3d1a78bfd4..0050584be9342 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -1,5 +1,6 @@ package jdk.incubator.vector; +import jdk.internal.util.StaticProperty; import jdk.internal.vm.annotation.DontInline; import jdk.internal.vm.annotation.ForceInline; import jdk.internal.vm.annotation.Stable; @@ -16,6 +17,67 @@ private static final SymbolLookup LOOKUP = SymbolLookup.loaderLookup(); + interface Library { + String symbolName(Operator op, VectorSpecies vspecies); + boolean isSupported(Operator op, VectorSpecies vspecies); + + String SVML = "svml"; + String SLEEF = "sleef"; + String JAVA = "java"; + + static Library getInstance() { + String libraryName = System.getProperty("jdk.incubator.vector.VectorMathLib", getDefaultName()); + try { + switch (libraryName) { + case SVML: return new SVML(); + case SLEEF: return new SLEEF(); + case JAVA: return new Java(); + + default: return new Java(); + } + } catch (ExceptionInInitializerError e) { + if (DEBUG) { + System.out.printf("DEBUG: VectorMathLibrary: Error during initialization of %s library: %s\n", + libraryName, e); + e.printStackTrace(System.out); + } + return new Java(); // fallback + } + } + + static String getDefaultName() { + switch (StaticProperty.osArch()) { + case "amd64": + case "x86_64": + return SVML; + case "aarch64": + return SLEEF; + default: + return JAVA; + } + } + } + + private static final Library LIBRARY = Library.getInstance(); + + static { + if (DEBUG) { + System.out.printf("DEBUG: VectorMathLibrary: %s is used\n", LIBRARY.getClass().getSimpleName()); + } + } + + private static class Java implements Library { + @Override + public String symbolName(Operator op, VectorSpecies vspecies) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isSupported(Operator op, VectorSpecies vspecies) { + return false; // always use default implementation + } + } + // SVML method naming convention // All the methods are named as __jsvml__ha_ // Where: @@ -30,7 +92,7 @@ // z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2 // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns - private static class SVML { + private static class SVML implements Library { static { loadNativeLibrary(); } @@ -40,34 +102,45 @@ private static void loadNativeLibrary() { System.loadLibrary("jsvml"); } - static int AVX = 0; - static String suffix(VectorShape vshape) { + static int AVX = 0; // FIXME + private static String suffix(VectorShape vshape) { String avx_sse_str = (AVX >= 2) ? "l9" : ((AVX == 1) ? "e9" : "ex"); - return switch (vshape) { - case S_64_BIT -> avx_sse_str; - case S_128_BIT -> avx_sse_str; - case S_256_BIT -> avx_sse_str; - case S_512_BIT -> "z0"; - case S_Max_BIT -> throw new InternalError("NYI"); + return switch (vshape.vectorBitSize()) { + case 64 -> avx_sse_str; // ex + case 128 -> avx_sse_str; // l2 / e9 / ex + case 256 -> avx_sse_str; // l9 / e9 + case 512 -> "z0"; + default -> throw new InternalError("not supported: " + vshape); }; } - static String symbolName(Operator op, VectorSpecies vspecies) { + @Override + public String symbolName(Operator op, VectorSpecies vspecies) { String suffix = suffix(vspecies.vectorShape()); return String.format("__jsvml_%s%s%d_%s", op.operatorName(), vspecies.elementType(), vspecies.length(), suffix); } - // VectorSupport::VEC_SIZE_512: - // if ((!VM_Version::supports_avx512dq()) && - // (vop == VectorSupport::VECTOR_OP_LOG || vop == VectorSupport::VECTOR_OP_LOG10 || vop == VectorSupport::VECTOR_OP_POW)) { - // continue; - // } - // if (vop == VectorSupport::VECTOR_OP_POW) { - // continue; - // } + @Override + public boolean isSupported(Operator op, VectorSpecies vspecies) { + Class etype = vspecies.elementType(); + if (etype != float.class && etype != double.class) { + return false; // only FP types are supported + } + int maxLaneCount = VectorSupport.getMaxLaneCount(vspecies.elementType()); + if (vspecies.length() > maxLaneCount) { + return false; // lacking vector support + } + if (op == POW) { + return false; // not supported + } + if (vspecies.vectorBitSize() == 512 && (op == LOG || op == LOG10)) { + return false; // FIXME: requires VM_Version::supports_avx512dq()) + } + return true; + } } - private static class SLEEF { + private static class SLEEF implements Library { static { loadNativeLibrary(); } @@ -77,11 +150,11 @@ private static void loadNativeLibrary() { System.loadLibrary("sleef"); } - static String suffix(VectorShape vshape) { - return "advsimd"; // FIXME + private static String suffix(VectorShape vshape) { + return (vshape.vectorBitSize() > 128 ? "sve" : "advsimd"); } - static String precisionLevel(Operator op) { + private static String precisionLevel(Operator op) { return (op == HYPOT ? "u05" : "u10"); } @@ -101,20 +174,40 @@ static String precisionLevel(Operator op) { // "sve/advsimd" for sve/neon implementations // e.g. sinfx_u10sve is the method for computing vector float sin using SVE instructions // cosd2_u10advsimd is the method for computing 2 elements vector double cos using NEON instructions - static String symbolName(Operator op, VectorSpecies vspecies) { + @Override + public String symbolName(Operator op, VectorSpecies vspecies) { return String.format("%s%s%d_%s%s", op.operatorName(), (vspecies.elementType() == float.class ? "f" : "d"), vspecies.length(), precisionLevel(op), suffix(vspecies.vectorShape())); } + + @Override + public boolean isSupported(Operator op, VectorSpecies vspecies) { + Class etype = vspecies.elementType(); + if (etype != float.class && etype != double.class) { + return false; // only FP element types are supported + } + int maxLaneCount = VectorSupport.getMaxLaneCount(vspecies.elementType()); + if (vspecies.length() > maxLaneCount) { + return false; // lacking vector support + } + if (vspecies.vectorBitSize() < 128) { + return false; // 64-bit vectors are not supported + } + if (op == TANH) { + return false; // skip due to performance considerations + } + return true; + } } private static final int SIZE = VectorSupport.VECTOR_OP_MATHLIB_LAST - VectorSupport.VECTOR_OP_MATHLIB_FIRST + 1; private record Entry (String name, MemorySegment entry, T impl) {} - private static final @Stable Entry[][][] LIBRARY = new Entry[SIZE][LaneType.SK_LIMIT][VectorShape.SK_LIMIT]; // OP x SHAPE x TYPE + private static final @Stable Entry[][][] LIBRARY_ENTRIES = new Entry[SIZE][LaneType.SK_LIMIT][VectorShape.SK_LIMIT]; // OP x SHAPE x TYPE @ForceInline private static Entry lookup(Operator op, int opc, VectorSpecies vspecies, IntFunction implSupplier) { @@ -122,10 +215,10 @@ private static Entry lookup(Operator op, int opc, VectorSpecies vspeci int elem_idx = ((AbstractSpecies)vspecies).laneType.switchKey; int shape_idx = vspecies.vectorShape().switchKey; @SuppressWarnings({"unchecked"}) - Entry entry = (Entry)LIBRARY[idx][elem_idx][shape_idx]; + Entry entry = (Entry)LIBRARY_ENTRIES[idx][elem_idx][shape_idx]; if (entry == null) { entry = constructEntry(op, opc, vspecies, implSupplier); - LIBRARY[idx][elem_idx][shape_idx] = entry; // FIXME: CAS + LIBRARY_ENTRIES[idx][elem_idx][shape_idx] = entry; // FIXME: CAS } return entry; } @@ -134,19 +227,22 @@ private static Entry lookup(Operator op, int opc, VectorSpecies vspeci private static , T> Entry constructEntry(Operator op, int opc, VectorSpecies vspecies, IntFunction implSupplier) { - String symbol = SLEEF.symbolName(op, vspecies); // FIXME - MemorySegment addr = LOOKUP.find(symbol).orElse(MemorySegment.NULL); // FIXME - if (DEBUG) { - System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, addr.address()); + String symbol = LIBRARY.symbolName(op, vspecies); + MemorySegment addr = MemorySegment.NULL; + if (LIBRARY.isSupported(op, vspecies)) { + addr = LOOKUP.find(symbol).orElseThrow(() -> new InternalError("not supported: " + op + " " + vspecies + " " + symbol)); + if (DEBUG) { + System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, addr.address()); + } } - T impl = implSupplier.apply(opc); // FIXME: should call into the same native impl + T impl = implSupplier.apply(opc); // FIXME: should call the very same native impl return new Entry<>(symbol, addr, impl); } @ForceInline /*package-private*/ static > - V unaryMathOp(VectorOperators.Unary op, int opc, VectorSpecies vspecies, + V unaryMathOp(Unary op, int opc, VectorSpecies vspecies, IntFunction> implSupplier, V v) { var entry = lookup(op, opc, vspecies, implSupplier); @@ -168,7 +264,7 @@ V unaryMathOp(VectorOperators.Unary op, int opc, VectorSpecies vspecies, @ForceInline /*package-private*/ static > - V binaryMathOp(VectorOperators.Binary op, int opc, VectorSpecies vspecies, + V binaryMathOp(Binary op, int opc, VectorSpecies vspecies, IntFunction> implSupplier, V v1, V v2) { var entry = lookup(op, opc, vspecies, implSupplier); From def34341f9511e8802652c87004a3a07e72dbf25 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Thu, 21 Nov 2024 14:36:05 -0800 Subject: [PATCH 07/25] fixes --- .../incubator/vector/VectorMathLibrary.java | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index 0050584be9342..299dfe9e24f77 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -35,7 +35,7 @@ static Library getInstance() { default: return new Java(); } - } catch (ExceptionInInitializerError e) { + } catch (Throwable e) { if (DEBUG) { System.out.printf("DEBUG: VectorMathLibrary: Error during initialization of %s library: %s\n", libraryName, e); @@ -62,7 +62,7 @@ static String getDefaultName() { static { if (DEBUG) { - System.out.printf("DEBUG: VectorMathLibrary: %s is used\n", LIBRARY.getClass().getSimpleName()); + System.out.printf("DEBUG: VectorMathLibrary: %s library is used\n", LIBRARY.getClass().getSimpleName()); } } @@ -102,21 +102,26 @@ private static void loadNativeLibrary() { System.loadLibrary("jsvml"); } - static int AVX = 0; // FIXME - private static String suffix(VectorShape vshape) { - String avx_sse_str = (AVX >= 2) ? "l9" : ((AVX == 1) ? "e9" : "ex"); - return switch (vshape.vectorBitSize()) { - case 64 -> avx_sse_str; // ex - case 128 -> avx_sse_str; // l2 / e9 / ex - case 256 -> avx_sse_str; // l9 / e9 - case 512 -> "z0"; - default -> throw new InternalError("not supported: " + vshape); - }; + private static String suffix(VectorSpecies vspecies) { + assert vspecies.vectorBitSize() <= VectorShape.getMaxVectorBitSize(vspecies.elementType()); + + boolean hasAVX2 = VectorShape.getMaxVectorBitSize(byte.class) >= 256; + boolean hasAVX1 = VectorShape.getMaxVectorBitSize(float.class) >= 256; + + if (vspecies.vectorBitSize() == 512) { + return "z0"; + } else if (hasAVX2) { + return "l9"; + } else if (hasAVX1) { + return "e9"; + } else { + return "ex"; + } } @Override public String symbolName(Operator op, VectorSpecies vspecies) { - String suffix = suffix(vspecies.vectorShape()); + String suffix = suffix(vspecies); return String.format("__jsvml_%s%s%d_%s", op.operatorName(), vspecies.elementType(), vspecies.length(), suffix); } @@ -225,18 +230,19 @@ private static Entry lookup(Operator op, int opc, VectorSpecies vspeci @DontInline private static - , T> + Entry constructEntry(Operator op, int opc, VectorSpecies vspecies, IntFunction implSupplier) { - String symbol = LIBRARY.symbolName(op, vspecies); - MemorySegment addr = MemorySegment.NULL; if (LIBRARY.isSupported(op, vspecies)) { - addr = LOOKUP.find(symbol).orElseThrow(() -> new InternalError("not supported: " + op + " " + vspecies + " " + symbol)); + String symbol = LIBRARY.symbolName(op, vspecies); + MemorySegment addr = LOOKUP.find(symbol).orElseThrow(() -> new InternalError("not supported: " + op + " " + vspecies + " " + symbol)); if (DEBUG) { System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, addr.address()); } + T impl = implSupplier.apply(opc); // FIXME: should call the very same native impl + return new Entry<>(symbol, addr, impl); + } else { + return new Entry<>(null, MemorySegment.NULL, implSupplier.apply(opc)); } - T impl = implSupplier.apply(opc); // FIXME: should call the very same native impl - return new Entry<>(symbol, addr, impl); } @ForceInline From 2efdac99cbb6c82ddc0b338c94b174347ce2c927 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Thu, 21 Nov 2024 15:16:42 -0800 Subject: [PATCH 08/25] Update templates --- .../jdk/incubator/vector/ByteVector.java | 7 ++- .../jdk/incubator/vector/DoubleVector.java | 14 +++--- .../jdk/incubator/vector/FloatVector.java | 25 ++++++----- .../jdk/incubator/vector/IntVector.java | 7 ++- .../jdk/incubator/vector/LongVector.java | 7 ++- .../jdk/incubator/vector/ShortVector.java | 7 ++- .../incubator/vector/X-Vector.java.template | 43 ++++++++++++++++++- 7 files changed, 84 insertions(+), 26 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java index ed8d273ff37db..2b1cc879e6638 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java @@ -689,7 +689,7 @@ ByteVector lanewiseTemplate(VectorOperators.Unary op) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); } - if (op == NOT) { + else if (op == NOT) { return broadcast(-1).lanewise(XOR, this); } } @@ -717,7 +717,7 @@ ByteVector lanewiseTemplate(VectorOperators.Unary op, if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); } - if (op == NOT) { + else if (op == NOT) { return lanewise(XOR, broadcast(-1), m); } } @@ -728,6 +728,7 @@ opc, getClass(), maskClass, byte.class, length(), UN_IMPL.find(op, opc, ByteVector::unaryOperations)); } + private static final ImplCache>> UN_IMPL = new ImplCache<>(Unary.class, ByteVector.class); @@ -824,6 +825,7 @@ ByteVector lanewiseTemplate(VectorOperators.Binary op, = this.compare(EQ, (byte) 0, m); return this.blend(that, mask); } + if (opKind(op, VO_SHIFT)) { // As per shift specification for Java, mask the shift count. // This allows the JIT to ignore some ISA details. @@ -850,6 +852,7 @@ opc, getClass(), maskClass, byte.class, length(), BIN_IMPL.find(op, opc, ByteVector::binaryOperations)); } + private static final ImplCache>> BIN_IMPL = new ImplCache<>(Binary.class, ByteVector.class); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index 6eb3fe82299b1..48446c6fa0129 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -677,7 +677,8 @@ DoubleVector lanewiseTemplate(VectorOperators.Unary op) { if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); - } else if (opKind(op, VO_MATHLIB)) { + } + else if (opKind(op, VO_MATHLIB)) { return unaryMathOp(op); } } @@ -704,7 +705,8 @@ DoubleVector lanewiseTemplate(VectorOperators.Unary op, if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); - } else if (opKind(op, VO_MATHLIB)) { + } + else if (opKind(op, VO_MATHLIB)) { return blend(unaryMathOp(op), m); } } @@ -715,7 +717,6 @@ opc, getClass(), maskClass, double.class, length(), UN_IMPL.find(op, opc, DoubleVector::unaryOperations)); } - @ForceInline final DoubleVector unaryMathOp(VectorOperators.Unary op) { @@ -792,7 +793,8 @@ DoubleVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = this.viewAsIntegralLanes().compare(EQ, (long) 0); return this.blend(that, mask.cast(vspecies())); - } else if (opKind(op, VO_MATHLIB)) { + } + else if (opKind(op, VO_MATHLIB)) { return binaryMathOp(op, that); } } @@ -828,9 +830,11 @@ DoubleVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = bits.compare(EQ, (long) 0, m.cast(bits.vspecies())); return this.blend(that, mask.cast(vspecies())); - } else if (opKind(op, VO_MATHLIB)) { + } + else if (opKind(op, VO_MATHLIB)) { return this.blend(binaryMathOp(op, that), m); } + } int opc = opCode(op); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index 733b08db55e67..1e0829a3b1c0c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -677,7 +677,8 @@ FloatVector lanewiseTemplate(VectorOperators.Unary op) { if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); - } else if (opKind(op, VO_MATHLIB)) { + } + else if (opKind(op, VO_MATHLIB)) { return unaryMathOp(op); } } @@ -704,7 +705,8 @@ FloatVector lanewiseTemplate(VectorOperators.Unary op, if (opKind(op, VO_SPECIAL)) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); - } else if (opKind(op, VO_MATHLIB)) { + } + else if (opKind(op, VO_MATHLIB)) { return blend(unaryMathOp(op), m); } } @@ -715,11 +717,11 @@ opc, getClass(), maskClass, float.class, length(), UN_IMPL.find(op, opc, FloatVector::unaryOperations)); } - @ForceInline final FloatVector unaryMathOp(VectorOperators.Unary op) { - return VectorMathLibrary.unaryMathOp(op, opCode(op), species(), FloatVector::unaryOperations, this); + return VectorMathLibrary.unaryMathOp(op, opCode(op), species(), FloatVector::unaryOperations, + this); } private static final @@ -791,7 +793,8 @@ FloatVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = this.viewAsIntegralLanes().compare(EQ, (int) 0); return this.blend(that, mask.cast(vspecies())); - } else if (opKind(op, VO_MATHLIB)) { + } + else if (opKind(op, VO_MATHLIB)) { return binaryMathOp(op, that); } } @@ -815,8 +818,8 @@ FloatVector lanewise(VectorOperators.Binary op, @ForceInline final FloatVector lanewiseTemplate(VectorOperators.Binary op, - Class> maskClass, - Vector v, VectorMask m) { + Class> maskClass, + Vector v, VectorMask m) { FloatVector that = (FloatVector) v; that.check(this); m.check(maskClass, this); @@ -827,9 +830,11 @@ FloatVector lanewiseTemplate(VectorOperators.Binary op, VectorMask mask = bits.compare(EQ, (int) 0, m.cast(bits.vspecies())); return this.blend(that, mask.cast(vspecies())); - } else if (opKind(op, VO_MATHLIB)) { + } + else if (opKind(op, VO_MATHLIB)) { return this.blend(binaryMathOp(op, that), m); } + } int opc = opCode(op); @@ -839,11 +844,9 @@ opc, getClass(), maskClass, float.class, length(), BIN_IMPL.find(op, opc, FloatVector::binaryOperations)); } - @ForceInline final - FloatVector binaryMathOp(VectorOperators.Binary op, - FloatVector that) { + FloatVector binaryMathOp(VectorOperators.Binary op, FloatVector that) { return VectorMathLibrary.binaryMathOp(op, opCode(op), species(), FloatVector::binaryOperations, this, that); } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index 076a66ed6a543..b691527bec680 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -689,7 +689,7 @@ IntVector lanewiseTemplate(VectorOperators.Unary op) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); } - if (op == NOT) { + else if (op == NOT) { return broadcast(-1).lanewise(XOR, this); } } @@ -717,7 +717,7 @@ IntVector lanewiseTemplate(VectorOperators.Unary op, if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); } - if (op == NOT) { + else if (op == NOT) { return lanewise(XOR, broadcast(-1), m); } } @@ -728,6 +728,7 @@ opc, getClass(), maskClass, int.class, length(), UN_IMPL.find(op, opc, IntVector::unaryOperations)); } + private static final ImplCache>> UN_IMPL = new ImplCache<>(Unary.class, IntVector.class); @@ -824,6 +825,7 @@ IntVector lanewiseTemplate(VectorOperators.Binary op, = this.compare(EQ, (int) 0, m); return this.blend(that, mask); } + if (opKind(op, VO_SHIFT)) { // As per shift specification for Java, mask the shift count. // This allows the JIT to ignore some ISA details. @@ -850,6 +852,7 @@ opc, getClass(), maskClass, int.class, length(), BIN_IMPL.find(op, opc, IntVector::binaryOperations)); } + private static final ImplCache>> BIN_IMPL = new ImplCache<>(Binary.class, IntVector.class); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index 21903aa6794e8..9e4dcd23d677a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -647,7 +647,7 @@ LongVector lanewiseTemplate(VectorOperators.Unary op) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); } - if (op == NOT) { + else if (op == NOT) { return broadcast(-1).lanewise(XOR, this); } } @@ -675,7 +675,7 @@ LongVector lanewiseTemplate(VectorOperators.Unary op, if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); } - if (op == NOT) { + else if (op == NOT) { return lanewise(XOR, broadcast(-1), m); } } @@ -686,6 +686,7 @@ opc, getClass(), maskClass, long.class, length(), UN_IMPL.find(op, opc, LongVector::unaryOperations)); } + private static final ImplCache>> UN_IMPL = new ImplCache<>(Unary.class, LongVector.class); @@ -782,6 +783,7 @@ LongVector lanewiseTemplate(VectorOperators.Binary op, = this.compare(EQ, (long) 0, m); return this.blend(that, mask); } + if (opKind(op, VO_SHIFT)) { // As per shift specification for Java, mask the shift count. // This allows the JIT to ignore some ISA details. @@ -808,6 +810,7 @@ opc, getClass(), maskClass, long.class, length(), BIN_IMPL.find(op, opc, LongVector::binaryOperations)); } + private static final ImplCache>> BIN_IMPL = new ImplCache<>(Binary.class, LongVector.class); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java index 0bb97da824459..46df27309ae3b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java @@ -689,7 +689,7 @@ ShortVector lanewiseTemplate(VectorOperators.Unary op) { if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0)); } - if (op == NOT) { + else if (op == NOT) { return broadcast(-1).lanewise(XOR, this); } } @@ -717,7 +717,7 @@ ShortVector lanewiseTemplate(VectorOperators.Unary op, if (op == ZOMO) { return blend(broadcast(-1), compare(NE, 0, m)); } - if (op == NOT) { + else if (op == NOT) { return lanewise(XOR, broadcast(-1), m); } } @@ -728,6 +728,7 @@ opc, getClass(), maskClass, short.class, length(), UN_IMPL.find(op, opc, ShortVector::unaryOperations)); } + private static final ImplCache>> UN_IMPL = new ImplCache<>(Unary.class, ShortVector.class); @@ -824,6 +825,7 @@ ShortVector lanewiseTemplate(VectorOperators.Binary op, = this.compare(EQ, (short) 0, m); return this.blend(that, mask); } + if (opKind(op, VO_SHIFT)) { // As per shift specification for Java, mask the shift count. // This allows the JIT to ignore some ISA details. @@ -850,6 +852,7 @@ opc, getClass(), maskClass, short.class, length(), BIN_IMPL.find(op, opc, ShortVector::binaryOperations)); } + private static final ImplCache>> BIN_IMPL = new ImplCache<>(Binary.class, ShortVector.class); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index 8084cc307e867..6d9db65a1ba83 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -712,10 +712,15 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { return blend(broadcast(-1), compare(NE, 0)); } #if[BITWISE] - if (op == NOT) { + else if (op == NOT) { return broadcast(-1).lanewise(XOR, this); } #end[BITWISE] +#if[FP] + else if (opKind(op, VO_MATHLIB)) { + return unaryMathOp(op); + } +#end[FP] } int opc = opCode(op); return VectorSupport.unaryOp( @@ -742,10 +747,15 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { return blend(broadcast(-1), compare(NE, 0, m)); } #if[BITWISE] - if (op == NOT) { + else if (op == NOT) { return lanewise(XOR, broadcast(-1), m); } #end[BITWISE] +#if[FP] + else if (opKind(op, VO_MATHLIB)) { + return blend(unaryMathOp(op), m); + } +#end[FP] } int opc = opCode(op); return VectorSupport.unaryOp( @@ -754,6 +764,15 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations)); } +#if[FP] + @ForceInline + final + $abstractvectortype$ unaryMathOp(VectorOperators.Unary op) { + return VectorMathLibrary.unaryMathOp(op, opCode(op), species(), $abstractvectortype$::unaryOperations, + this); + } +#end[FP] + private static final ImplCache>> UN_IMPL = new ImplCache<>(Unary.class, $Type$Vector.class); @@ -856,6 +875,11 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = this{#if[FP]?.viewAsIntegralLanes()}.compare(EQ, ($bitstype$) 0); return this.blend(that, mask{#if[FP]?.cast(vspecies())}); } +#if[FP] + else if (opKind(op, VO_MATHLIB)) { + return binaryMathOp(op, that); + } +#end[FP] #if[BITWISE] #if[!FP] if (opKind(op, VO_SHIFT)) { @@ -915,6 +939,12 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { return this.blend(that, mask); #end[FP] } +#if[FP] + else if (opKind(op, VO_MATHLIB)) { + return this.blend(binaryMathOp(op, that), m); + } +#end[FP] + #if[BITWISE] #if[!FP] if (opKind(op, VO_SHIFT)) { @@ -945,6 +975,15 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { BIN_IMPL.find(op, opc, $abstractvectortype$::binaryOperations)); } +#if[FP] + @ForceInline + final + $abstractvectortype$ binaryMathOp(VectorOperators.Binary op, $abstractvectortype$ that) { + return VectorMathLibrary.binaryMathOp(op, opCode(op), species(), $abstractvectortype$::binaryOperations, + this, that); + } +#end[FP] + private static final ImplCache>> BIN_IMPL = new ImplCache<>(Binary.class, $Type$Vector.class); From c74739e66192ebe32d8fdc718e9ef9def68912ed Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Thu, 21 Nov 2024 15:29:39 -0800 Subject: [PATCH 09/25] SVML fixes --- .../classes/jdk/incubator/vector/VectorMathLibrary.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index 299dfe9e24f77..3567ed436f924 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -122,7 +122,8 @@ private static String suffix(VectorSpecies vspecies) { @Override public String symbolName(Operator op, VectorSpecies vspecies) { String suffix = suffix(vspecies); - return String.format("__jsvml_%s%s%d_%s", op.operatorName(), vspecies.elementType(), vspecies.length(), suffix); + String elemType = (vspecies.elementType() == float.class ? "f" : ""); + return String.format("__jsvml_%s%s%d_ha_%s", op.operatorName(), elemType, vspecies.length(), suffix); } @Override @@ -135,6 +136,9 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { if (vspecies.length() > maxLaneCount) { return false; // lacking vector support } + if (vspecies.vectorBitSize() < 128) { + return false; // 64-bit vectors are not supported + } if (op == POW) { return false; // not supported } From 2a7006bbb6486a2f0336f86844f47d0a6f527414 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Thu, 21 Nov 2024 16:30:41 -0800 Subject: [PATCH 10/25] TODO list --- .../classes/jdk/incubator/vector/VectorMathLibrary.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index 3567ed436f924..8f1596319596b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -12,6 +12,11 @@ import static jdk.incubator.vector.VectorOperators.*; +// TODO: +// * refactor native library API: a single registerNatives() call which returns a vector of entry points +// * SVML: // FIXME: requires VM_Version::supports_avx512dq()) +// * invoke native library as a fallback implementation +// /*package-private*/ class VectorMathLibrary { private static final boolean DEBUG = Boolean.getBoolean("jdk.incubator.vector.VectorMathLibrary.DEBUG"); From 70cdd15ab179a2dca6d317c1d1dc3191b19888b6 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Mon, 31 Mar 2025 15:31:30 -0700 Subject: [PATCH 11/25] Cleanup --- src/hotspot/share/prims/vectorSupport.cpp | 17 +++++++++++- .../jdk/internal/vm/vector/VectorSupport.java | 5 ++++ .../incubator/vector/VectorMathLibrary.java | 26 ++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index 27e52966c8309..921ab462f739d 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -39,6 +39,7 @@ #include "runtime/stackValue.hpp" #ifdef COMPILER2 #include "opto/matcher.hpp" +#include "opto/vectornode.hpp" #endif // COMPILER2 bool VectorSupport::is_vector(Klass* klass) { @@ -613,6 +614,19 @@ JVM_ENTRY(jint, VectorSupport_GetMaxLaneCount(JNIEnv *env, jclass vsclazz, jobje return -1; } JVM_END +JVM_ENTRY(jint, VectorSupport_IsVectorOperationSupported(JNIEnv *env, jclass vsclazz, jint opr, jobject clazz, jint vlen)) { +#ifdef COMPILER2 + oop mirror = JNIHandles::resolve_non_null(clazz); + if (java_lang_Class::is_primitive(mirror)) { + BasicType elem_bt = java_lang_Class::primitive_type(mirror); + int opc = VectorSupport::vop2ideal(opr, elem_bt); + int sopc = VectorNode::opcode(opc, elem_bt); + return Matcher::match_rule_supported_vector(sopc, vlen, elem_bt); + } +#endif // COMPILER2 + return JNI_FALSE; + } JVM_END + // JVM_RegisterVectorSupportMethods #define LANG "Ljava/lang/" @@ -622,7 +636,8 @@ JVM_ENTRY(jint, VectorSupport_GetMaxLaneCount(JNIEnv *env, jclass vsclazz, jobje #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f) static JNINativeMethod jdk_internal_vm_vector_VectorSupport_methods[] = { - {CC "getMaxLaneCount", CC "(" CLS ")I", FN_PTR(VectorSupport_GetMaxLaneCount)} + {CC "getMaxLaneCount", CC "(" CLS ")I", FN_PTR(VectorSupport_GetMaxLaneCount)}, + {CC "isVectorOperationSupported", CC "(I" CLS "I)I", FN_PTR(VectorSupport_IsVectorOperationSupported)} }; #undef CC diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index 07d41c279da38..fbe21e0986435 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -775,6 +775,11 @@ long maskReductionCoerced(int oper, /* ============================================================================ */ + // query the JVM's supported vector operations + public static native boolean isSupported(int opr, Class etype, int length); + + /* ============================================================================ */ + public static boolean isNonCapturingLambda(Object o) { return o.getClass().getDeclaredFields().length == 0; } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index 8f1596319596b..be4f0e6eb4e38 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ package jdk.incubator.vector; import jdk.internal.util.StaticProperty; @@ -247,7 +271,7 @@ Entry constructEntry(Operator op, int opc, VectorSpecies vspecies, IntFunc if (DEBUG) { System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, addr.address()); } - T impl = implSupplier.apply(opc); // FIXME: should call the very same native impl + T impl = implSupplier.apply(opc); // TODO: should call the very same native implementation return new Entry<>(symbol, addr, impl); } else { return new Entry<>(null, MemorySegment.NULL, implSupplier.apply(opc)); From 0a1bba885cb323131ac126ef7d5a84308632b500 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Tue, 1 Apr 2025 15:27:52 -0700 Subject: [PATCH 12/25] CPU features support --- src/hotspot/share/include/jvm.h | 3 + src/hotspot/share/prims/jvm.cpp | 6 ++ .../share/classes/jdk/internal/misc/VM.java | 5 ++ src/java.base/share/native/libjava/VM.c | 5 ++ .../incubator/vector/VectorMathLibrary.java | 59 +++++++++++++------ 5 files changed, 61 insertions(+), 17 deletions(-) diff --git a/src/hotspot/share/include/jvm.h b/src/hotspot/share/include/jvm.h index 61857ac3ccd24..db2c35a0f932e 100644 --- a/src/hotspot/share/include/jvm.h +++ b/src/hotspot/share/include/jvm.h @@ -221,6 +221,9 @@ JVM_DumpDynamicArchive(JNIEnv* env, jstring archiveName); JNIEXPORT jboolean JNICALL JVM_NeedsClassInitBarrierForCDS(JNIEnv* env, jclass cls); +JNIEXPORT jstring JNICALL +JVM_GetCPUFeatures(JNIEnv* env); + /* * java.lang.Throwable */ diff --git a/src/hotspot/share/prims/jvm.cpp b/src/hotspot/share/prims/jvm.cpp index 3b05806dc37ac..84437324289d5 100644 --- a/src/hotspot/share/prims/jvm.cpp +++ b/src/hotspot/share/prims/jvm.cpp @@ -3836,3 +3836,9 @@ JVM_END JVM_LEAF(jboolean, JVM_PrintWarningAtDynamicAgentLoad(void)) return (EnableDynamicAgentLoading && !FLAG_IS_CMDLINE(EnableDynamicAgentLoading)) ? JNI_TRUE : JNI_FALSE; JVM_END + +JVM_ENTRY(jstring, JVM_GetCPUFeatures(JNIEnv* env)) + const char* features = VM_Version::features_string(); + ThreadToNativeFromVM ttn(thread); + return env->NewStringUTF(features); +JVM_END diff --git a/src/java.base/share/classes/jdk/internal/misc/VM.java b/src/java.base/share/classes/jdk/internal/misc/VM.java index db57b1f9da520..03f6962e7fa90 100644 --- a/src/java.base/share/classes/jdk/internal/misc/VM.java +++ b/src/java.base/share/classes/jdk/internal/misc/VM.java @@ -445,6 +445,11 @@ public static boolean isSetUID() { */ public static native String[] getRuntimeArguments(); + /** + * Return a string containing a list of CPU features VM detected. + */ + public static native String getCPUFeatures(); + static { initialize(); } diff --git a/src/java.base/share/native/libjava/VM.c b/src/java.base/share/native/libjava/VM.c index 099ad46fff15d..cb668749b64b2 100644 --- a/src/java.base/share/native/libjava/VM.c +++ b/src/java.base/share/native/libjava/VM.c @@ -55,3 +55,8 @@ JNIEXPORT jobjectArray JNICALL Java_jdk_internal_misc_VM_getRuntimeArguments(JNIEnv *env, jclass cls) { return JVM_GetVmArguments(env); } + +JNIEXPORT jobject JNICALL +Java_jdk_internal_misc_VM_getCPUFeatures(JNIEnv *env, jclass cls) { + return JVM_GetCPUFeatures(env); +} diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index be4f0e6eb4e38..1493b5e901612 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -24,6 +24,7 @@ */ package jdk.incubator.vector; +import jdk.internal.misc.VM; import jdk.internal.util.StaticProperty; import jdk.internal.vm.annotation.DontInline; import jdk.internal.vm.annotation.ForceInline; @@ -32,6 +33,9 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SymbolLookup; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import java.util.function.IntFunction; import static jdk.incubator.vector.VectorOperators.*; @@ -46,6 +50,29 @@ private static final SymbolLookup LOOKUP = SymbolLookup.loaderLookup(); + static class CPUFeatures { + static final Set features = getFeatureSet(); + + private static Set getFeatureSet() { + String[] features = VM.getCPUFeatures().split(" "); + if (DEBUG) { + System.out.printf("DEBUG: CPUFeatures: %s\n", Arrays.deepToString(features)); + } + return Set.copyOf(Arrays.asList(features)); + } + + public static boolean hasFeature(String feature) { + return features.contains(feature); + } + + public static boolean isX64() { + return switch (StaticProperty.osArch()) { + case "amd64", "x86_64" -> true; + default -> false; + }; + } + } + interface Library { String symbolName(Operator op, VectorSpecies vspecies); boolean isSupported(Operator op, VectorSpecies vspecies); @@ -57,13 +84,12 @@ interface Library { static Library getInstance() { String libraryName = System.getProperty("jdk.incubator.vector.VectorMathLib", getDefaultName()); try { - switch (libraryName) { - case SVML: return new SVML(); - case SLEEF: return new SLEEF(); - case JAVA: return new Java(); - - default: return new Java(); - } + return switch (libraryName) { + case SVML -> new SVML(); + case SLEEF -> new SLEEF(); + case JAVA -> new Java(); + default -> new Java(); + }; } catch (Throwable e) { if (DEBUG) { System.out.printf("DEBUG: VectorMathLibrary: Error during initialization of %s library: %s\n", @@ -75,15 +101,11 @@ static Library getInstance() { } static String getDefaultName() { - switch (StaticProperty.osArch()) { - case "amd64": - case "x86_64": - return SVML; - case "aarch64": - return SLEEF; - default: - return JAVA; - } + return switch (StaticProperty.osArch()) { + case "amd64", "x86_64" -> SVML; + case "aarch64" -> SLEEF; + default -> JAVA; + }; } } @@ -107,6 +129,7 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { } } + // SVML method naming convention // All the methods are named as __jsvml__ha_ // Where: @@ -122,6 +145,8 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns private static class SVML implements Library { + static boolean SUPPORTS_AVX512DQ = CPUFeatures.isX64() && CPUFeatures.hasFeature("avx512dq"); + static { loadNativeLibrary(); } @@ -172,7 +197,7 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { return false; // not supported } if (vspecies.vectorBitSize() == 512 && (op == LOG || op == LOG10)) { - return false; // FIXME: requires VM_Version::supports_avx512dq()) + return SUPPORTS_AVX512DQ; } return true; } From fc27aee573a0e4fa6852b4981085b3fabe6ccc38 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Wed, 2 Apr 2025 12:35:01 -0700 Subject: [PATCH 13/25] Misc fixes and cleanups --- .../cpu/aarch64/vm_version_aarch64.cpp | 7 +- src/hotspot/cpu/x86/vm_version_x86.cpp | 12 +- .../linux_riscv/vm_version_linux_riscv.cpp | 9 +- src/hotspot/share/include/jvm.h | 3 - src/hotspot/share/prims/jvm.cpp | 6 - src/hotspot/share/prims/vectorSupport.cpp | 25 ++- src/hotspot/share/prims/whitebox.cpp | 2 +- .../share/runtime/abstract_vm_version.cpp | 14 ++ .../share/runtime/abstract_vm_version.hpp | 6 + src/hotspot/share/runtime/os.cpp | 2 +- .../share/classes/jdk/internal/misc/VM.java | 5 - .../classes/jdk/internal/vm/vector/Utils.java | 49 ++++++ .../jdk/internal/vm/vector/VectorSupport.java | 16 +- src/java.base/share/native/libjava/VM.c | 5 - .../jdk/incubator/vector/CPUFeatures.java | 81 +++++++++ .../classes/jdk/incubator/vector/Util.java | 33 ++++ .../incubator/vector/VectorMathLibrary.java | 166 +++++++----------- .../jdk/incubator/vector/VectorOperators.java | 4 +- 18 files changed, 294 insertions(+), 151 deletions(-) create mode 100644 src/java.base/share/classes/jdk/internal/vm/vector/Utils.java create mode 100644 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java create mode 100644 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Util.java diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index 0f04fee79220a..d734b62cc1c80 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -628,6 +628,7 @@ void VM_Version::initialize() { if (_model2) { os::snprintf_checked(buf + buf_used_len, sizeof(buf) - buf_used_len, "(0x%03x)", _model2); } + int features_offset = strnlen(buf, sizeof(buf)); #define ADD_FEATURE_IF_SUPPORTED(id, name, bit) \ do { \ if (VM_Version::supports_##name()) strcat(buf, ", " #name); \ @@ -635,7 +636,11 @@ void VM_Version::initialize() { CPU_FEATURE_FLAGS(ADD_FEATURE_IF_SUPPORTED) #undef ADD_FEATURE_IF_SUPPORTED - _features_string = os::strdup(buf); + _cpu_info_string = os::strdup(buf); + + _features_string = extract_features_string(_cpu_info_string, + strnlen(_cpu_info_string, sizeof(buf)), + features_offset); } #if defined(LINUX) diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp index 4b9c1c3416a5d..32823cfd858cd 100644 --- a/src/hotspot/cpu/x86/vm_version_x86.cpp +++ b/src/hotspot/cpu/x86/vm_version_x86.cpp @@ -1090,15 +1090,19 @@ void VM_Version::get_processor_features() { } char buf[1024]; - int res = jio_snprintf( + int cpu_info_size = jio_snprintf( buf, sizeof(buf), "(%u cores per cpu, %u threads per core) family %d model %d stepping %d microcode 0x%x", cores_per_cpu(), threads_per_core(), cpu_family(), _model, _stepping, os::cpu_microcode_revision()); - assert(res > 0, "not enough temporary space allocated"); - insert_features_names(buf + res, sizeof(buf) - res, _features_names); + assert(cpu_info_size > 0, "not enough temporary space allocated"); + insert_features_names(buf + cpu_info_size, sizeof(buf) - cpu_info_size, _features_names); - _features_string = os::strdup(buf); + _cpu_info_string = os::strdup(buf); + + _features_string = extract_features_string(_cpu_info_string, + strnlen(_cpu_info_string, sizeof(buf)), + cpu_info_size); // Use AES instructions if available. if (supports_aes()) { diff --git a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp index b6095c279cbf0..484a2a645aaae 100644 --- a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp +++ b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp @@ -129,6 +129,9 @@ void VM_Version::setup_cpu_available_features() { snprintf(buf, sizeof(buf)/2, "%s ", uarch); } os::free((void*) uarch); + + int features_offset = strnlen(buf, sizeof(buf)); + strcat(buf, "rv64"); int i = 0; while (_feature_list[i] != nullptr) { @@ -191,7 +194,11 @@ void VM_Version::setup_cpu_available_features() { } } - _features_string = os::strdup(buf); + _cpu_info_string = os::strdup(buf); + + _features_string = extract_features_string(_cpu_info_string, + strnlen(_cpu_info_string, sizeof(buf)), + features_offset); } void VM_Version::os_aux_features() { diff --git a/src/hotspot/share/include/jvm.h b/src/hotspot/share/include/jvm.h index db2c35a0f932e..61857ac3ccd24 100644 --- a/src/hotspot/share/include/jvm.h +++ b/src/hotspot/share/include/jvm.h @@ -221,9 +221,6 @@ JVM_DumpDynamicArchive(JNIEnv* env, jstring archiveName); JNIEXPORT jboolean JNICALL JVM_NeedsClassInitBarrierForCDS(JNIEnv* env, jclass cls); -JNIEXPORT jstring JNICALL -JVM_GetCPUFeatures(JNIEnv* env); - /* * java.lang.Throwable */ diff --git a/src/hotspot/share/prims/jvm.cpp b/src/hotspot/share/prims/jvm.cpp index 84437324289d5..3b05806dc37ac 100644 --- a/src/hotspot/share/prims/jvm.cpp +++ b/src/hotspot/share/prims/jvm.cpp @@ -3836,9 +3836,3 @@ JVM_END JVM_LEAF(jboolean, JVM_PrintWarningAtDynamicAgentLoad(void)) return (EnableDynamicAgentLoading && !FLAG_IS_CMDLINE(EnableDynamicAgentLoading)) ? JNI_TRUE : JNI_FALSE; JVM_END - -JVM_ENTRY(jstring, JVM_GetCPUFeatures(JNIEnv* env)) - const char* features = VM_Version::features_string(); - ThreadToNativeFromVM ttn(thread); - return env->NewStringUTF(features); -JVM_END diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index 921ab462f739d..2a40116c8be68 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -614,30 +614,26 @@ JVM_ENTRY(jint, VectorSupport_GetMaxLaneCount(JNIEnv *env, jclass vsclazz, jobje return -1; } JVM_END -JVM_ENTRY(jint, VectorSupport_IsVectorOperationSupported(JNIEnv *env, jclass vsclazz, jint opr, jobject clazz, jint vlen)) { -#ifdef COMPILER2 - oop mirror = JNIHandles::resolve_non_null(clazz); - if (java_lang_Class::is_primitive(mirror)) { - BasicType elem_bt = java_lang_Class::primitive_type(mirror); - int opc = VectorSupport::vop2ideal(opr, elem_bt); - int sopc = VectorNode::opcode(opc, elem_bt); - return Matcher::match_rule_supported_vector(sopc, vlen, elem_bt); - } -#endif // COMPILER2 - return JNI_FALSE; - } JVM_END +JVM_ENTRY(jstring, VectorSupport_GetCPUFeatures(JNIEnv* env, jclass ignored)) + const char* features_string = VM_Version::features_string(); + assert(features_string != nullptr, "missing cpu features info"); + + ThreadToNativeFromVM ttn(thread); + return env->NewStringUTF(features_string); +JVM_END // JVM_RegisterVectorSupportMethods #define LANG "Ljava/lang/" #define CLS LANG "Class;" +#define LSTR LANG "String;" #define CC (char*) /*cast a literal from (const char*)*/ #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f) static JNINativeMethod jdk_internal_vm_vector_VectorSupport_methods[] = { - {CC "getMaxLaneCount", CC "(" CLS ")I", FN_PTR(VectorSupport_GetMaxLaneCount)}, - {CC "isVectorOperationSupported", CC "(I" CLS "I)I", FN_PTR(VectorSupport_IsVectorOperationSupported)} + {CC "getMaxLaneCount", CC "(" CLS ")I", FN_PTR(VectorSupport_GetMaxLaneCount)}, + {CC "getCPUFeatures", CC "()" LSTR, FN_PTR(VectorSupport_GetCPUFeatures)} }; #undef CC @@ -645,6 +641,7 @@ static JNINativeMethod jdk_internal_vm_vector_VectorSupport_methods[] = { #undef LANG #undef CLS +#undef LSTR // This function is exported, used by NativeLookup. diff --git a/src/hotspot/share/prims/whitebox.cpp b/src/hotspot/share/prims/whitebox.cpp index e08a5ba5ebd12..41f32729cd432 100644 --- a/src/hotspot/share/prims/whitebox.cpp +++ b/src/hotspot/share/prims/whitebox.cpp @@ -1533,7 +1533,7 @@ WB_ENTRY(void, WB_ReadReservedMemory(JNIEnv* env, jobject o)) WB_END WB_ENTRY(jstring, WB_GetCPUFeatures(JNIEnv* env, jobject o)) - const char* features = VM_Version::features_string(); + const char* features = VM_Version::cpu_info_string(); ThreadToNativeFromVM ttn(thread); jstring features_string = env->NewStringUTF(features); diff --git a/src/hotspot/share/runtime/abstract_vm_version.cpp b/src/hotspot/share/runtime/abstract_vm_version.cpp index 763e441fe541e..d88a7cd9566d0 100644 --- a/src/hotspot/share/runtime/abstract_vm_version.cpp +++ b/src/hotspot/share/runtime/abstract_vm_version.cpp @@ -34,6 +34,7 @@ const char* Abstract_VM_Version::_s_internal_vm_info_string = Abstract_VM_Versio uint64_t Abstract_VM_Version::_features = 0; const char* Abstract_VM_Version::_features_string = ""; +const char* Abstract_VM_Version::_cpu_info_string = ""; uint64_t Abstract_VM_Version::_cpu_features = 0; #ifndef SUPPORTS_NATIVE_CX8 @@ -336,6 +337,19 @@ void Abstract_VM_Version::insert_features_names(char* buf, size_t buflen, const } } +const char* Abstract_VM_Version::extract_features_string(const char* cpu_info_string, + size_t cpu_info_string_len, + size_t features_offset) { + assert(features_offset <= cpu_info_string_len, ""); + if (features_offset < cpu_info_string_len) { + assert(cpu_info_string[features_offset + 0] == ',', ""); + assert(cpu_info_string[features_offset + 1] == ' ', ""); + return cpu_info_string + features_offset + 2; // skip initial ", " + } else { + return ""; // empty + } +} + bool Abstract_VM_Version::print_matching_lines_from_file(const char* filename, outputStream* st, const char* keywords_to_match[]) { char line[500]; FILE* fp = os::fopen(filename, "r"); diff --git a/src/hotspot/share/runtime/abstract_vm_version.hpp b/src/hotspot/share/runtime/abstract_vm_version.hpp index 8cfc7031f97ae..6f1b886bc9878 100644 --- a/src/hotspot/share/runtime/abstract_vm_version.hpp +++ b/src/hotspot/share/runtime/abstract_vm_version.hpp @@ -58,6 +58,8 @@ class Abstract_VM_Version: AllStatic { static uint64_t _features; static const char* _features_string; + static const char* _cpu_info_string; + // Original CPU feature flags, not affected by VM settings. static uint64_t _cpu_features; @@ -128,7 +130,11 @@ class Abstract_VM_Version: AllStatic { static uint64_t features() { return _features; } static const char* features_string() { return _features_string; } + static const char* cpu_info_string() { return _cpu_info_string; } static void insert_features_names(char* buf, size_t buflen, const char* features_names[]); + static const char* extract_features_string(const char* cpu_info_string, + size_t cpu_info_string_len, + size_t features_offset); static VirtualizationType get_detected_virtualization() { return _detected_virtualization; diff --git a/src/hotspot/share/runtime/os.cpp b/src/hotspot/share/runtime/os.cpp index cbe40ca214bba..997a94d6da6da 100644 --- a/src/hotspot/share/runtime/os.cpp +++ b/src/hotspot/share/runtime/os.cpp @@ -1166,7 +1166,7 @@ void os::print_cpu_info(outputStream* st, char* buf, size_t buflen) { // We access the raw value here because the assert in the accessor will // fail if the crash occurs before initialization of this value. st->print(" (initial active %d)", _initial_active_processor_count); - st->print(" %s", VM_Version::features_string()); + st->print(" %s", VM_Version::cpu_info_string()); st->cr(); pd_print_cpu_info(st, buf, buflen); } diff --git a/src/java.base/share/classes/jdk/internal/misc/VM.java b/src/java.base/share/classes/jdk/internal/misc/VM.java index 03f6962e7fa90..db57b1f9da520 100644 --- a/src/java.base/share/classes/jdk/internal/misc/VM.java +++ b/src/java.base/share/classes/jdk/internal/misc/VM.java @@ -445,11 +445,6 @@ public static boolean isSetUID() { */ public static native String[] getRuntimeArguments(); - /** - * Return a string containing a list of CPU features VM detected. - */ - public static native String getCPUFeatures(); - static { initialize(); } diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/Utils.java b/src/java.base/share/classes/jdk/internal/vm/vector/Utils.java new file mode 100644 index 0000000000000..de0bb83d39d99 --- /dev/null +++ b/src/java.base/share/classes/jdk/internal/vm/vector/Utils.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.internal.vm.vector; + +import jdk.internal.reflect.CallerSensitive; +import jdk.internal.reflect.Reflection; + +/** + * Miscellaneous utility methods. + */ +public class Utils { + public static final boolean DEBUG = Boolean.getBoolean("jdk.incubator.vector.DEBUG"); + + public static boolean isNonCapturingLambda(Object o) { + return o.getClass().getDeclaredFields().length == 0; + } + + @CallerSensitive + public static void debug(String format, Object... args) { + if (DEBUG) { + Class caller = Reflection.getCallerClass(); + System.out.printf("DEBUG: %s: ", caller.getSimpleName()); + System.out.printf(format, args); + System.out.println(); + } + } +} diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index fbe21e0986435..81b047743e47f 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -28,9 +28,10 @@ import jdk.internal.vm.annotation.IntrinsicCandidate; import jdk.internal.misc.Unsafe; -import java.lang.foreign.MemorySegment; import java.util.function.*; +import static jdk.internal.vm.vector.Utils.isNonCapturingLambda; + public class VectorSupport { static { registerNatives(); @@ -770,18 +771,19 @@ long maskReductionCoerced(int oper, /* ============================================================================ */ - // query the JVM's supported vector sizes and types - public static native int getMaxLaneCount(Class etype); + // Returns a string containing a list of CPU features VM detected. + public static native String getCPUFeatures(); /* ============================================================================ */ - // query the JVM's supported vector operations - public static native boolean isSupported(int opr, Class etype, int length); + // query the JVM's supported vector sizes and types + public static native int getMaxLaneCount(Class etype); /* ============================================================================ */ - public static boolean isNonCapturingLambda(Object o) { - return o.getClass().getDeclaredFields().length == 0; + @SuppressWarnings({"restricted"}) + public static void loadNativeLibrary(String name) { + System.loadLibrary(name); } /* ============================================================================ */ diff --git a/src/java.base/share/native/libjava/VM.c b/src/java.base/share/native/libjava/VM.c index cb668749b64b2..099ad46fff15d 100644 --- a/src/java.base/share/native/libjava/VM.c +++ b/src/java.base/share/native/libjava/VM.c @@ -55,8 +55,3 @@ JNIEXPORT jobjectArray JNICALL Java_jdk_internal_misc_VM_getRuntimeArguments(JNIEnv *env, jclass cls) { return JVM_GetVmArguments(env); } - -JNIEXPORT jobject JNICALL -Java_jdk_internal_misc_VM_getCPUFeatures(JNIEnv *env, jclass cls) { - return JVM_GetCPUFeatures(env); -} diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java new file mode 100644 index 0000000000000..3fc0f5e6fdbb6 --- /dev/null +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.incubator.vector; + +import jdk.internal.vm.vector.VectorSupport; + +import java.util.Set; + +import static jdk.incubator.vector.Util.requires; +import static jdk.internal.util.Architecture.isX64; +import static jdk.internal.vm.vector.Utils.debug; + +/** + * Enumerates CPU ISA extensions supported by the JVM on the current hardware. + */ +/*package-private*/ class CPUFeatures { + private static final Set features = getCPUFeatures(); + + private static Set getCPUFeatures() { + String featuresString = VectorSupport.getCPUFeatures(); + debug(featuresString); + String[] features = featuresString.toLowerCase().split(", "); // ", " is used as a delimiter + assert validateFeatures(features); + return Set.of(features); + } + + private static boolean validateFeatures(String[] features) { + for (String s : features) { + assert s != null && s.matches("[a-z0-9._]+") : String.format("Invalid CPU feature name: '%s'", s); + } + return true; + } + + private static boolean hasFeature(String feature) { + return features.contains(feature.toLowerCase()); + } + + public static class X64 { + public static boolean SUPPORTS_AVX = hasFeature("avx"); + public static boolean SUPPORTS_AVX2 = hasFeature("avx2"); + public static boolean SUPPORTS_AVX512F = hasFeature("avx512f"); + public static boolean SUPPORTS_AVX512DQ = hasFeature("avx512dq"); + + static { + requires(isX64(), "unsupported platform"); + + debug("AVX=%b; AVX2=%b; AVX512F=%b; AVX512DQ=%b", + SUPPORTS_AVX, SUPPORTS_AVX2, SUPPORTS_AVX512F, SUPPORTS_AVX512DQ); + + assert SUPPORTS_AVX512F == (VectorShape.getMaxVectorBitSize(int.class) == 512); + assert SUPPORTS_AVX2 == (VectorShape.getMaxVectorBitSize(byte.class) >= 256); + assert SUPPORTS_AVX == (VectorShape.getMaxVectorBitSize(float.class) >= 256); + } + } + + public static Set features() { + return features; + } +} \ No newline at end of file diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Util.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Util.java new file mode 100644 index 0000000000000..8562d4b5d7a2a --- /dev/null +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Util.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.incubator.vector; + +/*package-private*/ class Util { + public static void requires(boolean cond, String message) { + if (!cond) { + throw new InternalError(message); + } + } +} diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index 1493b5e901612..4cfb5e9a9b053 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -24,7 +24,6 @@ */ package jdk.incubator.vector; -import jdk.internal.misc.VM; import jdk.internal.util.StaticProperty; import jdk.internal.vm.annotation.DontInline; import jdk.internal.vm.annotation.ForceInline; @@ -33,46 +32,20 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SymbolLookup; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; import java.util.function.IntFunction; +import static jdk.incubator.vector.Util.requires; import static jdk.incubator.vector.VectorOperators.*; +import static jdk.internal.util.Architecture.isX64; +import static jdk.internal.vm.vector.Utils.debug; -// TODO: -// * refactor native library API: a single registerNatives() call which returns a vector of entry points -// * SVML: // FIXME: requires VM_Version::supports_avx512dq()) -// * invoke native library as a fallback implementation -// +/** + * A wrapper for native vector math libraries bundled with the JDK (SVML and SLEEF). + * Binds vector operations to native implementations provided by the libraries. + */ /*package-private*/ class VectorMathLibrary { - private static final boolean DEBUG = Boolean.getBoolean("jdk.incubator.vector.VectorMathLibrary.DEBUG"); - private static final SymbolLookup LOOKUP = SymbolLookup.loaderLookup(); - static class CPUFeatures { - static final Set features = getFeatureSet(); - - private static Set getFeatureSet() { - String[] features = VM.getCPUFeatures().split(" "); - if (DEBUG) { - System.out.printf("DEBUG: CPUFeatures: %s\n", Arrays.deepToString(features)); - } - return Set.copyOf(Arrays.asList(features)); - } - - public static boolean hasFeature(String feature) { - return features.contains(feature); - } - - public static boolean isX64() { - return switch (StaticProperty.osArch()) { - case "amd64", "x86_64" -> true; - default -> false; - }; - } - } - interface Library { String symbolName(Operator op, VectorSpecies vspecies); boolean isSupported(Operator op, VectorSpecies vspecies); @@ -82,20 +55,16 @@ interface Library { String JAVA = "java"; static Library getInstance() { - String libraryName = System.getProperty("jdk.incubator.vector.VectorMathLib", getDefaultName()); + String libraryName = System.getProperty("jdk.incubator.vector.VectorMathLibrary", getDefaultName()); try { return switch (libraryName) { case SVML -> new SVML(); case SLEEF -> new SLEEF(); case JAVA -> new Java(); - default -> new Java(); + default -> throw new IllegalArgumentException("Unsupported vector math library: " + libraryName); }; } catch (Throwable e) { - if (DEBUG) { - System.out.printf("DEBUG: VectorMathLibrary: Error during initialization of %s library: %s\n", - libraryName, e); - e.printStackTrace(System.out); - } + debug("Error during initialization of %s library: %s", libraryName, e); return new Java(); // fallback } } @@ -112,9 +81,7 @@ static String getDefaultName() { private static final Library LIBRARY = Library.getInstance(); static { - if (DEBUG) { - System.out.printf("DEBUG: VectorMathLibrary: %s library is used\n", LIBRARY.getClass().getSimpleName()); - } + debug("%s library is used (cpu features: %s)", LIBRARY.getClass().getSimpleName(), CPUFeatures.features()); } private static class Java implements Library { @@ -129,44 +96,40 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { } } - - // SVML method naming convention - // All the methods are named as __jsvml__ha_ - // Where: - // ha stands for high accuracy - // is optional to indicate float/double - // Set to f for vector float operation - // Omitted for vector double operation - // is the number of elements in the vector - // 1, 2, 4, 8, 16 - // e.g. 128 bit float vector has 4 float elements - // indicates the avx/sse level: - // z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2 - // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns - // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns + /** + * Naming convention in SVML vector math library. + * All the methods are named as __jsvml__ha_ where: + * ha stands for high accuracy + * is optional to indicate float/double + * Set to f for vector float operation + * Omitted for vector double operation + * is the number of elements in the vector + * 1, 2, 4, 8, 16 + * e.g. 128 bit float vector has 4 float elements + * indicates the avx/sse level: + * z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2 + * e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns + * __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns + */ private static class SVML implements Library { - static boolean SUPPORTS_AVX512DQ = CPUFeatures.isX64() && CPUFeatures.hasFeature("avx512dq"); - static { loadNativeLibrary(); } - @SuppressWarnings({"removal", "restricted"}) private static void loadNativeLibrary() { - System.loadLibrary("jsvml"); + requires(isX64(), "SVML library is x64-specific"); + VectorSupport.loadNativeLibrary("jsvml"); } private static String suffix(VectorSpecies vspecies) { assert vspecies.vectorBitSize() <= VectorShape.getMaxVectorBitSize(vspecies.elementType()); - boolean hasAVX2 = VectorShape.getMaxVectorBitSize(byte.class) >= 256; - boolean hasAVX1 = VectorShape.getMaxVectorBitSize(float.class) >= 256; - if (vspecies.vectorBitSize() == 512) { + assert CPUFeatures.X64.SUPPORTS_AVX512F; return "z0"; - } else if (hasAVX2) { + } else if (CPUFeatures.X64.SUPPORTS_AVX2) { return "l9"; - } else if (hasAVX1) { + } else if (CPUFeatures.X64.SUPPORTS_AVX) { return "e9"; } else { return "ex"; @@ -192,25 +155,41 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { } if (vspecies.vectorBitSize() < 128) { return false; // 64-bit vectors are not supported - } - if (op == POW) { - return false; // not supported - } - if (vspecies.vectorBitSize() == 512 && (op == LOG || op == LOG10)) { - return SUPPORTS_AVX512DQ; + } else if (vspecies.vectorBitSize() < 512) { + if (op == POW) { + return false; // not supported + } + } else if (vspecies.vectorBitSize() == 512) { + if (op == LOG || op == LOG10 || op == POW) { + return CPUFeatures.X64.SUPPORTS_AVX512DQ; // requires AVX512DQ CPU support + } + } else { + throw new InternalError("unsupported vector shape: " + vspecies); } return true; } } + /** + * Naming convention in SLEEF-based vector math library . + * All the methods are named as _ where: + * is the operation name, e.g. sin + * is optional to indicate float/double + * "f/d" for vector float/double operation + * is the number of elements in the vector + * "2/4" for neon, and "x" for sve + * is the precision level + * "u10/u05" represents 1.0/0.5 ULP error bounds + * We use "u10" for all operations by default + * But for those functions do not have u10 support, we use "u05" instead + * indicates neon/sve + * "sve/advsimd" for sve/neon implementations + * e.g. sinfx_u10sve is the method for computing vector float sin using SVE instructions + * cosd2_u10advsimd is the method for computing 2 elements vector double cos using NEON instructions + */ private static class SLEEF implements Library { static { - loadNativeLibrary(); - } - - @SuppressWarnings({"removal", "restricted"}) - private static void loadNativeLibrary() { - System.loadLibrary("sleef"); + VectorSupport.loadNativeLibrary("sleef"); } private static String suffix(VectorShape vshape) { @@ -221,22 +200,6 @@ private static String precisionLevel(Operator op) { return (op == HYPOT ? "u05" : "u10"); } - // Method naming convention - // All the methods are named as _ - // Where: - // is the operation name, e.g. sin - // is optional to indicate float/double - // "f/d" for vector float/double operation - // is the number of elements in the vector - // "2/4" for neon, and "x" for sve - // is the precision level - // "u10/u05" represents 1.0/0.5 ULP error bounds - // We use "u10" for all operations by default - // But for those functions do not have u10 support, we use "u05" instead - // indicates neon/sve - // "sve/advsimd" for sve/neon implementations - // e.g. sinfx_u10sve is the method for computing vector float sin using SVE instructions - // cosd2_u10advsimd is the method for computing 2 elements vector double cos using NEON instructions @Override public String symbolName(Operator op, VectorSpecies vspecies) { return String.format("%s%s%d_%s%s", op.operatorName(), @@ -281,7 +244,7 @@ private static Entry lookup(Operator op, int opc, VectorSpecies vspeci Entry entry = (Entry)LIBRARY_ENTRIES[idx][elem_idx][shape_idx]; if (entry == null) { entry = constructEntry(op, opc, vspecies, implSupplier); - LIBRARY_ENTRIES[idx][elem_idx][shape_idx] = entry; // FIXME: CAS + LIBRARY_ENTRIES[idx][elem_idx][shape_idx] = entry; } return entry; } @@ -292,11 +255,10 @@ private static Entry lookup(Operator op, int opc, VectorSpecies vspeci Entry constructEntry(Operator op, int opc, VectorSpecies vspecies, IntFunction implSupplier) { if (LIBRARY.isSupported(op, vspecies)) { String symbol = LIBRARY.symbolName(op, vspecies); - MemorySegment addr = LOOKUP.find(symbol).orElseThrow(() -> new InternalError("not supported: " + op + " " + vspecies + " " + symbol)); - if (DEBUG) { - System.out.printf("DEBUG: VectorMathLibrary: %s %s => 0x%016x\n", op, symbol, addr.address()); - } - T impl = implSupplier.apply(opc); // TODO: should call the very same native implementation + MemorySegment addr = LOOKUP.find(symbol) + .orElseThrow(() -> new InternalError("not supported: " + op + " " + vspecies + " " + symbol)); + debug("%s %s => 0x%016x\n", op, symbol, addr.address()); + T impl = implSupplier.apply(opc); // TODO: should call the very same native implementation eventually (once FFM API supports vectors) return new Entry<>(symbol, addr, impl); } else { return new Entry<>(null, MemorySegment.NULL, implSupplier.apply(opc)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java index c8cb5e0a904cc..0e231bd5174f0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java @@ -33,6 +33,8 @@ import jdk.internal.vm.vector.VectorSupport; +import static jdk.internal.vm.vector.Utils.isNonCapturingLambda; + /** * This class consists solely of static constants * that describe lane-wise vector operations, plus nested interfaces @@ -1374,7 +1376,7 @@ public T find(OP op, int opc, IntFunction supplier) { if (fn != null) return fn; fn = supplier.apply(opc); if (fn == null) throw badOp(op); - assert(VectorSupport.isNonCapturingLambda(fn)) : fn; + assert(isNonCapturingLambda(fn)) : fn; // The JIT can see into this cache: cache[opc] = fn; return fn; From 368b943eeec75f1bfe2498a987ae651b67aec653 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Mon, 7 Apr 2025 14:31:27 -0700 Subject: [PATCH 14/25] Reviews and Float64Vector-related fix --- .../jdk/incubator/vector/CPUFeatures.java | 5 ++-- .../incubator/vector/VectorMathLibrary.java | 28 +++++++++---------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java index 3fc0f5e6fdbb6..290b3b7c63491 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java @@ -26,6 +26,7 @@ import jdk.internal.vm.vector.VectorSupport; +import java.util.Locale; import java.util.Set; import static jdk.incubator.vector.Util.requires; @@ -41,7 +42,7 @@ private static Set getCPUFeatures() { String featuresString = VectorSupport.getCPUFeatures(); debug(featuresString); - String[] features = featuresString.toLowerCase().split(", "); // ", " is used as a delimiter + String[] features = featuresString.toLowerCase(Locale.ROOT).split(", "); // ", " is used as a delimiter assert validateFeatures(features); return Set.of(features); } @@ -54,7 +55,7 @@ private static boolean validateFeatures(String[] features) { } private static boolean hasFeature(String feature) { - return features.contains(feature.toLowerCase()); + return features.contains(feature.toLowerCase(Locale.ROOT)); } public static class X64 { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index 4cfb5e9a9b053..a5c05a1f1265b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -140,7 +140,8 @@ private static String suffix(VectorSpecies vspecies) { public String symbolName(Operator op, VectorSpecies vspecies) { String suffix = suffix(vspecies); String elemType = (vspecies.elementType() == float.class ? "f" : ""); - return String.format("__jsvml_%s%s%d_ha_%s", op.operatorName(), elemType, vspecies.length(), suffix); + int vlen = (vspecies == FloatVector.SPECIES_64 ? 4 : vspecies.length()); // reuse 128-bit variant for 64-bit float vectors + return String.format("__jsvml_%s%s%d_ha_%s", op.operatorName(), elemType, vlen, suffix); } @Override @@ -153,18 +154,12 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { if (vspecies.length() > maxLaneCount) { return false; // lacking vector support } - if (vspecies.vectorBitSize() < 128) { - return false; // 64-bit vectors are not supported - } else if (vspecies.vectorBitSize() < 512) { - if (op == POW) { - return false; // not supported - } - } else if (vspecies.vectorBitSize() == 512) { + if (vspecies.vectorBitSize() == 512) { if (op == LOG || op == LOG10 || op == POW) { return CPUFeatures.X64.SUPPORTS_AVX512DQ; // requires AVX512DQ CPU support } - } else { - throw new InternalError("unsupported vector shape: " + vspecies); + } else if (op == POW) { + return false; // not supported } return true; } @@ -255,11 +250,14 @@ private static Entry lookup(Operator op, int opc, VectorSpecies vspeci Entry constructEntry(Operator op, int opc, VectorSpecies vspecies, IntFunction implSupplier) { if (LIBRARY.isSupported(op, vspecies)) { String symbol = LIBRARY.symbolName(op, vspecies); - MemorySegment addr = LOOKUP.find(symbol) - .orElseThrow(() -> new InternalError("not supported: " + op + " " + vspecies + " " + symbol)); - debug("%s %s => 0x%016x\n", op, symbol, addr.address()); - T impl = implSupplier.apply(opc); // TODO: should call the very same native implementation eventually (once FFM API supports vectors) - return new Entry<>(symbol, addr, impl); + try { + MemorySegment addr = LOOKUP.findOrThrow(symbol); + debug("%s %s => 0x%016x\n", op, symbol, addr.address()); + T impl = implSupplier.apply(opc); // TODO: should call the very same native implementation eventually (once FFM API supports vectors) + return new Entry<>(symbol, addr, impl); + } catch (RuntimeException e) { + throw new InternalError("not supported: " + op + " " + vspecies + " " + symbol, e); + } } else { return new Entry<>(null, MemorySegment.NULL, implSupplier.apply(opc)); } From 9a8f62005e57d7103f2b76eaea9936fc29c548bc Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Mon, 7 Apr 2025 15:59:18 -0700 Subject: [PATCH 15/25] features_string -> cpu_info_string --- .../cpu/aarch64/vm_version_aarch64.cpp | 2 +- src/hotspot/cpu/arm/vm_version_arm_32.cpp | 4 +-- src/hotspot/cpu/ppc/vm_version_ppc.cpp | 6 ++-- src/hotspot/cpu/riscv/vm_version_riscv.cpp | 2 +- src/hotspot/cpu/s390/vm_version_s390.cpp | 32 +++++++++---------- src/hotspot/cpu/s390/vm_version_s390.hpp | 2 +- src/hotspot/cpu/zero/vm_version_zero.cpp | 2 +- src/hotspot/share/runtime/vmStructs.cpp | 1 + 8 files changed, 26 insertions(+), 25 deletions(-) diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index d734b62cc1c80..cdaafd0e137a0 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -707,7 +707,7 @@ void VM_Version::initialize_cpu_information(void) { int desc_len = snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "AArch64 "); get_compatible_board(_cpu_desc + desc_len, CPU_DETAILED_DESC_BUF_SIZE - desc_len); desc_len = (int)strlen(_cpu_desc); - snprintf(_cpu_desc + desc_len, CPU_DETAILED_DESC_BUF_SIZE - desc_len, " %s", _features_string); + snprintf(_cpu_desc + desc_len, CPU_DETAILED_DESC_BUF_SIZE - desc_len, " %s", _cpu_info_string); _initialized = true; } diff --git a/src/hotspot/cpu/arm/vm_version_arm_32.cpp b/src/hotspot/cpu/arm/vm_version_arm_32.cpp index 148786a55da41..d094193603567 100644 --- a/src/hotspot/cpu/arm/vm_version_arm_32.cpp +++ b/src/hotspot/cpu/arm/vm_version_arm_32.cpp @@ -295,7 +295,7 @@ void VM_Version::initialize() { (has_multiprocessing_extensions() ? ", mp_ext" : "")); // buf is started with ", " or is empty - _features_string = os::strdup(buf); + _cpu_info_string = os::strdup(buf); if (has_simd()) { if (FLAG_IS_DEFAULT(UsePopCountInstruction)) { @@ -363,6 +363,6 @@ void VM_Version::initialize_cpu_information(void) { _no_of_threads = _no_of_cores; _no_of_sockets = _no_of_cores; snprintf(_cpu_name, CPU_TYPE_DESC_BUF_SIZE - 1, "ARM%d", _arm_arch); - snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "%s", _features_string); + snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "%s", _cpu_info_string); _initialized = true; } diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.cpp b/src/hotspot/cpu/ppc/vm_version_ppc.cpp index 8ec69bffe15ea..3cb0bf9bf720b 100644 --- a/src/hotspot/cpu/ppc/vm_version_ppc.cpp +++ b/src/hotspot/cpu/ppc/vm_version_ppc.cpp @@ -219,7 +219,7 @@ void VM_Version::initialize() { (has_brw() ? " brw" : "") // Make sure number of %s matches num_features! ); - _features_string = os::strdup(buf); + _cpu_info_string = os::strdup(buf); if (Verbose) { print_features(); } @@ -519,7 +519,7 @@ void VM_Version::print_platform_virtualization_info(outputStream* st) { } void VM_Version::print_features() { - tty->print_cr("Version: %s L1_data_cache_line_size=%d", features_string(), L1_data_cache_line_size()); + tty->print_cr("Version: %s L1_data_cache_line_size=%d", cpu_info_string(), L1_data_cache_line_size()); if (Verbose) { if (ContendedPaddingWidth > 0) { @@ -726,6 +726,6 @@ void VM_Version::initialize_cpu_information(void) { _no_of_threads = _no_of_cores; _no_of_sockets = _no_of_cores; snprintf(_cpu_name, CPU_TYPE_DESC_BUF_SIZE, "PowerPC POWER%lu", PowerArchitecturePPC64); - snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "PPC %s", features_string()); + snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "PPC %s", cpu_info_string()); _initialized = true; } diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp index a0de9d767bfb2..7aa257d493e03 100644 --- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp +++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp @@ -476,7 +476,7 @@ void VM_Version::initialize_cpu_information(void) { _no_of_threads = _no_of_cores; _no_of_sockets = _no_of_cores; snprintf(_cpu_name, CPU_TYPE_DESC_BUF_SIZE - 1, "RISCV64"); - snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "RISCV64 %s", features_string()); + snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "RISCV64 %s", cpu_info_string()); _initialized = true; } diff --git a/src/hotspot/cpu/s390/vm_version_s390.cpp b/src/hotspot/cpu/s390/vm_version_s390.cpp index 157b945e6e1a4..8261fbd083aae 100644 --- a/src/hotspot/cpu/s390/vm_version_s390.cpp +++ b/src/hotspot/cpu/s390/vm_version_s390.cpp @@ -90,7 +90,7 @@ static const char* z_features[] = {" ", void VM_Version::initialize() { determine_features(); // Get processor capabilities. - set_features_string(); // Set a descriptive feature indication. + set_cpu_info_string(); // Set a descriptive feature indication. if (Verbose || PrintAssembly || PrintStubCode) { print_features_internal("CPU Version as detected internally:", PrintAssembly || PrintStubCode); @@ -388,9 +388,9 @@ int VM_Version::get_model_index() { } -void VM_Version::set_features_string() { - // A note on the _features_string format: - // There are jtreg tests checking the _features_string for various properties. +void VM_Version::set_cpu_info_string() { + // A note on the _cpu_info_string format: + // There are jtreg tests checking the _cpu_info_string for various properties. // For some strange reason, these tests require the string to contain // only _lowercase_ characters. Keep that in mind when being surprised // about the unusual notation of features - and when adding new ones. @@ -412,29 +412,29 @@ void VM_Version::set_features_string() { _model_string = "unknown model"; strcpy(buf, "z/Architecture (ambiguous detection)"); } - _features_string = os::strdup(buf); + _cpu_info_string = os::strdup(buf); if (has_Crypto_AES()) { - assert(strlen(_features_string) + 3*8 < sizeof(buf), "increase buffer size"); + assert(strlen(_cpu_info_string) + 3*8 < sizeof(buf), "increase buffer size"); jio_snprintf(buf, sizeof(buf), "%s%s%s%s", - _features_string, + _cpu_info_string, has_Crypto_AES128() ? ", aes128" : "", has_Crypto_AES192() ? ", aes192" : "", has_Crypto_AES256() ? ", aes256" : ""); - os::free((void *)_features_string); - _features_string = os::strdup(buf); + os::free((void *)_cpu_info_string); + _cpu_info_string = os::strdup(buf); } if (has_Crypto_SHA()) { - assert(strlen(_features_string) + 6 + 2*8 + 7 < sizeof(buf), "increase buffer size"); + assert(strlen(_cpu_info_string) + 6 + 2*8 + 7 < sizeof(buf), "increase buffer size"); jio_snprintf(buf, sizeof(buf), "%s%s%s%s%s", - _features_string, + _cpu_info_string, has_Crypto_SHA1() ? ", sha1" : "", has_Crypto_SHA256() ? ", sha256" : "", has_Crypto_SHA512() ? ", sha512" : "", has_Crypto_GHASH() ? ", ghash" : ""); - os::free((void *)_features_string); - _features_string = os::strdup(buf); + os::free((void *)_cpu_info_string); + _cpu_info_string = os::strdup(buf); } } @@ -464,7 +464,7 @@ bool VM_Version::test_feature_bit(unsigned long* featureBuffer, int featureNum, } void VM_Version::print_features_internal(const char* text, bool print_anyway) { - tty->print_cr("%s %s", text, features_string()); + tty->print_cr("%s %s", text, cpu_info_string()); tty->cr(); if (Verbose || print_anyway) { @@ -906,7 +906,7 @@ void VM_Version::set_features_from(const char* march) { err = true; } if (!err) { - set_features_string(); + set_cpu_info_string(); if (prt || PrintAssembly) { print_features_internal("CPU Version as set by cmdline option:", prt); } @@ -1542,6 +1542,6 @@ void VM_Version::initialize_cpu_information(void) { _no_of_threads = _no_of_cores; _no_of_sockets = _no_of_cores; snprintf(_cpu_name, CPU_TYPE_DESC_BUF_SIZE, "s390 %s", VM_Version::get_model_string()); - snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "s390 %s", features_string()); + snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "s390 %s", cpu_info_string()); _initialized = true; } diff --git a/src/hotspot/cpu/s390/vm_version_s390.hpp b/src/hotspot/cpu/s390/vm_version_s390.hpp index 49e6f5686f60a..6c6eb76bf7b03 100644 --- a/src/hotspot/cpu/s390/vm_version_s390.hpp +++ b/src/hotspot/cpu/s390/vm_version_s390.hpp @@ -148,7 +148,7 @@ class VM_Version: public Abstract_VM_Version { static bool test_feature_bit(unsigned long* featureBuffer, int featureNum, unsigned int bufLen); static int get_model_index(); - static void set_features_string(); + static void set_cpu_info_string(); static void print_features_internal(const char* text, bool print_anyway=false); static void determine_features(); static long call_getFeatures(unsigned long* buffer, int buflen, int functionCode); diff --git a/src/hotspot/cpu/zero/vm_version_zero.cpp b/src/hotspot/cpu/zero/vm_version_zero.cpp index e38561e19c571..3ce9227c1939c 100644 --- a/src/hotspot/cpu/zero/vm_version_zero.cpp +++ b/src/hotspot/cpu/zero/vm_version_zero.cpp @@ -151,6 +151,6 @@ void VM_Version::initialize_cpu_information(void) { _no_of_threads = _no_of_cores; _no_of_sockets = _no_of_cores; snprintf(_cpu_name, CPU_TYPE_DESC_BUF_SIZE - 1, "Zero VM"); - snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "%s", _features_string); + snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "%s", _cpu_info_string); _initialized = true; } diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp index c95dd709c844e..f982e8423a503 100644 --- a/src/hotspot/share/runtime/vmStructs.cpp +++ b/src/hotspot/share/runtime/vmStructs.cpp @@ -704,6 +704,7 @@ static_field(Abstract_VM_Version, _s_internal_vm_info_string, const char*) \ static_field(Abstract_VM_Version, _features, uint64_t) \ static_field(Abstract_VM_Version, _features_string, const char*) \ + static_field(Abstract_VM_Version, _cpu_info_string, const char*) \ static_field(Abstract_VM_Version, _vm_major_version, int) \ static_field(Abstract_VM_Version, _vm_minor_version, int) \ static_field(Abstract_VM_Version, _vm_security_version, int) \ From bb1a11dbc25dd65036acd313f7c25e91ee7305e4 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Mon, 7 Apr 2025 16:27:36 -0700 Subject: [PATCH 16/25] Fix windows-aarch64 build failure --- src/hotspot/cpu/aarch64/vm_version_aarch64.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index cdaafd0e137a0..2b951f2c3a5b7 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -628,7 +628,7 @@ void VM_Version::initialize() { if (_model2) { os::snprintf_checked(buf + buf_used_len, sizeof(buf) - buf_used_len, "(0x%03x)", _model2); } - int features_offset = strnlen(buf, sizeof(buf)); + size_t features_offset = strnlen(buf, sizeof(buf)); #define ADD_FEATURE_IF_SUPPORTED(id, name, bit) \ do { \ if (VM_Version::supports_##name()) strcat(buf, ", " #name); \ From 22754c80a8c51ae5341be820d60ba77d3343bb46 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Fri, 11 Apr 2025 14:18:10 -0700 Subject: [PATCH 17/25] RVV and SVE adjustments --- .../incubator/vector/VectorMathLibrary.java | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index a5c05a1f1265b..d4878c2a116c7 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -36,7 +36,7 @@ import static jdk.incubator.vector.Util.requires; import static jdk.incubator.vector.VectorOperators.*; -import static jdk.internal.util.Architecture.isX64; +import static jdk.internal.util.Architecture.*; import static jdk.internal.vm.vector.Utils.debug; /** @@ -154,6 +154,9 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { if (vspecies.length() > maxLaneCount) { return false; // lacking vector support } + if (vspecies == DoubleVector.SPECIES_64) { + return false; // 64-bit double vectors are not supported + } if (vspecies.vectorBitSize() == 512) { if (op == LOG || op == LOG10 || op == POW) { return CPUFeatures.X64.SUPPORTS_AVX512DQ; // requires AVX512DQ CPU support @@ -172,13 +175,13 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { * is optional to indicate float/double * "f/d" for vector float/double operation * is the number of elements in the vector - * "2/4" for neon, and "x" for sve + * "2/4" for neon, and "x" for sve/rvv * is the precision level * "u10/u05" represents 1.0/0.5 ULP error bounds * We use "u10" for all operations by default * But for those functions do not have u10 support, we use "u05" instead - * indicates neon/sve - * "sve/advsimd" for sve/neon implementations + * indicates neon/sve/rvv + * "sve/advsimd/rvv" for sve/neon/rvv implementations * e.g. sinfx_u10sve is the method for computing vector float sin using SVE instructions * cosd2_u10advsimd is the method for computing 2 elements vector double cos using NEON instructions */ @@ -188,7 +191,19 @@ private static class SLEEF implements Library { } private static String suffix(VectorShape vshape) { - return (vshape.vectorBitSize() > 128 ? "sve" : "advsimd"); + if (isAARCH64()) { + if (vshape.vectorBitSize() <= 128) { + return "advsimd"; + } else { + assert (vshape == VectorShape.S_Max_BIT) : "not supported: " + vshape; + return "sve"; + } + } else if (isRISCV64()) { + assert (vshape == VectorShape.S_Max_BIT) : "not supported: " + vshape; + return "rvv"; + } else { + throw new InternalError("unsupported platform"); + } } private static String precisionLevel(Operator op) { @@ -197,9 +212,11 @@ private static String precisionLevel(Operator op) { @Override public String symbolName(Operator op, VectorSpecies vspecies) { - return String.format("%s%s%d_%s%s", op.operatorName(), + int vlen = (vspecies == FloatVector.SPECIES_64 ? 4 : vspecies.length()); // reuse 128-bit variant for 64-bit float vectors + boolean isShapeAgnostic = isRISCV64() || (isAARCH64() && vspecies.vectorBitSize() > 128); + return String.format("%s%s%s_%s%s", op.operatorName(), (vspecies.elementType() == float.class ? "f" : "d"), - vspecies.length(), + (isShapeAgnostic ? "x" : Integer.toString(vlen)), precisionLevel(op), suffix(vspecies.vectorShape())); } @@ -214,8 +231,16 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { if (vspecies.length() > maxLaneCount) { return false; // lacking vector support } - if (vspecies.vectorBitSize() < 128) { - return false; // 64-bit vectors are not supported + if (vspecies.vectorShape() != VectorShape.S_Max_BIT) { + if (isRISCV64()) { + return false; // FIXME: only MAX shapes are supported on RISC-V + } + if (isAARCH64() && vspecies.vectorBitSize() > 128) { + return false; // FIXME: SVE support only for MAX shapes + } + if (vspecies == DoubleVector.SPECIES_64) { + return false; // 64-bit double vectors are not supported + } } if (op == TANH) { return false; // skip due to performance considerations From 84d02cb3389a707bbd4155466a9fbe651ec00000 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Wed, 16 Apr 2025 12:13:57 -0700 Subject: [PATCH 18/25] Fix debugName handling --- src/hotspot/share/classfile/vmIntrinsics.hpp | 8 ++-- src/hotspot/share/opto/vectorIntrinsics.cpp | 46 ++++++++++--------- .../jdk/internal/vm/vector/VectorSupport.java | 10 ++-- .../incubator/vector/VectorMathLibrary.java | 10 ++-- 4 files changed, 36 insertions(+), 38 deletions(-) diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 8e70b9a0aaf76..b5ad109b3e7e8 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -1007,9 +1007,9 @@ class methodHandle; "Ljava/lang/Class;" \ "Ljava/lang/Class;" \ "I" \ + "Ljava/lang/String;" \ "Ljdk/internal/vm/vector/VectorSupport$Vector;" \ - "Ljdk/internal/vm/vector/VectorSupport$UnaryOperation;" \ - "Ljava/lang/String;)" \ + "Ljdk/internal/vm/vector/VectorSupport$UnaryOperation;)" \ "Ljdk/internal/vm/vector/VectorSupport$Vector;") \ do_name(vector_unary_lib_op_name, "libraryUnaryOp") \ \ @@ -1018,10 +1018,10 @@ class methodHandle; "Ljava/lang/Class;" \ "Ljava/lang/Class;" \ "I" \ + "Ljava/lang/String;" \ "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \ "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \ - "Ljdk/internal/vm/vector/VectorSupport$BinaryOperation;" \ - "Ljava/lang/String;)" \ + "Ljdk/internal/vm/vector/VectorSupport$BinaryOperation;)" \ "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;") \ do_name(vector_binary_lib_op_name, "libraryBinaryOp") \ \ diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index 9b37d97948073..13acc0469eb29 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -477,30 +477,35 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { // public static // , E> -// V unaryOp(long address, Class vClass, Class elementType, int length, -// V v, -// UnaryOperation defaultImpl) +// V libraryUnaryOp(long address, Class vClass, Class elementType, int length, String debugName, +// V v, +// UnaryOperation defaultImpl) // // public static // -// V binaryOp(long address, Class vClass, Class elementType, int length, +// V libraryBinaryOp(long address, Class vClass, Class elementType, int length, String debugName, // V v1, V v2, // BinaryOperation defaultImpl) bool LibraryCallKit::inline_vector_call(int arity) { assert(Matcher::supports_vector_calling_convention(), "required"); - const TypeLong* entry = gvn().type(argument(0))->isa_long(); - const TypeInstPtr* vector_klass = gvn().type(argument(2))->isa_instptr(); - const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr(); - const TypeInt* vlen = gvn().type(argument(4))->isa_int(); - - if (entry == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || - !entry->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) { - log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", + const TypeLong* entry = gvn().type(argument(0))->isa_long(); + const TypeInstPtr* vector_klass = gvn().type(argument(2))->isa_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr(); + const TypeInt* vlen = gvn().type(argument(4))->isa_int(); + const TypeInstPtr* debug_name_oop = gvn().type(argument(5))->isa_instptr(); + + if (entry == nullptr || !entry->is_con() || + vector_klass == nullptr || vector_klass->const_oop() == nullptr || + elem_klass == nullptr || elem_klass->const_oop() == nullptr || + vlen == nullptr || !vlen->is_con() || + debug_name_oop == nullptr || debug_name_oop->const_oop() == nullptr) { + log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s debug_name=%s", NodeClassNames[argument(0)->Opcode()], NodeClassNames[argument(2)->Opcode()], NodeClassNames[argument(3)->Opcode()], - NodeClassNames[argument(4)->Opcode()]); + NodeClassNames[argument(4)->Opcode()], + NodeClassNames[argument(5)->Opcode()]); return false; // not enough info for intrinsification } @@ -530,30 +535,27 @@ bool LibraryCallKit::inline_vector_call(int arity) { ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); - Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); + Node* opd1 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); if (opd1 == nullptr) { - log_if_needed(" ** unbox failed v1=%s", - NodeClassNames[argument(5)->Opcode()]); + log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(6)->Opcode()]); return false; } Node* opd2 = nullptr; if (arity > 1) { - opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); + opd2 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem); if (opd2 == nullptr) { - log_if_needed(" ** unbox failed v2=%s", - NodeClassNames[argument(6)->Opcode()]); + log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(7)->Opcode()]); return false; } } - assert(arity == 1 || arity == 2, "not supported"); + assert(arity == 1 || arity == 2, "arity %d not supported", arity); const TypeVect* vt = TypeVect::make(elem_bt, num_elem); const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(arity, vt, vt); address entry_addr = (address)entry->get_con(); const char* debug_name = ""; - const TypeInstPtr* debug_name_oop = gvn().type(argument(8))->isa_instptr(); - if (debug_name_oop != nullptr && debug_name_oop->const_oop() && !debug_name_oop->const_oop()->is_null_object()) { + if (!debug_name_oop->const_oop()->is_null_object()) { size_t buflen = 100; char* buf = NEW_ARENA_ARRAY(C->comp_arena(), char, buflen); debug_name = debug_name_oop->const_oop()->as_instance()->java_lang_String_str(buf, buflen); diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index 81b047743e47f..e37f5186e4ea4 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -347,10 +347,9 @@ V unaryOp(int oprId, @IntrinsicCandidate public static , E> - V libraryUnaryOp(long addr, Class vClass, Class eClass, int length, + V libraryUnaryOp(long addr, Class vClass, Class eClass, int length, String debugName, V v, - UnaryOperation defaultImpl, - String debugName) { + UnaryOperation defaultImpl) { assert isNonCapturingLambda(defaultImpl) : defaultImpl; return defaultImpl.apply(v, null); } @@ -386,10 +385,9 @@ VM binaryOp(int oprId, @IntrinsicCandidate public static - V libraryBinaryOp(long addr, Class vClass, Class eClass, int length, + V libraryBinaryOp(long addr, Class vClass, Class eClass, int length, String debugName, V v1, V v2, - BinaryOperation defaultImpl, - String debugName) { + BinaryOperation defaultImpl) { assert isNonCapturingLambda(defaultImpl) : defaultImpl; return defaultImpl.apply(v1, v2, null); } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index d4878c2a116c7..e0c4c0317c8ec 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -301,10 +301,9 @@ V unaryMathOp(Unary op, int opc, VectorSpecies vspecies, @SuppressWarnings({"unchecked"}) Class vt = (Class)vspecies.vectorType(); return VectorSupport.libraryUnaryOp( - entry.entry.address(), vt, vspecies.elementType(), vspecies.length(), + entry.entry.address(), vt, vspecies.elementType(), vspecies.length(), entry.name, v, - entry.impl, - entry.name); + entry.impl); } else { return entry.impl.apply(v, null); } @@ -323,10 +322,9 @@ V binaryMathOp(Binary op, int opc, VectorSpecies vspecies, @SuppressWarnings({"unchecked"}) Class vt = (Class)vspecies.vectorType(); return VectorSupport.libraryBinaryOp( - entry.entry.address(), vt, vspecies.elementType(), vspecies.length(), + entry.entry.address(), vt, vspecies.elementType(), vspecies.length(), entry.name, v1, v2, - entry.impl, - entry.name); + entry.impl); } else { return entry.impl.apply(v1, v2, null); } From 1ade1ffd2ed18fe120aefb7be8a607beb0563fb2 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Wed, 16 Apr 2025 18:00:58 -0700 Subject: [PATCH 19/25] fix broken merge --- src/hotspot/cpu/aarch64/vm_version_aarch64.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index 1ba245e4bb84f..0db41035169f1 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -718,7 +718,7 @@ void VM_Version::initialize_cpu_information(void) { int desc_len = snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "AArch64 "); get_compatible_board(_cpu_desc + desc_len, CPU_DETAILED_DESC_BUF_SIZE - desc_len); desc_len = (int)strlen(_cpu_desc); - snprintf(_cpu_desc + desc_len, cpu_detailed_desc_buf_size - desc_len, " %s", _cpu_info_string); + snprintf(_cpu_desc + desc_len, CPU_DETAILED_DESC_BUF_SIZE - desc_len, " %s", _cpu_info_string); _initialized = true; } From e2b762ec1a2c8ec838d467693bc6703dbf4233ab Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Thu, 17 Apr 2025 10:50:38 -0700 Subject: [PATCH 20/25] RVV and SVE adjustments --- .../incubator/vector/VectorMathLibrary.java | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index e0c4c0317c8ec..e6b959e025bd2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -190,16 +190,15 @@ private static class SLEEF implements Library { VectorSupport.loadNativeLibrary("sleef"); } - private static String suffix(VectorShape vshape) { + private static String suffix(VectorShape vshape, boolean isShapeAgnostic) { if (isAARCH64()) { - if (vshape.vectorBitSize() <= 128) { - return "advsimd"; - } else { - assert (vshape == VectorShape.S_Max_BIT) : "not supported: " + vshape; + if (isShapeAgnostic) { return "sve"; + } else { + return "advsimd"; } } else if (isRISCV64()) { - assert (vshape == VectorShape.S_Max_BIT) : "not supported: " + vshape; + assert isShapeAgnostic : "not supported"; return "rvv"; } else { throw new InternalError("unsupported platform"); @@ -218,7 +217,7 @@ public String symbolName(Operator op, VectorSpecies vspecies) { (vspecies.elementType() == float.class ? "f" : "d"), (isShapeAgnostic ? "x" : Integer.toString(vlen)), precisionLevel(op), - suffix(vspecies.vectorShape())); + suffix(vspecies.vectorShape(), isShapeAgnostic)); } @Override @@ -231,16 +230,8 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { if (vspecies.length() > maxLaneCount) { return false; // lacking vector support } - if (vspecies.vectorShape() != VectorShape.S_Max_BIT) { - if (isRISCV64()) { - return false; // FIXME: only MAX shapes are supported on RISC-V - } - if (isAARCH64() && vspecies.vectorBitSize() > 128) { - return false; // FIXME: SVE support only for MAX shapes - } - if (vspecies == DoubleVector.SPECIES_64) { - return false; // 64-bit double vectors are not supported - } + if (vspecies == DoubleVector.SPECIES_64) { + return false; // 64-bit double vectors are not supported } if (op == TANH) { return false; // skip due to performance considerations From 3d1adff2e76eca03013c2739678ce5613c430982 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Tue, 22 Apr 2025 16:57:28 -0700 Subject: [PATCH 21/25] riscv fix --- .../share/classes/jdk/incubator/vector/CPUFeatures.java | 2 +- .../share/classes/jdk/incubator/vector/VectorMathLibrary.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java index 290b3b7c63491..e68b58489bc02 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java @@ -79,4 +79,4 @@ public static class X64 { public static Set features() { return features; } -} \ No newline at end of file +} diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index e6b959e025bd2..59baa6179406d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -72,7 +72,7 @@ static Library getInstance() { static String getDefaultName() { return switch (StaticProperty.osArch()) { case "amd64", "x86_64" -> SVML; - case "aarch64" -> SLEEF; + case "aarch64", "riscv64" -> SLEEF; default -> JAVA; }; } From 42ed9baac3d927fa79bc00ebd243f21324f83b78 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Tue, 22 Apr 2025 17:58:37 -0700 Subject: [PATCH 22/25] Avoid thread state transition in VectorSupport_GetCPUFeatures --- src/hotspot/share/prims/vectorSupport.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index 2a40116c8be68..c907ddb48857e 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -618,8 +618,8 @@ JVM_ENTRY(jstring, VectorSupport_GetCPUFeatures(JNIEnv* env, jclass ignored)) const char* features_string = VM_Version::features_string(); assert(features_string != nullptr, "missing cpu features info"); - ThreadToNativeFromVM ttn(thread); - return env->NewStringUTF(features_string); + oop result = java_lang_String::create_oop_from_str(features_string, CHECK_NULL); + return (jstring) JNIHandles::make_local(THREAD, result); JVM_END // JVM_RegisterVectorSupportMethods From 585312ae4ce114e13efef425634808fb16b3da29 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Wed, 23 Apr 2025 16:49:35 -0700 Subject: [PATCH 23/25] CPUFeatures: RISC-V support --- src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp | 4 +--- .../share/classes/jdk/incubator/vector/CPUFeatures.java | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp index 484a2a645aaae..506c78cacca18 100644 --- a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp +++ b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp @@ -196,9 +196,7 @@ void VM_Version::setup_cpu_available_features() { _cpu_info_string = os::strdup(buf); - _features_string = extract_features_string(_cpu_info_string, - strnlen(_cpu_info_string, sizeof(buf)), - features_offset); + _features_string = _cpu_info_string + features_offset; } void VM_Version::os_aux_features() { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java index e68b58489bc02..1d44860ab86eb 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/CPUFeatures.java @@ -42,7 +42,8 @@ private static Set getCPUFeatures() { String featuresString = VectorSupport.getCPUFeatures(); debug(featuresString); - String[] features = featuresString.toLowerCase(Locale.ROOT).split(", "); // ", " is used as a delimiter + String[] features = featuresString.toLowerCase(Locale.ROOT) + .split(",? "); // " " or ", " are used as a delimiter by JVM assert validateFeatures(features); return Set.of(features); } From 541c4d7fd8919c811ac2483fbd7e0bed4489f942 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Thu, 24 Apr 2025 16:21:24 -0700 Subject: [PATCH 24/25] Improve comments --- .../share/classes/jdk/incubator/vector/VectorMathLibrary.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java index 59baa6179406d..1dfd6f131d20f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMathLibrary.java @@ -152,7 +152,7 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { } int maxLaneCount = VectorSupport.getMaxLaneCount(vspecies.elementType()); if (vspecies.length() > maxLaneCount) { - return false; // lacking vector support + return false; // lacking vector support (either hardware or disabled on JVM side) } if (vspecies == DoubleVector.SPECIES_64) { return false; // 64-bit double vectors are not supported @@ -228,7 +228,7 @@ public boolean isSupported(Operator op, VectorSpecies vspecies) { } int maxLaneCount = VectorSupport.getMaxLaneCount(vspecies.elementType()); if (vspecies.length() > maxLaneCount) { - return false; // lacking vector support + return false; // lacking vector support (either hardware or disabled on JVM side) } if (vspecies == DoubleVector.SPECIES_64) { return false; // 64-bit double vectors are not supported From f4373e4101550f8afd39e7707e08b5e0ea607ab4 Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Thu, 24 Apr 2025 22:22:23 -0700 Subject: [PATCH 25/25] Remove UseVectorStubs usage in riscv.ad --- src/hotspot/cpu/riscv/riscv.ad | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 9d92f078a5479..646cef818667e 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1948,7 +1948,7 @@ bool Matcher::supports_vector_calling_convention(void) { } OptoRegPair Matcher::vector_return_value(uint ideal_reg) { - assert(EnableVectorSupport && UseVectorStubs, "sanity"); + assert(EnableVectorSupport, "sanity"); assert(ideal_reg == Op_VecA, "sanity"); // check more info at https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc int lo = V8_num;