diff --git a/sycl/include/CL/__spirv/spirv_ops.hpp b/sycl/include/CL/__spirv/spirv_ops.hpp index aa07bda9f4282..8518b37c4a43f 100644 --- a/sycl/include/CL/__spirv/spirv_ops.hpp +++ b/sycl/include/CL/__spirv/spirv_ops.hpp @@ -24,19 +24,23 @@ #ifdef __SYCL_DEVICE_ONLY__ #if (SYCL_EXT_ONEAPI_MATRIX_VERSION > 1) -template -extern SYCL_EXTERNAL __spv::__spirv_JointMatrixINTEL * +extern SYCL_EXTERNAL __spv::__spirv_JointMatrixINTEL * __spirv_JointMatrixLoadINTEL(T *Ptr, std::size_t Stride, __spv::MatrixLayout Layout = L, __spv::Scope::Flag Sc = S, int MemOperand = 0); -template extern SYCL_EXTERNAL void __spirv_JointMatrixStoreINTEL( - T *Ptr, __spv::__spirv_JointMatrixINTEL *Object, + T *Ptr, __spv::__spirv_JointMatrixINTEL *Object, std::size_t Stride, __spv::MatrixLayout Layout = L, __spv::Scope::Flag Sc = S, int MemOperand = 0); @@ -95,10 +99,11 @@ __spirv_JointMatrixSUMadINTEL( __spv::__spirv_JointMatrixINTEL *C, __spv::Scope::Flag Sc = __spv::Scope::Flag::Subgroup); -template -extern SYCL_EXTERNAL __spv::__spirv_JointMatrixINTEL * +extern SYCL_EXTERNAL __spv::__spirv_JointMatrixINTEL * __spirv_CompositeConstruct(const T v); template *); -template -extern SYCL_EXTERNAL T __spirv_VectorExtractDynamic( +extern SYCL_EXTERNAL Ts __spirv_VectorExtractDynamic( __spv::__spirv_JointMatrixINTEL *, size_t i); -template extern SYCL_EXTERNAL __spv::__spirv_JointMatrixINTEL * __spirv_VectorInsertDynamic(__spv::__spirv_JointMatrixINTEL *, - T val, size_t i); + Ts val, size_t i); #else template struct joint_matrix; +// Differentiating between the "element type" and the "storage element type" +template struct helper_traits { + using element_type = T; + using storage_element_type = T; + using fill_argument_type = T; +}; + +template <> struct helper_traits { + using element_type = precision::tf32; + using storage_element_type = float; + using fill_argument_type = float; +}; + template class wi_element { @@ -72,12 +85,19 @@ class wi_element { std::size_t idx; public: + using storage_element_type = typename helper_traits::storage_element_type; wi_element(joint_matrix &Mat, std::size_t i) : M(Mat), idx(i) {} - operator T() { + operator storage_element_type() { #ifdef __SYCL_DEVICE_ONLY__ - return __spirv_VectorExtractDynamic(M.spvm, idx); + storage_element_type elem = + __spirv_VectorExtractDynamic::value, + spv_matrix_layout_traits::value, + spv_scope_traits::value>(M.spvm, + idx); + return elem; #else throw runtime_error("joint matrix is not supported on host device.", PI_ERROR_INVALID_DEVICE); @@ -86,7 +106,12 @@ class wi_element { explicit operator bool() { #ifdef __SYCL_DEVICE_ONLY__ - return __spirv_VectorExtractDynamic(M.spvm, idx) != static_cast(0); + return __spirv_VectorExtractDynamic::value, + spv_matrix_layout_traits::value, + spv_scope_traits::value>( + M.spvm, idx) != static_cast(0); #else throw runtime_error("joint matrix is not supported on host device.", PI_ERROR_INVALID_DEVICE); @@ -95,7 +120,8 @@ class wi_element { template wi_element &operator=(const T2 &rhs) { #ifdef __SYCL_DEVICE_ONLY__ - M.spvm = __spirv_VectorInsertDynamic(M.spvm, static_cast(rhs), idx); + M.spvm = __spirv_VectorInsertDynamic( + M.spvm, static_cast(rhs), idx); return *this; #else (void)rhs; @@ -108,7 +134,13 @@ class wi_element { operator=(const wi_element &rhs) { #ifdef __SYCL_DEVICE_ONLY__ M.spvm = __spirv_VectorInsertDynamic( - M.spvm, __spirv_VectorExtractDynamic(rhs.M.spvm, rhs.idx), idx); + M.spvm, + __spirv_VectorExtractDynamic::value, + spv_matrix_layout_traits::value, + spv_scope_traits::value>(rhs.M.spvm, + rhs.idx), + idx); return *this; #else (void)rhs; @@ -122,8 +154,13 @@ class wi_element { template wi_element &operator op##=(const T2 &rhs) { \ M.spvm = __spirv_VectorInsertDynamic( \ M.spvm, \ - static_cast(__spirv_VectorExtractDynamic(M.spvm, idx) \ - op static_cast(rhs)), \ + static_cast( \ + __spirv_VectorExtractDynamic< \ + storage_element_type, T, NumRows, NumCols, \ + spv_matrix_use_traits::value, \ + spv_matrix_layout_traits::value, \ + spv_scope_traits::value>(M.spvm, idx) \ + op static_cast(rhs)), \ idx); \ return *this; \ } @@ -157,7 +194,11 @@ class wi_element::value, + spv_matrix_layout_traits::value, + spv_scope_traits::value>(M.spvm, idx); #else throw runtime_error("joint matrix is not supported on host device.", PI_ERROR_INVALID_DEVICE); @@ -166,8 +207,13 @@ class wi_element(__spirv_VectorExtractDynamic( - M.spvm, idx))) >= std::numeric_limits::epsilon(); + return std::fabs(static_cast( + __spirv_VectorExtractDynamic< + sycl::ext::oneapi::bfloat16, sycl::ext::oneapi::bfloat16, + NumRows, NumCols, spv_matrix_use_traits::value, + spv_matrix_layout_traits::value, + spv_scope_traits::value>(M.spvm, idx))) >= + std::numeric_limits::epsilon(); #else throw runtime_error("joint matrix is not supported on host device.", PI_ERROR_INVALID_DEVICE); @@ -189,7 +235,14 @@ class wi_element &rhs) { #ifdef __SYCL_DEVICE_ONLY__ M.spvm = __spirv_VectorInsertDynamic( - M.spvm, __spirv_VectorExtractDynamic(rhs.M.spvm, rhs.idx), idx); + M.spvm, + __spirv_VectorExtractDynamic::value, + spv_matrix_layout_traits::value, + spv_scope_traits::value>(rhs.M.spvm, + rhs.idx), + idx); return *this; #else (void)rhs; @@ -202,7 +255,13 @@ class wi_element::value, \ + spv_matrix_layout_traits::value, \ + spv_scope_traits::value>(M.spvm, idx) op rhs, \ + idx); \ return *this; \ } #else // __SYCL_DEVICE_ONLY__ @@ -225,13 +284,21 @@ class wi_element &lhs, \ const sycl::ext::oneapi::bfloat16 &rhs) { \ - return __spirv_VectorExtractDynamic(lhs.M.spvm, lhs.idx) op rhs; \ + return __spirv_VectorExtractDynamic< \ + sycl::ext::oneapi::bfloat16, sycl::ext::oneapi::bfloat16, NumRows, \ + NumCols, spv_matrix_use_traits::value, \ + spv_matrix_layout_traits::value, \ + spv_scope_traits::value>(lhs.M.spvm, lhs.idx) op rhs; \ } \ friend type operator op( \ const sycl::ext::oneapi::bfloat16 &lhs, \ const wi_element &rhs) { \ - return __spirv_VectorExtractDynamic(rhs.M.spvm, rhs.idx) op lhs; \ + return __spirv_VectorExtractDynamic< \ + sycl::ext::oneapi::bfloat16, sycl::ext::oneapi::bfloat16, NumRows, \ + NumCols, spv_matrix_use_traits::value, \ + spv_matrix_layout_traits::value, \ + spv_scope_traits::value>(rhs.M.spvm, rhs.idx) op lhs; \ } OP(sycl::ext::oneapi::bfloat16, +) OP(sycl::ext::oneapi::bfloat16, -) @@ -243,15 +310,25 @@ class wi_element &lhs, \ const sycl::ext::oneapi::bfloat16 &rhs) { \ - return type{static_cast(__spirv_VectorExtractDynamic( \ - lhs.M.spvm, lhs.idx)) op static_cast(rhs)}; \ + return type{static_cast( \ + __spirv_VectorExtractDynamic< \ + sycl::ext::oneapi::bfloat16, sycl::ext::oneapi::bfloat16, NumRows, \ + NumCols, spv_matrix_use_traits::value, \ + spv_matrix_layout_traits::value, \ + spv_scope_traits::value>(lhs.M.spvm, lhs.idx)) \ + op static_cast(rhs)}; \ } \ friend type operator op( \ const sycl::ext::oneapi::bfloat16 &lhs, \ const wi_element &rhs) { \ - return type{static_cast(__spirv_VectorExtractDynamic( \ - rhs.M.spvm, rhs.idx)) op static_cast(lhs)}; \ + return type{static_cast( \ + __spirv_VectorExtractDynamic< \ + sycl::ext::oneapi::bfloat16, sycl::ext::oneapi::bfloat16, NumRows, \ + NumCols, spv_matrix_use_traits::value, \ + spv_matrix_layout_traits::value, \ + spv_scope_traits::value>(rhs.M.spvm, rhs.idx)) \ + op static_cast(lhs)}; \ } OP(bool, ==) OP(bool, !=) @@ -296,7 +373,7 @@ class wi_element &src, + Group, Tp, Use, NumRows, NumCols, Layout> &src, multi_ptr dst, size_t stride) { #if defined(__SYCL_DEVICE_ONLY__) #if defined(__NVPTX__) @@ -321,7 +398,7 @@ joint_matrix_store(Group sg, #else // intel's impl T *Ptr = dst.get(); - __spirv_JointMatrixStoreINTEL::value, sycl::ext::oneapi::experimental::matrix:: diff --git a/sycl/include/sycl/ext/oneapi/matrix/matrix-tensorcores.hpp b/sycl/include/sycl/ext/oneapi/matrix/matrix-tensorcores.hpp index a871b9709ae66..018c66cf5213d 100644 --- a/sycl/include/sycl/ext/oneapi/matrix/matrix-tensorcores.hpp +++ b/sycl/include/sycl/ext/oneapi/matrix/matrix-tensorcores.hpp @@ -18,12 +18,6 @@ namespace oneapi { namespace experimental { namespace matrix { -namespace precision { -class tf32 { - tf32() = delete; -}; -} // namespace precision - template struct joint_matrix; diff --git a/sycl/include/sycl/ext/oneapi/matrix/matrix-unified-utils.hpp b/sycl/include/sycl/ext/oneapi/matrix/matrix-unified-utils.hpp index 718411b22ddbb..6f820fb82575a 100644 --- a/sycl/include/sycl/ext/oneapi/matrix/matrix-unified-utils.hpp +++ b/sycl/include/sycl/ext/oneapi/matrix/matrix-unified-utils.hpp @@ -18,6 +18,12 @@ enum class use { a, b, accumulator }; enum class layout { row_major = 0, col_major = 1, dynamic = 3 }; +namespace precision { +class tf32 { + tf32() = delete; +}; +} // namespace precision + } // namespace matrix } // namespace experimental } // namespace oneapi diff --git a/sycl/include/sycl/ext/oneapi/matrix/matrix-unified.hpp b/sycl/include/sycl/ext/oneapi/matrix/matrix-unified.hpp index 5b43e11564a18..8f98c55944ea0 100644 --- a/sycl/include/sycl/ext/oneapi/matrix/matrix-unified.hpp +++ b/sycl/include/sycl/ext/oneapi/matrix/matrix-unified.hpp @@ -123,11 +123,12 @@ joint_matrix_fill(Group sg, std::ignore = sg; res.cuda_impl.wi_marray = v; #else + using storage_element_type = typename helper_traits::storage_element_type; res.spvm = - __spirv_CompositeConstruct::value, spv_matrix_layout_traits::value>( - static_cast(v)); + static_cast(v)); #endif // defined(__NVPTX__) #else std::ignore = sg; @@ -161,21 +162,21 @@ inline __SYCL_ALWAYS_INLINE void joint_matrix_load( assert(false && "Invalid Memory Layout!"); case layout::row_major: res.spvm = __spirv_JointMatrixLoadINTEL< - T, NumRows, NumCols, spv_matrix_use_traits::value, + T, S, NumRows, NumCols, spv_matrix_use_traits::value, spv_matrix_layout_traits::value>( Ptr, stride, __spv::MatrixLayout::RowMajor, spv_scope_traits::value); break; case layout::col_major: res.spvm = __spirv_JointMatrixLoadINTEL< - T, NumRows, NumCols, spv_matrix_use_traits::value, + T, S, NumRows, NumCols, spv_matrix_use_traits::value, spv_matrix_layout_traits::value>( Ptr, stride, __spv::MatrixLayout::ColumnMajor, spv_scope_traits::value); break; case sycl::ext::intel::experimental::matrix::layout::packed: res.spvm = __spirv_JointMatrixLoadINTEL< - T, NumRows, NumCols, spv_matrix_use_traits::value, + T, S, NumRows, NumCols, spv_matrix_use_traits::value, spv_matrix_layout_traits::value>( Ptr, stride, __spv::MatrixLayout::Packed, spv_scope_traits::value); @@ -214,7 +215,7 @@ joint_matrix_load(Group sg, #else T *Ptr = src.get(); res.spvm = - __spirv_JointMatrixLoadINTEL::value, spv_matrix_layout_traits::value>( Ptr, stride, spv_matrix_layout_traits::value, @@ -251,21 +252,21 @@ inline __SYCL_ALWAYS_INLINE void joint_matrix_store( assert(false && "Invalid Memory Layout!"); case layout::row_major: __spirv_JointMatrixStoreINTEL< - T, NumRows, NumCols, spv_matrix_use_traits::value, + T, T, NumRows, NumCols, spv_matrix_use_traits::value, spv_matrix_layout_traits::value>( Ptr, src.spvm, stride, __spv::MatrixLayout::RowMajor, spv_scope_traits::value); break; case layout::col_major: __spirv_JointMatrixStoreINTEL< - T, NumRows, NumCols, spv_matrix_use_traits::value, + T, T, NumRows, NumCols, spv_matrix_use_traits::value, spv_matrix_layout_traits::value>( Ptr, src.spvm, stride, __spv::MatrixLayout::ColumnMajor, spv_scope_traits::value); break; case sycl::ext::intel::experimental::matrix::layout::packed: __spirv_JointMatrixStoreINTEL< - T, NumRows, NumCols, spv_matrix_use_traits::value, + T, T, NumRows, NumCols, spv_matrix_use_traits::value, spv_matrix_layout_traits::value>( Ptr, src.spvm, stride, __spv::MatrixLayout::Packed, spv_scope_traits::value); @@ -337,18 +338,22 @@ inline __SYCL_ALWAYS_INLINE // This function rounds the bottom 13 bits up or down, and then zeros out the // bottom bits -inline __SYCL_ALWAYS_INLINE float round_to_tf32(float &a) { -#if defined(__SYCL_DEVICE_ONLY__) && defined(__NVPTX__) +inline __SYCL_ALWAYS_INLINE float round_to_tf32(const float &a) { +#if defined(__SYCL_DEVICE_ONLY__) +#if defined(__NVPTX__) int32_t tmp_int = __nvvm_f2tf32_rna(a); return __nvvm_bitcast_i2f(tmp_int); #else - uint32_t tmp_uint = reinterpret_cast(a); + return __spirv_ConvertFToTF32INTEL(a); +#endif // defined(__NVPTX__) +#else + uint32_t tmp_uint = reinterpret_cast(a); tmp_uint += 0x1000u; tmp_uint &= 0xFFFFE000u; float ret = 0; std::memcpy(&ret, &tmp_uint, sizeof(float)); return ret; -#endif // defined(__SYCL_DEVICE_ONLY__) && defined(__NVPTX__) +#endif // defined(__SYCL_DEVICE_ONLY__) } } // namespace matrix diff --git a/sycl/test/matrix/matrix-tf32-test.cpp b/sycl/test/matrix/matrix-tf32-test.cpp new file mode 100644 index 0000000000000..e4c6daddf1624 --- /dev/null +++ b/sycl/test/matrix/matrix-tf32-test.cpp @@ -0,0 +1,160 @@ +// RUN: %clangxx -fsycl -O2 -DSYCL_EXT_ONEAPI_MATRIX_VERSION=4 %s -o %t.out +#include +#include + +using namespace sycl; +using namespace sycl::ext::oneapi::experimental::matrix; + +auto constexpr SG_SZ = 16; + +#define TM 8 +#define TN SG_SZ +#define TK 16 + +template struct big_matrix { +public: + T *mat; + +public: + T *get_data() { return mat; } + void set_data(T *data) { mat = data; } + big_matrix(T *data) : mat(data) {} +}; + +template +void matrix_multiply(big_matrix &C, + big_matrix &A, + big_matrix &B) { + size_t M = NUM_ROWS_C; + size_t N = NUM_COLS_C; + size_t K = NUM_COLS_A; + + assert(NUM_ROWS_C == NUM_ROWS_A && NUM_COLS_A == NUM_ROWS_B); + size_t NDRangeM = M / TM; + size_t NDRangeN = N / TN; + buffer bufA(A.get_data(), range<2>(M, K)); + buffer bufB(B.get_data(), range<2>(K, N)); + buffer bufC((float *)C.get_data(), range<2>(M, N)); + + queue q; + q.submit([&](handler &cgh) { + auto accC = bufC.get_access(cgh); + auto accA = bufA.get_access(cgh); + auto accB = bufB.get_access(cgh); + + cgh.parallel_for( + nd_range<2>({NDRangeM, NDRangeN * SG_SZ}, {1, 1 * SG_SZ}), + [=](nd_item<2> spmd_item) [[intel::reqd_sub_group_size(SG_SZ)]] + + { + // The matrix API has to be accessed by all the workitems in a + // subgroup these functions will be called once by the subgroup no + // code divergence between the workitems + const auto global_idx = spmd_item.get_global_id(0); + const auto global_idy = spmd_item.get_global_id(1); + const auto sg_startx = global_idx - spmd_item.get_local_id(0); + const auto sg_starty = global_idy - spmd_item.get_local_id(1); + + sub_group sg = spmd_item.get_sub_group(); + joint_matrix + sub_a; + joint_matrix + sub_b; + joint_matrix sub_c; + joint_matrix_load(sg, sub_c, + accC.get_pointer() + (sg_startx * TM) * N + + sg_starty / SG_SZ * TN, + N, layout::row_major); + for (int k = 0; k < K; k += TK) { + joint_matrix_load( + sg, sub_a, accA.get_pointer() + (sg_startx * TM) * K + k, K); + joint_matrix_load( + sg, sub_b, + accB.get_pointer() + (k) * (N) + sg_starty / SG_SZ * TN, N); + // If no rounding to tf32 function is called, joint_matrix_mad + // function will work on truncated floats. + for (int i = 0; i < get_wi_data(sg, sub_a).length(); i++) { + get_wi_data(sg, sub_a)[i] = + round_to_tf32(get_wi_data(sg, sub_a)[i]); + } + for (int i = 0; i < get_wi_data(sg, sub_b).length(); i++) { + get_wi_data(sg, sub_b)[i] = + round_to_tf32(get_wi_data(sg, sub_b)[i]); + } + sub_c = joint_matrix_mad(sg, sub_a, sub_b, sub_c); + } + auto wi_slice_a = get_wi_data(sg, sub_a); + for (int i = 0; i < wi_slice_a.length(); i++) { + float elem = wi_slice_a[i]; + wi_slice_a[i] *= 2; + } + joint_matrix_store(sg, sub_c, + accC.get_pointer() + (sg_startx * TM) * N + + sg_starty / SG_SZ * TN, + N, layout::row_major); + }); // parallel for + }).wait(); +} + +static constexpr size_t MATRIX_M = TM * 2; +static constexpr size_t MATRIX_N = TN * 2; +static constexpr size_t MATRIX_K = TK * 2; +float A[MATRIX_M][MATRIX_K]; +float B[MATRIX_K][MATRIX_N]; +float C[MATRIX_M][MATRIX_N]; +float D[MATRIX_M][MATRIX_N]; + +void matrix_multiply_ref(float *A_mem, float *B_mem, float *C_mem, int M, int N, + int K) { + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + for (int k = 0; k < K; k++) { + float va = A_mem[m * K + k]; + float vb = B_mem[k * N + n]; + C_mem[m * N + n] += va * vb; + } + } +} + +int main() { + for (int i = 0; i < MATRIX_M; i++) { + for (int j = 0; j < MATRIX_K; j++) { + A[i][j] = 1.0f * (i + j); + } + } + for (int i = 0; i < MATRIX_K; i++) { + for (int j = 0; j < MATRIX_N; j++) { + B[i][j] = 2.0f * i + 3.0f * j; + } + } + for (int i = 0; i < MATRIX_M; i++) { + for (int j = 0; j < MATRIX_N; j++) { + C[i][j] = 1.0; + D[i][j] = 1.0; + } + } + + big_matrix MC((float *)&C); + big_matrix MD((float *)&D); + big_matrix MA((float *)&A); + big_matrix MB((float *)&B); + matrix_multiply(MC, MA, MB); + matrix_multiply_ref((float *)A, (float *)B, (float *)D, MATRIX_M, MATRIX_N, + MATRIX_K); + + bool res = true; + for (int i = 0; i < MATRIX_M; i++) { + for (int j = 0; j < MATRIX_N; j++) { + if (C[i][j] != D[i][j]) + res = false; + } + } + if (res) + std::cout << "passed\n"; + else + std::cout << "failed\n"; +}