From 0556328ee5adec96f38c2105536e33d852ab216f Mon Sep 17 00:00:00 2001 From: "Sidorov, Dmitry" Date: Tue, 3 Dec 2024 18:14:06 -0800 Subject: [PATCH 1/6] [SPIR-V] Add SPV_INTEL_joint_matrix extension The spec is available here: https://github.com/intel/llvm/pull/12497 The PR doesn't add OpCooperativeMatrixApplyFunctionINTEL instruction as it's still experimental and not properly tested E2E. The PR also fixes few bugs in the related code: 1. CooperativeMatrixMulAddKHR optional operand must be literal, not a constant; 2. Fixed available capabilities table creation for a case, when a single extension adds few capabilities, that occupy not contiguous op codes. Signed-off-by: Sidorov, Dmitry --- llvm/docs/SPIRVUsage.rst | 2 + .../SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp | 8 +- .../Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h | 10 ++ .../SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp | 27 +++++ llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 45 +++++++- llvm/lib/Target/SPIRV/SPIRVBuiltins.td | 7 ++ llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp | 2 + llvm/lib/Target/SPIRV/SPIRVInstrInfo.td | 17 +++ llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 104 ++++++++++++++++++ .../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 68 ++++++++++++ .../cooperative_matrix_bf16.ll | 46 ++++++++ .../cooperative_matrix_checked.ll | 59 ++++++++++ .../cooperative_matrix_get_coord.ll | 35 ++++++ .../cooperative_matrix_packed.ll | 78 +++++++++++++ .../cooperative_matrix_prefetch.ll | 34 ++++++ .../cooperative_matrix_tf32.ll | 46 ++++++++ .../cooperative_matrix.ll | 2 +- 17 files changed, 582 insertions(+), 8 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst index 28e919fdf516a..8f7ac71f8026b 100644 --- a/llvm/docs/SPIRVUsage.rst +++ b/llvm/docs/SPIRVUsage.rst @@ -179,6 +179,8 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na - Introduces two new storage classes that are subclasses of the CrossWorkgroup storage class that provides additional information that can enable optimization. * - ``SPV_INTEL_variable_length_array`` - Allows to allocate local arrays whose number of elements is unknown at compile time. + * - ``SPV_INTEL_joint_matrix`` + - Adds few matrix capabilities on top of SPV_KHR_cooperative_matrix extension, such as matrix prefetch, get element coordinate and checked load/store/construct instructions, tensor float 32 and bfloat type interpretations for multuply-add instruction. * - ``SPV_KHR_bit_instructions`` - Enables bit instructions to be used by SPIR-V modules without requiring the Shader capability. * - ``SPV_KHR_expect_assume`` diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp index 0f9a2a69e0739..67bf459615249 100644 --- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp +++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp @@ -137,8 +137,12 @@ getCapabilitiesEnabledByExtension(SPIRV::Extension::Extension Extension) { CapabilityList Capabilities; while (Entry && - Entry->Category == SPIRV::OperandCategory::CapabilityOperand && - Entry->ReqExtension == Extension) { + Entry->Category == SPIRV::OperandCategory::CapabilityOperand) { + // Some capabilities' codes might go not in order. + if (Entry->ReqExtension != Extension) { + ++Entry; + continue; + } Capabilities.push_back( static_cast(Entry->Value)); ++Entry; diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h index 44625793e9413..823c33ecb6bd3 100644 --- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h +++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h @@ -207,6 +207,16 @@ namespace Opcode { #include "SPIRVGenTables.inc" } // namespace Opcode +namespace CooperativeMatrixLayout { +#define GET_CooperativeMatrixLayout_DECL +#include "SPIRVGenTables.inc" +} // namespace Opcode + +namespace CooperativeMatrixOperands { +#define GET_CooperativeMatrixOperands_DECL +#include "SPIRVGenTables.inc" +} // namespace Opcode + struct ExtendedBuiltin { StringRef Name; InstructionSet::InstructionSet Set; diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp index ff8759755e517..d05a0e87ca870 100644 --- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp +++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp @@ -211,6 +211,33 @@ void SPIRVInstPrinter::printInst(const MCInst *MI, uint64_t Address, // are part of the variable value. printOpConstantVarOps(MI, NumFixedOps - 1, OS); break; + case SPIRV::OpCooperativeMatrixMulAddKHR: { + const unsigned NumOps = MI->getNumOperands(); + if (NumFixedOps == NumOps) + break; + + OS << ' '; + const unsigned MulAddOp = MI->getOperand(FirstVariableIndex).getImm(); + if (MulAddOp == 0) { + printSymbolicOperand< + OperandCategory::CooperativeMatrixOperandsOperand>( + MI, FirstVariableIndex, OS); + } else { + std::string Buffer; + for (unsigned Mask = 0x1; + Mask != SPIRV::CooperativeMatrixOperands::MatrixResultBFloat16ComponentsINTEL; + Mask <<= 1) { + if (MulAddOp & Mask) { + if (!Buffer.empty()) + Buffer += '|'; + Buffer += getSymbolicOperandMnemonic( + OperandCategory::CooperativeMatrixOperandsOperand, Mask); + } + } + OS << Buffer; + } + break; + } default: printRemainingVariableOps(MI, NumFixedOps, OS); break; diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index 45a49674d4ca2..28fcbda2d1df8 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -1969,15 +1969,50 @@ static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call, const SPIRV::DemangledBuiltin *Builtin = Call->Builtin; unsigned Opcode = SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode; - bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR; + bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR && + Opcode != SPIRV::OpCooperativeMatrixStoreCheckedINTEL && + Opcode != SPIRV::OpCooperativeMatrixPrefetchINTEL; unsigned ArgSz = Call->Arguments.size(); unsigned LiteralIdx = 0; - if (Opcode == SPIRV::OpCooperativeMatrixLoadKHR && ArgSz > 3) - LiteralIdx = 3; - else if (Opcode == SPIRV::OpCooperativeMatrixStoreKHR && ArgSz > 4) - LiteralIdx = 4; + switch(Opcode) { + // Memory operand is optional and is literal. + case SPIRV::OpCooperativeMatrixLoadKHR: + LiteralIdx = ArgSz > 3 ? 3 : 0; + break; + case SPIRV::OpCooperativeMatrixStoreKHR: + LiteralIdx = ArgSz > 4 ? 4 : 0; + break; + case SPIRV::OpCooperativeMatrixLoadCheckedINTEL: + LiteralIdx = ArgSz > 7 ? 7 : 0; + break; + case SPIRV::OpCooperativeMatrixStoreCheckedINTEL: + LiteralIdx = ArgSz > 8 ? 8 : 0; + break; + // Cooperative Matrix Operands operand is optional and is literal. + case SPIRV::OpCooperativeMatrixMulAddKHR: + LiteralIdx = ArgSz > 3 ? 3 : 0; + break; + }; + SmallVector ImmArgs; MachineRegisterInfo *MRI = MIRBuilder.getMRI(); + if (Opcode == SPIRV::OpCooperativeMatrixPrefetchINTEL) { + const uint32_t CacheLevel = + getConstFromIntrinsic(Call->Arguments[3], MRI); + auto MIB = MIRBuilder.buildInstr(SPIRV::OpCooperativeMatrixPrefetchINTEL) + .addUse(Call->Arguments[0]) // pointer + .addUse(Call->Arguments[1]) // rows + .addUse(Call->Arguments[2]) // columns + .addImm(CacheLevel) // cache level + .addUse(Call->Arguments[4]); // memory layout + if (ArgSz > 5) + MIB.addUse(Call->Arguments[5]); // stride + if (ArgSz > 6) { + const uint32_t MemOp = getConstFromIntrinsic(Call->Arguments[6], MRI); + MIB.addImm(MemOp); // memory operand + } + return true; + } if (LiteralIdx > 0) ImmArgs.push_back(getConstFromIntrinsic(Call->Arguments[LiteralIdx], MRI)); Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType); diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td index dc2da4a3a5647..e29013d28aafe 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td @@ -695,6 +695,13 @@ defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixStoreKHR", OpenCL_std, C defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixMulAddKHR", OpenCL_std, CoopMatr, 3, 4, OpCooperativeMatrixMulAddKHR>; defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLengthKHR", OpenCL_std, CoopMatr, 1, 1, OpCooperativeMatrixLengthKHR>; +// Cooperative Matrix Intel builtin records: +defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixPrefetchINTEL", OpenCL_std, CoopMatr, 5, 7, OpCooperativeMatrixPrefetchINTEL>; +defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLoadCheckedINTEL", OpenCL_std, CoopMatr, 6, 8, OpCooperativeMatrixLoadCheckedINTEL>; +defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixStoreCheckedINTEL", OpenCL_std, CoopMatr, 7, 9, OpCooperativeMatrixStoreCheckedINTEL>; +defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixConstructCheckedINTEL", OpenCL_std, CoopMatr, 5, 5, OpCooperativeMatrixConstructCheckedINTEL>; +defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixGetElementCoordINTEL", OpenCL_std, CoopMatr, 2, 2, OpCooperativeMatrixGetElementCoordINTEL>; + //===----------------------------------------------------------------------===// // Class defining a work/sub group builtin that should be translated into a // SPIR-V instruction using the defined properties. diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp index e78fc5ce18707..fb05c1fdbd1e3 100644 --- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp @@ -51,6 +51,8 @@ static const std::map> SPIRV::Extension::Extension::SPV_INTEL_subgroups}, {"SPV_INTEL_media_block_io", SPIRV::Extension::Extension::SPV_INTEL_media_block_io}, + {"SPV_INTEL_joint_matrix", + SPIRV::Extension::Extension::SPV_INTEL_joint_matrix}, {"SPV_KHR_uniform_group_instructions", SPIRV::Extension::Extension::SPV_KHR_uniform_group_instructions}, {"SPV_KHR_no_integer_wrap_decoration", diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td index 53f1b644a9498..d95803fea56a5 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td +++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td @@ -895,6 +895,23 @@ def OpCooperativeMatrixMulAddKHR: Op<4459, (outs ID:$res), def OpCooperativeMatrixLengthKHR: Op<4460, (outs ID:$res), (ins TYPE:$type, ID:$coop_matr_type), "$res = OpCooperativeMatrixLengthKHR $type $coop_matr_type">; +// SPV_INTEL_joint_matrix +def OpCooperativeMatrixLoadCheckedINTEL: Op<6193, (outs ID:$res), + (ins TYPE:$resType, ID:$pointer, ID:$xOffset, ID:$yOffset, ID:$memory_layout, ID:$height, ID:$width, variable_ops), + "$res = OpCooperativeMatrixLoadCheckedINTEL $resType $pointer $xOffset $yOffset $memory_layout $height $width">; +def OpCooperativeMatrixStoreCheckedINTEL: Op<6194, (outs), + (ins ID:$pointer, ID:$xOffset, ID:$yOffset, ID:$objectToStore, ID:$memory_layout, ID:$height, ID:$width, variable_ops), + "OpCooperativeMatrixStoreCheckedINTEL $pointer $xOffset $yOffset $objectToStore $memory_layout $height $width">; +def OpCooperativeMatrixConstructCheckedINTEL: Op<6195, (outs ID:$res), + (ins TYPE:$resType, ID:$xOffset, ID:$yOffset, ID:$height, ID:$width, ID:$value), + "$res = OpCooperativeMatrixConstructCheckedINTEL $resType $xOffset $yOffset $height $width $value">; +def OpCooperativeMatrixGetElementCoordINTEL: Op<6440, (outs ID:$res), + (ins TYPE:$resType, ID:$matrix, ID:$index), + "$res = OpCooperativeMatrixGetElementCoordINTEL $resType $matrix $index">; +def OpCooperativeMatrixPrefetchINTEL: Op<6449, (outs), + (ins ID:$pointer, ID:$rows, ID:$columns, i32imm:$cacheLevel, ID:$memory_layout, variable_ops), + "OpCooperativeMatrixPrefetchINTEL $pointer $rows $columns $cacheLevel $memory_layout">; + // SPV_EXT_arithmetic_fence def OpArithmeticFenceEXT: Op<6145, (outs ID:$res), (ins TYPE:$type, ID:$target), "$res = OpArithmeticFenceEXT $type $target">; diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index 2054081476315..ee7c08d324bb0 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -1437,6 +1437,110 @@ void addInstrRequirements(const MachineInstr &MI, Reqs.addCapability(SPIRV::Capability::SplitBarrierINTEL); } break; + case SPIRV::OpCooperativeMatrixMulAddKHR: { + if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix)) + report_fatal_error("Cooperative matrix instructions require the " + "following SPIR-V extension: " + "SPV_KHR_cooperative_matrix", false); + Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix); + Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR); + constexpr unsigned MulAddMaxSize = 6; + if (MI.getNumOperands() != MulAddMaxSize) + break; + const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm(); + if (CoopOperands & + SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) { + Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); + Reqs.addCapability( + SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL); + } + if (CoopOperands & SPIRV::CooperativeMatrixOperands:: + MatrixAAndBBFloat16ComponentsINTEL || + CoopOperands & SPIRV::CooperativeMatrixOperands:: + MatrixCBFloat16ComponentsINTEL || + CoopOperands & SPIRV::CooperativeMatrixOperands:: + MatrixResultBFloat16ComponentsINTEL) { + Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); + Reqs.addCapability( + SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL); + } + break; + } + case SPIRV::OpCooperativeMatrixLoadKHR: + case SPIRV::OpCooperativeMatrixStoreKHR: + case SPIRV::OpCooperativeMatrixLoadCheckedINTEL: + case SPIRV::OpCooperativeMatrixStoreCheckedINTEL: + case SPIRV::OpCooperativeMatrixPrefetchINTEL: { + if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix)) + report_fatal_error("Cooperative matrix instructions require the " + "following SPIR-V extension: " + "SPV_KHR_cooperative_matrix", false); + Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix); + Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR); + + // Check Layout operand in case if it's not a standart one and add the + // appropriate capability. + std::unordered_map LayoutToInstMap = { + {SPIRV::OpCooperativeMatrixLoadKHR, 3}, + {SPIRV::OpCooperativeMatrixStoreKHR, 2}, + {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5}, + {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4}, + {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4} + }; + + const auto OpCode = MI.getOpcode(); + const unsigned LayoutNum = LayoutToInstMap[OpCode]; + Register RegLayout = MI.getOperand(LayoutNum).getReg(); + const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); + MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout); + if (MILayout->getOpcode() == SPIRV::OpConstantI) { + const unsigned LayoutVal = MILayout->getOperand(2).getImm(); + if (LayoutVal == static_cast( + SPIRV::CooperativeMatrixLayout::PackedINTEL)) { + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) + report_fatal_error("PackedINTEL layout require the following SPIR-V " + "extension: SPV_INTEL_joint_matrix", false); + Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); + Reqs.addCapability(SPIRV::Capability::PackedCooperativeMatrixINTEL); + } + } + + // Nothing to do. + if (OpCode == SPIRV::OpCooperativeMatrixLoadKHR || + OpCode == SPIRV::OpCooperativeMatrixStoreKHR) + break; + + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) + report_fatal_error("OpCooperativeMatrix[Load/Store]CheckedINTEL " + "instructions require the following SPIR-V extension: " + "SPV_INTEL_joint_matrix", false); + Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); + if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) { + Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL); + break; + } + Reqs.addCapability( + SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL); + break; + } + case SPIRV::OpCooperativeMatrixConstructCheckedINTEL: + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) + report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL" + " instructions require the following SPIR-V extension:" + " SPV_INTEL_joint_matrix", false); + Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); + Reqs.addCapability( + SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL); + break; + case SPIRV::OpCooperativeMatrixGetElementCoordINTEL: + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) + report_fatal_error("OpCooperativeMatrixGetElementCoordINTEL requires the " + "following SPIR-V extension: SPV_INTEL_joint_matrix", + false); + Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); + Reqs.addCapability( + SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL); + break; case SPIRV::OpKill: { Reqs.addCapability(SPIRV::Capability::Shader); } break; diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index a3a88acdd6c6a..745d1e1aec67a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -170,6 +170,8 @@ def GroupOperationOperand : OperandCategory; def KernelEnqueueFlagsOperand : OperandCategory; def KernelProfilingInfoOperand : OperandCategory; def OpcodeOperand : OperandCategory; +def CooperativeMatrixLayoutOperand : OperandCategory; +def CooperativeMatrixOperandsOperand : OperandCategory; //===----------------------------------------------------------------------===// // Multiclass used to define Extesions enum values and at the same time @@ -305,6 +307,7 @@ defm SPV_INTEL_global_variable_fpga_decorations : ExtensionOperand<110>; defm SPV_KHR_cooperative_matrix : ExtensionOperand<111>; defm SPV_EXT_arithmetic_fence : ExtensionOperand<112>; defm SPV_EXT_optnone : ExtensionOperand<113>; +defm SPV_INTEL_joint_matrix : ExtensionOperand<114>; //===----------------------------------------------------------------------===// // Multiclass used to define Capabilities enum values and at the same time @@ -492,6 +495,12 @@ defm CacheControlsINTEL : CapabilityOperand<6441, 0, 0, [SPV_INTEL_cache_control defm CooperativeMatrixKHR : CapabilityOperand<6022, 0, 0, [SPV_KHR_cooperative_matrix], []>; defm ArithmeticFenceEXT : CapabilityOperand<6144, 0, 0, [SPV_EXT_arithmetic_fence], []>; defm SplitBarrierINTEL : CapabilityOperand<6141, 0, 0, [SPV_INTEL_split_barrier], []>; +defm CooperativeMatrixCheckedInstructionsINTEL : CapabilityOperand<6192, 0, 0, [SPV_INTEL_joint_matrix], []>; +defm CooperativeMatrixPrefetchINTEL : CapabilityOperand<6411, 0, 0, [SPV_INTEL_joint_matrix], []>; +defm PackedCooperativeMatrixINTEL : CapabilityOperand<6434, 0, 0, [SPV_INTEL_joint_matrix], []>; +defm CooperativeMatrixInvocationInstructionsINTEL : CapabilityOperand<6435, 0, 0, [SPV_INTEL_joint_matrix], []>; +defm CooperativeMatrixTF32ComponentTypeINTEL : CapabilityOperand<6436, 0, 0, [SPV_INTEL_joint_matrix], []>; +defm CooperativeMatrixBFloat16ComponentTypeINTEL : CapabilityOperand<6437, 0, 0, [SPV_INTEL_joint_matrix], []>; //===----------------------------------------------------------------------===// // Multiclass used to define SourceLanguage enum values and at the same time @@ -1649,3 +1658,62 @@ defm GenericCastToPtr : OpcodeOperand<122>; defm Bitcast : OpcodeOperand<124>; defm ConvertPtrToU : OpcodeOperand<117>; defm ConvertUToPtr : OpcodeOperand<120>; + +//===----------------------------------------------------------------------===// +// Multiclass used to define Cooperative Matrix Layout enum values and at the +// same time SymbolicOperand entries extensions and capabilities. +//===----------------------------------------------------------------------===// + +def CooperativeMatrixLayout : GenericEnum, Operand { + let FilterClass = "CooperativeMatrixLayout"; + let NameField = "Name"; + let ValueField = "Value"; +} + +class CooperativeMatrixLayout value> { + string Name = name; + bits<32> Value = value; +} + +multiclass CooperativeMatrixLayoutOperand value, list reqExtensions, list reqCapabilities> { + def : CooperativeMatrixLayout; + defm : SymbolicOperandWithRequirements; +} + +defm RowMajorKHR : CooperativeMatrixLayoutOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>; +defm ColumnMajorKHR : CooperativeMatrixLayoutOperand<0x1, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>; +defm PackedINTEL : CooperativeMatrixLayoutOperand<0x2, [SPV_INTEL_joint_matrix], [PackedCooperativeMatrixINTEL]>; + +//===----------------------------------------------------------------------===// +// Multiclass used to define Cooperative Matrix Operands enum values and at the +// same time SymbolicOperand entries with string mnemonics, extensions and +// capabilities. +//===----------------------------------------------------------------------===// + +def CooperativeMatrixOperands : GenericEnum, Operand { + let FilterClass = "CooperativeMatrixOperands"; + let NameField = "Name"; + let ValueField = "Value"; + let PrintMethod = !strconcat("printSymbolicOperand"); +} + +class CooperativeMatrixOperands value> { + string Name = name; + bits<32> Value = value; +} + +multiclass CooperativeMatrixOperandsOperand value, list reqExtensions, list reqCapabilities> { + def : CooperativeMatrixOperands; + defm : SymbolicOperandWithRequirements; +} + +defm NoneKHR : CooperativeMatrixOperandsOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>; +defm MatrixASignedComponentsKHR : CooperativeMatrixOperandsOperand<0x1, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>; +defm MatrixBSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x2, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>; +defm MatrixCSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x4, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>; +defm MatrixResultSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x8, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>; +defm SaturatingAccumulationKHR : CooperativeMatrixOperandsOperand<0x10, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>; +defm MatrixAAndBTF32ComponentsINTEL : CooperativeMatrixOperandsOperand<0x20, [SPV_INTEL_joint_matrix], [CooperativeMatrixTF32ComponentTypeINTEL]>; +defm MatrixAAndBBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x40, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>; +defm MatrixCBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x80, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>; +defm MatrixResultBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x100, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>; diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll new file mode 100644 index 0000000000000..192d769b8a3ce --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll @@ -0,0 +1,46 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s + +; CHECK-DAG: Capability CooperativeMatrixKHR +; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix" +; CHECK-DAG: Extension "SPV_INTEL_joint_matrix" +; CHECK-DAG: CooperativeMatrixBFloat16ComponentTypeINTEL + +; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBBFloat16ComponentsINTEL + +; ModuleID = 'test-matrix-opaque.bc' +source_filename = "matrix-int8-test.cpp" +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" +target triple = "spir64-unknown-unknown" + +$_ZTSZZ15matrix_multiply = comdat any + +; Function Attrs: convergent norecurse +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +entry: + %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0) + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2 + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2 + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 64) #2 + tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2 + ret void +} + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1 + +; Function Attrs: convergent +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1 + +; Function Attrs: convergent +declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2), i32, i64, i32) #1 + + +attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } +attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { convergent } diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll new file mode 100644 index 0000000000000..73ddaad0dbad1 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll @@ -0,0 +1,59 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s + +; CHECK-DAG: Capability CooperativeMatrixKHR +; CHECK-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL +; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix" +; CHECK-DAG: Extension "SPV_INTEL_joint_matrix" +; CHECK-DAG: %[[#Int8Ty:]] = OpTypeInt 8 0 +; CHECK-DAG: %[[#Int32Ty:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#Const12:]] = OpConstant %[[#Int32Ty]] 12 +; CHECK-DAG: %[[#Const48:]] = OpConstant %[[#Int32Ty]] 48 +; CHECK-DAG: %[[#Const0:]] = OpConstant %[[#Int32Ty]] 0 +; CHECK-DAG: %[[#Const3:]] = OpConstant %[[#Int32Ty]] 3 +; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32Ty]] 2 +; CHECK-DAG: %[[#Const1:]] = OpConstant %[[#Int32Ty]] 1 +; CHECK-DAG: %[[#MatTy1:]] = OpTypeCooperativeMatrixKHR %[[#Int32Ty]] %[[#Const3]] %[[#Const12]] %[[#Const12]] %[[#Const2]] +; CHECK-DAG: %[[#MatTy2:]] = OpTypeCooperativeMatrixKHR %[[#Int8Ty]] %[[#Const3]] %[[#Const12]] %[[#Const48]] %[[#Const0]] +; CHECK-DAG: %[[#MatTy3:]] = OpTypeCooperativeMatrixKHR %[[#Int8Ty]] %[[#Const2]] %[[#Const48]] %[[#Const12]] %[[#Const1]] +; CHECK: OpCooperativeMatrixConstructCheckedINTEL %[[#MatTy1]] +; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#MatTy2]] +; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#MatTy3]] +; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]] +; CHECK: OpCooperativeMatrixStoreCheckedINTEL + +; ModuleID = 'test-matrix-opaque.bc' +source_filename = "matrix-int8-test.cpp" +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" +target triple = "spir64-unknown-unknown" + +$_ZTSZZ15matrix_multiply = comdat any + +; Function Attrs: convergent norecurse +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +entry: + %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #2 + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #2 + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 1, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #2 + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #2 + tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 0, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #2 + ret void +} + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 + +attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } +attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { convergent } diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll new file mode 100644 index 0000000000000..4af85beb401dd --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll @@ -0,0 +1,35 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s + +; CHECK-DAG: Capability CooperativeMatrixKHR +; CHECK-DAG: Capability CooperativeMatrixInvocationInstructionsINTEL +; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix" +; CHECK-DAG: Extension "SPV_INTEL_joint_matrix" +; CHECK-DAG: %[[#MatrixTy:]] = OpTypeCooperativeMatrixKHR + +; CHECK: %[[#Matrix:]] = OpCompositeConstruct %[[#MatrixTy]] +; CHECK: %[[#]] = OpCooperativeMatrixGetElementCoordINTEL %[[#]] %[[#Matrix]] %[[#]] + +; ModuleID = 'test-matrix-opaque.bc' +source_filename = "matrix-int8-test.cpp" +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" +target triple = "spir64-unknown-unknown" + +$_ZTSZZ15matrix_multiply = comdat any + +$_ZTSZZ23matrix_multiply_checked = comdat any + +; Function Attrs: convergent norecurse +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(i32 noundef %_idx, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +entry: + %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0) + %coord = tail call spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) %matrixC, i32 noundef %_idx) + ret void +} +; Function Attrs: convergent +declare dso_local spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32 noundef) + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr #1 + +attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } +attributes #1 = { convergent } diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll new file mode 100644 index 0000000000000..50461af93084e --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll @@ -0,0 +1,78 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s + +; CHECK-DAG: Capability CooperativeMatrixKHR +; CHECK-DAG: Capability PackedCooperativeMatrixINTEL +; CHECK-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL +; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix" +; CHECK-DAG: Extension "SPV_INTEL_joint_matrix" +; CHECK-DAG: %[[#Int32Ty:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32Ty]] 2 + +; CHECK: OpCooperativeMatrixLoadKHR %[[#]] %[[#]] %[[#Const2]] +; CHECK: OpCooperativeMatrixLoadKHR %[[#]] %[[#]] %[[#Const2]] +; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#]] %[[#Const2]] +; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]] +; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]] +; CHECK: OpCooperativeMatrixStoreCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]] + +; ModuleID = 'test-matrix-opaque.bc' +source_filename = "matrix-int8-test.cpp" +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" +target triple = "spir64-unknown-unknown" + +$_ZTSZZ15matrix_multiply = comdat any + +$_ZTSZZ23matrix_multiply_checked = comdat any + +; Function Attrs: convergent norecurse +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +entry: + %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 %_arg_Initvalue) #1 + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #1 + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #1 + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1 + tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #1 + ret void +} + +; Function Attrs: convergent norecurse +define weak_odr dso_local spir_kernel void @_ZTSZZ23matrix_multiply_checked(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +entry: + %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #1 + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #1 + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #1 + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1 + tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #1 + ret void +} + +; Function Attrs: convergent +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) #1 + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1 + +; Function Attrs: convergent +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1 + +; Function Attrs: convergent +declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1 + + +attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } +attributes #1 = { convergent } diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll new file mode 100644 index 0000000000000..e68a5a2383e85 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll @@ -0,0 +1,34 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s + +; CHECK-DAG: Capability CooperativeMatrixPrefetchINTEL +; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix" +; CHECK-DAG: OpExtension "SPV_INTEL_joint_matrix" + +; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] +; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]] +; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]] 1 + +; ModuleID = 'test-matrix-opaque.bc' +source_filename = "matrix-int8-test.cpp" +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" +target triple = "spir64-unknown-unknown" + +$_ZTSZZ15matrix_multiply = comdat any + +; Function Attrs: convergent norecurse +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_K) local_unnamed_addr #0 comdat { +entry: + tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef %_arg_accA, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0) #1 + tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef %_arg_accB, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K) #1 + tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef %_arg_accC, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K, i32 1) #1 + ret void +} + +declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1 + +declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1 + +declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 + +attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } +attributes #1 = { convergent } diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll new file mode 100644 index 0000000000000..4fafca457110f --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll @@ -0,0 +1,46 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s + +; CHECK-DAG: Capability CooperativeMatrixKHR +; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix" +; CHECK-DAG: Extension "SPV_INTEL_joint_matrix" +; CHECK-DAG: OpCapability CooperativeMatrixTF32ComponentTypeINTEL + +; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBTF32ComponentsINTEL + +; ModuleID = 'test-matrix-opaque.bc' +source_filename = "matrix-int8-test.cpp" +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" +target triple = "spir64-unknown-unknown" + +$_ZTSZZ15matrix_multiply = comdat any + +; Function Attrs: convergent norecurse +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, float noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +entry: + %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float %_arg_Initvalue) + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2 + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2 + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 32) #2 + tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2 + ret void +} + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1 + +; Function Attrs: convergent +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1 + +; Function Attrs: convergent +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1 + +; Function Attrs: convergent +declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1 + + +attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } +attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { convergent } diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll index 1c41c7331cda8..e290c1eaeabad 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll @@ -21,7 +21,7 @@ ; CHECK: %[[#Load1:]] = OpCooperativeMatrixLoadKHR %[[#MatTy2]] ; CHECK: OpCooperativeMatrixLengthKHR %[[#Int32Ty]] %[[#MatTy2:]] ; CHECK: OpCooperativeMatrixLoadKHR %[[#MatTy3]] -; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]] +; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]] %[[#]] %[[#]] %[[#]] MatrixCSignedComponentsKHR|MatrixResultSignedComponentsKHR ; CHECK: OpCooperativeMatrixStoreKHR define spir_kernel void @matr_mult(ptr addrspace(1) align 1 %_arg_accA, ptr addrspace(1) align 1 %_arg_accB, ptr addrspace(1) align 4 %_arg_accC, i64 %_arg_N, i64 %_arg_K) { From 5f459b8cd737fdad936db945848a03a590e025dc Mon Sep 17 00:00:00 2001 From: "Sidorov, Dmitry" Date: Wed, 4 Dec 2024 03:12:44 -0800 Subject: [PATCH 2/6] format Signed-off-by: Sidorov, Dmitry --- .../Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h | 4 +- .../SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp | 7 +-- llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 51 +++++++++---------- llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 38 +++++++------- 4 files changed, 52 insertions(+), 48 deletions(-) diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h index 823c33ecb6bd3..2437fbb820a36 100644 --- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h +++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h @@ -210,12 +210,12 @@ namespace Opcode { namespace CooperativeMatrixLayout { #define GET_CooperativeMatrixLayout_DECL #include "SPIRVGenTables.inc" -} // namespace Opcode +} // namespace CooperativeMatrixLayout namespace CooperativeMatrixOperands { #define GET_CooperativeMatrixOperands_DECL #include "SPIRVGenTables.inc" -} // namespace Opcode +} // namespace CooperativeMatrixOperands struct ExtendedBuiltin { StringRef Name; diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp index d05a0e87ca870..2ee0c79b8f7c1 100644 --- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp +++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp @@ -220,12 +220,13 @@ void SPIRVInstPrinter::printInst(const MCInst *MI, uint64_t Address, const unsigned MulAddOp = MI->getOperand(FirstVariableIndex).getImm(); if (MulAddOp == 0) { printSymbolicOperand< - OperandCategory::CooperativeMatrixOperandsOperand>( - MI, FirstVariableIndex, OS); + OperandCategory::CooperativeMatrixOperandsOperand>( + MI, FirstVariableIndex, OS); } else { std::string Buffer; for (unsigned Mask = 0x1; - Mask != SPIRV::CooperativeMatrixOperands::MatrixResultBFloat16ComponentsINTEL; + Mask != SPIRV::CooperativeMatrixOperands:: + MatrixResultBFloat16ComponentsINTEL; Mask <<= 1) { if (MulAddOp & Mask) { if (!Buffer.empty()) diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index 28fcbda2d1df8..9b6c2a849edce 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -1974,42 +1974,41 @@ static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call, Opcode != SPIRV::OpCooperativeMatrixPrefetchINTEL; unsigned ArgSz = Call->Arguments.size(); unsigned LiteralIdx = 0; - switch(Opcode) { - // Memory operand is optional and is literal. - case SPIRV::OpCooperativeMatrixLoadKHR: - LiteralIdx = ArgSz > 3 ? 3 : 0; - break; - case SPIRV::OpCooperativeMatrixStoreKHR: - LiteralIdx = ArgSz > 4 ? 4 : 0; - break; - case SPIRV::OpCooperativeMatrixLoadCheckedINTEL: - LiteralIdx = ArgSz > 7 ? 7 : 0; - break; - case SPIRV::OpCooperativeMatrixStoreCheckedINTEL: - LiteralIdx = ArgSz > 8 ? 8 : 0; - break; - // Cooperative Matrix Operands operand is optional and is literal. - case SPIRV::OpCooperativeMatrixMulAddKHR: - LiteralIdx = ArgSz > 3 ? 3 : 0; - break; + switch (Opcode) { + // Memory operand is optional and is literal. + case SPIRV::OpCooperativeMatrixLoadKHR: + LiteralIdx = ArgSz > 3 ? 3 : 0; + break; + case SPIRV::OpCooperativeMatrixStoreKHR: + LiteralIdx = ArgSz > 4 ? 4 : 0; + break; + case SPIRV::OpCooperativeMatrixLoadCheckedINTEL: + LiteralIdx = ArgSz > 7 ? 7 : 0; + break; + case SPIRV::OpCooperativeMatrixStoreCheckedINTEL: + LiteralIdx = ArgSz > 8 ? 8 : 0; + break; + // Cooperative Matrix Operands operand is optional and is literal. + case SPIRV::OpCooperativeMatrixMulAddKHR: + LiteralIdx = ArgSz > 3 ? 3 : 0; + break; }; SmallVector ImmArgs; MachineRegisterInfo *MRI = MIRBuilder.getMRI(); if (Opcode == SPIRV::OpCooperativeMatrixPrefetchINTEL) { - const uint32_t CacheLevel = - getConstFromIntrinsic(Call->Arguments[3], MRI); + const uint32_t CacheLevel = getConstFromIntrinsic(Call->Arguments[3], MRI); auto MIB = MIRBuilder.buildInstr(SPIRV::OpCooperativeMatrixPrefetchINTEL) - .addUse(Call->Arguments[0]) // pointer - .addUse(Call->Arguments[1]) // rows - .addUse(Call->Arguments[2]) // columns - .addImm(CacheLevel) // cache level - .addUse(Call->Arguments[4]); // memory layout + .addUse(Call->Arguments[0]) // pointer + .addUse(Call->Arguments[1]) // rows + .addUse(Call->Arguments[2]) // columns + .addImm(CacheLevel) // cache level + .addUse(Call->Arguments[4]); // memory layout if (ArgSz > 5) MIB.addUse(Call->Arguments[5]); // stride if (ArgSz > 6) { const uint32_t MemOp = getConstFromIntrinsic(Call->Arguments[6], MRI); - MIB.addImm(MemOp); // memory operand + MIB.addImm(MemOp); // memory operand } return true; } diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index ee7c08d324bb0..616e9d348e15a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -1441,7 +1441,8 @@ void addInstrRequirements(const MachineInstr &MI, if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix)) report_fatal_error("Cooperative matrix instructions require the " "following SPIR-V extension: " - "SPV_KHR_cooperative_matrix", false); + "SPV_KHR_cooperative_matrix", + false); Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix); Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR); constexpr unsigned MulAddMaxSize = 6; @@ -1455,11 +1456,11 @@ void addInstrRequirements(const MachineInstr &MI, SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL); } if (CoopOperands & SPIRV::CooperativeMatrixOperands:: - MatrixAAndBBFloat16ComponentsINTEL || - CoopOperands & SPIRV::CooperativeMatrixOperands:: - MatrixCBFloat16ComponentsINTEL || + MatrixAAndBBFloat16ComponentsINTEL || + CoopOperands & + SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL || CoopOperands & SPIRV::CooperativeMatrixOperands:: - MatrixResultBFloat16ComponentsINTEL) { + MatrixResultBFloat16ComponentsINTEL) { Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); Reqs.addCapability( SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL); @@ -1474,19 +1475,19 @@ void addInstrRequirements(const MachineInstr &MI, if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix)) report_fatal_error("Cooperative matrix instructions require the " "following SPIR-V extension: " - "SPV_KHR_cooperative_matrix", false); + "SPV_KHR_cooperative_matrix", + false); Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix); Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR); // Check Layout operand in case if it's not a standart one and add the // appropriate capability. std::unordered_map LayoutToInstMap = { - {SPIRV::OpCooperativeMatrixLoadKHR, 3}, - {SPIRV::OpCooperativeMatrixStoreKHR, 2}, - {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5}, - {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4}, - {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4} - }; + {SPIRV::OpCooperativeMatrixLoadKHR, 3}, + {SPIRV::OpCooperativeMatrixStoreKHR, 2}, + {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5}, + {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4}, + {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}}; const auto OpCode = MI.getOpcode(); const unsigned LayoutNum = LayoutToInstMap[OpCode]; @@ -1495,11 +1496,12 @@ void addInstrRequirements(const MachineInstr &MI, MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout); if (MILayout->getOpcode() == SPIRV::OpConstantI) { const unsigned LayoutVal = MILayout->getOperand(2).getImm(); - if (LayoutVal == static_cast( - SPIRV::CooperativeMatrixLayout::PackedINTEL)) { + if (LayoutVal == + static_cast(SPIRV::CooperativeMatrixLayout::PackedINTEL)) { if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) report_fatal_error("PackedINTEL layout require the following SPIR-V " - "extension: SPV_INTEL_joint_matrix", false); + "extension: SPV_INTEL_joint_matrix", + false); Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); Reqs.addCapability(SPIRV::Capability::PackedCooperativeMatrixINTEL); } @@ -1513,7 +1515,8 @@ void addInstrRequirements(const MachineInstr &MI, if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) report_fatal_error("OpCooperativeMatrix[Load/Store]CheckedINTEL " "instructions require the following SPIR-V extension: " - "SPV_INTEL_joint_matrix", false); + "SPV_INTEL_joint_matrix", + false); Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) { Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL); @@ -1527,7 +1530,8 @@ void addInstrRequirements(const MachineInstr &MI, if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL" " instructions require the following SPIR-V extension:" - " SPV_INTEL_joint_matrix", false); + " SPV_INTEL_joint_matrix", + false); Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); Reqs.addCapability( SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL); From cc73812a71bcefae5f3e0ae230d634fae71f2564 Mon Sep 17 00:00:00 2001 From: "Sidorov, Dmitry" Date: Wed, 4 Dec 2024 06:20:45 -0800 Subject: [PATCH 3/6] apply comments, improve validation, add negative tests Signed-off-by: Sidorov, Dmitry --- llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 37 +++++++--- .../cooperative_matrix_bf16.ll | 42 ++++------- .../cooperative_matrix_checked.ll | 43 ++++-------- .../cooperative_matrix_get_coord.ll | 23 ++---- .../cooperative_matrix_packed.ll | 70 +++++++------------ .../cooperative_matrix_prefetch.ll | 29 +++----- .../cooperative_matrix_tf32.ll | 42 ++++------- 7 files changed, 113 insertions(+), 173 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index 616e9d348e15a..1a125c44750e6 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -1451,6 +1451,9 @@ void addInstrRequirements(const MachineInstr &MI, const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm(); if (CoopOperands & SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) { + report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation " + "require the following SPIR-V extension: " + "SPV_INTEL_joint_matrix", false); Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); Reqs.addCapability( SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL); @@ -1461,6 +1464,9 @@ void addInstrRequirements(const MachineInstr &MI, SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL || CoopOperands & SPIRV::CooperativeMatrixOperands:: MatrixResultBFloat16ComponentsINTEL) { + report_fatal_error("***BF16ComponentsINTEL type interpretations require " + "the following SPIR-V extension: " + "SPV_INTEL_joint_matrix", false); Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); Reqs.addCapability( SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL); @@ -1480,7 +1486,7 @@ void addInstrRequirements(const MachineInstr &MI, Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix); Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR); - // Check Layout operand in case if it's not a standart one and add the + // Check Layout operand in case if it's not a standard one and add the // appropriate capability. std::unordered_map LayoutToInstMap = { {SPIRV::OpCooperativeMatrixLoadKHR, 3}, @@ -1512,11 +1518,24 @@ void addInstrRequirements(const MachineInstr &MI, OpCode == SPIRV::OpCooperativeMatrixStoreKHR) break; - if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) - report_fatal_error("OpCooperativeMatrix[Load/Store]CheckedINTEL " - "instructions require the following SPIR-V extension: " - "SPV_INTEL_joint_matrix", - false); + std::string InstName; + switch(OpCode) { + case SPIRV::OpCooperativeMatrixPrefetchINTEL: + InstName = "OpCooperativeMatrixPrefetchINTEL"; + break; + case SPIRV::OpCooperativeMatrixLoadCheckedINTEL: + InstName = "OpCooperativeMatrixLoadCheckedINTEL"; + break; + case SPIRV::OpCooperativeMatrixStoreCheckedINTEL: + InstName = "OpCooperativeMatrixStoreCheckedINTEL"; + break; + } + + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) { + const std::string ErrorMsg = InstName + " instruction requires the " + "following SPIR-V extension: SPV_INTEL_joint_matrix"; + report_fatal_error(ErrorMsg.c_str(), false); + } Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) { Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL); @@ -1528,9 +1547,9 @@ void addInstrRequirements(const MachineInstr &MI, } case SPIRV::OpCooperativeMatrixConstructCheckedINTEL: if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) - report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL" - " instructions require the following SPIR-V extension:" - " SPV_INTEL_joint_matrix", + report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL " + "instructions require the following SPIR-V extension: " + "SPV_INTEL_joint_matrix", false); Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); Reqs.addCapability( diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll index 192d769b8a3ce..6d448040059ec 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll @@ -1,3 +1,7 @@ +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +; CHECK-ERROR: LLVM ERROR: ***BF16ComponentsINTEL type interpretations require the following SPIR-V extension: SPV_INTEL_joint_matrix + ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s ; CHECK-DAG: Capability CooperativeMatrixKHR @@ -7,40 +11,22 @@ ; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBBFloat16ComponentsINTEL -; ModuleID = 'test-matrix-opaque.bc' -source_filename = "matrix-int8-test.cpp" -target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" -target triple = "spir64-unknown-unknown" - -$_ZTSZZ15matrix_multiply = comdat any - -; Function Attrs: convergent norecurse -define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) { entry: %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0) - %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2 - %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2 - %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 64) #2 - tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2 + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i64 noundef %_arg_K, i32 noundef 1) + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 1, i64 noundef %_arg_K) + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 64) + tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 0, i64 noundef %_arg_N, i32 noundef 1) ret void } -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1 - -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1 - -; Function Attrs: convergent -declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) -; Function Attrs: convergent -declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) -; Function Attrs: convergent -declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2), i32, i64, i32) #1 +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) -attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } -attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -attributes #2 = { convergent } +declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2), i32, i64, i32) diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll index 73ddaad0dbad1..9d89707fd3178 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll @@ -1,3 +1,7 @@ +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixConstructCheckedINTEL instructions require the following SPIR-V extension:SPV_INTEL_joint_matrix + ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s ; CHECK-DAG: Capability CooperativeMatrixKHR @@ -21,39 +25,22 @@ ; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]] ; CHECK: OpCooperativeMatrixStoreCheckedINTEL -; ModuleID = 'test-matrix-opaque.bc' -source_filename = "matrix-int8-test.cpp" -target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" -target triple = "spir64-unknown-unknown" - -$_ZTSZZ15matrix_multiply = comdat any - -; Function Attrs: convergent norecurse -define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) { entry: - %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #2 - %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #2 - %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 1, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #2 - %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #2 - tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 0, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #2 + %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 1, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) + tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 0, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) ret void } -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1 - -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) -; Function Attrs: convergent -declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) -attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } -attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -attributes #2 = { convergent } +declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll index 4af85beb401dd..c9cbe239c7cee 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll @@ -1,3 +1,7 @@ +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixGetElementCoordINTEL requires the following SPIR-V extension: SPV_INTEL_joint_matrix + ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s ; CHECK-DAG: Capability CooperativeMatrixKHR @@ -9,27 +13,12 @@ ; CHECK: %[[#Matrix:]] = OpCompositeConstruct %[[#MatrixTy]] ; CHECK: %[[#]] = OpCooperativeMatrixGetElementCoordINTEL %[[#]] %[[#Matrix]] %[[#]] -; ModuleID = 'test-matrix-opaque.bc' -source_filename = "matrix-int8-test.cpp" -target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" -target triple = "spir64-unknown-unknown" - -$_ZTSZZ15matrix_multiply = comdat any - -$_ZTSZZ23matrix_multiply_checked = comdat any - -; Function Attrs: convergent norecurse -define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(i32 noundef %_idx, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(i32 noundef %_idx, i32 noundef %_arg_Initvalue) { entry: %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0) %coord = tail call spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) %matrixC, i32 noundef %_idx) ret void } -; Function Attrs: convergent declare dso_local spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32 noundef) -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr #1 - -attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } -attributes #1 = { convergent } +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll index 50461af93084e..806641a29520b 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll @@ -1,3 +1,7 @@ +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +; CHECK-ERROR: LLVM ERROR: PackedINTEL layout require the following SPIR-V extension: SPV_INTEL_joint_matrix + ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s ; CHECK-DAG: Capability CooperativeMatrixKHR @@ -15,64 +19,40 @@ ; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]] ; CHECK: OpCooperativeMatrixStoreCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]] -; ModuleID = 'test-matrix-opaque.bc' -source_filename = "matrix-int8-test.cpp" -target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" -target triple = "spir64-unknown-unknown" - -$_ZTSZZ15matrix_multiply = comdat any - -$_ZTSZZ23matrix_multiply_checked = comdat any - -; Function Attrs: convergent norecurse -define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) { entry: - %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 %_arg_Initvalue) #1 - %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #1 - %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #1 - %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1 - tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #1 + %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 %_arg_Initvalue) + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) + tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) ret void } -; Function Attrs: convergent norecurse -define weak_odr dso_local spir_kernel void @_ZTSZZ23matrix_multiply_checked(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +define weak_odr dso_local spir_kernel void @_ZTSZZ23matrix_multiply_checked(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) { entry: - %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #1 - %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #1 - %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #1 - %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1 - tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #1 + %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) + tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) ret void } -; Function Attrs: convergent -declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32) local_unnamed_addr #1 - -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) #1 - -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32) -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) -; Function Attrs: convergent -declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) -; Function Attrs: convergent -declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) -; Function Attrs: convergent -declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1 +declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) -; Function Attrs: convergent -declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1 +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) -attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } -attributes #1 = { convergent } +declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll index e68a5a2383e85..90c71ebfb660e 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll @@ -1,3 +1,7 @@ +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixPrefetchINTEL instruction requires the following SPIR-V extension: SPV_INTEL_joint_matrix + ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s ; CHECK-DAG: Capability CooperativeMatrixPrefetchINTEL @@ -8,27 +12,16 @@ ; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]] ; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]] 1 -; ModuleID = 'test-matrix-opaque.bc' -source_filename = "matrix-int8-test.cpp" -target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" -target triple = "spir64-unknown-unknown" - -$_ZTSZZ15matrix_multiply = comdat any - -; Function Attrs: convergent norecurse -define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_K) local_unnamed_addr #0 comdat { +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_K) { entry: - tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef %_arg_accA, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0) #1 - tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef %_arg_accB, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K) #1 - tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef %_arg_accC, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K, i32 1) #1 + tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef %_arg_accA, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0) + tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef %_arg_accB, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K) + tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef %_arg_accC, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K, i32 1) ret void } -declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1 - -declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1 +declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) -declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1 +declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) -attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } -attributes #1 = { convergent } +declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll index 4fafca457110f..15a57621ccde0 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll @@ -1,3 +1,7 @@ +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +; CHECK-ERROR: LLVM ERROR: MatrixAAndBTF32ComponentsINTEL type interpretation require the following SPIR-V extension: SPV_INTEL_joint_matrix + ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s ; CHECK-DAG: Capability CooperativeMatrixKHR @@ -7,40 +11,22 @@ ; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBTF32ComponentsINTEL -; ModuleID = 'test-matrix-opaque.bc' -source_filename = "matrix-int8-test.cpp" -target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" -target triple = "spir64-unknown-unknown" - -$_ZTSZZ15matrix_multiply = comdat any - -; Function Attrs: convergent norecurse -define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, float noundef %_arg_Initvalue) local_unnamed_addr #0 comdat { +define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, float noundef %_arg_Initvalue) { entry: %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float %_arg_Initvalue) - %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2 - %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2 - %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 32) #2 - tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2 + %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i64 noundef %_arg_K, i32 noundef 1) + %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 1, i64 noundef %_arg_K) + %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 32) + tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 0, i64 noundef %_arg_N, i32 noundef 1) ret void } -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1 - -; Function Attrs: convergent -declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1 - -; Function Attrs: convergent -declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) -; Function Attrs: convergent -declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1 +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) -; Function Attrs: convergent -declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1 +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) +declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) -attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" } -attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -attributes #2 = { convergent } +declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) From b88548217c6fe22266b1c12a11953c038c59ef94 Mon Sep 17 00:00:00 2001 From: "Sidorov, Dmitry" Date: Wed, 4 Dec 2024 06:23:07 -0800 Subject: [PATCH 4/6] fix tests Signed-off-by: Sidorov, Dmitry --- .../SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll | 2 +- .../SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll | 2 +- .../SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll | 2 +- .../SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll | 2 +- .../SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll | 2 +- .../SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll index 6d448040059ec..c0b23a324992e 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll @@ -1,4 +1,4 @@ -; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR ; CHECK-ERROR: LLVM ERROR: ***BF16ComponentsINTEL type interpretations require the following SPIR-V extension: SPV_INTEL_joint_matrix diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll index 9d89707fd3178..f5e1907792fce 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll @@ -1,4 +1,4 @@ -; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR ; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixConstructCheckedINTEL instructions require the following SPIR-V extension:SPV_INTEL_joint_matrix diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll index c9cbe239c7cee..e7ec5e186d16a 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll @@ -1,4 +1,4 @@ -; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR ; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixGetElementCoordINTEL requires the following SPIR-V extension: SPV_INTEL_joint_matrix diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll index 806641a29520b..8b336df1c2c20 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll @@ -1,4 +1,4 @@ -; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR ; CHECK-ERROR: LLVM ERROR: PackedINTEL layout require the following SPIR-V extension: SPV_INTEL_joint_matrix diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll index 90c71ebfb660e..8573e09284403 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll @@ -1,4 +1,4 @@ -; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR ; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixPrefetchINTEL instruction requires the following SPIR-V extension: SPV_INTEL_joint_matrix diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll index 15a57621ccde0..d3ada306d2c2f 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll @@ -1,4 +1,4 @@ -; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR ; CHECK-ERROR: LLVM ERROR: MatrixAAndBTF32ComponentsINTEL type interpretation require the following SPIR-V extension: SPV_INTEL_joint_matrix From 2c8b86ab6b488483e739e86944a2435377293e12 Mon Sep 17 00:00:00 2001 From: "Sidorov, Dmitry" Date: Wed, 4 Dec 2024 06:49:05 -0800 Subject: [PATCH 5/6] format and fix Signed-off-by: Sidorov, Dmitry --- llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 41 +++++++++++-------- .../cooperative_matrix_checked.ll | 2 +- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index 1a125c44750e6..a9829ead0cf1f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -1451,9 +1451,11 @@ void addInstrRequirements(const MachineInstr &MI, const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm(); if (CoopOperands & SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) { - report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation " - "require the following SPIR-V extension: " - "SPV_INTEL_joint_matrix", false); + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) + report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation " + "require the following SPIR-V extension: " + "SPV_INTEL_joint_matrix", + false); Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); Reqs.addCapability( SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL); @@ -1464,9 +1466,11 @@ void addInstrRequirements(const MachineInstr &MI, SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL || CoopOperands & SPIRV::CooperativeMatrixOperands:: MatrixResultBFloat16ComponentsINTEL) { - report_fatal_error("***BF16ComponentsINTEL type interpretations require " - "the following SPIR-V extension: " - "SPV_INTEL_joint_matrix", false); + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) + report_fatal_error("***BF16ComponentsINTEL type interpretations " + "require the following SPIR-V extension: " + "SPV_INTEL_joint_matrix", + false); Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); Reqs.addCapability( SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL); @@ -1519,21 +1523,22 @@ void addInstrRequirements(const MachineInstr &MI, break; std::string InstName; - switch(OpCode) { - case SPIRV::OpCooperativeMatrixPrefetchINTEL: - InstName = "OpCooperativeMatrixPrefetchINTEL"; - break; - case SPIRV::OpCooperativeMatrixLoadCheckedINTEL: - InstName = "OpCooperativeMatrixLoadCheckedINTEL"; - break; - case SPIRV::OpCooperativeMatrixStoreCheckedINTEL: - InstName = "OpCooperativeMatrixStoreCheckedINTEL"; - break; + switch (OpCode) { + case SPIRV::OpCooperativeMatrixPrefetchINTEL: + InstName = "OpCooperativeMatrixPrefetchINTEL"; + break; + case SPIRV::OpCooperativeMatrixLoadCheckedINTEL: + InstName = "OpCooperativeMatrixLoadCheckedINTEL"; + break; + case SPIRV::OpCooperativeMatrixStoreCheckedINTEL: + InstName = "OpCooperativeMatrixStoreCheckedINTEL"; + break; } if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) { - const std::string ErrorMsg = InstName + " instruction requires the " - "following SPIR-V extension: SPV_INTEL_joint_matrix"; + const std::string ErrorMsg = + InstName + " instruction requires the " + "following SPIR-V extension: SPV_INTEL_joint_matrix"; report_fatal_error(ErrorMsg.c_str(), false); } Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll index f5e1907792fce..a4b2c4be5084b 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll @@ -1,6 +1,6 @@ ; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR -; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixConstructCheckedINTEL instructions require the following SPIR-V extension:SPV_INTEL_joint_matrix +; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixConstructCheckedINTEL instructions require the following SPIR-V extension: SPV_INTEL_joint_matrix ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s From cfacb9972c12e6bfd4b75b986f6fdf7ed3f3a693 Mon Sep 17 00:00:00 2001 From: "Sidorov, Dmitry" Date: Wed, 4 Dec 2024 07:16:15 -0800 Subject: [PATCH 6/6] patch Signed-off-by: Sidorov, Dmitry --- llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index a9829ead0cf1f..4ee71c703f90b 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -1466,11 +1466,11 @@ void addInstrRequirements(const MachineInstr &MI, SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL || CoopOperands & SPIRV::CooperativeMatrixOperands:: MatrixResultBFloat16ComponentsINTEL) { - if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) - report_fatal_error("***BF16ComponentsINTEL type interpretations " - "require the following SPIR-V extension: " - "SPV_INTEL_joint_matrix", - false); + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) + report_fatal_error("***BF16ComponentsINTEL type interpretations " + "require the following SPIR-V extension: " + "SPV_INTEL_joint_matrix", + false); Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); Reqs.addCapability( SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);