Repository navigation
[LV] Recognize safe loads through identical GEPs - #228797
Draft
davidbolvansky wants to merge 207 commits into
Draft
davidbolvansky wants to merge 207 commits into
davidbolvansky wants to merge 207 commits into
Conversation
|
@llvm/pr-subscribers-vectorizers @llvm/pr-subscribers-llvm-transforms Author: Dávid Bolvanský (davidbolvansky) ChangesRecognize predicated loads whose pointer SCEV matches an unconditional access of the same type and sufficient alignment. Identical GEPs also reuse that access's non-poison guarantee; other equivalent expressions retain the existing speculation checks. Motivated by LAME's Implemented with assistance from OpenAI Codex. Patch is 32.24 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/228797.diff 2 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 46f082604b5fb..1b07aa31a6cbb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1407,25 +1407,45 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// the value's type) unconditionally within the loop header without
// introducing a new fault.
SmallPtrSet<Value *, 8> SafePointers;
+ ScalarEvolution &SE = *PSE.getSE();
+ DenseMap<std::pair<const SCEV *, Type *>, Instruction *> SafeAccesses;
// Collect safe addresses.
for (BasicBlock *BB : TheLoop->blocks()) {
if (!blockNeedsPredication(BB)) {
- for (Instruction &I : *BB)
- if (auto *Ptr = getLoadStorePointerOperand(&I))
- SafePointers.insert(Ptr);
- continue;
+ for (Instruction &I : *BB) {
+ auto *Ptr = getLoadStorePointerOperand(&I);
+ if (!Ptr)
+ continue;
+ SafePointers.insert(Ptr);
+ bool IsSimple = isa<LoadInst>(I) ? cast<LoadInst>(I).isSimple()
+ : cast<StoreInst>(I).isSimple();
+ Type *Ty = getLoadStoreType(&I);
+ if (!IsSimple || Ty->isVectorTy())
+ continue;
+ auto Key = std::make_pair(SE.getSCEV(Ptr), Ty);
+ auto [It, Inserted] = SafeAccesses.try_emplace(Key, &I);
+ if (!Inserted &&
+ getLoadStoreAlignment(&I) > getLoadStoreAlignment(It->second))
+ It->second = &I;
+ }
}
+ }
+
+ for (BasicBlock *BB : TheLoop->blocks()) {
+ if (!blockNeedsPredication(BB))
+ continue;
// For a block which requires predication, a address may be safe to access
// in the loop w/o predication if we can prove dereferenceability facts
// sufficient to ensure it'll never fault within the loop. For the moment,
// we restrict this to loads; stores are more complicated due to
// concurrency restrictions.
- ScalarEvolution &SE = *PSE.getSE();
SmallVector<const SCEVPredicate *, 4> Predicates;
for (Instruction &I : *BB) {
LoadInst *LI = dyn_cast<LoadInst>(&I);
+ if (!LI || SafePointers.contains(LI->getPointerOperand()))
+ continue;
// Make sure we can execute all computations feeding into Ptr in the loop
// w/o triggering UB and that none of the out-of-loop operands are poison.
@@ -1434,7 +1454,14 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// because flags will be dropped when executing them unconditionally.
// TODO: Results could be improved by considering poison-propagation
// properties of visited ops.
- auto CanSpeculatePointerOp = [this](Value *Ptr) {
+ auto CanSpeculatePointerOp = [this](Value *Ptr, Value *KnownSafePtr) {
+ auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+ auto *SafeGEP = dyn_cast_or_null<GetElementPtrInst>(KnownSafePtr);
+ // Identical operands and flags inherit the unconditional access's
+ // non-poison guarantee, including operands defined outside the loop.
+ if (GEP && SafeGEP && GEP->isIdenticalTo(SafeGEP))
+ return true;
+
SmallVector<Value *> Worklist = {Ptr};
SmallPtrSet<Value *, 4> Visited;
while (!Worklist.empty()) {
@@ -1471,11 +1498,21 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// Pass the Predicates pointer to isDereferenceableAndAlignedInLoop so
// that it will consider loops that need guarding by SCEV checks. The
// vectoriser will generate these checks if we decide to vectorise.
- if (LI && !LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI) &&
- CanSpeculatePointerOp(LI->getPointerOperand()) &&
- isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
- &Predicates))
- SafePointers.insert(LI->getPointerOperand());
+ if (!LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI)) {
+ // Equivalent pointers need not be represented by the same SSA value.
+ // Require the same access type and sufficient alignment.
+ auto It = SafeAccesses.find(
+ {SE.getSCEV(LI->getPointerOperand()), LI->getType()});
+ bool HasSafeAccess =
+ LI->isSimple() && It != SafeAccesses.end() &&
+ getLoadStoreAlignment(It->second) >= LI->getAlign();
+ Value *KnownSafePtr =
+ HasSafeAccess ? getLoadStorePointerOperand(It->second) : nullptr;
+ if (CanSpeculatePointerOp(LI->getPointerOperand(), KnownSafePtr) &&
+ (HasSafeAccess || isDereferenceableAndAlignedInLoop(
+ LI, TheLoop, SE, *DT, AC, &Predicates)))
+ SafePointers.insert(LI->getPointerOperand());
+ }
Predicates.clear();
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
new file mode 100644
index 0000000000000..5a0ebcdef1080
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
@@ -0,0 +1,470 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+define void @equivalent_geps(ptr noalias noundef %xr, ptr noalias %ix, i64 %n, float %threshold) {
+; CHECK-LABEL: @equivalent_geps(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP25:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> zeroinitializer
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP28]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[PREHEADER]] ]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT: [[XP:%.*]] = getelementptr float, ptr [[XR]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load float, ptr [[XP]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = fcmp oge float [[X]], [[THRESHOLD]]
+; CHECK-NEXT: br i1 [[KEEP]], label [[READ:%.*]], label [[MERGE]]
+; CHECK: read:
+; CHECK-NEXT: [[LOAD_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: [[OLD:%.*]] = load i32, ptr [[LOAD_PTR]], align 4
+; CHECK-NEXT: br label [[MERGE]]
+; CHECK: merge:
+; CHECK-NEXT: [[V:%.*]] = phi i32 [ [[OLD]], [[READ]] ], [ 0, [[LOOP]] ]
+; CHECK-NEXT: [[STORE_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[V]], ptr [[STORE_PTR]], align 4
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT_LOOPEXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+preheader:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %xp = getelementptr float, ptr %xr, i64 %iv
+ %x = load float, ptr %xp, align 4
+ %keep = fcmp oge float %x, %threshold
+ br i1 %keep, label %read, label %merge
+read:
+ %load.ptr = getelementptr i32, ptr %ix, i64 %iv
+ %old = load i32, ptr %load.ptr, align 4
+ br label %merge
+merge:
+ %v = phi i32 [ %old, %read ], [ 0, %loop ]
+ %store.ptr = getelementptr i32, ptr %ix, i64 %iv
+ store i32 %v, ptr %store.ptr, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @stronger_load_alignment(ptr noalias noundef %xr, ptr noalias noundef %ix, i64 %n, float %threshold) {
+; CHECK-LABEL: @stronger_load_alignment(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK: pred.load.if:
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
+; CHECK: pred.load.continue:
+; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK: pred.load.if1:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 16
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP11]], i64 1
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
+; CHECK: pred.load.continue2:
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
+; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK: pred.load.if3:
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 16
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP17]], i64 2
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
+; CHECK: pred.load.continue4:
+; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
+; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.if5:
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP21]]
+; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 16
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP23]], i64 3
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.continue6:
+; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP26]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[PREHEADER]] ]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT: [[XP:%.*]] = getelementptr float, ptr [[XR]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load float, ptr [[XP]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = fcmp oge float [[X]], [[THRESHOLD]]
+; CHECK-NEXT: br i1 [[KEEP]], label [[READ:%.*]], label [[MERGE]]
+; CHECK: read:
+; CHECK-NEXT: [[LOAD_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: [[OLD:%.*]] = load i32, ptr [[LOAD_PTR]], align 16
+; CHECK-NEXT: br label [[MERGE]]
+; CHECK: merge:
+; CHECK-NEXT: [[V:%.*]] = phi i32 [ [[OLD]], [[READ]] ], [ 0, [[LOOP]] ]
+; CHECK-NEXT: [[STORE_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[V]], ptr [[STORE_PTR]], align 4
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT_LOOPEXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+preheader:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %xp = getelementptr float, ptr %xr, i64 %iv
+ %x = load float, ptr %xp, align 4
+ %keep = fcmp oge float %x, %threshold
+ br i1 %keep, label %read, label %merge
+read:
+ %load.ptr = getelementptr i32, ptr %ix, i64 %iv
+ %old = load i32, ptr %load.ptr, align 16
+ br label %merge
+merge:
+ %v = phi i32 [ %old, %read ], [ 0, %loop ]
+ %store.ptr = getelementptr i32, ptr %ix, i64 %iv
+ store i32 %v, ptr %store.ptr, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @narrower_store(ptr noalias noundef %xr, ptr noalias noundef %ix, i64 %n, float %threshold) {
+; CHECK-LABEL: @narrower_store(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK: vector.scevcheck:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 4, i64 [[TMP0]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[IX:%.*]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[IX]]
+; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0
+; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK: pred.load.if:
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
+; CHECK: pred.load.continue:
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP13]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP9]], i64 1
+; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK: pred.load.if1:
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP17]], i64 1
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
+; CHECK: pred.load.continue2:
+; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP18]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP9]], i64 2
+; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK: pred.load.if3:
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP22]], i64 2
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
+; CHECK: pred.load.continue4:
+; CHECK-NEXT: [[TMP24:%.*]] = ...
[truncated]
|
artagnon
self-requested a review
October 4, 2026 09:07
artagnon
reviewed
Oct 5, 2026
The DAGCombiner was folding (A + vscale(C1)) + vscale(C2) into A + vscale(C1+C2) unconditionally. On RISC-V, It is not profitable when inner add had multiple uses, because it could introduce an extra add instruction. This patch restricts the fold to the case where the inner add has only one use. Fixes llvm#222933
Replace the manual block-splitting in expandExtendStackPseudo with MachineBasicBlock::splitAt. Reduces boilerplate, but there's some block renumbering churn in the output. Co-authored-by: Claude (Claude-Opus-4.8)
…instructions" (llvm#229071) Reverts llvm#225097 breaks build of hipCUB LCOMPILER-2914
V_BFE masks its width operand to 5 bits, so extracting the high half with width 64-32 zeroed it instead of passing it through
This PR expands the set of supported instruction formats for the `.insn` directive to all published instruction formats. This includes all of the sub-formats (e.g., `RI-a`, `RI-b`, etc.). Sub-formats are indicated by an underscore `_`, followed by the subscript letter of the sub-format in the assembly (e.g. `ri_a` for `RI-a`). Newly supported formats are: - `I` - `IE` - `MII` - `RI-[abc]` - `RIE-[abcdefg]` - `RIL-[abc]` - `RRD` - `RRF-[abcde]` - `RS-[ab]` - `RSL-[ab]` - `RSY-[ab]` - `RX-[ab]` - `RXY-[abc]` - `SMI` - `SS-[abcdef]` - `VRI-[abcdefghijkl]` - `VRR-[abcdefghijk]` - `VRS-[abcd]` Further, the way the `.insn` directives are implemented is changed in order to reduce maintenance overhead in keeping them up to date. Specifically: - a new TableGen backend autogenerates the match table in `SystemZAsmParser.cpp` - The `DirectiveInsn*` intermediate TableGen class layer was removed in favor of having the relevant definitions in a marker class, as well as the individual `Insn*` records themselves. Note that some of the newly supported formats have immediate operands which are interpreted inconsistently by the instructions using the format, i.e. some instructions use them as signed immediates, while some use them as unsigned immediates. This PR adds assembler operand types (`X(8|16|32)Imm`) to support a range check on these immediates that uses the union of the signed and unsigned ranges. So, an 8-bit immediate in a format that is used in both signed and unsigned forms will accept the range `[-128..255]`. This PR also reorganizes the tests for the `.insn` directives to group them by machine, to mirror the regular assembly tests, and expands them to include all supported instruction formats. Finally, the PR introduces a rule for picking the operand order for the `.insn` directives: since the individual supported instructions themselves have varying operand orders, the operands to the `.insn` directives are ordered by their index, e.g. `R1` > `I2` > `R3` > `BDL4`. So far, this has not been applied to the formats that are already supported, so as not to potentially break existing uses of the `.insn` format directives. This PR was assisted by IBM Bob (for the `SystemZInsnDirectiveEmitter`, and some enhanced test coverage). I manually reviewed the autogenerated code to the best of my ability.
davidbolvansky
changed the title
…9052) Add new constructor overloads that accept just an insertion point iterator, without the redundant basic block argument. The overload with the basic block will be deprecated in a followup. This is llvm#228476, but for the constructor instead of `SetInsertPoint()`. Mass replacement was AI assisted.
…m#226379) Split off from llvm#207695 - this reduces a movdqa to xmm0 on CodeGen/X86/inc-of-add.ll in for SSE by swapping x and y, as sub writes to the first operand.
…lvm#222247) SIModeRegister set a non-default mode for `llvm.fptrunc.round` and left it live across the call boundary, where the caller assumes the default
llvm#219362) When vectorizing loops with a **FindIV reduction** that includes a sunk expression, LLVM miscompiles the code if an epilogue vector loop is triggered by later decisions Specifically, later optimization passes like `simplifyRecipe` can transform the sunk operation (here, converting `mul` to a bit-shift `shl`), converting it into a `VPInstruction`. This tricks the safety check `hasUnsupportedHeaderPhiRecipe`, which inspects recipe types 2nd time wrongly. This triggers epilogue vectorization to be enabled causing the sunk expression to be calculated **twice** (once after the main loop and once after the epilogue), producing incorrect results. **Approach** - Added an `ExpressionSunk` bool flag to `VPReductionPHIRecipe` to track the sinking of the FindIV reduction expression. updated the `VPlanTransforms::optimizeFindIVReductions` to set `ExpressionSunk = true` when sinking the expression, and refactored hasUnsupportedHeaderPhiRecipe` to inspect `RedPhi->isExpressionSunk()` directly rather than relying on recipe class type checks fixes: llvm#219211 Assisted by `DeepWiki` for codebase exploration
Assisted-by: Claude Code. Signed-off-by: Tikhomirova, Kseniya <kseniya.tikhomirova@intel.com>
….cl (llvm#228621) Add a missing noundef in libclc/test/work-item/get_group_id.cl from a recent commit. Fixes: 552ea17 ([ValueTracking] Handle zext/sext in computeConstantRange (llvm#227726), 2026-09-30)
…8099) getDeviceKernelInfo() searched MDeviceKernelInfoMap without holding MDataCollectionMutex, so a kernel launch could race with the registerFatBin() or unregisterFatBin(). Fixing it by addition of lock. Report an unknown kernel name with sycl::errc::runtime instead of asserting. Unlike registration, which runs from a module's static initialization where nothing can catch, this runs from a user kernel submission, so the exception reaches the user's handler. Assited-by: Claude Code. Signed-off-by: Tikhomirova, Kseniya <kseniya.tikhomirova@intel.com>
This fixes 0e7e59d (llvm#222031). Buildkite error link: https://buildkite.com/llvm-project/upstream-bazel/builds?commit=0e7e59dcf4ac70f055195782cf88e637655f6fa3 Co-authored-by: Google Bazel Bot <google-bazel-bot@google.com>
llvm#228587) The z/OS runtime libcall table maps most math libcalls to the IEEE entry points of the C runtime (e.g. `cos` -> `@@FCOS@B`, `pow` -> `@@WPow@B`), but `tan`, `asin`, `acos`, `atan`, `atan2`, `sinh`, `cosh` and `tanh` had no entries. Calls generated for the corresponding intrinsics then used the plain names (`atan2f` etc.), which bind to entry points that cannot be used in IEEE mode: a program fails at run time with `CEE3728S The use of a function, which is not supported by this release of Language Environment was detected.` Reproducer and details are in llvm#228584. This adds the F32/F64/F128 entry points for these eight functions, with the names the z/OS C headers map them to in IEEE mode, and three more cases to `zos-intrinsics.ll`. Tests: the same change on a branch based on 85ac560, with SystemZ as the only target: `llvm-lit test/CodeGen/SystemZ` passes (1282 tests), and five programs of the NIST FORTRAN 77 test suite that failed with CEE3728S (they use `ATAN2`, `TANH`, `ASIN`, which flang lowers to these intrinsics) now give the same output on z/OS 3.1 as on the host. The patch applies unchanged to `main`; I could not rebuild `main` locally at the moment. Fixes llvm#228584. Assisted-by: Claude Code (Anthropic) 🤖 Generated with [Claude Code](https://claude.com/claude-code)
) The MIN/MAX reduction identities were computed with `APInt::getSigned{Min,Max}Value(bits).getSExtValue()` and passed to `createIntegerConstant(int64_t)`. For INTEGER(16) that value does not fit in `int64_t`. An assertions build aborts at `-O1` on any `MINVAL`/`MAXVAL`/`MINLOC`/`MAXLOC` of an INTEGER(16) array, and on OpenMP `reduction(min:)`/`reduction(max:)` of INTEGER(16) at any `-O`. A release build zero-extends the low 64 bits instead. The identity becomes `2**64-1` or `0`, so empty and fully masked reductions return the wrong value. This adds an `APInt` overload of `FirOpBuilder::createIntegerConstant` and uses it at the four affected sites: `SimplifyHLFIRIntrinsics`, the MAXVAL and MINLOC/MAXLOC `init` lambdas in `SimplifyIntrinsics`, and `ReductionProcessor::getReductionInitValue`. Fixes llvm#228509 Assisted-by: AI
…lLabel (llvm#227021) `NativePDB` creates function and method decls without an asm label or with the public symbol name as the label. The expression evaluator then has to find the callee by the name Clang mangles for the decl, which fails whenever that name is not in the symbol table: - Methods of class template specializations: the record is a plain `CXXRecordDecl` named `"A<int>"`, so Clang mangles `?foo@A<int>@@...` instead of `?foo@?$A@H@@...`. - Functions with internal linkage, which have no `S_PUB32`. - Functions renamed with `__asm__`. All of the above fail with: ``` error: Couldn't look up symbols: int A<int>::foo(void) ``` --- DWARF avoids this by labelling decls with a `$__lldb_func` `FunctionCallLabel` which identifies the `DIE`. `IRExecutionUnit` returns it back to `SymbolFile::ResolveFunctionCallLabel`. This patch does the same for `NativePDB`. The label identifies the `S_GPROC32`/`S_LPROC32` record and carries the public (mangled) name, or the procedure name for functions without one. `UdtRecordCompleter` finds the definition of each method in the field list by its qualified name and `LF_MFUNCTION` type. The methods which do not have a definition in the module have no label. Therefore they are still found by their Clang-mangled name like before. `ResolveFunctionCallLabel` rejects constructor closures and deleting destructors, which the procedure record doesn't describe. With PDB debug info this fixes: - `TestCallStdStringFunction` - `TestClassTemplateSpecializationParametersHandling` - `TestExprsBug35310` - `TestClassTemplateParameterPack` - `TestOverloadedFunctions` - `TestTemplateArgs` rdar://189180045
…nidump.py (llvm#229059) This API will never return None, instead it will be an invalid SBValue. I noticed this because I'm checking what can be run on an AArch64 host, and this part of the test passed due to this mistake.
fhahn
reviewed
Oct 6, 2026
Comment on lines
+177
to
+178
| %empty = icmp eq i64 %n, 0 | ||
| br i1 %empty, label %exit, label %preheader |
| @@ -0,0 +1,470 @@ | |||
| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py | |||
Contributor
There was a problem hiding this comment.
please generate with latest UTC version, fix formatting in the tests
Comment on lines
+1444
to
+1445
| Align Alignment = getLoadStoreAlignment(&I); | ||
| auto [It, Inserted] = SafeAccesses.try_emplace({GEP, Ty}, Alignment); |
Contributor
There was a problem hiding this comment.
please split off the changes to include Ty and Alignment as separate changes; AFAICT they fix existing mis-compiles
Contributor
Author
There was a problem hiding this comment.
Extracted the miscompilation fix as requested into #229379.
davidbolvansky
requested review from
a team,
JDevlieghere,
Jianhui-Li,
Pierre-vh,
aaronmondal,
andykaylor,
banach-space,
bcardosolopes,
charithaintc,
dcaballe,
fabianmcg,
googlewalt,
jhuber6,
kaladron,
keith,
labath,
nicolasvasilache,
nikic,
ritter-x2a,
rupprecht,
vangthao95 and
xlauko
as code owners
October 6, 2026 11:04
davidbolvansky
marked this pull request as draft
October 6, 2026 11:08
You can test this locally with the following command:git-clang-format --diff origin/main HEAD --extensions h,cpp -- llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp --diff_from_common_commit
View the diff from clang-format here.diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 7037d7574..27f24717b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1426,8 +1426,7 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// introducing a new fault.
SafeAccessesTy SafePointers;
ScalarEvolution &SE = *PSE.getSE();
- SmallDenseMap<GEPAccessDenseMapInfo::KeyTy, Align, 8,
- GEPAccessDenseMapInfo>
+ SmallDenseMap<GEPAccessDenseMapInfo::KeyTy, Align, 8, GEPAccessDenseMapInfo>
SafeAccesses;
// Collect safe addresses.
@@ -1437,8 +1436,9 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
auto *Ptr = getLoadStorePointerOperand(&I);
if (!Ptr)
continue;
- auto [PtrIt, PtrInserted] = SafePointers.try_emplace(
- std::make_pair(Ptr, getLoadStoreType(&I)), getLoadStoreAlignment(&I));
+ auto [PtrIt, PtrInserted] =
+ SafePointers.try_emplace(std::make_pair(Ptr, getLoadStoreType(&I)),
+ getLoadStoreAlignment(&I));
if (!PtrInserted)
PtrIt->second = std::max(PtrIt->second, getLoadStoreAlignment(&I));
auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
@@ -1527,7 +1527,8 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
isDereferenceableAndAlignedInLoop(
LI, TheLoop, SE, *DT, AC, &Predicates))) {
auto [PtrIt, PtrInserted] = SafePointers.try_emplace(
- std::make_pair(LI->getPointerOperand(), LI->getType()), LI->getAlign());
+ std::make_pair(LI->getPointerOperand(), LI->getType()),
+ LI->getAlign());
if (!PtrInserted)
PtrIt->second = std::max(PtrIt->second, LI->getAlign());
}
|
This branch has not been deployed
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Recognize identical GEPs used by unconditional accesses so predicated loads can reuse their safety information. This helps LAME's
count_bitsloop avoid masked loads. Depends on #229379; implemented with assistance from OpenAI Codex.