Repository navigation
[LV] Respect access type and alignment when speculating loads - #229379
Merged
fhahn merged 7 commits intoNov 8, 2026
Merged
Conversation
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
|
@llvm/pr-subscribers-llvm-loopvectorize @llvm/pr-subscribers-vectorizers Author: Dávid Bolvanský (davidbolvansky) ChangesAn unconditional access currently marks its pointer as safe for any predicated load, regardless of the load's type or alignment. This can speculate a wider load past the end of an object, or a load requiring stronger alignment than the unconditional access provides. Track the access type and maximum known safe alignment along with the pointer. Add regression tests for both cases and a positive case with sufficient alignment. Split from #228797 following review. Implemented with assistance from OpenAI Codex. The #include <sys/mman.h>
#include <unistd.h>
#include <stdint.h>
#include <stdio.h>
// Generated from wider_predicated_load in the LLVM regression test.
// It reads one byte per iteration, and reads an i64 only if that byte is nonzero.
extern void wider_predicated_load(void *, void *, uint64_t);
int main(void) {
enum { Iterations = 4, Stride = 8,
AccessibleBytes = (Iterations - 1) * Stride + 1 }; // 25 bytes
long size = sysconf(_SC_PAGESIZE);
if (size <= 0) return 2;
// Anonymous mappings start zeroed. Protect the second page so that an
// out-of-bounds read fails deterministically instead of reading adjacent data.
char *p = mmap(0, size * 2, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANON, -1, 0);
if (p == MAP_FAILED || mprotect(p + size, size, PROT_NONE)) return 2;
// Place the last accessible byte immediately before the protected page.
// The byte loads at offsets 0, 8, 16 and 24 are all valid and return zero,
// so none of the conditional i64 loads should execute.
// Speculating the last i64 load reads offsets 24..31: seven protected bytes.
int64_t out[Iterations] = {1, 1, 1, 1};
wider_predicated_load(p + size - AccessibleBytes, out, Iterations);
for (int i = 0; i < Iterations; i++) if (out[i]) return 3;
munmap(p, size * 2);
puts("PASS: no out-of-bounds speculative load");
return 0;
}On Apple Silicon, linking this harness with the vectorized test produces SIGBUS before the fix and prints Patch is 26.06 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/229379.diff 3 Files Affected:
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 6dd44f8a71538..8011424bf8f5e 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -26,8 +26,10 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -621,15 +623,18 @@ class LoopVectorizationLegality {
/// for it.
bool canUncountableExitConditionLoadBeMoved(BasicBlock *ExitingBlock);
+ /// Maps a pointer and access type to its known safe alignment.
+ using SafeAccessesTy = DenseMap<std::pair<Value *, Type *>, Align>;
+
/// Return true if all of the instructions in the block can be speculatively
/// executed, and record the loads/stores that require masking.
- /// \p SafePtrs is a list of addresses that are known to be legal and we know
- /// that we can read from them without segfault.
+ /// \p SafePtrs records pointers that can be accessed unconditionally,
+ /// together with the access type and alignment justified for each pointer.
/// \p MaskedOp is a list of instructions that have to be transformed into
/// calls to the appropriate masked intrinsic when the loop is vectorized
/// or dropped if the instruction is a conditional assume intrinsic.
bool
- blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs,
+ blockCanBePredicated(BasicBlock *BB, SafeAccessesTy &SafePtrs,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const;
/// Updates the vectorization state by adding \p Phi to the inductions list.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 46f082604b5fb..f8239da6cf18c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1342,7 +1342,7 @@ bool LoopVectorizationLegality::blockNeedsPredication(
}
bool LoopVectorizationLegality::blockCanBePredicated(
- BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs,
+ BasicBlock *BB, SafeAccessesTy &SafePtrs,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const {
for (Instruction &I : *BB) {
// We can predicate blocks with calls to assume, as long as we drop them in
@@ -1370,7 +1370,8 @@ bool LoopVectorizationLegality::blockCanBePredicated(
// Loads are handled via masking (or speculated if safe to do so.)
if (auto *LI = dyn_cast<LoadInst>(&I)) {
- if (!SafePtrs.count(LI->getPointerOperand()))
+ auto It = SafePtrs.find({LI->getPointerOperand(), LI->getType()});
+ if (It == SafePtrs.end() || It->second < LI->getAlign())
MaskedOp.insert(LI);
continue;
}
@@ -1406,14 +1407,20 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// the memory pointed to can be dereferenced (with the access size implied by
// the value's type) unconditionally within the loop header without
// introducing a new fault.
- SmallPtrSet<Value *, 8> SafePointers;
+ SafeAccessesTy SafePointers;
// Collect safe addresses.
for (BasicBlock *BB : TheLoop->blocks()) {
if (!blockNeedsPredication(BB)) {
- for (Instruction &I : *BB)
- if (auto *Ptr = getLoadStorePointerOperand(&I))
- SafePointers.insert(Ptr);
+ for (Instruction &I : *BB) {
+ if (auto *Ptr = getLoadStorePointerOperand(&I)) {
+ auto [It, Inserted] = SafePointers.try_emplace(
+ std::make_pair(Ptr, getLoadStoreType(&I)),
+ getLoadStoreAlignment(&I));
+ if (!Inserted)
+ It->second = std::max(It->second, getLoadStoreAlignment(&I));
+ }
+ }
continue;
}
@@ -1474,8 +1481,12 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
if (LI && !LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI) &&
CanSpeculatePointerOp(LI->getPointerOperand()) &&
isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
- &Predicates))
- SafePointers.insert(LI->getPointerOperand());
+ &Predicates)) {
+ auto [It, Inserted] = SafePointers.try_emplace(
+ std::make_pair(LI->getPointerOperand(), LI->getType()), LI->getAlign());
+ if (!Inserted)
+ It->second = std::max(It->second, LI->getAlign());
+ }
Predicates.clear();
}
}
@@ -2047,7 +2058,7 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
// The list of pointers that we can safely read and write to remains empty.
- SmallPtrSet<Value *, 8> SafePointers;
+ SafeAccessesTy SafePointers;
// Check all blocks for predication, including those that ordinarily do not
// need predication such as the header block.
@@ -2066,7 +2077,7 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
void LoopVectorizationLegality::prepareToFoldTailByMasking() {
// The list of pointers that we can safely read and write to remains empty.
- SmallPtrSet<Value *, 8> SafePointers;
+ SafeAccessesTy SafePointers;
// Mark all blocks for predication, including those that ordinarily do not
// need predication such as the header block, and collect instructions needing
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
new file mode 100644
index 0000000000000..51b945b58ba5b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
@@ -0,0 +1,369 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+; A byte load does not make a wider load at the same pointer safe.
+define void @wider_predicated_load(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @wider_predicated_load(
+; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i8> poison, i8 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[TMP10]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP11]], i64 3
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i8> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP16]], i64 0
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> poison, i64 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP19]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP16]], i64 1
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP20]], i64 [[TMP22]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i64> [ [[TMP20]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP23]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP16]], i64 2
+; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP26]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i64> [ [[TMP24]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP27]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP16]], i64 3
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP30]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP32:%.*]] = phi <4 x i64> [ [[TMP28]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP31]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i64> [[TMP32]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[PREDPHI]], ptr [[TMP33]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[GUARD:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i8 [[GUARD]], 0
+; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[VALUE:%.*]] = load i64, ptr [[P]], align 1
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i64 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
+; CHECK-NEXT: [[OUT:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i64 [[RESULT]], ptr [[OUT]], align 1
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+
+preheader:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %p = getelementptr i64, ptr %src, i64 %iv
+ %guard = load i8, ptr %p, align 1
+ %keep = icmp ne i8 %guard, 0
+ br i1 %keep, label %read, label %merge
+
+read:
+ %value = load i64, ptr %p, align 1
+ br label %merge
+
+merge:
+ %result = phi i64 [ %value, %read ], [ 0, %loop ]
+ %out = getelementptr i64, ptr %dst, i64 %iv
+ store i64 %result, ptr %out, align 1
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A weaker alignment does not justify the predicated load alignment.
+define void @stronger_predicated_alignment(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @stronger_predicated_alignment(
+; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i64 3
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP16]], i64 0
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP19]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP16]], i64 1
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP5]], align 16
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP22]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP23]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP16]], i64 2
+; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP6]], align 16
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP26]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i32> [ [[TMP24]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP27]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP16]], i64 3
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP7]], align 16
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP30]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP32:%.*]] = phi <4 x i32> [ [[TMP28]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP31]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP32]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP33]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[GUARD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i32 [[GUARD]], 0
+; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[VALUE:%.*]] = load i32, ptr [[P]], align 16
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
+; CHECK-NEXT: [[OUT:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[RESULT]], ptr [[OUT]], align 1
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+
+preheader:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %p = getelementptr [4 x i32], ptr %src, i64 %iv
+ %guard = load i32, ptr %p, align 4
+ %keep = icmp ne i32 %guard, 0
+ br i1 %keep, label %read, label %merge
+
+read:
+ %value = load i32, ptr %p, align 16
+ br label %merge
+
+merge:
+ %result = phi i32 [ %value, %read ], [ 0, %loop ]
+ %out = getelementptr i32, ptr %dst, i64 %i...
[truncated]
|
artagnon
reviewed
Oct 6, 2026
Contributor
Author
|
Thank you for taking a quick look. |
davidbolvansky
force-pushed
the
perf/lv-predicated-load-access-safety
branch
2 times, most recently
from
October 6, 2026 13:07
31cde4f to
42341e9
Compare
artagnon
reviewed
Oct 6, 2026
davidbolvansky
force-pushed
the
perf/lv-predicated-load-access-safety
branch
2 times, most recently
from
October 6, 2026 16:25
3d7c5b0 to
19d02f0
Compare
davidbolvansky
force-pushed
the
perf/lv-predicated-load-access-safety
branch
from
October 8, 2026 11:33
19d02f0 to
cd71c14
Compare
Contributor
Author
|
Rebased on main now that the baseline tests from #229468 are merged. The diff now shows the fix and updated checks. |
artagnon
reviewed
Oct 8, 2026
artagnon
approved these changes
Oct 8, 2026
artagnon
left a comment
artagnon
left a comment
Contributor
There was a problem hiding this comment.
LGTM, thanks. Would be good to wait a bit for other reviewers as well.
artagnon
reviewed
Oct 8, 2026
artagnon
left a comment
artagnon
left a comment
Contributor
There was a problem hiding this comment.
Ah, this would unfortunately be a regression in the type -- I think we want to keep Align; it's alright if my suggestions were unworkable -- they were just nits, but kindly revert to using Align? :)
davidbolvansky
force-pushed
the
perf/lv-predicated-load-access-safety
branch
from
October 8, 2026 13:49
c2fed33 to
cd71c14
Compare
Contributor
Author
|
@artagnon Done, you are right. Keeping Align and the explicit find check makes sense here. |
davidbolvansky
force-pushed
the
perf/lv-predicated-load-access-safety
branch
from
October 8, 2026 13:50
cd71c14 to
18fff35
Compare
Contributor
Author
|
@fhahn any comment from your side? |
fhahn
reviewed
Oct 12, 2026
fhahn
reviewed
Oct 12, 2026
fhahn
left a comment
fhahn
left a comment
Contributor
There was a problem hiding this comment.
As a general comment on workflow, using a perf/ branch is probably not ideal unless you are specifically interested in measuring compile-time for each commit. Otherwise it mostly uses resources unnecessarily
Contributor
Author
Yeah, sry for that, wont use it for future. |
davidbolvansky
force-pushed
the
perf/lv-predicated-load-access-safety
branch
from
October 15, 2026 21:57
a2e86b6 to
47dc935
Compare
Contributor
Author
|
Rebased on latest main and resolved the merge conflict. |
fhahn
reviewed
Oct 16, 2026
fhahn
approved these changes
Nov 1, 2026
Contributor
Author
|
@fhahn you can merge this on my behalf |
fhahn
reviewed
Nov 6, 2026
fhahn
left a comment
fhahn
left a comment
Contributor
There was a problem hiding this comment.
@fhahn you can merge this on my behalf
Sure but can you clean up the PR description a bit? I think you can remove the linux-specific C code example. It may be worth checking if it can be added as an end-to-end test to llvm-test-suite
Contributor
Author
|
Done, removed the C example and cleaned up the description. |
Contributor
Author
|
I prepared the runtime test in llvm/llvm-test-suite#532, with a small helper for POSIX and Windows guarded pages. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
An unconditional access does not make a wider or more strongly aligned load at the same pointer safe. Track the maximum fixed access size and alignment for each pointer, skipping scalable accesses. Implemented with assistance from OpenAI Codex.