Skip to content

[LV] Add baseline tests for predicated load access safety #229468

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fhahn merged 3 commits into from
Oct 8, 2026
+228 −0
Merged

[LV] Add baseline tests for predicated load access safety #229468

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
66a9738
[LV] Add tests for predicated load access safety
davidbolvansky Oct 6, 2026
2584448
[LV] Make access safety tests target independent
davidbolvansky Oct 6, 2026
8cf3c07
Merge branch 'main' into test/lv-predicated-load-access-safety
fhahn Oct 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
228 changes: 228 additions & 0 deletions llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s

; A byte load does not make a wider load at the same pointer safe.
define void @wider_predicated_load(ptr noalias %src, ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @wider_predicated_load(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP4]], align 1
; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP5]], align 1
; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP6]], align 1
; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP7]], align 1
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i8> poison, i8 [[TMP8]], i64 0
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[TMP9]], i64 1
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[TMP10]], i64 2
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP11]], i64 3
; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i8> [[TMP15]], zeroinitializer
; CHECK-NEXT: [[TMP32:%.*]] = load <4 x i64>, ptr [[TMP4]], align 1
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i64> [[TMP32]], <4 x i64> zeroinitializer
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i64, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i64> [[PREDPHI]], ptr [[TMP33]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
entry:
br label %loop.header

loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%p = getelementptr i64, ptr %src, i64 %iv
%guard = load i8, ptr %p, align 1
%keep = icmp ne i8 %guard, 0
br i1 %keep, label %read, label %loop.latch

read:
%value = load i64, ptr %p, align 1
br label %loop.latch

loop.latch:
%result = phi i64 [ %value, %read ], [ 0, %loop.header ]
%out = getelementptr i64, ptr %dst, i64 %iv
store i64 %result, ptr %out, align 1
%iv.next = add nuw i64 %iv, 1
%done = icmp eq i64 %iv.next, %n
br i1 %done, label %exit, label %loop.header

exit:
ret void
}

; A weaker alignment does not justify the predicated load alignment.
define void @stronger_predicated_alignment(ptr noalias %src, ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @stronger_predicated_alignment(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4
; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 1
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 2
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i64 3
; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i32> [[TMP15]], zeroinitializer
; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP4]], align 16
; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP5]], align 16
; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP6]], align 16
; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP7]], align 16
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> poison, i32 [[TMP18]], i64 0
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP22]], i64 1
; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP26]], i64 2
; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP30]], i64 3
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP31]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP33]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
entry:
br label %loop.header

loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%p = getelementptr [4 x i32], ptr %src, i64 %iv
%guard = load i32, ptr %p, align 4
%keep = icmp ne i32 %guard, 0
br i1 %keep, label %read, label %loop.latch

read:
%value = load i32, ptr %p, align 16
br label %loop.latch

loop.latch:
%result = phi i32 [ %value, %read ], [ 0, %loop.header ]
%out = getelementptr i32, ptr %dst, i64 %iv
store i32 %result, ptr %out, align 1
%iv.next = add nuw i64 %iv, 1
%done = icmp eq i64 %iv.next, %n
br i1 %done, label %exit, label %loop.header

exit:
ret void
}

; Keep the strongest known alignment when several accesses share a pointer.
define void @sufficient_alignment(ptr noalias %src, ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @sufficient_alignment(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 16
; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 16
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 16
; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 16
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 1
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 2
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i64 3
; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP4]], align 4
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP7]], align 4
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> poison, i32 [[TMP16]], i64 0
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP17]], i64 1
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i64 2
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP19]], i64 3
; CHECK-NEXT: [[TMP24:%.*]] = add <4 x i32> [[TMP15]], [[TMP23]]
; CHECK-NEXT: [[TMP25:%.*]] = icmp ne <4 x i32> [[TMP24]], zeroinitializer
; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP4]], align 16
; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP5]], align 16
; CHECK-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP6]], align 16
; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP7]], align 16
; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP26]], i64 0
; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP27]], i64 1
; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i64 2
; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i64 3
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP25]], <4 x i32> [[TMP33]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP34]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
entry:
br label %loop.header

loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%p = getelementptr [4 x i32], ptr %src, i64 %iv
%guard = load i32, ptr %p, align 16
%weak = load i32, ptr %p, align 4
%sum = add i32 %guard, %weak
%keep = icmp ne i32 %sum, 0
br i1 %keep, label %read, label %loop.latch

read:
%value = load i32, ptr %p, align 16
br label %loop.latch

loop.latch:
%result = phi i32 [ %value, %read ], [ 0, %loop.header ]
%out = getelementptr i32, ptr %dst, i64 %iv
store i32 %result, ptr %out, align 1
%iv.next = add nuw i64 %iv, 1
%done = icmp eq i64 %iv.next, %n
br i1 %done, label %exit, label %loop.header

exit:
ret void
}
Loading