From 32b2e724de8c4bb899681085b82faf47d5a11ef6 Mon Sep 17 00:00:00 2001 From: vg0204 Date: Fri, 6 Nov 2026 12:37:34 +0530 Subject: [PATCH] [AMDGPU] Teach isMaskedByExec that S_CSELECT x, 0 is masked iff x is S_CSELECT x, 0 produces x or 0, so inactive-lane bits are already clear iff x is exec-masked. tryFoldAndExec can then drop the redundant s_and exec that GISel emits after s_cselect exec, 0 in lane-mask merges. --- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 15 +++++++++++-- ...-divergent-i1-phis-no-lane-mask-merging.ll | 3 +-- ...vergence-divergent-i1-used-outside-loop.ll | 5 ++--- .../GlobalISel/divergence-structurizer.ll | 20 +++++++----------- .../divergence-temporal-divergent-i1.ll | 7 ++----- .../AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll | 6 ------ llvm/test/CodeGen/AMDGPU/fold-and-exec.mir | 21 +++++++++++++++++++ 7 files changed, 47 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 7895c1c8a6a8fe..7736769ea26926 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -5218,8 +5218,19 @@ bool SIInstrInfo::isMaskedByExec(Register Reg, const MachineInstr &Use, return true; if ((Opc == LMC.OrOpc || Opc == LMC.XorOpc) && Recurse(1) && Recurse(2)) return true; - // TODO: Sometimes we encounter "reg = S_CSELECT -1, 0". If Reg has no other - // uses this could be optimized to "reg = S_CSELECT $exec, 0". + // S_CSELECT x, 0 and S_CSELECT 0, x produce either x or 0. 0 has no bits set + // for inactive lanes, so the result is masked iff x is. An immediate x (e.g. + // -1) is rejected by Recurse, since it may set bits for inactive lanes. + if (Opc == LMC.CSelectOpc) { + std::optional FalseImm = + getImmOrMaterializedImm(MRI, Def->getOperand(2)); + if (FalseImm && *FalseImm == 0) + return Recurse(1); + std::optional TrueImm = + getImmOrMaterializedImm(MRI, Def->getOperand(1)); + if (TrueImm && *TrueImm == 0) + return Recurse(2); + } return false; } diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll index 399ff0136c4fe4..73ca282bd57758 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll @@ -112,11 +112,10 @@ define void @divergent_i1_phi_used_inside_loop(float %val, ptr %addr) { ; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0 ; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0 ; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo -; GFX10-NEXT: s_and_b32 s8, exec_lo, s8 ; GFX10-NEXT: s_xor_b32 s5, s5, 1 ; GFX10-NEXT: s_add_i32 s6, s6, 1 -; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4 ; GFX10-NEXT: s_or_b32 s7, s7, s8 +; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4 ; GFX10-NEXT: s_cbranch_execnz .LBB2_1 ; GFX10-NEXT: ; %bb.2: ; %exit diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll index 82d9080c892e22..2b10858a653f84 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll @@ -552,13 +552,12 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a ; GFX10-NEXT: ; %bb.3: ; %loop.body ; GFX10-NEXT: ; in Loop: Header=BB7_2 Depth=1 ; GFX10-NEXT: global_load_dword v6, v[0:1], off -; GFX10-NEXT: s_add_i32 s1, s1, 1 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 4 -; GFX10-NEXT: s_cmpk_lt_u32 s1, 0x64 +; GFX10-NEXT: s_add_i32 s1, s1, 1 ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo +; GFX10-NEXT: s_cmpk_lt_u32 s1, 0x64 ; GFX10-NEXT: s_cselect_b32 s6, exec_lo, 0 ; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo -; GFX10-NEXT: s_and_b32 s6, exec_lo, s6 ; GFX10-NEXT: s_andn2_b32 s3, s3, exec_lo ; GFX10-NEXT: s_or_b32 s4, s4, s6 ; GFX10-NEXT: s_waitcnt vmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll index e92c02b9b528b3..8781c06cbcb879 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll @@ -126,13 +126,12 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a ; GFX10-NEXT: ; %bb.3: ; %loop.body ; GFX10-NEXT: ; in Loop: Header=BB2_2 Depth=1 ; GFX10-NEXT: global_load_dword v4, v[0:1], off -; GFX10-NEXT: s_add_i32 s0, s0, 1 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 4 -; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64 +; GFX10-NEXT: s_add_i32 s0, s0, 1 ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo +; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64 ; GFX10-NEXT: s_cselect_b32 s4, exec_lo, 0 ; GFX10-NEXT: s_andn2_b32 s2, s2, exec_lo -; GFX10-NEXT: s_and_b32 s4, exec_lo, s4 ; GFX10-NEXT: s_or_b32 s2, s2, s4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v4 @@ -207,13 +206,12 @@ define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) % ; GFX10-NEXT: global_load_dword v6, v[0:1], off ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 4 ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo -; GFX10-NEXT: s_add_i32 s0, s0, 1 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v4, 4 -; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64 +; GFX10-NEXT: s_add_i32 s0, s0, 1 ; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v5, vcc_lo +; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64 ; GFX10-NEXT: s_cselect_b32 s6, exec_lo, 0 ; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo -; GFX10-NEXT: s_and_b32 s6, exec_lo, s6 ; GFX10-NEXT: s_or_b32 s4, s4, s6 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_add_nc_u32_e32 v6, 1, v6 @@ -310,13 +308,12 @@ define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) % ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v4, 4 ; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v5, vcc_lo -; GFX10-NEXT: s_add_i32 s0, s0, 1 ; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v6, 4 -; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64 +; GFX10-NEXT: s_add_i32 s0, s0, 1 ; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo +; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64 ; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0 ; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo -; GFX10-NEXT: s_and_b32 s8, exec_lo, s8 ; GFX10-NEXT: s_or_b32 s6, s6, s8 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_add_nc_u32_e32 v8, 1, v8 @@ -398,13 +395,12 @@ define amdgpu_cs void @loop_with_div_break_with_body(ptr addrspace(1) %x, ptr ad ; GFX10-NEXT: ; %bb.3: ; %loop.body ; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1 ; GFX10-NEXT: global_load_dword v6, v[0:1], off -; GFX10-NEXT: s_add_i32 s1, s1, 1 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 4 -; GFX10-NEXT: s_cmpk_lt_u32 s1, 0x64 +; GFX10-NEXT: s_add_i32 s1, s1, 1 ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo +; GFX10-NEXT: s_cmpk_lt_u32 s1, 0x64 ; GFX10-NEXT: s_cselect_b32 s6, exec_lo, 0 ; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo -; GFX10-NEXT: s_and_b32 s6, exec_lo, s6 ; GFX10-NEXT: s_andn2_b32 s3, s3, exec_lo ; GFX10-NEXT: s_or_b32 s4, s4, s6 ; GFX10-NEXT: s_waitcnt vmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll index ce21c834006d7d..211062e5eab066 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll @@ -17,11 +17,10 @@ define void @temporal_divergent_i1_phi(float %val, ptr %addr) { ; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0 ; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0 ; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo -; GFX10-NEXT: s_and_b32 s8, exec_lo, s8 ; GFX10-NEXT: s_xor_b32 s5, s5, 1 ; GFX10-NEXT: s_add_i32 s6, s6, 1 -; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4 ; GFX10-NEXT: s_or_b32 s7, s7, s8 +; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4 ; GFX10-NEXT: s_cbranch_execnz .LBB0_1 ; GFX10-NEXT: ; %bb.2: ; %exit @@ -64,11 +63,10 @@ define void @temporal_divergent_i1_non_phi(float %val, ptr %addr) { ; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0 ; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0 ; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo -; GFX10-NEXT: s_and_b32 s8, exec_lo, s8 ; GFX10-NEXT: s_xor_b32 s5, s5, 1 ; GFX10-NEXT: s_add_i32 s6, s6, 1 -; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4 ; GFX10-NEXT: s_or_b32 s7, s7, s8 +; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4 ; GFX10-NEXT: s_cbranch_execnz .LBB1_1 ; GFX10-NEXT: ; %bb.2: ; %exit @@ -127,7 +125,6 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr ad ; GFX10-NEXT: s_cselect_b32 s5, exec_lo, 0 ; GFX10-NEXT: s_and_b32 s6, exec_lo, s10 ; GFX10-NEXT: s_andn2_b32 s7, s9, exec_lo -; GFX10-NEXT: s_and_b32 s5, exec_lo, s5 ; GFX10-NEXT: s_or_b32 s8, s6, s8 ; GFX10-NEXT: s_or_b32 s9, s7, s5 ; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll index 680334dd4bae49..025b4160a66597 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll @@ -1304,7 +1304,6 @@ define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(ptr addrspace(1) %out, [ ; GFX7-NEXT: s_cmp_lg_u32 s0, 0 ; GFX7-NEXT: s_cselect_b64 s[0:1], exec, 0 ; GFX7-NEXT: s_andn2_b64 s[8:9], 0, exec -; GFX7-NEXT: s_and_b64 s[0:1], exec, s[0:1] ; GFX7-NEXT: s_or_b64 vcc, s[8:9], s[0:1] ; GFX7-NEXT: .LBB13_2: ; %exit ; GFX7-NEXT: s_or_b64 exec, exec, s[6:7] @@ -1338,7 +1337,6 @@ define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(ptr addrspace(1) %out, [ ; GFX8-NEXT: s_cmp_lg_u32 s0, 0 ; GFX8-NEXT: s_cselect_b64 s[0:1], exec, 0 ; GFX8-NEXT: s_andn2_b64 s[6:7], 0, exec -; GFX8-NEXT: s_and_b64 s[0:1], exec, s[0:1] ; GFX8-NEXT: s_or_b64 vcc, s[6:7], s[0:1] ; GFX8-NEXT: .LBB13_2: ; %exit ; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] @@ -1371,7 +1369,6 @@ define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(ptr addrspace(1) %out, [ ; GFX10_W32-NEXT: s_cmp_lg_u32 s0, 0 ; GFX10_W32-NEXT: s_cselect_b32 s0, exec_lo, 0 ; GFX10_W32-NEXT: s_andn2_b32 s2, 0, exec_lo -; GFX10_W32-NEXT: s_and_b32 s0, exec_lo, s0 ; GFX10_W32-NEXT: s_or_b32 vcc_lo, s2, s0 ; GFX10_W32-NEXT: .LBB13_2: ; %exit ; GFX10_W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 @@ -1401,7 +1398,6 @@ define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(ptr addrspace(1) %out, [ ; GFX10_W64-NEXT: s_cmp_lg_u32 s0, 0 ; GFX10_W64-NEXT: s_cselect_b64 s[0:1], exec, 0 ; GFX10_W64-NEXT: s_andn2_b64 s[6:7], 0, exec -; GFX10_W64-NEXT: s_and_b64 s[0:1], exec, s[0:1] ; GFX10_W64-NEXT: s_or_b64 vcc, s[6:7], s[0:1] ; GFX10_W64-NEXT: .LBB13_2: ; %exit ; GFX10_W64-NEXT: s_or_b64 exec, exec, s[2:3] @@ -1432,7 +1428,6 @@ define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(ptr addrspace(1) %out, [ ; GFX11_W32-NEXT: s_cmp_lg_u32 s0, 0 ; GFX11_W32-NEXT: s_cselect_b32 s0, exec_lo, 0 ; GFX11_W32-NEXT: s_and_not1_b32 s2, 0, exec_lo -; GFX11_W32-NEXT: s_and_b32 s0, exec_lo, s0 ; GFX11_W32-NEXT: s_or_b32 vcc_lo, s2, s0 ; GFX11_W32-NEXT: .LBB13_2: ; %exit ; GFX11_W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 @@ -1463,7 +1458,6 @@ define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(ptr addrspace(1) %out, [ ; GFX11_W64-NEXT: s_cmp_lg_u32 s0, 0 ; GFX11_W64-NEXT: s_cselect_b64 s[0:1], exec, 0 ; GFX11_W64-NEXT: s_and_not1_b64 s[6:7], 0, exec -; GFX11_W64-NEXT: s_and_b64 s[0:1], exec, s[0:1] ; GFX11_W64-NEXT: s_or_b64 vcc, s[6:7], s[0:1] ; GFX11_W64-NEXT: .LBB13_2: ; %exit ; GFX11_W64-NEXT: s_or_b64 exec, exec, s[2:3] diff --git a/llvm/test/CodeGen/AMDGPU/fold-and-exec.mir b/llvm/test/CodeGen/AMDGPU/fold-and-exec.mir index da9711f0e6d248..8354a937016091 100644 --- a/llvm/test/CodeGen/AMDGPU/fold-and-exec.mir +++ b/llvm/test/CodeGen/AMDGPU/fold-and-exec.mir @@ -258,3 +258,24 @@ body: | S_ENDPGM 0, implicit $vcc ... +# S_CSELECT EXEC, 0 is itself masked by EXEC, so ANDing it with EXEC is a no-op. +--- +name: cselect_exec_zero +tracksRegLiveness: true +body: | + bb.0: + liveins: $sgpr0 + ; CHECK-LABEL: name: cselect_exec_zero + ; CHECK: liveins: $sgpr0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, 0, implicit-def $scc + ; CHECK-NEXT: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 $exec, 0, implicit $scc + ; CHECK-NEXT: $vcc = COPY [[S_CSELECT_B64_]] + ; CHECK-NEXT: S_ENDPGM 0, implicit $vcc + S_CMP_LG_U32 $sgpr0, 0, implicit-def $scc + %0:sreg_64_xexec = S_CSELECT_B64 $exec, 0, implicit $scc + %1:sreg_64_xexec = S_AND_B64 $exec, %0, implicit-def dead $scc + $vcc = COPY %1 + S_ENDPGM 0, implicit $vcc +... +