Repository navigation
[DivRemPairs] Recognize another "expanded rem" pattern - #240002
Open
s-barannikov wants to merge 3 commits into
Open
s-barannikov wants to merge 3 commits into
s-barannikov wants to merge 3 commits into
Conversation
The pass recognizes expanded remainder in the form `X - (X / Y) * Y)`. Teach it to recognize a similar pattern: `X ==/!= (X / Y) * Y`. The transformation is to rewrite it to: `X % Y ==/!= 0`. Proof: https://alive2.llvm.org/ce/z/o2hi6F Fixes llvm#168749.
|
@llvm/pr-subscribers-llvm-transforms Author: Sergei Barannikov (s-barannikov) ChangesThe pass recognizes expanded remainder in the form Proof: https://alive2.llvm.org/ce/z/o2hi6F Fixes #168749. Full diff: https://github.com/llvm/llvm-project/pull/240002.diff 2 Files Affected:
diff --git a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp
index 3adb3539f5890c..55f00fa89b9525 100644
--- a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp
+++ b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp
@@ -44,30 +44,33 @@ struct ExpandedMatch {
} // namespace
/// See if we can match: (which is the form we expand into)
-/// X - ((X ?/ Y) * Y)
+/// X - ((X / Y) * Y)
/// which is equivalent to:
-/// X ?% Y
+/// X % Y
+/// Also look for:
+/// X ==/!= ((X / Y) * Y)
+/// which is equivalent to:
+/// X % Y ==/!= 0
static std::optional<ExpandedMatch> matchExpandedRem(Instruction &I) {
- Value *Dividend, *XroundedDownToMultipleOfY;
- if (!match(&I, m_Sub(m_Value(Dividend), m_Value(XroundedDownToMultipleOfY))))
- return std::nullopt;
-
- Value *Divisor;
+ Value *Dividend = nullptr, *Divisor;
Instruction *Div;
- // Look for ((X / Y) * Y)
- if (!match(
- XroundedDownToMultipleOfY,
- m_c_Mul(m_CombineAnd(m_IDiv(m_Specific(Dividend), m_Value(Divisor)),
- m_Instruction(Div)),
- m_Deferred(Divisor))))
- return std::nullopt;
-
- ExpandedMatch M;
- M.Key.SignedOp = Div->getOpcode() == Instruction::SDiv;
- M.Key.Dividend = Dividend;
- M.Key.Divisor = Divisor;
- M.Value = &I;
- return M;
+
+ auto DivMul = m_c_Mul(
+ m_Instruction(Div, m_IDiv(m_Deferred(Dividend), m_Value(Divisor))),
+ m_Deferred(Divisor));
+
+ if (match(&I, m_Sub(m_Value(Dividend), DivMul)) ||
+ (match(&I, m_c_ICmp(m_Value(Dividend), DivMul)) &&
+ cast<ICmpInst>(I).isEquality())) {
+ ExpandedMatch M;
+ M.Key.SignedOp = Div->getOpcode() == Instruction::SDiv;
+ M.Key.Dividend = Dividend;
+ M.Key.Divisor = Divisor;
+ M.Value = &I;
+ return M;
+ }
+
+ return std::nullopt;
}
namespace {
@@ -86,7 +89,9 @@ struct DivRemPairWorklistEntry {
assert((DivInst->getOpcode() == Instruction::UDiv ||
DivInst->getOpcode() == Instruction::SDiv) &&
"Not a division.");
- assert(DivInst->getType() == RemInst->getType() && "Types should match.");
+ assert((RemInst->getOpcode() == Instruction::ICmp ||
+ DivInst->getType() == RemInst->getType()) &&
+ "Types should match.");
// We can't check anything else about remainder instruction,
// it's not strictly required to be a urem/srem.
}
@@ -209,14 +214,28 @@ static bool optimizeDivRem(Function &F, const TargetTransformInfo &TTI,
// Note that we place it right next to the original expanded instruction,
// and letting further handling to move it if needed.
RealRem->setName(RemInst->getName() + ".recomposed");
+ RealRem->setDebugLoc(RemInst->getDebugLoc());
RealRem->insertAfter(RemInst->getIterator());
Instruction *OrigRemInst = RemInst;
// Update AssertingVH<> with new instruction so it doesn't assert.
RemInst = RealRem;
- // And replace the original instruction with the new one.
- OrigRemInst->replaceAllUsesWith(RealRem);
- RealRem->setDebugLoc(OrigRemInst->getDebugLoc());
- OrigRemInst->eraseFromParent();
+
+ if (auto *OrigCmp = dyn_cast<ICmpInst>(OrigRemInst)) {
+ // The expanded rem instruction is actually an icmp:
+ // X ==/!= (X / Y) * Y. Replace it with X % Y ==/!= 0.
+ Instruction *NewCmp =
+ new ICmpInst(OrigCmp->getPredicate(), RealRem,
+ Constant::getNullValue(RealRem->getType()));
+ NewCmp->takeName(OrigCmp);
+ NewCmp->setDebugLoc(OrigCmp->getDebugLoc());
+ NewCmp->insertAfter(RealRem->getIterator());
+ OrigCmp->replaceAllUsesWith(NewCmp);
+ OrigCmp->eraseFromParent();
+ } else {
+ // Replace the expanded rem instruction with the new one.
+ OrigRemInst->replaceAllUsesWith(RealRem);
+ OrigRemInst->eraseFromParent();
+ }
NumRecomposed++;
// Note that we have left ((X / Y) * Y) around.
// If it had other uses we could rewrite it as X - X % Y
diff --git a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll
index 5340527a046a54..35e16780aec302 100644
--- a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll
+++ b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll
@@ -271,8 +271,8 @@ define i64 @remainder_triangle_i64_multiple_rem_edges(i64 %a, i64 %b, i64 %c, pt
; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A]], [[B]]
; CHECK-NEXT: switch i64 [[C:%.*]], label [[SW_DEFAULT:%.*]] [
-; CHECK-NEXT: i64 0, label [[SW_BB:%.*]]
-; CHECK-NEXT: i64 2, label [[SW_BB]]
+; CHECK-NEXT: i64 0, label [[SW_BB:%.*]]
+; CHECK-NEXT: i64 2, label [[SW_BB]]
; CHECK-NEXT: ]
; CHECK: sw.bb:
; CHECK-NEXT: store i64 [[REM]], ptr [[RP:%.*]], align 8
@@ -302,8 +302,8 @@ define i64 @remainder_triangle_i64_multiple_div_edges(i64 %a, i64 %b, i64 %c, pt
; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A]], [[B]]
; CHECK-NEXT: switch i64 [[C:%.*]], label [[SW_DEFAULT:%.*]] [
-; CHECK-NEXT: i64 0, label [[SW_BB:%.*]]
-; CHECK-NEXT: i64 2, label [[SW_BB]]
+; CHECK-NEXT: i64 0, label [[SW_BB:%.*]]
+; CHECK-NEXT: i64 2, label [[SW_BB]]
; CHECK-NEXT: ]
; CHECK: sw.default:
; CHECK-NEXT: store i64 [[REM]], ptr [[RP:%.*]], align 8
@@ -459,8 +459,8 @@ define i64 @remainder_not_triangle_i32(i64 %a, i64 %b, i64 %c, ptr %rp) {
; CHECK-LABEL: @remainder_not_triangle_i32(
; CHECK-NEXT: entry:
; CHECK-NEXT: switch i64 [[C:%.*]], label [[RETURN:%.*]] [
-; CHECK-NEXT: i64 0, label [[SW_BB:%.*]]
-; CHECK-NEXT: i64 1, label [[SW_BB1:%.*]]
+; CHECK-NEXT: i64 0, label [[SW_BB:%.*]]
+; CHECK-NEXT: i64 1, label [[SW_BB1:%.*]]
; CHECK-NEXT: ]
; CHECK: sw.bb:
; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A:%.*]], [[B:%.*]]
@@ -561,3 +561,93 @@ divbb:
%div = sdiv i32 7, %t0
ret i32 %div
}
+
+; Recompose rem from equality comparison.
+; X == (X sdiv Y) * Y --> (X srem Y) == 0
+define i32 @recompose_srem_eq(i32 %x, i32 %y) {
+; CHECK-LABEL: @recompose_srem_eq(
+; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[DIV]], [[Y]]
+; CHECK-NEXT: [[IS_MULTIPLE_RECOMPOSED:%.*]] = srem i32 [[X]], [[Y]]
+; CHECK-NEXT: [[IS_MULTIPLE:%.*]] = icmp eq i32 [[IS_MULTIPLE_RECOMPOSED]], 0
+; CHECK-NEXT: [[ZEXT:%.*]] = zext i1 [[IS_MULTIPLE]] to i32
+; CHECK-NEXT: [[RES:%.*]] = add i32 [[DIV]], [[ZEXT]]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %div = sdiv i32 %x, %y
+ %mul = mul i32 %div, %y
+ %is_multiple = icmp eq i32 %x, %mul
+ %zext = zext i1 %is_multiple to i32
+ %res = add i32 %div, %zext
+ ret i32 %res
+}
+
+; Recompose rem from inequality comparison. mul and icmp are commuted.
+; Y * (X sdiv Y) != X --> (X srem Y) != 0
+define i32 @recompose_urem_ne_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @recompose_urem_ne_commuted(
+; CHECK-NEXT: [[DIV:%.*]] = udiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[Y]], [[DIV]]
+; CHECK-NEXT: [[IS_NOT_MULTIPLE_RECOMPOSED:%.*]] = urem i32 [[X]], [[Y]]
+; CHECK-NEXT: [[IS_NOT_MULTIPLE:%.*]] = icmp ne i32 [[IS_NOT_MULTIPLE_RECOMPOSED]], 0
+; CHECK-NEXT: [[ZEXT:%.*]] = zext i1 [[IS_NOT_MULTIPLE]] to i32
+; CHECK-NEXT: [[RES:%.*]] = add i32 [[DIV]], [[ZEXT]]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %div = udiv i32 %x, %y
+ %mul = mul i32 %y, %div
+ %is_not_multiple = icmp ne i32 %mul, %x
+ %zext = zext i1 %is_not_multiple to i32
+ %res = add i32 %div, %zext
+ ret i32 %res
+}
+
+; Negative test. icmp has wrong predicate.
+define i32 @recompose_srem_wrong_predicate(i32 %x, i32 %y) {
+; CHECK-LABEL: @recompose_srem_wrong_predicate(
+; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[DIV]], [[Y]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i32 [[X]], [[MUL]]
+; CHECK-NEXT: [[ZEXT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT: [[RES:%.*]] = add i32 [[DIV]], [[ZEXT]]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %div = sdiv i32 %x, %y
+ %mul = mul i32 %div, %y
+ %cmp = icmp ugt i32 %x, %mul
+ %zext = zext i1 %cmp to i32
+ %res = add i32 %div, %zext
+ ret i32 %res
+}
+
+; Recompose rem from equality comparison and hoist it.
+; X == (X sdiv Y) * Y --> (X srem Y) == 0
+define i1 @recompose_hoist_srem_eq(i32 %x, i32 %y) {
+; CHECK-LABEL: @recompose_hoist_srem_eq(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[IS_MULTIPLE_RECOMPOSED:%.*]] = srem i32 [[X]], [[Y]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[DIV]], 42
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[DIV]], [[Y]]
+; CHECK-NEXT: [[IS_MULTIPLE:%.*]] = icmp eq i32 [[IS_MULTIPLE_RECOMPOSED]], 0
+; CHECK-NEXT: br label [[IF_END]]
+; CHECK: if.end:
+; CHECK-NEXT: [[RES:%.*]] = phi i1 [ [[IS_MULTIPLE]], [[IF_THEN]] ], [ false, [[ENTRY:%.*]] ]
+; CHECK-NEXT: ret i1 [[RES]]
+;
+entry:
+ %div = sdiv i32 %x, %y
+ %cmp = icmp eq i32 %div, 42
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+ %mul = mul i32 %div, %y
+ %is_multiple = icmp eq i32 %x, %mul
+ br label %if.end
+
+if.end:
+ %res = phi i1 [ %is_multiple, %if.then ], [ false, %entry ]
+ ret i1 %res
+}
|
This branch has not been deployed
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
The pass recognizes expanded remainder in the form
X - (X / Y) * Y).Teach it to recognize a similar pattern:
X ==/!= (X / Y) * Y.The transformation is to rewrite it to:
X % Y ==/!= 0.Proof: https://alive2.llvm.org/ce/z/drmNUV
Fixes #168749.