summaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms')
-rw-r--r--llvm/lib/Transforms/Vectorize/LoopVectorize.cpp29
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlan.cpp1
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlan.h45
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp18
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp27
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp8
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanValue.h5
7 files changed, 35 insertions, 98 deletions
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4762e0534b67..e1fd1593654e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4084,7 +4084,6 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPDef::VPEVLBasedIVPHISC:
case VPDef::VPPredInstPHISC:
case VPDef::VPBranchOnMaskSC:
- case VPDef::VPMonotonicPHISC:
continue;
case VPDef::VPReductionSC:
case VPDef::VPActiveLaneMaskPHISC:
@@ -8115,7 +8114,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range)))
return Recipe;
- VPHeaderPHIRecipe *PhiRecipe = nullptr;
+ VPSingleDefRecipe *PhiRecipe = nullptr;
assert((Legal->isMonotonicPHI(Phi) || Legal->isReductionVariable(Phi) ||
Legal->isFixedOrderRecurrence(Phi)) &&
"can only widen monotonic phis, reductions and fixed-order "
@@ -8124,10 +8123,9 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
Value *IncomingVal =
Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader());
if (Legal->isMonotonicPHI(Phi)) {
- const MonotonicDescriptor &Desc =
- Legal->getMonotonicPHIs().find(Phi)->second;
- assert(Desc.getExpr()->getStart() == PSE.getSCEV(IncomingVal));
- PhiRecipe = new VPMonotonicPHIRecipe(Phi, Desc, StartV);
+ PhiRecipe = new VPPhi({StartV}, Phi->getDebugLoc(),
+ Phi->getName() + ".monotonic");
+ PhiRecipe->setUnderlyingValue(Phi);
} else if (Legal->isReductionVariable(Phi)) {
const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi);
assert(RdxDesc.getRecurrenceStartValue() ==
@@ -8480,10 +8478,19 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// ---------------------------------------------------------------------------
// Adjust the recipes for any monotonic phis.
+ auto &MonotonicPHIs = Legal->getMonotonicPHIs();
for (VPRecipeBase &R : HeaderVPBB->phis()) {
- auto *MonotonicPhi = dyn_cast<VPMonotonicPHIRecipe>(&R);
+ auto *MonotonicPhi = dyn_cast<VPPhi>(&R);
if (!MonotonicPhi)
continue;
+ assert(MonotonicPhi->getNumIncoming() == 2 &&
+ MonotonicPhi->getIncomingBlock(0) == Plan->getVectorPreheader());
+
+ auto It =
+ MonotonicPHIs.find(cast<PHINode>(MonotonicPhi->getUnderlyingValue()));
+ if (It == MonotonicPHIs.end())
+ continue;
+ auto &Desc = It->second;
// Prohibit scalarization of monotonic phis.
if (!all_of(Range, [&](ElementCount VF) {
@@ -8494,7 +8501,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// Obtain mask value for the predicate edge from the last VPBlendRecipe in
// chain.
- VPValue *Chain = MonotonicPhi->getBackedgeValue();
+ VPValue *Chain = MonotonicPhi->getIncomingValue(1);
VPValue *Mask = nullptr;
while (auto *BlendR = dyn_cast<VPBlendRecipe>(Chain))
for (unsigned I = 0, E = BlendR->getNumIncomingValues(); I != E; ++I)
@@ -8506,7 +8513,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
}
assert(Mask);
- auto &Desc = MonotonicPhi->getDescriptor();
auto &SE = *PSE.getSE();
auto *Step = vputils::getOrCreateVPValueForSCEVExpr(
*Plan, Desc.getExpr()->getStepRecurrence(SE));
@@ -8514,9 +8520,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
auto *MonotonicI =
new VPInstruction(VPInstruction::ComputeMonotonicResult,
{MonotonicPhi, Mask, Step}, *Desc.getStepInst());
- auto *InsertBlock = MonotonicPhi->getBackedgeRecipe().getParent();
+ auto *BackedgeVal = MonotonicPhi->getIncomingValue(1);
+ auto *InsertBlock = BackedgeVal->getDefiningRecipe()->getParent();
InsertBlock->insert(MonotonicI, InsertBlock->getFirstNonPhi());
- MonotonicPhi->getBackedgeValue()->replaceAllUsesWith(MonotonicI);
+ BackedgeVal->replaceAllUsesWith(MonotonicI);
}
// Adjust the recipes for any inloop reductions.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 87fd6b543c36..dd26a059d56a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -984,7 +984,6 @@ void VPlan::execute(VPTransformState *State) {
auto *PhiR = cast<VPSingleDefRecipe>(&R);
// VPInstructions currently model scalar Phis only.
bool NeedsScalar = isa<VPInstruction>(PhiR) ||
- isa<VPMonotonicPHIRecipe>(PhiR) ||
(isa<VPReductionPHIRecipe>(PhiR) &&
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d442b906884f..a763e82725d9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -553,7 +553,6 @@ public:
case VPRecipeBase::VPWidenIntOrFpInductionSC:
case VPRecipeBase::VPWidenPointerInductionSC:
case VPRecipeBase::VPReductionPHISC:
- case VPRecipeBase::VPMonotonicPHISC:
case VPRecipeBase::VPPartialReductionSC:
return true;
case VPRecipeBase::VPBranchOnMaskSC:
@@ -2408,50 +2407,6 @@ public:
}
};
-/// A recipe for handling monotonic phis. The start value is the first operand
-/// of the recipe and the incoming value from the backedge is the second
-/// operand.
-class VPMonotonicPHIRecipe : public VPHeaderPHIRecipe {
- MonotonicDescriptor Desc;
-
-public:
- VPMonotonicPHIRecipe(PHINode *Phi, const MonotonicDescriptor &Desc,
- VPValue *Start)
- : VPHeaderPHIRecipe(VPDef::VPMonotonicPHISC, Phi, Start), Desc(Desc) {}
-
- ~VPMonotonicPHIRecipe() override = default;
-
- VPMonotonicPHIRecipe *clone() override {
- auto *R = new VPMonotonicPHIRecipe(cast<PHINode>(getUnderlyingInstr()),
- Desc, getStartValue());
- R->addOperand(getBackedgeValue());
- return R;
- }
-
- VP_CLASSOF_IMPL(VPDef::VPMonotonicPHISC)
-
- static inline bool classof(const VPHeaderPHIRecipe *R) {
- return R->getVPDefID() == VPDef::VPMonotonicPHISC;
- }
-
- void execute(VPTransformState &State) override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-
- const MonotonicDescriptor &getDescriptor() const { return Desc; }
-
- /// Returns true if the recipe only uses the first lane of operand \p Op.
- bool usesFirstLaneOnly(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
-};
-
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
/// instructions.
class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPSingleDefRecipe {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index da9e1f27310e..94b4d5fe7e49 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -98,7 +98,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
return inferScalarType(R->getOperand(0));
}
case VPInstruction::ComputeMonotonicResult: {
- auto *PhiR = cast<VPMonotonicPHIRecipe>(R->getOperand(0));
+ auto *PhiR = cast<VPPhi>(R->getOperand(0));
auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
return OrigPhi->getType();
}
@@ -281,14 +281,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
- VPMonotonicPHIRecipe, VPWidenPointerInductionRecipe,
- VPEVLBasedIVPHIRecipe>([this](const auto *R) {
- // Handle header phi recipes, except VPWidenIntOrFpInduction
- // which needs special handling due it being possibly truncated.
- // TODO: consider inferring/caching type of siblings, e.g.,
- // backedge value, here and in cases below.
- return inferScalarType(R->getStartValue());
- })
+ VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
+ [this](const auto *R) {
+ // Handle header phi recipes, except VPWidenIntOrFpInduction
+ // which needs special handling due it being possibly truncated.
+ // TODO: consider inferring/caching type of siblings, e.g.,
+ // backedge value, here and in cases below.
+ return inferScalarType(R->getStartValue());
+ })
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
[](const auto *R) { return R->getScalarType(); })
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 437e5c987a10..132e9a03b813 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1217,8 +1217,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
default:
// TODO: Compute cost other VPInstructions once the legacy cost model has
// been retired.
- assert(!getUnderlyingValue() &&
- "unexpected VPInstruction witht underlying value");
+ assert((getOpcode() == Instruction::PHI || !getUnderlyingValue()) &&
+ "unexpected VPInstruction with underlying value");
return 0;
}
}
@@ -4495,29 +4495,6 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
}
#endif
-void VPMonotonicPHIRecipe::execute(VPTransformState &State) {
- assert(getParent()->getPlan()->getUF() == 1 && "Expected unroll factor 1.");
- Value *Start = getStartValue()->getLiveInIRValue();
- BasicBlock *VectorPH =
- State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));
- PHINode *MonotonicPHI =
- State.Builder.CreatePHI(Start->getType(), 2, "monotonic.iv");
- MonotonicPHI->addIncoming(Start, VectorPH);
- MonotonicPHI->setDebugLoc(getDebugLoc());
- State.set(this, MonotonicPHI, /*IsScalar=*/true);
-}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPMonotonicPHIRecipe::print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "MONOTONIC-PHI ";
-
- printAsOperand(O, SlotTracker);
- O << " = phi ";
- printOperands(O, SlotTracker);
-}
-#endif
-
void VPWidenPHIRecipe::execute(VPTransformState &State) {
Value *Op0 = State.get(getOperand(0));
Type *VecTy = Op0->getType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cdfbc531ebfa..bc305f5fbff6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4448,8 +4448,7 @@ void VPlanTransforms::addScalarResumePhis(
// TODO: Extract final value from induction recipe initially, optimize to
// pre-computed end value together in optimizeInductionExitUsers.
- auto *VectorPhiR =
- cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
+ VPRecipeBase *VectorPhiR = Builder.getRecipe(&ScalarPhiIRI->getIRPhi());
if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction(
WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
@@ -4471,7 +4470,8 @@ void VPlanTransforms::addScalarResumePhis(
// which for FORs is a vector whose last element needs to be extracted. The
// start value provides the value if the loop is bypassed.
bool IsFOR = isa<VPFirstOrderRecurrencePHIRecipe>(VectorPhiR);
- auto *ResumeFromVectorLoop = VectorPhiR->getBackedgeValue();
+ auto *PhiAccessor = cast<VPPhiAccessors>(VectorPhiR);
+ auto *ResumeFromVectorLoop = PhiAccessor->getIncomingValue(1);
assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
"Cannot handle loops with uncountable early exits");
if (IsFOR)
@@ -4480,7 +4480,7 @@ void VPlanTransforms::addScalarResumePhis(
"vector.recur.extract");
StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx";
auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
- {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name);
+ {ResumeFromVectorLoop, PhiAccessor->getIncomingValue(0)}, {}, Name);
ScalarPhiIRI->addOperand(ResumePhiR);
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 4fd2398c6499..5da74630ef62 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -371,13 +371,12 @@ public:
VPWidenIntOrFpInductionSC,
VPWidenPointerInductionSC,
VPReductionPHISC,
- VPMonotonicPHISC,
// END: SubclassID for recipes that inherit VPHeaderPHIRecipe
// END: Phi-like recipes
VPFirstPHISC = VPWidenPHISC,
VPFirstHeaderPHISC = VPCanonicalIVPHISC,
- VPLastHeaderPHISC = VPMonotonicPHISC,
- VPLastPHISC = VPMonotonicPHISC,
+ VPLastHeaderPHISC = VPReductionPHISC,
+ VPLastPHISC = VPReductionPHISC,
};
VPDef(const unsigned char SC) : SubclassID(SC) {}
generated by cgit v1.2.3 (git 2.25.1) at 2026-05-31 23:08:21 +0000