summaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms')
-rw-r--r--llvm/lib/Transforms/Vectorize/LoopVectorize.cpp17
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlan.h5
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp9
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp47
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanTransforms.h12
5 files changed, 42 insertions, 48 deletions
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5b8725178521..277e43a38018 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8532,7 +8532,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// failures.
VPlanTransforms::addExitUsersForFirstOrderRecurrences(*Plan, Range);
DenseMap<VPValue *, VPValue *> IVEndValues;
- VPlanTransforms::addScalarResumePhis(*Plan, RecipeBuilder, IVEndValues);
+ VPlanTransforms::updateScalarResumePhis(*Plan, IVEndValues);
// ---------------------------------------------------------------------------
// Transform initial VPlan: Apply previously taken decisions, in order, to
@@ -8630,23 +8630,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
*TLI))
return nullptr;
- // Collect mapping of IR header phis to header phi recipes, to be used in
- // addScalarResumePhis.
- DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
- VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
- Builder, BlockMaskCache);
- for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
- if (isa<VPCanonicalIVPHIRecipe>(&R))
- continue;
- auto *HeaderR = cast<VPHeaderPHIRecipe>(&R);
- RecipeBuilder.setRecipe(HeaderR->getUnderlyingInstr(), HeaderR);
- }
- DenseMap<VPValue *, VPValue *> IVEndValues;
// TODO: IVEndValues are not used yet in the native path, to optimize exit
// values.
// TODO: We can't call runPass on the transform yet, due to verifier
// failures.
- VPlanTransforms::addScalarResumePhis(*Plan, RecipeBuilder, IVEndValues);
+ DenseMap<VPValue *, VPValue *> IVEndValues;
+ VPlanTransforms::updateScalarResumePhis(*Plan, IVEndValues);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
return Plan;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 0fc527e07e63..405f83a6ce8e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1134,7 +1134,7 @@ private:
OpcodeTy Opcode;
/// An optional name that can be used for the generated IR instruction.
- const std::string Name;
+ std::string Name;
/// Returns true if we can generate a scalar for the first lane only if
/// needed.
@@ -1225,6 +1225,9 @@ public:
/// Returns the symbolic name assigned to the VPInstruction.
StringRef getName() const { return Name; }
+ /// Set the symbolic name for the VPInstruction.
+ void setName(StringRef NewName) { Name = NewName.str(); }
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the VPInstruction to \p O.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index dbbde1cafa9f..5fbd61a929fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -554,6 +554,15 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
Plan.getEntry()->swapSuccessors();
createExtractsForLiveOuts(Plan, MiddleVPBB);
+
+ VPBuilder ScalarPHBuilder(ScalarPH);
+ for (const auto &[PhiR, ScalarPhiR] : zip_equal(
+ drop_begin(HeaderVPBB->phis()), Plan.getScalarHeader()->phis())) {
+ auto *VectorPhiR = cast<VPPhi>(&PhiR);
+ auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
+ {VectorPhiR, VectorPhiR->getOperand(0)}, VectorPhiR->getDebugLoc());
+ cast<VPIRPhi>(&ScalarPhiR)->addOperand(ResumePhiR);
+ }
}
std::unique_ptr<VPlan>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a6a03c0d805f..dc2d013a2402 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4526,12 +4526,13 @@ void VPlanTransforms::addBranchWeightToMiddleTerminator(
MiddleTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
}
-/// Create and return a ResumePhi for \p WideIV, unless it is truncated. If the
-/// induction recipe is not canonical, creates a VPDerivedIVRecipe to compute
-/// the end value of the induction.
-static VPInstruction *addResumePhiRecipeForInduction(
- VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder,
- VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC) {
+/// Compute and return the end value for \p WideIV, unless it is truncated. If
+/// the induction recipe is not canonical, creates a VPDerivedIVRecipe to
+/// compute the end value of the induction.
+static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
+ VPBuilder &VectorPHBuilder,
+ VPTypeAnalysis &TypeInfo,
+ VPValue *VectorTC) {
auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(WideIV);
// Truncated wide inductions resume from the last lane of their vector value
// in the last vector iteration which is handled elsewhere.
@@ -4557,14 +4558,11 @@ static VPInstruction *addResumePhiRecipeForInduction(
WideIV->getDebugLoc());
}
- auto *ResumePhiRecipe = ScalarPHBuilder.createScalarPhi(
- {EndValue, Start}, WideIV->getDebugLoc(), "bc.resume.val");
- return ResumePhiRecipe;
+ return EndValue;
}
-void VPlanTransforms::addScalarResumePhis(
- VPlan &Plan, VPRecipeBuilder &Builder,
- DenseMap<VPValue *, VPValue *> &IVEndValues) {
+void VPlanTransforms::updateScalarResumePhis(
+ VPlan &Plan, DenseMap<VPValue *, VPValue *> &IVEndValues) {
VPTypeAnalysis TypeInfo(Plan);
auto *ScalarPH = Plan.getScalarPreheader();
auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getPredecessors()[0]);
@@ -4572,21 +4570,18 @@ void VPlanTransforms::addScalarResumePhis(
VPBuilder VectorPHBuilder(
cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
- VPBuilder ScalarPHBuilder(ScalarPH);
- for (VPRecipeBase &ScalarPhiR : Plan.getScalarHeader()->phis()) {
- auto *ScalarPhiIRI = cast<VPIRPhi>(&ScalarPhiR);
+ for (VPRecipeBase &PhiR : Plan.getScalarPreheader()->phis()) {
+ auto *ResumePhiR = cast<VPPhi>(&PhiR);
// TODO: Extract final value from induction recipe initially, optimize to
// pre-computed end value together in optimizeInductionExitUsers.
- auto *VectorPhiR =
- cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
+ auto *VectorPhiR = cast<VPHeaderPHIRecipe>(ResumePhiR->getOperand(0));
if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
- if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction(
- WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
- &Plan.getVectorTripCount())) {
- assert(isa<VPPhi>(ResumePhi) && "Expected a phi");
- IVEndValues[WideIVR] = ResumePhi->getOperand(0);
- ScalarPhiIRI->addOperand(ResumePhi);
+ if (VPValue *EndValue = tryToComputeEndValueForInduction(
+ WideIVR, VectorPHBuilder, TypeInfo, &Plan.getVectorTripCount())) {
+ IVEndValues[WideIVR] = EndValue;
+ ResumePhiR->setOperand(0, EndValue);
+ ResumePhiR->setName("bc.resume.val");
continue;
}
// TODO: Also handle truncated inductions here. Computing end-values
@@ -4608,10 +4603,8 @@ void VPlanTransforms::addScalarResumePhis(
ResumeFromVectorLoop = MiddleBuilder.createNaryOp(
VPInstruction::ExtractLastElement, {ResumeFromVectorLoop}, {},
"vector.recur.extract");
- StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx";
- auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
- {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name);
- ScalarPhiIRI->addOperand(ResumePhiR);
+ ResumePhiR->setName(IsFOR ? "scalar.recur.init" : "bc.merge.rdx");
+ ResumePhiR->setOperand(0, ResumeFromVectorLoop);
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 708ea4185e1c..5fd3f756c55e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -372,12 +372,12 @@ struct VPlanTransforms {
addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF,
std::optional<unsigned> VScaleForTuning);
- /// Create resume phis in the scalar preheader for first-order recurrences,
- /// reductions and inductions, and update the VPIRInstructions wrapping the
- /// original phis in the scalar header. End values for inductions are added to
- /// \p IVEndValues.
- static void addScalarResumePhis(VPlan &Plan, VPRecipeBuilder &Builder,
- DenseMap<VPValue *, VPValue *> &IVEndValues);
+ /// Update the resume phis in the scalar preheader after creating wide recipes
+ /// for first-order recurrences, reductions and inductions. End values for
+ /// inductions are added to \p IVEndValues.
+ static void
+ updateScalarResumePhis(VPlan &Plan,
+ DenseMap<VPValue *, VPValue *> &IVEndValues);
/// Handle users in the exit block for first order reductions in the original
/// exit block. The penultimate value of recurrences is fed to their LCSSA phi
generated by cgit v1.2.3 (git 2.25.1) at 2026-05-31 14:06:59 +0000