[review-suggestions] Replace VPMonotonicPHIRecipe with VPPhiusers/skachkov-sc/monotonic-vectorization

7 files changed, 35 insertions, 98 deletions

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4762e0534b67..e1fd1593654e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4084,7 +4084,6 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
       case VPDef::VPEVLBasedIVPHISC:
       case VPDef::VPPredInstPHISC:
       case VPDef::VPBranchOnMaskSC:
-      case VPDef::VPMonotonicPHISC:
         continue;
       case VPDef::VPReductionSC:
       case VPDef::VPActiveLaneMaskPHISC:
@@ -8115,7 +8114,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
     if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range)))
       return Recipe;
 
-    VPHeaderPHIRecipe *PhiRecipe = nullptr;
+    VPSingleDefRecipe *PhiRecipe = nullptr;
     assert((Legal->isMonotonicPHI(Phi) || Legal->isReductionVariable(Phi) ||
             Legal->isFixedOrderRecurrence(Phi)) &&
            "can only widen monotonic phis, reductions and fixed-order "
@@ -8124,10 +8123,9 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
     Value *IncomingVal =
         Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader());
     if (Legal->isMonotonicPHI(Phi)) {
-      const MonotonicDescriptor &Desc =
-          Legal->getMonotonicPHIs().find(Phi)->second;
-      assert(Desc.getExpr()->getStart() == PSE.getSCEV(IncomingVal));
-      PhiRecipe = new VPMonotonicPHIRecipe(Phi, Desc, StartV);
+      PhiRecipe = new VPPhi({StartV}, Phi->getDebugLoc(),
+                            Phi->getName() + ".monotonic");
+      PhiRecipe->setUnderlyingValue(Phi);
     } else if (Legal->isReductionVariable(Phi)) {
       const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi);
       assert(RdxDesc.getRecurrenceStartValue() ==
@@ -8480,10 +8478,19 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
   // ---------------------------------------------------------------------------
 
   // Adjust the recipes for any monotonic phis.
+  auto &MonotonicPHIs = Legal->getMonotonicPHIs();
   for (VPRecipeBase &R : HeaderVPBB->phis()) {
-    auto *MonotonicPhi = dyn_cast<VPMonotonicPHIRecipe>(&R);
+    auto *MonotonicPhi = dyn_cast<VPPhi>(&R);
     if (!MonotonicPhi)
       continue;
+    assert(MonotonicPhi->getNumIncoming() == 2 &&
+           MonotonicPhi->getIncomingBlock(0) == Plan->getVectorPreheader());
+
+    auto It =
+        MonotonicPHIs.find(cast<PHINode>(MonotonicPhi->getUnderlyingValue()));
+    if (It == MonotonicPHIs.end())
+      continue;
+    auto &Desc = It->second;
 
     // Prohibit scalarization of monotonic phis.
     if (!all_of(Range, [&](ElementCount VF) {
@@ -8494,7 +8501,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
 
     // Obtain mask value for the predicate edge from the last VPBlendRecipe in
     // chain.
-    VPValue *Chain = MonotonicPhi->getBackedgeValue();
+    VPValue *Chain = MonotonicPhi->getIncomingValue(1);
     VPValue *Mask = nullptr;
     while (auto *BlendR = dyn_cast<VPBlendRecipe>(Chain))
       for (unsigned I = 0, E = BlendR->getNumIncomingValues(); I != E; ++I)
@@ -8506,7 +8513,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
         }
     assert(Mask);
 
-    auto &Desc = MonotonicPhi->getDescriptor();
     auto &SE = *PSE.getSE();
     auto *Step = vputils::getOrCreateVPValueForSCEVExpr(
         *Plan, Desc.getExpr()->getStepRecurrence(SE));
@@ -8514,9 +8520,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
     auto *MonotonicI =
         new VPInstruction(VPInstruction::ComputeMonotonicResult,
                           {MonotonicPhi, Mask, Step}, *Desc.getStepInst());
-    auto *InsertBlock = MonotonicPhi->getBackedgeRecipe().getParent();
+    auto *BackedgeVal = MonotonicPhi->getIncomingValue(1);
+    auto *InsertBlock = BackedgeVal->getDefiningRecipe()->getParent();
     InsertBlock->insert(MonotonicI, InsertBlock->getFirstNonPhi());
-    MonotonicPhi->getBackedgeValue()->replaceAllUsesWith(MonotonicI);
+    BackedgeVal->replaceAllUsesWith(MonotonicI);
   }
 
   // Adjust the recipes for any inloop reductions.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 87fd6b543c36..dd26a059d56a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -984,7 +984,6 @@ void VPlan::execute(VPTransformState *State) {
     auto *PhiR = cast<VPSingleDefRecipe>(&R);
     // VPInstructions currently model scalar Phis only.
     bool NeedsScalar = isa<VPInstruction>(PhiR) ||
-                       isa<VPMonotonicPHIRecipe>(PhiR) ||
                        (isa<VPReductionPHIRecipe>(PhiR) &&
                         cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d442b906884f..a763e82725d9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -553,7 +553,6 @@ public:
     case VPRecipeBase::VPWidenIntOrFpInductionSC:
     case VPRecipeBase::VPWidenPointerInductionSC:
     case VPRecipeBase::VPReductionPHISC:
-    case VPRecipeBase::VPMonotonicPHISC:
     case VPRecipeBase::VPPartialReductionSC:
       return true;
     case VPRecipeBase::VPBranchOnMaskSC:
@@ -2408,50 +2407,6 @@ public:
   }
 };
 
-/// A recipe for handling monotonic phis. The start value is the first operand
-/// of the recipe and the incoming value from the backedge is the second
-/// operand.
-class VPMonotonicPHIRecipe : public VPHeaderPHIRecipe {
-  MonotonicDescriptor Desc;
-
-public:
-  VPMonotonicPHIRecipe(PHINode *Phi, const MonotonicDescriptor &Desc,
-                       VPValue *Start)
-      : VPHeaderPHIRecipe(VPDef::VPMonotonicPHISC, Phi, Start), Desc(Desc) {}
-
-  ~VPMonotonicPHIRecipe() override = default;
-
-  VPMonotonicPHIRecipe *clone() override {
-    auto *R = new VPMonotonicPHIRecipe(cast<PHINode>(getUnderlyingInstr()),
-                                       Desc, getStartValue());
-    R->addOperand(getBackedgeValue());
-    return R;
-  }
-
-  VP_CLASSOF_IMPL(VPDef::VPMonotonicPHISC)
-
-  static inline bool classof(const VPHeaderPHIRecipe *R) {
-    return R->getVPDefID() == VPDef::VPMonotonicPHISC;
-  }
-
-  void execute(VPTransformState &State) override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  /// Print the recipe.
-  void print(raw_ostream &O, const Twine &Indent,
-             VPSlotTracker &SlotTracker) const override;
-#endif
-
-  const MonotonicDescriptor &getDescriptor() const { return Desc; }
-
-  /// Returns true if the recipe only uses the first lane of operand \p Op.
-  bool usesFirstLaneOnly(const VPValue *Op) const override {
-    assert(is_contained(operands(), Op) &&
-           "Op must be an operand of the recipe");
-    return true;
-  }
-};
-
 /// A recipe for vectorizing a phi-node as a sequence of mask-based select
 /// instructions.
 class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPSingleDefRecipe {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index da9e1f27310e..94b4d5fe7e49 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -98,7 +98,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
     return inferScalarType(R->getOperand(0));
   }
   case VPInstruction::ComputeMonotonicResult: {
-    auto *PhiR = cast<VPMonotonicPHIRecipe>(R->getOperand(0));
+    auto *PhiR = cast<VPPhi>(R->getOperand(0));
     auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
     return OrigPhi->getType();
   }
@@ -281,14 +281,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
       TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
           .Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
                 VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
-                VPMonotonicPHIRecipe, VPWidenPointerInductionRecipe,
-                VPEVLBasedIVPHIRecipe>([this](const auto *R) {
-            // Handle header phi recipes, except VPWidenIntOrFpInduction
-            // which needs special handling due it being possibly truncated.
-            // TODO: consider inferring/caching type of siblings, e.g.,
-            // backedge value, here and in cases below.
-            return inferScalarType(R->getStartValue());
-          })
+                VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
+              [this](const auto *R) {
+                // Handle header phi recipes, except VPWidenIntOrFpInduction
+                // which needs special handling due it being possibly truncated.
+                // TODO: consider inferring/caching type of siblings, e.g.,
+                // backedge value, here and in cases below.
+                return inferScalarType(R->getStartValue());
+              })
           .Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
               [](const auto *R) { return R->getScalarType(); })
           .Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 437e5c987a10..132e9a03b813 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1217,8 +1217,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
   default:
     // TODO: Compute cost other VPInstructions once the legacy cost model has
     // been retired.
-    assert(!getUnderlyingValue() &&
-           "unexpected VPInstruction witht underlying value");
+    assert((getOpcode() == Instruction::PHI || !getUnderlyingValue()) &&
+           "unexpected VPInstruction with underlying value");
     return 0;
   }
 }
@@ -4495,29 +4495,6 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
-void VPMonotonicPHIRecipe::execute(VPTransformState &State) {
-  assert(getParent()->getPlan()->getUF() == 1 && "Expected unroll factor 1.");
-  Value *Start = getStartValue()->getLiveInIRValue();
-  BasicBlock *VectorPH =
-      State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));
-  PHINode *MonotonicPHI =
-      State.Builder.CreatePHI(Start->getType(), 2, "monotonic.iv");
-  MonotonicPHI->addIncoming(Start, VectorPH);
-  MonotonicPHI->setDebugLoc(getDebugLoc());
-  State.set(this, MonotonicPHI, /*IsScalar=*/true);
-}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPMonotonicPHIRecipe::print(raw_ostream &O, const Twine &Indent,
-                                 VPSlotTracker &SlotTracker) const {
-  O << Indent << "MONOTONIC-PHI ";
-
-  printAsOperand(O, SlotTracker);
-  O << " = phi ";
-  printOperands(O, SlotTracker);
-}
-#endif
-
 void VPWidenPHIRecipe::execute(VPTransformState &State) {
   Value *Op0 = State.get(getOperand(0));
   Type *VecTy = Op0->getType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cdfbc531ebfa..bc305f5fbff6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4448,8 +4448,7 @@ void VPlanTransforms::addScalarResumePhis(
 
     // TODO: Extract final value from induction recipe initially, optimize to
     // pre-computed end value together in optimizeInductionExitUsers.
-    auto *VectorPhiR =
-        cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
+    VPRecipeBase *VectorPhiR = Builder.getRecipe(&ScalarPhiIRI->getIRPhi());
     if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
       if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction(
               WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
@@ -4471,7 +4470,8 @@ void VPlanTransforms::addScalarResumePhis(
     // which for FORs is a vector whose last element needs to be extracted. The
     // start value provides the value if the loop is bypassed.
     bool IsFOR = isa<VPFirstOrderRecurrencePHIRecipe>(VectorPhiR);
-    auto *ResumeFromVectorLoop = VectorPhiR->getBackedgeValue();
+    auto *PhiAccessor = cast<VPPhiAccessors>(VectorPhiR);
+    auto *ResumeFromVectorLoop = PhiAccessor->getIncomingValue(1);
     assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
            "Cannot handle loops with uncountable early exits");
     if (IsFOR)
@@ -4480,7 +4480,7 @@ void VPlanTransforms::addScalarResumePhis(
           "vector.recur.extract");
     StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx";
     auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
-        {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name);
+        {ResumeFromVectorLoop, PhiAccessor->getIncomingValue(0)}, {}, Name);
     ScalarPhiIRI->addOperand(ResumePhiR);
   }
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 4fd2398c6499..5da74630ef62 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -371,13 +371,12 @@ public:
     VPWidenIntOrFpInductionSC,
     VPWidenPointerInductionSC,
     VPReductionPHISC,
-    VPMonotonicPHISC,
     // END: SubclassID for recipes that inherit VPHeaderPHIRecipe
     // END: Phi-like recipes
     VPFirstPHISC = VPWidenPHISC,
     VPFirstHeaderPHISC = VPCanonicalIVPHISC,
-    VPLastHeaderPHISC = VPMonotonicPHISC,
-    VPLastPHISC = VPMonotonicPHISC,
+    VPLastHeaderPHISC = VPReductionPHISC,
+    VPLastPHISC = VPReductionPHISC,
   };
 
   VPDef(const unsigned char SC) : SubclassID(SC) {}