diff options
Diffstat (limited to 'llvm/lib/Transforms/Vectorize')
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 18 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlan.h | 7 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | 1 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 22 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp | 39 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 29 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 61 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 13 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 46 |
10 files changed, 32 insertions, 207 deletions
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index e522d2f617d8..03112c67dda7 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -2097,6 +2097,24 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const { for (const auto &Reduction : getReductionVars()) ReductionLiveOuts.insert(Reduction.second.getLoopExitInstr()); + // TODO: handle non-reduction outside users when tail is folded by masking. + for (auto *AE : AllowedExit) { + // Check that all users of allowed exit values are inside the loop or + // are the live-out of a reduction. + if (ReductionLiveOuts.count(AE)) + continue; + for (User *U : AE->users()) { + Instruction *UI = cast<Instruction>(U); + if (TheLoop->contains(UI)) + continue; + LLVM_DEBUG( + dbgs() + << "LV: Cannot fold tail by masking, loop has an outside user for " + << *UI << "\n"); + return false; + } + } + for (const auto &Entry : getInductionVars()) { PHINode *OrigPhi = Entry.first; for (User *U : OrigPhi->users()) { diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index f4629d22002d..83844287adc5 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8893,8 +8893,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (FinalReductionResult == U || Parent->getParent()) continue; U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult); - if (match(U, m_CombineOr(m_ExtractLastElement(m_VPValue()), - m_ExtractLane(m_VPValue(), m_VPValue())))) + if (match(U, m_ExtractLastElement(m_VPValue()))) cast<VPInstruction>(U)->replaceAllUsesWith(FinalReductionResult); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index a9fe06c79df1..a49536a211c7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1047,13 +1047,6 @@ public: // It produces the lane index across all unrolled iterations. Unrolling will // add all copies of its original operand as additional operands. FirstActiveLane, - // Calculates the last active lane index of the vector predicate operands. - // The predicates must be prefix-masks (all 1s before all 0s). Used when - // tail-folding to extract the correct live-out value from the last active - // iteration. It produces the lane index across all unrolled iterations. - // Unrolling will add all copies of its original operand as additional - // operands. - LastActiveLane, // The opcodes below are used for VPInstructionWithType. // diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index fb0b029de3d4..80a2e4bc3f75 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -115,7 +115,6 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) { case VPInstruction::ExtractLane: return inferScalarType(R->getOperand(1)); case VPInstruction::FirstActiveLane: - case VPInstruction::LastActiveLane: return Type::getIntNTy(Ctx, 64); case VPInstruction::ExtractLastElement: case VPInstruction::ExtractLastLanePerPart: diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index f34c99b84b1a..47d8cc260511 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -395,24 +395,12 @@ m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1) { return m_VPInstruction<Instruction::ExtractElement>(Op0, Op1); } -template <typename Op0_t, typename Op1_t> -inline VPInstruction_match<VPInstruction::ExtractLane, Op0_t, Op1_t> -m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1) { - return m_VPInstruction<VPInstruction::ExtractLane>(Op0, Op1); -} - template <typename Op0_t> inline VPInstruction_match<VPInstruction::ExtractLastLanePerPart, Op0_t> m_ExtractLastLanePerPart(const Op0_t &Op0) { return m_VPInstruction<VPInstruction::ExtractLastLanePerPart>(Op0); } -template <typename Op0_t> -inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t> -m_ExtractPenultimateElement(const Op0_t &Op0) { - return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0); -} - template <typename Op0_t, typename Op1_t, typename Op2_t> inline VPInstruction_match<VPInstruction::ActiveLaneMask, Op0_t, Op1_t, Op2_t> m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { @@ -441,16 +429,6 @@ m_FirstActiveLane(const Op0_t &Op0) { return m_VPInstruction<VPInstruction::FirstActiveLane>(Op0); } -template <typename Op0_t> -inline VPInstruction_match<VPInstruction::LastActiveLane, Op0_t> -m_LastActiveLane(const Op0_t &Op0) { - return m_VPInstruction<VPInstruction::LastActiveLane>(Op0); -} - -inline VPInstruction_match<VPInstruction::StepVector> m_StepVector() { - return m_VPInstruction<VPInstruction::StepVector>(); -} - template <unsigned Opcode, typename Op0_t> inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) { return AllRecipe_match<Opcode, Op0_t>(Op0); diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp index 3579af21d8b0..fb17d5dd62b9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp @@ -44,6 +44,11 @@ class VPPredicator { /// possibly inserting new recipes at \p Dst (using Builder's insertion point) VPValue *createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst); + /// Returns the *entry* mask for \p VPBB. + VPValue *getBlockInMask(VPBasicBlock *VPBB) const { + return BlockMaskCache.lookup(VPBB); + } + /// Record \p Mask as the *entry* mask of \p VPBB, which is expected to not /// already have a mask. void setBlockInMask(VPBasicBlock *VPBB, VPValue *Mask) { @@ -63,11 +68,6 @@ class VPPredicator { } public: - /// Returns the *entry* mask for \p VPBB. - VPValue *getBlockInMask(VPBasicBlock *VPBB) const { - return BlockMaskCache.lookup(VPBB); - } - /// Returns the precomputed predicate of the edge from \p Src to \p Dst. VPValue *getEdgeMask(const VPBasicBlock *Src, const VPBasicBlock *Dst) const { return EdgeMaskCache.lookup({Src, Dst}); @@ -301,34 +301,5 @@ VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) { PrevVPBB = VPBB; } - - // If we folded the tail and introduced a header mask, any extract of the - // last element must be updated to extract from the last active lane of the - // header mask instead (i.e., the lane corresponding to the last active - // iteration). - if (FoldTail) { - assert(Plan.getExitBlocks().size() == 1 && - "only a single-exit block is supported currently"); - VPBasicBlock *EB = Plan.getExitBlocks().front(); - assert(EB->getSinglePredecessor() == Plan.getMiddleBlock() && - "the exit block must have middle block as single predecessor"); - - VPBuilder B(Plan.getMiddleBlock()->getTerminator()); - for (auto &P : EB->phis()) { - auto *ExitIRI = cast<VPIRPhi>(&P); - VPValue *Inc = ExitIRI->getIncomingValue(0); - VPValue *Op; - if (!match(Inc, m_ExtractLastElement(m_VPValue(Op)))) - continue; - - // Compute the index of the last active lane. - VPValue *HeaderMask = Predicator.getBlockInMask(Header); - VPValue *LastActiveLane = - B.createNaryOp(VPInstruction::LastActiveLane, HeaderMask); - auto *Ext = - B.createNaryOp(VPInstruction::ExtractLane, {LastActiveLane, Op}); - Inc->replaceAllUsesWith(Ext); - } - } return Predicator.getBlockMaskCache(); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 724b93215088..e0e6990c56ec 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -547,7 +547,6 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) { case VPInstruction::ExtractLastLanePerPart: case VPInstruction::ExtractPenultimateElement: case VPInstruction::FirstActiveLane: - case VPInstruction::LastActiveLane: case VPInstruction::Not: case VPInstruction::Unpack: return 1; @@ -1157,29 +1156,6 @@ InstructionCost VPInstruction::computeCost(ElementCount VF, {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)}); return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind); } - case VPInstruction::LastActiveLane: { - Type *ScalarTy = Ctx.Types.inferScalarType(getOperand(0)); - if (VF.isScalar()) - return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy, - CmpInst::makeCmpResultType(ScalarTy), - CmpInst::ICMP_EQ, Ctx.CostKind); - // Calculate the cost of determining the lane index: NOT + cttz_elts + SUB. - auto *PredTy = toVectorTy(ScalarTy, VF); - IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts, - Type::getInt64Ty(Ctx.LLVMCtx), - {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)}); - InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind); - // Add cost of NOT operation on the predicate. - Cost += Ctx.TTI.getArithmeticInstrCost( - Instruction::Xor, PredTy, Ctx.CostKind, - {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None}, - {TargetTransformInfo::OK_UniformConstantValue, - TargetTransformInfo::OP_None}); - // Add cost of SUB operation on the index. - Cost += Ctx.TTI.getArithmeticInstrCost( - Instruction::Sub, Type::getInt64Ty(Ctx.LLVMCtx), Ctx.CostKind); - return Cost; - } case VPInstruction::FirstOrderRecurrenceSplice: { assert(VF.isVector() && "Scalar FirstOrderRecurrenceSplice?"); SmallVector<int> Mask(VF.getKnownMinValue()); @@ -1234,7 +1210,6 @@ bool VPInstruction::isVectorToScalar() const { getOpcode() == Instruction::ExtractElement || getOpcode() == VPInstruction::ExtractLane || getOpcode() == VPInstruction::FirstActiveLane || - getOpcode() == VPInstruction::LastActiveLane || getOpcode() == VPInstruction::ComputeAnyOfResult || getOpcode() == VPInstruction::ComputeFindIVResult || getOpcode() == VPInstruction::ComputeReductionResult || @@ -1301,7 +1276,6 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const { case VPInstruction::ActiveLaneMask: case VPInstruction::ExplicitVectorLength: case VPInstruction::FirstActiveLane: - case VPInstruction::LastActiveLane: case VPInstruction::FirstOrderRecurrenceSplice: case VPInstruction::LogicalAnd: case VPInstruction::Not: @@ -1478,9 +1452,6 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent, case VPInstruction::FirstActiveLane: O << "first-active-lane"; break; - case VPInstruction::LastActiveLane: - O << "last-active-lane"; - break; case VPInstruction::ReductionStartVector: O << "reduction-start-vector"; break; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 7fe40c3e0490..6c32bd1c64c8 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -805,8 +805,8 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan, VPValue *Op, ScalarEvolution &SE) { VPValue *Incoming, *Mask; - if (!match(Op, m_ExtractLane(m_FirstActiveLane(m_VPValue(Mask)), - m_VPValue(Incoming)))) + if (!match(Op, m_VPInstruction<VPInstruction::ExtractLane>( + m_FirstActiveLane(m_VPValue(Mask)), m_VPValue(Incoming)))) return nullptr; auto *WideIV = getOptimizableIVOf(Incoming, SE); @@ -1274,7 +1274,8 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) { } // Look through ExtractPenultimateElement (BuildVector ....). - if (match(Def, m_ExtractPenultimateElement(m_BuildVector()))) { + if (match(Def, m_VPInstruction<VPInstruction::ExtractPenultimateElement>( + m_BuildVector()))) { auto *BuildVector = cast<VPInstruction>(Def->getOperand(0)); Def->replaceAllUsesWith( BuildVector->getOperand(BuildVector->getNumOperands() - 2)); @@ -2085,32 +2086,6 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan, // Set the first operand of RecurSplice to FOR again, after replacing // all users. RecurSplice->setOperand(0, FOR); - - // Check for users extracting at the penultimate active lane of the FOR. - // If only a single lane is active in the current iteration, we need to - // select the last element from the previous iteration (from the FOR phi - // directly). - for (VPUser *U : RecurSplice->users()) { - if (!match(U, m_ExtractLane(m_LastActiveLane(m_VPValue()), - m_Specific(RecurSplice)))) - continue; - - VPBuilder B(cast<VPInstruction>(U)); - VPValue *LastActiveLane = cast<VPInstruction>(U)->getOperand(0); - Type *I64Ty = Type::getInt64Ty(Plan.getContext()); - VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 0)); - VPValue *One = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 1)); - VPValue *PenultimateIndex = - B.createNaryOp(Instruction::Sub, {LastActiveLane, One}); - VPValue *PenultimateLastIter = - B.createNaryOp(VPInstruction::ExtractLane, - {PenultimateIndex, FOR->getBackedgeValue()}); - VPValue *LastPrevIter = - B.createNaryOp(VPInstruction::ExtractLastElement, FOR); - VPValue *Cmp = B.createICmp(CmpInst::ICMP_EQ, LastActiveLane, Zero); - VPValue *Sel = B.createSelect(Cmp, LastPrevIter, PenultimateLastIter); - cast<VPInstruction>(U)->replaceAllUsesWith(Sel); - } } return true; } @@ -3500,34 +3475,6 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) { ToRemove.push_back(Expr); } - // Expand LastActiveLane into Not + FirstActiveLane + Sub. - auto *LastActiveL = dyn_cast<VPInstruction>(&R); - if (LastActiveL && - LastActiveL->getOpcode() == VPInstruction::LastActiveLane) { - // Create Not(Mask) for all operands. - SmallVector<VPValue *, 2> NotMasks; - for (VPValue *Op : LastActiveL->operands()) { - VPValue *NotMask = Builder.createNot(Op, LastActiveL->getDebugLoc()); - NotMasks.push_back(NotMask); - } - - // Create FirstActiveLane on the inverted masks. - VPValue *FirstInactiveLane = Builder.createNaryOp( - VPInstruction::FirstActiveLane, NotMasks, - LastActiveL->getDebugLoc(), "first.inactive.lane"); - - // Subtract 1 to get the last active lane. - VPValue *One = Plan.getOrAddLiveIn( - ConstantInt::get(Type::getInt64Ty(Plan.getContext()), 1)); - VPValue *LastLane = Builder.createNaryOp( - Instruction::Sub, {FirstInactiveLane, One}, - LastActiveL->getDebugLoc(), "last.active.lane"); - - LastActiveL->replaceAllUsesWith(LastLane); - ToRemove.push_back(LastActiveL); - continue; - } - VPValue *VectorStep; VPValue *ScalarStep; if (!match(&R, m_VPInstruction<VPInstruction::WideIVStep>( diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index 221ca4ab0537..d4b8b72beb94 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -352,7 +352,6 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) { VPValue *Op1; if (match(&R, m_VPInstruction<VPInstruction::AnyOf>(m_VPValue(Op1))) || match(&R, m_FirstActiveLane(m_VPValue(Op1))) || - match(&R, m_LastActiveLane(m_VPValue(Op1))) || match(&R, m_VPInstruction<VPInstruction::ComputeAnyOfResult>( m_VPValue(), m_VPValue(), m_VPValue(Op1))) || match(&R, m_VPInstruction<VPInstruction::ComputeReductionResult>( @@ -365,21 +364,17 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) { continue; } VPValue *Op0; - if (match(&R, m_ExtractLane(m_VPValue(Op0), m_VPValue(Op1)))) { + if (match(&R, m_VPInstruction<VPInstruction::ExtractLane>( + m_VPValue(Op0), m_VPValue(Op1)))) { addUniformForAllParts(cast<VPInstruction>(&R)); for (unsigned Part = 1; Part != UF; ++Part) R.addOperand(getValueForPart(Op1, Part)); continue; } if (match(&R, m_ExtractLastElement(m_VPValue(Op0))) || - match(&R, m_ExtractPenultimateElement(m_VPValue(Op0)))) { + match(&R, m_VPInstruction<VPInstruction::ExtractPenultimateElement>( + m_VPValue(Op0)))) { addUniformForAllParts(cast<VPSingleDefRecipe>(&R)); - if (isa<VPFirstOrderRecurrencePHIRecipe>(Op0)) { - assert(match(&R, m_ExtractLastElement(m_VPValue())) && - "can only extract last element of FOR"); - continue; - } - if (Plan.hasScalarVFOnly()) { auto *I = cast<VPInstruction>(&R); // Extracting from end with VF = 1 implies retrieving the last or diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 2d63d2a787f8..34754a1ea399 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -18,7 +18,6 @@ #include "VPlanDominatorTree.h" #include "VPlanHelpers.h" #include "VPlanPatternMatch.h" -#include "VPlanUtils.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/TypeSwitch.h" @@ -45,9 +44,6 @@ class VPlanVerifier { /// incoming value into EVL's recipe. bool verifyEVLRecipe(const VPInstruction &EVL) const; - /// Verify that \p LastActiveLane's operand is guaranteed to be a prefix-mask. - bool verifyLastActiveLaneRecipe(const VPInstruction &LastActiveLane) const; - bool verifyVPBasicBlock(const VPBasicBlock *VPBB); bool verifyBlock(const VPBlockBase *VPB); @@ -225,44 +221,6 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const { }); } -bool VPlanVerifier::verifyLastActiveLaneRecipe( - const VPInstruction &LastActiveLane) const { - assert(LastActiveLane.getOpcode() == VPInstruction::LastActiveLane && - "must be called with VPInstruction::LastActiveLane"); - - if (LastActiveLane.getNumOperands() < 1) { - errs() << "LastActiveLane must have at least one operand\n"; - return false; - } - - const VPlan &Plan = *LastActiveLane.getParent()->getPlan(); - // All operands must be prefix-mask. Currently we check for header masks or - // EVL-derived masks, as those are currently the only operands in practice, - // but this may need updating in the future. - for (VPValue *Op : LastActiveLane.operands()) { - if (vputils::isHeaderMask(Op, Plan)) - continue; - - // Masks derived from EVL are also fine. - auto BroadcastOrEVL = - m_CombineOr(m_Broadcast(m_EVL(m_VPValue())), m_EVL(m_VPValue())); - if (match(Op, m_CombineOr(m_ICmp(m_StepVector(), BroadcastOrEVL), - m_ICmp(BroadcastOrEVL, m_StepVector())))) - continue; - - errs() << "LastActiveLane operand "; -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - VPSlotTracker Tracker(&Plan); - Op->printAsOperand(errs(), Tracker); -#endif - errs() << " must be prefix mask (a header mask or an " - "EVL-derived mask currently)\n"; - return false; - } - - return true; -} - bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { if (!verifyPhiRecipes(VPBB)) return false; @@ -355,10 +313,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { return false; } break; - case VPInstruction::LastActiveLane: - if (!verifyLastActiveLaneRecipe(*VPI)) - return false; - break; default: break; } |