diff options
Diffstat (limited to 'llvm/lib/CodeGen')
22 files changed, 459 insertions, 381 deletions
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 631cc26d6022..3e09fbad6ab1 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -7274,7 +7274,7 @@ bool CodeGenPrepare::optimizeShiftInst(BinaryOperator *Shift) { // We can't do this effectively in SDAG because we may not be able to // determine if the select operands are splats from within a basic block. Type *Ty = Shift->getType(); - if (!Ty->isVectorTy() || !TLI->isVectorShiftByScalarCheap(Ty)) + if (!Ty->isVectorTy() || !TTI->isVectorShiftByScalarCheap(Ty)) return false; Value *Cond, *TVal, *FVal; if (!match(Shift->getOperand(1), @@ -7309,7 +7309,7 @@ bool CodeGenPrepare::optimizeFunnelShift(IntrinsicInst *Fsh) { // We can't do this effectively in SDAG because we may not be able to // determine if the select operands are splats from within a basic block. Type *Ty = Fsh->getType(); - if (!Ty->isVectorTy() || !TLI->isVectorShiftByScalarCheap(Ty)) + if (!Ty->isVectorTy() || !TTI->isVectorShiftByScalarCheap(Ty)) return false; Value *Cond, *TVal, *FVal; if (!match(Fsh->getOperand(2), @@ -7566,7 +7566,7 @@ bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) { // If the operands of I can be folded into a target instruction together with // I, duplicate and sink them. SmallVector<Use *, 4> OpsToSink; - if (!TLI->shouldSinkOperands(I, OpsToSink)) + if (!TTI->isProfitableToSinkOperands(I, OpsToSink)) return false; // OpsToSink can contain multiple uses in a use chain (e.g. diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp index c827d5bdcf55..8d9813edd7e5 100644 --- a/llvm/lib/CodeGen/EarlyIfConversion.cpp +++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp @@ -163,7 +163,8 @@ private: void rewritePHIOperands(); public: - SSAIfConv(MachineFunction &MF) { + /// init - Initialize per-function data structures. + void init(MachineFunction &MF) { TII = MF.getSubtarget().getInstrInfo(); TRI = MF.getSubtarget().getRegisterInfo(); MRI = &MF.getRegInfo(); @@ -768,6 +769,7 @@ class EarlyIfConverter : public MachineFunctionPass { MachineLoopInfo *Loops = nullptr; MachineTraceMetrics *Traces = nullptr; MachineTraceMetrics::Ensemble *MinInstr = nullptr; + SSAIfConv IfConv; public: static char ID; @@ -777,9 +779,9 @@ public: StringRef getPassName() const override { return "Early If-Conversion"; } private: - bool tryConvertIf(SSAIfConv &IfConv, MachineBasicBlock *); - void invalidateTraces(SSAIfConv &IfConv); - bool shouldConvertIf(SSAIfConv &IfConv); + bool tryConvertIf(MachineBasicBlock *); + void invalidateTraces(); + bool shouldConvertIf(); }; } // end anonymous namespace @@ -835,7 +837,7 @@ void updateLoops(MachineLoopInfo *Loops, } // namespace /// Invalidate MachineTraceMetrics before if-conversion. -void EarlyIfConverter::invalidateTraces(SSAIfConv &IfConv) { +void EarlyIfConverter::invalidateTraces() { Traces->verifyAnalysis(); Traces->invalidate(IfConv.Head); Traces->invalidate(IfConv.Tail); @@ -865,7 +867,7 @@ template <typename Remark> Remark &operator<<(Remark &R, Cycles C) { /// Apply cost model and heuristics to the if-conversion in IfConv. /// Return true if the conversion is a good idea. /// -bool EarlyIfConverter::shouldConvertIf(SSAIfConv &IfConv) { +bool EarlyIfConverter::shouldConvertIf() { // Stress testing mode disables all cost considerations. if (Stress) return true; @@ -1058,11 +1060,11 @@ bool EarlyIfConverter::shouldConvertIf(SSAIfConv &IfConv) { /// Attempt repeated if-conversion on MBB, return true if successful. /// -bool EarlyIfConverter::tryConvertIf(SSAIfConv &IfConv, MachineBasicBlock *MBB) { +bool EarlyIfConverter::tryConvertIf(MachineBasicBlock *MBB) { bool Changed = false; - while (IfConv.canConvertIf(MBB) && shouldConvertIf(IfConv)) { + while (IfConv.canConvertIf(MBB) && shouldConvertIf()) { // If-convert MBB and update analyses. - invalidateTraces(IfConv); + invalidateTraces(); SmallVector<MachineBasicBlock *, 4> RemoveBlocks; IfConv.convertIf(RemoveBlocks); Changed = true; @@ -1095,14 +1097,14 @@ bool EarlyIfConverter::runOnMachineFunction(MachineFunction &MF) { MinInstr = nullptr; bool Changed = false; - SSAIfConv IfConv(MF); + IfConv.init(MF); // Visit blocks in dominator tree post-order. The post-order enables nested // if-conversion in a single pass. The tryConvertIf() function may erase // blocks, but only blocks dominated by the head block. This makes it safe to // update the dominator tree while the post-order iterator is still active. for (auto *DomNode : post_order(DomTree)) - if (tryConvertIf(IfConv, DomNode->getBlock())) + if (tryConvertIf(DomNode->getBlock())) Changed = true; return Changed; @@ -1121,6 +1123,7 @@ class EarlyIfPredicator : public MachineFunctionPass { MachineDominatorTree *DomTree = nullptr; MachineBranchProbabilityInfo *MBPI = nullptr; MachineLoopInfo *Loops = nullptr; + SSAIfConv IfConv; public: static char ID; @@ -1130,8 +1133,8 @@ public: StringRef getPassName() const override { return "Early If-predicator"; } protected: - bool tryConvertIf(SSAIfConv &IfConv, MachineBasicBlock *); - bool shouldConvertIf(SSAIfConv &IfConv); + bool tryConvertIf(MachineBasicBlock *); + bool shouldConvertIf(); }; } // end anonymous namespace @@ -1158,7 +1161,7 @@ void EarlyIfPredicator::getAnalysisUsage(AnalysisUsage &AU) const { } /// Apply the target heuristic to decide if the transformation is profitable. -bool EarlyIfPredicator::shouldConvertIf(SSAIfConv &IfConv) { +bool EarlyIfPredicator::shouldConvertIf() { auto TrueProbability = MBPI->getEdgeProbability(IfConv.Head, IfConv.TBB); if (IfConv.isTriangle()) { MachineBasicBlock &IfBlock = @@ -1198,14 +1201,12 @@ bool EarlyIfPredicator::shouldConvertIf(SSAIfConv &IfConv) { /// Attempt repeated if-conversion on MBB, return true if successful. /// -bool EarlyIfPredicator::tryConvertIf(SSAIfConv &IfConv, - MachineBasicBlock *MBB) { +bool EarlyIfPredicator::tryConvertIf(MachineBasicBlock *MBB) { bool Changed = false; - while (IfConv.canConvertIf(MBB, /*Predicate=*/true) && - shouldConvertIf(IfConv)) { + while (IfConv.canConvertIf(MBB, /*Predicate*/ true) && shouldConvertIf()) { // If-convert MBB and update analyses. SmallVector<MachineBasicBlock *, 4> RemoveBlocks; - IfConv.convertIf(RemoveBlocks, /*Predicate=*/true); + IfConv.convertIf(RemoveBlocks, /*Predicate*/ true); Changed = true; updateDomTree(DomTree, IfConv, RemoveBlocks); for (MachineBasicBlock *MBB : RemoveBlocks) @@ -1231,14 +1232,14 @@ bool EarlyIfPredicator::runOnMachineFunction(MachineFunction &MF) { MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI(); bool Changed = false; - SSAIfConv IfConv(MF); + IfConv.init(MF); // Visit blocks in dominator tree post-order. The post-order enables nested // if-conversion in a single pass. The tryConvertIf() function may erase // blocks, but only blocks dominated by the head block. This makes it safe to // update the dominator tree while the post-order iterator is still active. for (auto *DomNode : post_order(DomTree)) - if (tryConvertIf(IfConv, DomNode->getBlock())) + if (tryConvertIf(DomNode->getBlock())) Changed = true; return Changed; diff --git a/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp b/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp index 11f123aa5bed..0a3d0cf8ec93 100644 --- a/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp +++ b/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp @@ -356,7 +356,7 @@ static void expandIToFP(Instruction *IToFP) { Entry->getTerminator()->eraseFromParent(); Function *CTLZ = - Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, IntTy); + Intrinsic::getOrInsertDeclaration(F->getParent(), Intrinsic::ctlz, IntTy); ConstantInt *True = Builder.getTrue(); // entry: diff --git a/llvm/lib/CodeGen/ExpandMemCmp.cpp b/llvm/lib/CodeGen/ExpandMemCmp.cpp index 04222d5b4afd..6d626de0b4e6 100644 --- a/llvm/lib/CodeGen/ExpandMemCmp.cpp +++ b/llvm/lib/CodeGen/ExpandMemCmp.cpp @@ -355,7 +355,7 @@ MemCmpExpansion::LoadPair MemCmpExpansion::getLoadPair(Type *LoadSizeType, // Swap bytes if required. if (BSwapSizeType) { - Function *Bswap = Intrinsic::getDeclaration( + Function *Bswap = Intrinsic::getOrInsertDeclaration( CI->getModule(), Intrinsic::bswap, BSwapSizeType); Lhs = Builder.CreateCall(Bswap, Lhs); Rhs = Builder.CreateCall(Bswap, Rhs); diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp index ffe879ff0496..32ba3e91822d 100644 --- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp +++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp @@ -237,7 +237,7 @@ Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder, if (ElemCount.isScalable()) { auto *M = Builder.GetInsertBlock()->getModule(); Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount); - Function *ActiveMaskFunc = Intrinsic::getDeclaration( + Function *ActiveMaskFunc = Intrinsic::getOrInsertDeclaration( M, Intrinsic::get_active_lane_mask, {BoolVecTy, EVLParam->getType()}); // `get_active_lane_mask` performs an implicit less-than comparison. Value *ConstZero = Builder.getInt32(0); @@ -299,7 +299,7 @@ Value *CachingVPExpander::expandPredicationToIntCall( case Intrinsic::umin: { Value *Op0 = VPI.getOperand(0); Value *Op1 = VPI.getOperand(1); - Function *Fn = Intrinsic::getDeclaration( + Function *Fn = Intrinsic::getOrInsertDeclaration( VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()}); Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName()); replaceOperation(*NewOp, VPI); @@ -308,7 +308,7 @@ Value *CachingVPExpander::expandPredicationToIntCall( case Intrinsic::bswap: case Intrinsic::bitreverse: { Value *Op = VPI.getOperand(0); - Function *Fn = Intrinsic::getDeclaration( + Function *Fn = Intrinsic::getOrInsertDeclaration( VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()}); Value *NewOp = Builder.CreateCall(Fn, {Op}, VPI.getName()); replaceOperation(*NewOp, VPI); @@ -327,7 +327,7 @@ Value *CachingVPExpander::expandPredicationToFPCall( case Intrinsic::fabs: case Intrinsic::sqrt: { Value *Op0 = VPI.getOperand(0); - Function *Fn = Intrinsic::getDeclaration( + Function *Fn = Intrinsic::getOrInsertDeclaration( VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()}); Value *NewOp = Builder.CreateCall(Fn, {Op0}, VPI.getName()); replaceOperation(*NewOp, VPI); @@ -337,7 +337,7 @@ Value *CachingVPExpander::expandPredicationToFPCall( case Intrinsic::minnum: { Value *Op0 = VPI.getOperand(0); Value *Op1 = VPI.getOperand(1); - Function *Fn = Intrinsic::getDeclaration( + Function *Fn = Intrinsic::getOrInsertDeclaration( VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()}); Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName()); replaceOperation(*NewOp, VPI); @@ -350,7 +350,7 @@ Value *CachingVPExpander::expandPredicationToFPCall( Value *Op0 = VPI.getOperand(0); Value *Op1 = VPI.getOperand(1); Value *Op2 = VPI.getOperand(2); - Function *Fn = Intrinsic::getDeclaration( + Function *Fn = Intrinsic::getOrInsertDeclaration( VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()}); Value *NewOp; if (Intrinsic::isConstrainedFPIntrinsic(UnpredicatedIntrinsicID)) @@ -594,7 +594,7 @@ bool CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) { // TODO add caching auto *M = VPI.getModule(); Function *VScaleFunc = - Intrinsic::getDeclaration(M, Intrinsic::vscale, Int32Ty); + Intrinsic::getOrInsertDeclaration(M, Intrinsic::vscale, Int32Ty); IRBuilder<> Builder(VPI.getParent(), VPI.getIterator()); Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue()); Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale"); diff --git a/llvm/lib/CodeGen/HardwareLoops.cpp b/llvm/lib/CodeGen/HardwareLoops.cpp index 9205eabcf568..c8a63304a3b6 100644 --- a/llvm/lib/CodeGen/HardwareLoops.cpp +++ b/llvm/lib/CodeGen/HardwareLoops.cpp @@ -512,7 +512,7 @@ Value* HardwareLoop::InsertIterationSetup(Value *LoopCountInit) { : Intrinsic::test_set_loop_iterations) : (UsePhi ? Intrinsic::start_loop_iterations : Intrinsic::set_loop_iterations); - Function *LoopIter = Intrinsic::getDeclaration(M, ID, Ty); + Function *LoopIter = Intrinsic::getOrInsertDeclaration(M, ID, Ty); Value *LoopSetup = Builder.CreateCall(LoopIter, LoopCountInit); // Use the return value of the intrinsic to control the entry of the loop. @@ -541,9 +541,8 @@ void HardwareLoop::InsertLoopDec() { Attribute::StrictFP)) CondBuilder.setIsFPConstrained(true); - Function *DecFunc = - Intrinsic::getDeclaration(M, Intrinsic::loop_decrement, - LoopDecrement->getType()); + Function *DecFunc = Intrinsic::getOrInsertDeclaration( + M, Intrinsic::loop_decrement, LoopDecrement->getType()); Value *Ops[] = { LoopDecrement }; Value *NewCond = CondBuilder.CreateCall(DecFunc, Ops); Value *OldCond = ExitBranch->getCondition(); @@ -566,9 +565,8 @@ Instruction* HardwareLoop::InsertLoopRegDec(Value *EltsRem) { Attribute::StrictFP)) CondBuilder.setIsFPConstrained(true); - Function *DecFunc = - Intrinsic::getDeclaration(M, Intrinsic::loop_decrement_reg, - { EltsRem->getType() }); + Function *DecFunc = Intrinsic::getOrInsertDeclaration( + M, Intrinsic::loop_decrement_reg, {EltsRem->getType()}); Value *Ops[] = { EltsRem, LoopDecrement }; Value *Call = CondBuilder.CreateCall(DecFunc, Ops); diff --git a/llvm/lib/CodeGen/IntrinsicLowering.cpp b/llvm/lib/CodeGen/IntrinsicLowering.cpp index 256c081b46e2..f799a8cfc1ba 100644 --- a/llvm/lib/CodeGen/IntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/IntrinsicLowering.cpp @@ -474,7 +474,7 @@ bool IntrinsicLowering::LowerToByteSwap(CallInst *CI) { // Okay, we can do this xform, do so now. Module *M = CI->getModule(); - Function *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Ty); + Function *Int = Intrinsic::getOrInsertDeclaration(M, Intrinsic::bswap, Ty); Value *Op = CI->getArgOperand(0); Op = CallInst::Create(Int, Op, CI->getName(), CI->getIterator()); diff --git a/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp b/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp index 0c0a4e13c7c9..a2b166227194 100644 --- a/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp +++ b/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp @@ -102,14 +102,6 @@ LiveDebugValues::LiveDebugValues() : MachineFunctionPass(ID) { } bool LiveDebugValues::runOnMachineFunction(MachineFunction &MF) { - // Except for Wasm, all targets should be only using physical register at this - // point. Wasm only use virtual registers throught its pipeline, but its - // virtual registers don't participate in this LiveDebugValues analysis; only - // its target indices do. - assert(MF.getTarget().getTargetTriple().isWasm() || - MF.getProperties().hasProperty( - MachineFunctionProperties::Property::NoVRegs)); - bool InstrRefBased = MF.useDebugInstrRef(); // Allow the user to force selection of InstrRef LDV. InstrRefBased |= ForceInstrRefLDV; diff --git a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp index 7e7d90f24fcc..c80b54a4f912 100644 --- a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp +++ b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp @@ -239,6 +239,10 @@ struct LocIndex { /// becomes a problem. static constexpr u32_location_t kWasmLocation = kFirstInvalidRegLocation + 2; + /// The first location that is reserved for VarLocs with locations of kind + /// VirtualRegisterKind. + static constexpr u32_location_t kFirstVirtualRegLocation = 1 << 31; + LocIndex(u32_location_t Location, u32_index_t Index) : Location(Location), Index(Index) {} @@ -810,9 +814,10 @@ private: VL.getDescribingRegs(Locations); assert(all_of(Locations, [](auto RegNo) { - return RegNo < LocIndex::kFirstInvalidRegLocation; + return (RegNo < LocIndex::kFirstInvalidRegLocation) || + (LocIndex::kFirstVirtualRegLocation <= RegNo); }) && - "Physreg out of range?"); + "Physical or virtual register out of range?"); if (VL.containsSpillLocs()) Locations.push_back(LocIndex::kSpillLocation); if (VL.containsWasmLocs()) @@ -1240,9 +1245,9 @@ void VarLocBasedLDV::getUsedRegs(const VarLocSet &CollectFrom, LocIndex::rawIndexForReg(LocIndex::kFirstRegLocation); uint64_t FirstInvalidIndex = LocIndex::rawIndexForReg(LocIndex::kFirstInvalidRegLocation); - for (auto It = CollectFrom.find(FirstRegIndex), - End = CollectFrom.find(FirstInvalidIndex); - It != End;) { + uint64_t FirstVirtualRegIndex = + LocIndex::rawIndexForReg(LocIndex::kFirstVirtualRegLocation); + auto doGetUsedRegs = [&](VarLocSet::const_iterator &It) { // We found a VarLoc ID for a VarLoc that lives in a register. Figure out // which register and add it to UsedRegs. uint32_t FoundReg = LocIndex::fromRawInteger(*It).Location; @@ -1255,6 +1260,16 @@ void VarLocBasedLDV::getUsedRegs(const VarLocSet &CollectFrom, // guaranteed to move on to the next register (or to end()). uint64_t NextRegIndex = LocIndex::rawIndexForReg(FoundReg + 1); It.advanceToLowerBound(NextRegIndex); + }; + for (auto It = CollectFrom.find(FirstRegIndex), + End = CollectFrom.find(FirstInvalidIndex); + It != End;) { + doGetUsedRegs(It); + } + for (auto It = CollectFrom.find(FirstVirtualRegIndex), + End = CollectFrom.end(); + It != End;) { + doGetUsedRegs(It); } } diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp index c42e63202c3b..dd5220b4599f 100644 --- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp +++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp @@ -3572,7 +3572,7 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) { if (UseExtTspForPerf || UseExtTspForSize) { assert( !(UseExtTspForPerf && UseExtTspForSize) && - "UseExtTspForPerf and UseExtTspForSize can not be set simultaneosly"); + "UseExtTspForPerf and UseExtTspForSize can not be set simultaneously"); applyExtTsp(/*OptForSize=*/UseExtTspForSize); createCFGChainExtTsp(); } @@ -3745,11 +3745,6 @@ void MachineBlockPlacement::assignBlockOrder( continue; MBB.updateTerminator(FTMBB); } - -#ifndef NDEBUG - // Make sure we correctly constructed all branches. - F->verify(this, "After optimized block reordering", &errs()); -#endif } void MachineBlockPlacement::createCFGChainExtTsp() { diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp index 8bcc437cbfb8..fb4da2c11cda 100644 --- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp +++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp @@ -886,8 +886,11 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) { "MachineCopyPropagation should be run after register allocation!"); if (MO.isDef() && !MO.isEarlyClobber()) { - Defs.push_back(Reg.asMCReg()); - continue; + // Skip invalidating constant registers. + if (!MRI->isConstantPhysReg(Reg)) { + Defs.push_back(Reg.asMCReg()); + continue; + } } else if (MO.readsReg()) ReadRegister(Reg.asMCReg(), MI, MO.isDebug() ? DebugUse : RegularUse); } diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 24a0f41775cc..e2c09fe25d55 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -77,8 +77,10 @@ #include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/ModRef.h" +#include "llvm/Support/Mutex.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include <algorithm> @@ -93,257 +95,300 @@ using namespace llvm; namespace { - struct MachineVerifier { - MachineVerifier(MachineFunctionAnalysisManager &MFAM, const char *b, - raw_ostream *OS) - : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b) {} - - MachineVerifier(Pass *pass, const char *b, raw_ostream *OS) - : PASS(pass), OS(OS ? *OS : nulls()), Banner(b) {} - - MachineVerifier(const char *b, LiveVariables *LiveVars, - LiveIntervals *LiveInts, LiveStacks *LiveStks, - SlotIndexes *Indexes, raw_ostream *OS) - : OS(OS ? *OS : nulls()), Banner(b), LiveVars(LiveVars), - LiveInts(LiveInts), LiveStks(LiveStks), Indexes(Indexes) {} - - unsigned verify(const MachineFunction &MF); - - MachineFunctionAnalysisManager *MFAM = nullptr; - Pass *const PASS = nullptr; - raw_ostream &OS; - const char *Banner; - const MachineFunction *MF = nullptr; - const TargetMachine *TM = nullptr; - const TargetInstrInfo *TII = nullptr; - const TargetRegisterInfo *TRI = nullptr; - const MachineRegisterInfo *MRI = nullptr; - const RegisterBankInfo *RBI = nullptr; - - unsigned foundErrors = 0; - - // Avoid querying the MachineFunctionProperties for each operand. - bool isFunctionRegBankSelected = false; - bool isFunctionSelected = false; - bool isFunctionTracksDebugUserValues = false; - - using RegVector = SmallVector<Register, 16>; - using RegMaskVector = SmallVector<const uint32_t *, 4>; - using RegSet = DenseSet<Register>; - using RegMap = DenseMap<Register, const MachineInstr *>; - using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>; - - const MachineInstr *FirstNonPHI = nullptr; - const MachineInstr *FirstTerminator = nullptr; - BlockSet FunctionBlocks; - - BitVector regsReserved; - RegSet regsLive; - RegVector regsDefined, regsDead, regsKilled; - RegMaskVector regMasks; - - SlotIndex lastIndex; - - // Add Reg and any sub-registers to RV - void addRegWithSubRegs(RegVector &RV, Register Reg) { - RV.push_back(Reg); - if (Reg.isPhysical()) - append_range(RV, TRI->subregs(Reg.asMCReg())); - } - - struct BBInfo { - // Is this MBB reachable from the MF entry point? - bool reachable = false; - - // Vregs that must be live in because they are used without being - // defined. Map value is the user. vregsLiveIn doesn't include regs - // that only are used by PHI nodes. - RegMap vregsLiveIn; - - // Regs killed in MBB. They may be defined again, and will then be in both - // regsKilled and regsLiveOut. - RegSet regsKilled; - - // Regs defined in MBB and live out. Note that vregs passing through may - // be live out without being mentioned here. - RegSet regsLiveOut; - - // Vregs that pass through MBB untouched. This set is disjoint from - // regsKilled and regsLiveOut. - RegSet vregsPassed; - - // Vregs that must pass through MBB because they are needed by a successor - // block. This set is disjoint from regsLiveOut. - RegSet vregsRequired; - - // Set versions of block's predecessor and successor lists. - BlockSet Preds, Succs; - - BBInfo() = default; - - // Add register to vregsRequired if it belongs there. Return true if - // anything changed. - bool addRequired(Register Reg) { - if (!Reg.isVirtual()) - return false; - if (regsLiveOut.count(Reg)) - return false; - return vregsRequired.insert(Reg).second; - } +/// Used the by the ReportedErrors class to guarantee only one error is reported +/// at one time. +static ManagedStatic<sys::SmartMutex<true>> ReportedErrorsLock; + +struct MachineVerifier { + MachineVerifier(MachineFunctionAnalysisManager &MFAM, const char *b, + raw_ostream *OS, bool AbortOnError = true) + : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b), + ReportedErrs(AbortOnError) {} + + MachineVerifier(Pass *pass, const char *b, raw_ostream *OS, + bool AbortOnError = true) + : PASS(pass), OS(OS ? *OS : nulls()), Banner(b), + ReportedErrs(AbortOnError) {} + + MachineVerifier(const char *b, LiveVariables *LiveVars, + LiveIntervals *LiveInts, LiveStacks *LiveStks, + SlotIndexes *Indexes, raw_ostream *OS, + bool AbortOnError = true) + : OS(OS ? *OS : nulls()), Banner(b), LiveVars(LiveVars), + LiveInts(LiveInts), LiveStks(LiveStks), Indexes(Indexes), + ReportedErrs(AbortOnError) {} + + /// \returns true if no problems were found. + bool verify(const MachineFunction &MF); + + MachineFunctionAnalysisManager *MFAM = nullptr; + Pass *const PASS = nullptr; + raw_ostream &OS; + const char *Banner; + const MachineFunction *MF = nullptr; + const TargetMachine *TM = nullptr; + const TargetInstrInfo *TII = nullptr; + const TargetRegisterInfo *TRI = nullptr; + const MachineRegisterInfo *MRI = nullptr; + const RegisterBankInfo *RBI = nullptr; + + // Avoid querying the MachineFunctionProperties for each operand. + bool isFunctionRegBankSelected = false; + bool isFunctionSelected = false; + bool isFunctionTracksDebugUserValues = false; + + using RegVector = SmallVector<Register, 16>; + using RegMaskVector = SmallVector<const uint32_t *, 4>; + using RegSet = DenseSet<Register>; + using RegMap = DenseMap<Register, const MachineInstr *>; + using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>; + + const MachineInstr *FirstNonPHI = nullptr; + const MachineInstr *FirstTerminator = nullptr; + BlockSet FunctionBlocks; + + BitVector regsReserved; + RegSet regsLive; + RegVector regsDefined, regsDead, regsKilled; + RegMaskVector regMasks; + + SlotIndex lastIndex; + + // Add Reg and any sub-registers to RV + void addRegWithSubRegs(RegVector &RV, Register Reg) { + RV.push_back(Reg); + if (Reg.isPhysical()) + append_range(RV, TRI->subregs(Reg.asMCReg())); + } + + struct BBInfo { + // Is this MBB reachable from the MF entry point? + bool reachable = false; + + // Vregs that must be live in because they are used without being + // defined. Map value is the user. vregsLiveIn doesn't include regs + // that only are used by PHI nodes. + RegMap vregsLiveIn; + + // Regs killed in MBB. They may be defined again, and will then be in both + // regsKilled and regsLiveOut. + RegSet regsKilled; + + // Regs defined in MBB and live out. Note that vregs passing through may + // be live out without being mentioned here. + RegSet regsLiveOut; + + // Vregs that pass through MBB untouched. This set is disjoint from + // regsKilled and regsLiveOut. + RegSet vregsPassed; + + // Vregs that must pass through MBB because they are needed by a successor + // block. This set is disjoint from regsLiveOut. + RegSet vregsRequired; + + // Set versions of block's predecessor and successor lists. + BlockSet Preds, Succs; + + BBInfo() = default; + + // Add register to vregsRequired if it belongs there. Return true if + // anything changed. + bool addRequired(Register Reg) { + if (!Reg.isVirtual()) + return false; + if (regsLiveOut.count(Reg)) + return false; + return vregsRequired.insert(Reg).second; + } - // Same for a full set. - bool addRequired(const RegSet &RS) { - bool Changed = false; - for (Register Reg : RS) - Changed |= addRequired(Reg); - return Changed; - } + // Same for a full set. + bool addRequired(const RegSet &RS) { + bool Changed = false; + for (Register Reg : RS) + Changed |= addRequired(Reg); + return Changed; + } - // Same for a full map. - bool addRequired(const RegMap &RM) { - bool Changed = false; - for (const auto &I : RM) - Changed |= addRequired(I.first); - return Changed; - } + // Same for a full map. + bool addRequired(const RegMap &RM) { + bool Changed = false; + for (const auto &I : RM) + Changed |= addRequired(I.first); + return Changed; + } - // Live-out registers are either in regsLiveOut or vregsPassed. - bool isLiveOut(Register Reg) const { - return regsLiveOut.count(Reg) || vregsPassed.count(Reg); - } - }; + // Live-out registers are either in regsLiveOut or vregsPassed. + bool isLiveOut(Register Reg) const { + return regsLiveOut.count(Reg) || vregsPassed.count(Reg); + } + }; - // Extra register info per MBB. - DenseMap<const MachineBasicBlock*, BBInfo> MBBInfoMap; - - bool isReserved(Register Reg) { - return Reg.id() < regsReserved.size() && regsReserved.test(Reg.id()); - } - - bool isAllocatable(Register Reg) const { - return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(Reg) && - !regsReserved.test(Reg.id()); - } - - // Analysis information if available - LiveVariables *LiveVars = nullptr; - LiveIntervals *LiveInts = nullptr; - LiveStacks *LiveStks = nullptr; - SlotIndexes *Indexes = nullptr; - - // This is calculated only when trying to verify convergence control tokens. - // Similar to the LLVM IR verifier, we calculate this locally instead of - // relying on the pass manager. - MachineDominatorTree DT; - - void visitMachineFunctionBefore(); - void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB); - void visitMachineBundleBefore(const MachineInstr *MI); - - /// Verify that all of \p MI's virtual register operands are scalars. - /// \returns True if all virtual register operands are scalar. False - /// otherwise. - bool verifyAllRegOpsScalar(const MachineInstr &MI, - const MachineRegisterInfo &MRI); - bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI); - - bool verifyGIntrinsicSideEffects(const MachineInstr *MI); - bool verifyGIntrinsicConvergence(const MachineInstr *MI); - void verifyPreISelGenericInstruction(const MachineInstr *MI); - - void visitMachineInstrBefore(const MachineInstr *MI); - void visitMachineOperand(const MachineOperand *MO, unsigned MONum); - void visitMachineBundleAfter(const MachineInstr *MI); - void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB); - void visitMachineFunctionAfter(); - - void report(const char *msg, const MachineFunction *MF); - void report(const char *msg, const MachineBasicBlock *MBB); - void report(const char *msg, const MachineInstr *MI); - void report(const char *msg, const MachineOperand *MO, unsigned MONum, - LLT MOVRegType = LLT{}); - void report(const Twine &Msg, const MachineInstr *MI); - - void report_context(const LiveInterval &LI) const; - void report_context(const LiveRange &LR, Register VRegUnit, - LaneBitmask LaneMask) const; - void report_context(const LiveRange::Segment &S) const; - void report_context(const VNInfo &VNI) const; - void report_context(SlotIndex Pos) const; - void report_context(MCPhysReg PhysReg) const; - void report_context_liverange(const LiveRange &LR) const; - void report_context_lanemask(LaneBitmask LaneMask) const; - void report_context_vreg(Register VReg) const; - void report_context_vreg_regunit(Register VRegOrUnit) const; - - void verifyInlineAsm(const MachineInstr *MI); - - void checkLiveness(const MachineOperand *MO, unsigned MONum); - void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum, - SlotIndex UseIdx, const LiveRange &LR, - Register VRegOrUnit, - LaneBitmask LaneMask = LaneBitmask::getNone()); - void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum, - SlotIndex DefIdx, const LiveRange &LR, - Register VRegOrUnit, bool SubRangeCheck = false, - LaneBitmask LaneMask = LaneBitmask::getNone()); - - void markReachable(const MachineBasicBlock *MBB); - void calcRegsPassed(); - void checkPHIOps(const MachineBasicBlock &MBB); - - void calcRegsRequired(); - void verifyLiveVariables(); - void verifyLiveIntervals(); - void verifyLiveInterval(const LiveInterval&); - void verifyLiveRangeValue(const LiveRange &, const VNInfo *, Register, - LaneBitmask); - void verifyLiveRangeSegment(const LiveRange &, - const LiveRange::const_iterator I, Register, - LaneBitmask); - void verifyLiveRange(const LiveRange &, Register, - LaneBitmask LaneMask = LaneBitmask::getNone()); + // Extra register info per MBB. + DenseMap<const MachineBasicBlock *, BBInfo> MBBInfoMap; + + bool isReserved(Register Reg) { + return Reg.id() < regsReserved.size() && regsReserved.test(Reg.id()); + } + + bool isAllocatable(Register Reg) const { + return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(Reg) && + !regsReserved.test(Reg.id()); + } + + // Analysis information if available + LiveVariables *LiveVars = nullptr; + LiveIntervals *LiveInts = nullptr; + LiveStacks *LiveStks = nullptr; + SlotIndexes *Indexes = nullptr; - void verifyStackFrame(); + /// A class to track the number of reported error and to guarantee that only + /// one error is reported at one time. + class ReportedErrors { + unsigned NumReported = 0; + bool AbortOnError; - void verifySlotIndexes() const; - void verifyProperties(const MachineFunction &MF); + public: + /// \param AbortOnError -- If set, abort after printing the first error. + ReportedErrors(bool AbortOnError) : AbortOnError(AbortOnError) {} + + ~ReportedErrors() { + if (!hasError()) + return; + if (AbortOnError) + report_fatal_error("Found " + Twine(NumReported) + + " machine code errors."); + // Since we haven't aborted, release the lock to allow other threads to + // report errors. + ReportedErrorsLock->unlock(); + } + + /// Increment the number of reported errors. + /// \returns true if this is the first reported error. + bool increment() { + // If this is the first error this thread has encountered, grab the lock + // to prevent other threads from reporting errors at the same time. + // Otherwise we assume we already have the lock. + if (!hasError()) + ReportedErrorsLock->lock(); + ++NumReported; + return NumReported == 1; + } + + /// \returns true if an error was reported. + bool hasError() { return NumReported; } }; + ReportedErrors ReportedErrs; + + // This is calculated only when trying to verify convergence control tokens. + // Similar to the LLVM IR verifier, we calculate this locally instead of + // relying on the pass manager. + MachineDominatorTree DT; + + void visitMachineFunctionBefore(); + void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB); + void visitMachineBundleBefore(const MachineInstr *MI); + + /// Verify that all of \p MI's virtual register operands are scalars. + /// \returns True if all virtual register operands are scalar. False + /// otherwise. + bool verifyAllRegOpsScalar(const MachineInstr &MI, + const MachineRegisterInfo &MRI); + bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI); + + bool verifyGIntrinsicSideEffects(const MachineInstr *MI); + bool verifyGIntrinsicConvergence(const MachineInstr *MI); + void verifyPreISelGenericInstruction(const MachineInstr *MI); + + void visitMachineInstrBefore(const MachineInstr *MI); + void visitMachineOperand(const MachineOperand *MO, unsigned MONum); + void visitMachineBundleAfter(const MachineInstr *MI); + void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB); + void visitMachineFunctionAfter(); + + void report(const char *msg, const MachineFunction *MF); + void report(const char *msg, const MachineBasicBlock *MBB); + void report(const char *msg, const MachineInstr *MI); + void report(const char *msg, const MachineOperand *MO, unsigned MONum, + LLT MOVRegType = LLT{}); + void report(const Twine &Msg, const MachineInstr *MI); + + void report_context(const LiveInterval &LI) const; + void report_context(const LiveRange &LR, Register VRegUnit, + LaneBitmask LaneMask) const; + void report_context(const LiveRange::Segment &S) const; + void report_context(const VNInfo &VNI) const; + void report_context(SlotIndex Pos) const; + void report_context(MCPhysReg PhysReg) const; + void report_context_liverange(const LiveRange &LR) const; + void report_context_lanemask(LaneBitmask LaneMask) const; + void report_context_vreg(Register VReg) const; + void report_context_vreg_regunit(Register VRegOrUnit) const; + + void verifyInlineAsm(const MachineInstr *MI); + + void checkLiveness(const MachineOperand *MO, unsigned MONum); + void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum, + SlotIndex UseIdx, const LiveRange &LR, + Register VRegOrUnit, + LaneBitmask LaneMask = LaneBitmask::getNone()); + void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum, + SlotIndex DefIdx, const LiveRange &LR, + Register VRegOrUnit, bool SubRangeCheck = false, + LaneBitmask LaneMask = LaneBitmask::getNone()); + + void markReachable(const MachineBasicBlock *MBB); + void calcRegsPassed(); + void checkPHIOps(const MachineBasicBlock &MBB); + + void calcRegsRequired(); + void verifyLiveVariables(); + void verifyLiveIntervals(); + void verifyLiveInterval(const LiveInterval &); + void verifyLiveRangeValue(const LiveRange &, const VNInfo *, Register, + LaneBitmask); + void verifyLiveRangeSegment(const LiveRange &, + const LiveRange::const_iterator I, Register, + LaneBitmask); + void verifyLiveRange(const LiveRange &, Register, + LaneBitmask LaneMask = LaneBitmask::getNone()); - struct MachineVerifierLegacyPass : public MachineFunctionPass { - static char ID; // Pass ID, replacement for typeid + void verifyStackFrame(); - const std::string Banner; + void verifySlotIndexes() const; + void verifyProperties(const MachineFunction &MF); +}; - MachineVerifierLegacyPass(std::string banner = std::string()) - : MachineFunctionPass(ID), Banner(std::move(banner)) { - initializeMachineVerifierLegacyPassPass(*PassRegistry::getPassRegistry()); - } +struct MachineVerifierLegacyPass : public MachineFunctionPass { + static char ID; // Pass ID, replacement for typeid - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addUsedIfAvailable<LiveStacks>(); - AU.addUsedIfAvailable<LiveVariablesWrapperPass>(); - AU.addUsedIfAvailable<SlotIndexesWrapperPass>(); - AU.addUsedIfAvailable<LiveIntervalsWrapperPass>(); - AU.setPreservesAll(); - MachineFunctionPass::getAnalysisUsage(AU); - } + const std::string Banner; - bool runOnMachineFunction(MachineFunction &MF) override { - // Skip functions that have known verification problems. - // FIXME: Remove this mechanism when all problematic passes have been - // fixed. - if (MF.getProperties().hasProperty( - MachineFunctionProperties::Property::FailsVerification)) - return false; + MachineVerifierLegacyPass(std::string banner = std::string()) + : MachineFunctionPass(ID), Banner(std::move(banner)) { + initializeMachineVerifierLegacyPassPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addUsedIfAvailable<LiveStacks>(); + AU.addUsedIfAvailable<LiveVariablesWrapperPass>(); + AU.addUsedIfAvailable<SlotIndexesWrapperPass>(); + AU.addUsedIfAvailable<LiveIntervalsWrapperPass>(); + AU.setPreservesAll(); + MachineFunctionPass::getAnalysisUsage(AU); + } - unsigned FoundErrors = - MachineVerifier(this, Banner.c_str(), &errs()).verify(MF); - if (FoundErrors) - report_fatal_error("Found "+Twine(FoundErrors)+" machine code errors."); + bool runOnMachineFunction(MachineFunction &MF) override { + // Skip functions that have known verification problems. + // FIXME: Remove this mechanism when all problematic passes have been + // fixed. + if (MF.getProperties().hasProperty( + MachineFunctionProperties::Property::FailsVerification)) return false; - } - }; + + MachineVerifier(this, Banner.c_str(), &errs()).verify(MF); + return false; + } +}; } // end anonymous namespace @@ -356,10 +401,7 @@ MachineVerifierPass::run(MachineFunction &MF, if (MF.getProperties().hasProperty( MachineFunctionProperties::Property::FailsVerification)) return PreservedAnalyses::all(); - unsigned FoundErrors = - MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF); - if (FoundErrors) - report_fatal_error("Found " + Twine(FoundErrors) + " machine code errors."); + MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF); return PreservedAnalyses::all(); } @@ -379,31 +421,20 @@ void llvm::verifyMachineFunction(const std::string &Banner, // LiveIntervals *LiveInts; // LiveStacks *LiveStks; // SlotIndexes *Indexes; - unsigned FoundErrors = - MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF); - if (FoundErrors) - report_fatal_error("Found " + Twine(FoundErrors) + " machine code errors."); + MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF); } bool MachineFunction::verify(Pass *p, const char *Banner, raw_ostream *OS, - bool AbortOnErrors) const { - MachineFunction &MF = const_cast<MachineFunction&>(*this); - unsigned FoundErrors = MachineVerifier(p, Banner, OS).verify(MF); - if (AbortOnErrors && FoundErrors) - report_fatal_error("Found "+Twine(FoundErrors)+" machine code errors."); - return FoundErrors == 0; + bool AbortOnError) const { + return MachineVerifier(p, Banner, OS, AbortOnError).verify(*this); } bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes, const char *Banner, raw_ostream *OS, - bool AbortOnErrors) const { - MachineFunction &MF = const_cast<MachineFunction &>(*this); - unsigned FoundErrors = - MachineVerifier(Banner, nullptr, LiveInts, nullptr, Indexes, OS) - .verify(MF); - if (AbortOnErrors && FoundErrors) - report_fatal_error("Found " + Twine(FoundErrors) + " machine code errors."); - return FoundErrors == 0; + bool AbortOnError) const { + return MachineVerifier(Banner, /*LiveVars=*/nullptr, LiveInts, + /*LiveStks=*/nullptr, Indexes, OS, AbortOnError) + .verify(*this); } void MachineVerifier::verifySlotIndexes() const { @@ -429,9 +460,7 @@ void MachineVerifier::verifyProperties(const MachineFunction &MF) { report("Function has NoVRegs property but there are VReg operands", &MF); } -unsigned MachineVerifier::verify(const MachineFunction &MF) { - foundErrors = 0; - +bool MachineVerifier::verify(const MachineFunction &MF) { this->MF = &MF; TM = &MF.getTarget(); TII = MF.getSubtarget().getInstrInfo(); @@ -446,7 +475,7 @@ unsigned MachineVerifier::verify(const MachineFunction &MF) { // it's expected that the MIR is somewhat broken but that's ok since we'll // reset it and clear the FailedISel attribute in ResetMachineFunctions. if (isFunctionFailedISel) - return foundErrors; + return true; isFunctionRegBankSelected = MF.getProperties().hasProperty( MachineFunctionProperties::Property::RegBankSelected); @@ -543,13 +572,13 @@ unsigned MachineVerifier::verify(const MachineFunction &MF) { regMasks.clear(); MBBInfoMap.clear(); - return foundErrors; + return !ReportedErrs.hasError(); } void MachineVerifier::report(const char *msg, const MachineFunction *MF) { assert(MF); OS << '\n'; - if (!foundErrors++) { + if (ReportedErrs.increment()) { if (Banner) OS << "# " << Banner << '\n'; @@ -3846,18 +3875,18 @@ namespace { // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the // value is zero. // We use a bool plus an integer to capture the stack state. - struct StackStateOfBB { - StackStateOfBB() = default; - StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup) : - EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup), - ExitIsSetup(ExitSetup) {} - - // Can be negative, which means we are setting up a frame. - int EntryValue = 0; - int ExitValue = 0; - bool EntryIsSetup = false; - bool ExitIsSetup = false; - }; +struct StackStateOfBB { + StackStateOfBB() = default; + StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup) + : EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup), + ExitIsSetup(ExitSetup) {} + + // Can be negative, which means we are setting up a frame. + int EntryValue = 0; + int ExitValue = 0; + bool EntryIsSetup = false; + bool ExitIsSetup = false; +}; } // end anonymous namespace diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp index 9fbb7b461364..7f3c5cf6cb44 100644 --- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp +++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp @@ -100,20 +100,34 @@ static void replaceWithTLIFunction(IntrinsicInst *II, VFInfo &Info, static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, IntrinsicInst *II) { assert(II != nullptr && "Intrinsic cannot be null"); + Intrinsic::ID IID = II->getIntrinsicID(); + Type *RetTy = II->getType(); + Type *ScalarRetTy = RetTy->getScalarType(); // At the moment VFABI assumes the return type is always widened unless it is // a void type. - auto *VTy = dyn_cast<VectorType>(II->getType()); + auto *VTy = dyn_cast<VectorType>(RetTy); ElementCount EC(VTy ? VTy->getElementCount() : ElementCount::getFixed(0)); + + // OloadTys collects types used in scalar intrinsic overload name. + SmallVector<Type *, 3> OloadTys; + if (!RetTy->isVoidTy() && isVectorIntrinsicWithOverloadTypeAtArg(IID, -1)) + OloadTys.push_back(ScalarRetTy); + // Compute the argument types of the corresponding scalar call and check that // all vector operands match the previously found EC. SmallVector<Type *, 8> ScalarArgTypes; - Intrinsic::ID IID = II->getIntrinsicID(); for (auto Arg : enumerate(II->args())) { auto *ArgTy = Arg.value()->getType(); + bool IsOloadTy = isVectorIntrinsicWithOverloadTypeAtArg(IID, Arg.index()); if (isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) { ScalarArgTypes.push_back(ArgTy); + if (IsOloadTy) + OloadTys.push_back(ArgTy); } else if (auto *VectorArgTy = dyn_cast<VectorType>(ArgTy)) { - ScalarArgTypes.push_back(VectorArgTy->getElementType()); + auto *ScalarArgTy = VectorArgTy->getElementType(); + ScalarArgTypes.push_back(ScalarArgTy); + if (IsOloadTy) + OloadTys.push_back(ScalarArgTy); // When return type is void, set EC to the first vector argument, and // disallow vector arguments with different ECs. if (EC.isZero()) @@ -129,7 +143,7 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, // using scalar argument types. std::string ScalarName = Intrinsic::isOverloaded(IID) - ? Intrinsic::getName(IID, ScalarArgTypes, II->getModule()) + ? Intrinsic::getName(IID, OloadTys, II->getModule()) : Intrinsic::getName(IID).str(); // Try to find the mapping for the scalar version of this intrinsic and the @@ -146,7 +160,6 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, // Replace the call to the intrinsic with a call to the vector library // function. - Type *ScalarRetTy = II->getType()->getScalarType(); FunctionType *ScalarFTy = FunctionType::get(ScalarRetTy, ScalarArgTypes, /*isVarArg*/ false); const std::string MangledName = VD->getVectorFunctionABIVariantString(); @@ -193,6 +206,8 @@ static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { for (auto &I : instructions(F)) { // Process only intrinsic calls that return void or a vector. if (auto *II = dyn_cast<IntrinsicInst>(&I)) { + if (II->getIntrinsicID() == Intrinsic::not_intrinsic) + continue; if (!II->getType()->isVectorTy() && !II->getType()->isVoidTy()) continue; diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp index e41d1bfb0e53..a50909af8bfc 100644 --- a/llvm/lib/CodeGen/SafeStack.cpp +++ b/llvm/lib/CodeGen/SafeStack.cpp @@ -368,7 +368,8 @@ Value *SafeStack::getStackGuard(IRBuilder<> &IRB, Function &F) { if (!StackGuardVar) { TL.insertSSPDeclarations(*M); - return IRB.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::stackguard)); + return IRB.CreateCall( + Intrinsic::getOrInsertDeclaration(M, Intrinsic::stackguard)); } return IRB.CreateLoad(StackPtrTy, StackGuardVar, "StackGuard"); diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index c6f6fc250805..810ca458bc87 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -11639,6 +11639,7 @@ static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG) { SDValue N1 = N->getOperand(1); SDValue N2 = N->getOperand(2); EVT VT = N->getValueType(0); + unsigned EltSizeInBits = VT.getScalarSizeInBits(); SDValue Cond0, Cond1; ISD::CondCode CC; @@ -11659,7 +11660,7 @@ static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG) { // (Cond0 s< 0) ? N1 : 0 --> (Cond0 s>> BW-1) & freeze(N1) if (isNullOrNullSplat(N2)) { SDLoc DL(N); - SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); + SDValue ShiftAmt = DAG.getShiftAmountConstant(EltSizeInBits - 1, VT, DL); SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); return DAG.getNode(ISD::AND, DL, VT, Sra, DAG.getFreeze(N1)); } @@ -11667,7 +11668,7 @@ static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG) { // (Cond0 s< 0) ? -1 : N2 --> (Cond0 s>> BW-1) | freeze(N2) if (isAllOnesOrAllOnesSplat(N1)) { SDLoc DL(N); - SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); + SDValue ShiftAmt = DAG.getShiftAmountConstant(EltSizeInBits - 1, VT, DL); SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); return DAG.getNode(ISD::OR, DL, VT, Sra, DAG.getFreeze(N2)); } @@ -11678,7 +11679,7 @@ static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (isNullOrNullSplat(N1) && TLI.hasAndNot(N1)) { SDLoc DL(N); - SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); + SDValue ShiftAmt = DAG.getShiftAmountConstant(EltSizeInBits - 1, VT, DL); SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); SDValue Not = DAG.getNOT(DL, Sra, VT); return DAG.getNode(ISD::AND, DL, VT, Not, DAG.getFreeze(N2)); @@ -20321,13 +20322,11 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) { !FastLD || !FastST) return SDValue(); - SDValue NewLD = - DAG.getLoad(IntVT, SDLoc(Value), LD->getChain(), LD->getBasePtr(), - LD->getPointerInfo(), LD->getAlign()); + SDValue NewLD = DAG.getLoad(IntVT, SDLoc(Value), LD->getChain(), + LD->getBasePtr(), LD->getMemOperand()); - SDValue NewST = - DAG.getStore(ST->getChain(), SDLoc(N), NewLD, ST->getBasePtr(), - ST->getPointerInfo(), ST->getAlign()); + SDValue NewST = DAG.getStore(ST->getChain(), SDLoc(N), NewLD, + ST->getBasePtr(), ST->getMemOperand()); AddToWorklist(NewLD.getNode()); AddToWorklist(NewST.getNode()); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 42d031310d5e..ea22b4670d6f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1657,7 +1657,8 @@ SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, SignMask); - // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X) + // If FABS is legal transform + // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x) EVT FloatVT = Mag.getValueType(); if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) { diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index ab734ffb25db..9674de773883 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -1287,6 +1287,8 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { case ISD::VP_FMINIMUM: case ISD::FMAXIMUM: case ISD::VP_FMAXIMUM: + case ISD::FMINIMUMNUM: + case ISD::FMAXIMUMNUM: case ISD::SDIV: case ISD::VP_SDIV: case ISD::UDIV: case ISD::VP_UDIV: case ISD::FDIV: case ISD::VP_FDIV: diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 25213f587116..a981e9cc7928 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4386,6 +4386,15 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { // it. IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType()); + SDNodeFlags ScaleFlags; + // The multiplication of an index by the type size does not wrap the + // pointer index type in a signed sense (mul nsw). + ScaleFlags.setNoSignedWrap(NW.hasNoUnsignedSignedWrap()); + + // The multiplication of an index by the type size does not wrap the + // pointer index type in an unsigned sense (mul nuw). + ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap()); + if (ElementScalable) { EVT VScaleTy = N.getValueType().getScalarType(); SDValue VScale = DAG.getNode( @@ -4393,27 +4402,34 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy)); if (IsVectorGEP) VScale = DAG.getSplatVector(N.getValueType(), dl, VScale); - IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale); + IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale, + ScaleFlags); } else { // If this is a multiply by a power of two, turn it into a shl // immediately. This is a very common case. if (ElementMul != 1) { if (ElementMul.isPowerOf2()) { unsigned Amt = ElementMul.logBase2(); - IdxN = DAG.getNode(ISD::SHL, dl, - N.getValueType(), IdxN, - DAG.getConstant(Amt, dl, IdxN.getValueType())); + IdxN = DAG.getNode(ISD::SHL, dl, N.getValueType(), IdxN, + DAG.getConstant(Amt, dl, IdxN.getValueType()), + ScaleFlags); } else { SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl, IdxN.getValueType()); - IdxN = DAG.getNode(ISD::MUL, dl, - N.getValueType(), IdxN, Scale); + IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, Scale, + ScaleFlags); } } } - N = DAG.getNode(ISD::ADD, dl, - N.getValueType(), N, IdxN); + // The successive addition of the current address, truncated to the + // pointer index type and interpreted as an unsigned number, and each + // offset, also interpreted as an unsigned number, does not wrap the + // pointer index type (add nuw). + SDNodeFlags AddFlags; + AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap()); + + N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, IdxN, AddFlags); } } diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 9df0f0adcc1e..793b8ff164c2 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -8424,6 +8424,11 @@ TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node, if (Node->getFlags().hasNoNaNs()) { ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT; + EVT VT = Node->getValueType(0); + if ((!isCondCodeLegal(Pred, VT.getSimpleVT()) || + !isOperationLegalOrCustom(ISD::VSELECT, VT)) && + VT.isVector()) + return SDValue(); SDValue Op1 = Node->getOperand(0); SDValue Op2 = Node->getOperand(1); SDValue SelCC = DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred); diff --git a/llvm/lib/CodeGen/SjLjEHPrepare.cpp b/llvm/lib/CodeGen/SjLjEHPrepare.cpp index 054f7d721596..c4ad9f0b2172 100644 --- a/llvm/lib/CodeGen/SjLjEHPrepare.cpp +++ b/llvm/lib/CodeGen/SjLjEHPrepare.cpp @@ -508,17 +508,19 @@ bool SjLjEHPrepareImpl::runOnFunction(Function &F) { PointerType *AllocaPtrTy = M.getDataLayout().getAllocaPtrType(M.getContext()); - FrameAddrFn = - Intrinsic::getDeclaration(&M, Intrinsic::frameaddress, {AllocaPtrTy}); - StackAddrFn = - Intrinsic::getDeclaration(&M, Intrinsic::stacksave, {AllocaPtrTy}); - StackRestoreFn = - Intrinsic::getDeclaration(&M, Intrinsic::stackrestore, {AllocaPtrTy}); + FrameAddrFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::frameaddress, + {AllocaPtrTy}); + StackAddrFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::stacksave, + {AllocaPtrTy}); + StackRestoreFn = Intrinsic::getOrInsertDeclaration( + &M, Intrinsic::stackrestore, {AllocaPtrTy}); BuiltinSetupDispatchFn = - Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_setup_dispatch); - LSDAAddrFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_lsda); - CallSiteFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_callsite); - FuncCtxFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_functioncontext); + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::eh_sjlj_setup_dispatch); + LSDAAddrFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::eh_sjlj_lsda); + CallSiteFn = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::eh_sjlj_callsite); + FuncCtxFn = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::eh_sjlj_functioncontext); bool Res = setupEntryBlockAndCallSites(F); return Res; diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp index 1f23838b2de0..a192161bbd94 100644 --- a/llvm/lib/CodeGen/StackProtector.cpp +++ b/llvm/lib/CodeGen/StackProtector.cpp @@ -519,7 +519,8 @@ static Value *getStackGuard(const TargetLoweringBase *TLI, Module *M, if (SupportsSelectionDAGSP) *SupportsSelectionDAGSP = true; TLI->insertSSPDeclarations(*M); - return B.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::stackguard)); + return B.CreateCall( + Intrinsic::getOrInsertDeclaration(M, Intrinsic::stackguard)); } /// Insert code into the entry block that stores the stack guard @@ -540,7 +541,7 @@ static bool CreatePrologue(Function *F, Module *M, Instruction *CheckLoc, AI = B.CreateAlloca(PtrTy, nullptr, "StackGuardSlot"); Value *GuardSlot = getStackGuard(TLI, M, B, &SupportsSelectionDAGSP); - B.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::stackprotector), + B.CreateCall(Intrinsic::getOrInsertDeclaration(M, Intrinsic::stackprotector), {GuardSlot, AI}); return SupportsSelectionDAGSP; } diff --git a/llvm/lib/CodeGen/WasmEHPrepare.cpp b/llvm/lib/CodeGen/WasmEHPrepare.cpp index 7514d49fb18a..1701b0d04425 100644 --- a/llvm/lib/CodeGen/WasmEHPrepare.cpp +++ b/llvm/lib/CodeGen/WasmEHPrepare.cpp @@ -196,7 +196,7 @@ bool WasmEHPrepareImpl::prepareThrows(Function &F) { bool Changed = false; // wasm.throw() intinsic, which will be lowered to wasm 'throw' instruction. - ThrowF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_throw); + ThrowF = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_throw); // Insert an unreachable instruction after a call to @llvm.wasm.throw and // delete all following instructions within the BB, and delete all the dead // children of the BB as well. @@ -260,18 +260,21 @@ bool WasmEHPrepareImpl::prepareEHPads(Function &F) { 0, 2, "selector_gep"); // wasm.landingpad.index() intrinsic, which is to specify landingpad index - LPadIndexF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_landingpad_index); + LPadIndexF = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_landingpad_index); // wasm.lsda() intrinsic. Returns the address of LSDA table for the current // function. - LSDAF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_lsda); + LSDAF = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_lsda); // wasm.get.exception() and wasm.get.ehselector() intrinsics. Calls to these // are generated in clang. - GetExnF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_get_exception); - GetSelectorF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_get_ehselector); + GetExnF = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_get_exception); + GetSelectorF = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_get_ehselector); // wasm.catch() will be lowered down to wasm 'catch' instruction in // instruction selection. - CatchF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_catch); + CatchF = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_catch); // _Unwind_CallPersonality() wrapper function, which calls the personality CallPersonalityF = M.getOrInsertFunction("_Unwind_CallPersonality", |