summaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms')
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroFrame.cpp190
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroInternal.h16
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroSplit.cpp35
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp9
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp45
-rw-r--r--llvm/lib/Transforms/Scalar/ConstantHoisting.cpp3
-rw-r--r--llvm/lib/Transforms/Scalar/ConstraintElimination.cpp63
-rw-r--r--llvm/lib/Transforms/Scalar/LICM.cpp19
-rw-r--r--llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp5
-rw-r--r--llvm/lib/Transforms/Utils/LoopRotationUtils.cpp5
-rw-r--r--llvm/lib/Transforms/Utils/MemoryOpRemark.cpp10
11 files changed, 276 insertions, 124 deletions
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 6b85de15947c..f37b4dc938d3 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -964,12 +964,17 @@ static void cacheDIVar(FrameDataInfo &FrameData,
continue;
SmallVector<DbgDeclareInst *, 1> DDIs;
- findDbgDeclares(DDIs, V);
- auto *I = llvm::find_if(DDIs, [](DbgDeclareInst *DDI) {
- return DDI->getExpression()->getNumElements() == 0;
- });
- if (I != DDIs.end())
- DIVarCache.insert({V, (*I)->getVariable()});
+ SmallVector<DPValue *, 1> DPVs;
+ findDbgDeclares(DDIs, V, &DPVs);
+ auto CacheIt = [&DIVarCache, V](auto &Container) {
+ auto *I = llvm::find_if(Container, [](auto *DDI) {
+ return DDI->getExpression()->getNumElements() == 0;
+ });
+ if (I != Container.end())
+ DIVarCache.insert({V, (*I)->getVariable()});
+ };
+ CacheIt(DDIs);
+ CacheIt(DPVs);
}
}
@@ -1121,15 +1126,25 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
"Coroutine with switch ABI should own Promise alloca");
SmallVector<DbgDeclareInst *, 1> DIs;
- findDbgDeclares(DIs, PromiseAlloca);
- if (DIs.empty())
+ SmallVector<DPValue *, 1> DPVs;
+ findDbgDeclares(DIs, PromiseAlloca, &DPVs);
+
+ DILocalVariable *PromiseDIVariable = nullptr;
+ DILocation *DILoc = nullptr;
+ if (!DIs.empty()) {
+ DbgDeclareInst *PromiseDDI = DIs.front();
+ PromiseDIVariable = PromiseDDI->getVariable();
+ DILoc = PromiseDDI->getDebugLoc().get();
+ } else if (!DPVs.empty()) {
+ DPValue *PromiseDPV = DPVs.front();
+ PromiseDIVariable = PromiseDPV->getVariable();
+ DILoc = PromiseDPV->getDebugLoc().get();
+ } else {
return;
+ }
- DbgDeclareInst *PromiseDDI = DIs.front();
- DILocalVariable *PromiseDIVariable = PromiseDDI->getVariable();
DILocalScope *PromiseDIScope = PromiseDIVariable->getScope();
DIFile *DFile = PromiseDIScope->getFile();
- DILocation *DILoc = PromiseDDI->getDebugLoc().get();
unsigned LineNum = PromiseDIVariable->getLine();
DICompositeType *FrameDITy = DBuilder.createStructType(
@@ -1243,7 +1258,7 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
auto *FrameDIVar = DBuilder.createAutoVariable(PromiseDIScope, "__coro_frame",
DFile, LineNum, FrameDITy,
true, DINode::FlagArtificial);
- assert(FrameDIVar->isValidLocationForIntrinsic(PromiseDDI->getDebugLoc()));
+ assert(FrameDIVar->isValidLocationForIntrinsic(DILoc));
// Subprogram would have ContainedNodes field which records the debug
// variables it contained. So we need to add __coro_frame to the
@@ -1261,9 +1276,17 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
7, (MDTuple::get(F.getContext(), RetainedNodesVec)));
}
- DBuilder.insertDeclare(Shape.FramePtr, FrameDIVar,
- DBuilder.createExpression(), DILoc,
- Shape.getInsertPtAfterFramePtr());
+ if (UseNewDbgInfoFormat) {
+ DPValue *NewDPV = new DPValue(ValueAsMetadata::get(Shape.FramePtr),
+ FrameDIVar, DBuilder.createExpression(),
+ DILoc, DPValue::LocationType::Declare);
+ BasicBlock::iterator It = Shape.getInsertPtAfterFramePtr();
+ It->getParent()->insertDPValueBefore(NewDPV, It);
+ } else {
+ DBuilder.insertDeclare(Shape.FramePtr, FrameDIVar,
+ DBuilder.createExpression(), DILoc,
+ &*Shape.getInsertPtAfterFramePtr());
+ }
}
// Build a struct that will keep state for an active coroutine.
@@ -1771,7 +1794,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
if (auto *Arg = dyn_cast<Argument>(Def)) {
// For arguments, we will place the store instruction right after
// the coroutine frame pointer instruction, i.e. coro.begin.
- InsertPt = Shape.getInsertPtAfterFramePtr()->getIterator();
+ InsertPt = Shape.getInsertPtAfterFramePtr();
// If we're spilling an Argument, make sure we clear 'nocapture'
// from the coroutine function.
@@ -1788,7 +1811,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
if (!DT.dominates(CB, I)) {
// If it is not dominated by CoroBegin, then spill should be
// inserted immediately after CoroFrame is computed.
- InsertPt = Shape.getInsertPtAfterFramePtr()->getIterator();
+ InsertPt = Shape.getInsertPtAfterFramePtr();
} else if (auto *II = dyn_cast<InvokeInst>(I)) {
// If we are spilling the result of the invoke instruction, split
// the normal edge and insert the spill in the new block.
@@ -1843,7 +1866,8 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
SpillAlignment, E.first->getName() + Twine(".reload"));
SmallVector<DbgDeclareInst *, 1> DIs;
- findDbgDeclares(DIs, Def);
+ SmallVector<DPValue *, 1> DPVs;
+ findDbgDeclares(DIs, Def, &DPVs);
// Try best to find dbg.declare. If the spill is a temp, there may not
// be a direct dbg.declare. Walk up the load chain to find one from an
// alias.
@@ -1858,24 +1882,36 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
if (!isa<AllocaInst, LoadInst>(CurDef))
break;
DIs.clear();
- findDbgDeclares(DIs, CurDef);
+ DPVs.clear();
+ findDbgDeclares(DIs, CurDef, &DPVs);
}
}
- for (DbgDeclareInst *DDI : DIs) {
+ auto SalvageOne = [&](auto *DDI) {
bool AllowUnresolved = false;
// This dbg.declare is preserved for all coro-split function
// fragments. It will be unreachable in the main function, and
// processed by coro::salvageDebugInfo() by CoroCloner.
- DIBuilder(*CurrentBlock->getParent()->getParent(), AllowUnresolved)
- .insertDeclare(CurrentReload, DDI->getVariable(),
- DDI->getExpression(), DDI->getDebugLoc(),
- &*Builder.GetInsertPoint());
+ if (UseNewDbgInfoFormat) {
+ DPValue *NewDPV =
+ new DPValue(ValueAsMetadata::get(CurrentReload),
+ DDI->getVariable(), DDI->getExpression(),
+ DDI->getDebugLoc(), DPValue::LocationType::Declare);
+ Builder.GetInsertPoint()->getParent()->insertDPValueBefore(
+ NewDPV, Builder.GetInsertPoint());
+ } else {
+ DIBuilder(*CurrentBlock->getParent()->getParent(), AllowUnresolved)
+ .insertDeclare(CurrentReload, DDI->getVariable(),
+ DDI->getExpression(), DDI->getDebugLoc(),
+ &*Builder.GetInsertPoint());
+ }
// This dbg.declare is for the main function entry point. It
// will be deleted in all coro-split functions.
- coro::salvageDebugInfo(ArgToAllocaMap, DDI, Shape.OptimizeFrame,
+ coro::salvageDebugInfo(ArgToAllocaMap, *DDI, Shape.OptimizeFrame,
false /*UseEntryValue*/);
- }
+ };
+ for_each(DIs, SalvageOne);
+ for_each(DPVs, SalvageOne);
}
// If we have a single edge PHINode, remove it and replace it with a
@@ -1893,6 +1929,10 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
// Replace all uses of CurrentValue in the current instruction with
// reload.
U->replaceUsesOfWith(Def, CurrentReload);
+ // Instructions are added to Def's user list if the attached
+ // debug records use Def. Update those now.
+ for (auto &DPV : U->getDbgValueRange())
+ DPV.replaceVariableLocationOp(Def, CurrentReload, true);
}
}
@@ -1943,9 +1983,12 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
G->setName(Alloca->getName() + Twine(".reload.addr"));
SmallVector<DbgVariableIntrinsic *, 4> DIs;
- findDbgUsers(DIs, Alloca);
+ SmallVector<DPValue *> DPValues;
+ findDbgUsers(DIs, Alloca, &DPValues);
for (auto *DVI : DIs)
DVI->replaceUsesOfWith(Alloca, G);
+ for (auto *DPV : DPValues)
+ DPV->replaceVariableLocationOp(Alloca, G);
for (Instruction *I : UsersToUpdate) {
// It is meaningless to retain the lifetime intrinsics refer for the
@@ -1959,7 +2002,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
I->replaceUsesOfWith(Alloca, G);
}
}
- Builder.SetInsertPoint(Shape.getInsertPtAfterFramePtr());
+ Builder.SetInsertPoint(&*Shape.getInsertPtAfterFramePtr());
for (const auto &A : FrameData.Allocas) {
AllocaInst *Alloca = A.Alloca;
if (A.MayWriteBeforeCoroBegin) {
@@ -2020,7 +2063,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
isa<BitCastInst>(Inst);
});
if (HasAccessingPromiseBeforeCB) {
- Builder.SetInsertPoint(Shape.getInsertPtAfterFramePtr());
+ Builder.SetInsertPoint(&*Shape.getInsertPtAfterFramePtr());
auto *G = GetFramePointer(PA);
auto *Value = Builder.CreateLoad(PA->getAllocatedType(), PA);
Builder.CreateStore(Value, G);
@@ -2802,21 +2845,16 @@ static void collectFrameAlloca(AllocaInst *AI, coro::Shape &Shape,
Visitor.getMayWriteBeforeCoroBegin());
}
-void coro::salvageDebugInfo(
- SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
- DbgVariableIntrinsic *DVI, bool OptimizeFrame, bool UseEntryValue) {
- Function *F = DVI->getFunction();
+static std::optional<std::pair<Value &, DIExpression &>>
+salvageDebugInfoImpl(SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
+ bool OptimizeFrame, bool UseEntryValue, Function *F,
+ Value *Storage, DIExpression *Expr,
+ bool SkipOutermostLoad) {
IRBuilder<> Builder(F->getContext());
auto InsertPt = F->getEntryBlock().getFirstInsertionPt();
while (isa<IntrinsicInst>(InsertPt))
++InsertPt;
Builder.SetInsertPoint(&F->getEntryBlock(), InsertPt);
- DIExpression *Expr = DVI->getExpression();
- // Follow the pointer arithmetic all the way to the incoming
- // function argument and convert into a DIExpression.
- bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
- Value *Storage = DVI->getVariableLocationOp(0);
- Value *OriginalStorage = Storage;
while (auto *Inst = dyn_cast_or_null<Instruction>(Storage)) {
if (auto *LdInst = dyn_cast<LoadInst>(Inst)) {
@@ -2848,7 +2886,7 @@ void coro::salvageDebugInfo(
SkipOutermostLoad = false;
}
if (!Storage)
- return;
+ return std::nullopt;
auto *StorageAsArg = dyn_cast<Argument>(Storage);
const bool IsSwiftAsyncArg =
@@ -2884,8 +2922,30 @@ void coro::salvageDebugInfo(
Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
}
- DVI->replaceVariableLocationOp(OriginalStorage, Storage);
- DVI->setExpression(Expr);
+ return {{*Storage, *Expr}};
+}
+
+void coro::salvageDebugInfo(
+ SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
+ DbgVariableIntrinsic &DVI, bool OptimizeFrame, bool UseEntryValue) {
+
+ Function *F = DVI.getFunction();
+ // Follow the pointer arithmetic all the way to the incoming
+ // function argument and convert into a DIExpression.
+ bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
+ Value *OriginalStorage = DVI.getVariableLocationOp(0);
+
+ auto SalvagedInfo = ::salvageDebugInfoImpl(
+ ArgToAllocaMap, OptimizeFrame, UseEntryValue, F, OriginalStorage,
+ DVI.getExpression(), SkipOutermostLoad);
+ if (!SalvagedInfo)
+ return;
+
+ Value *Storage = &SalvagedInfo->first;
+ DIExpression *Expr = &SalvagedInfo->second;
+
+ DVI.replaceVariableLocationOp(OriginalStorage, Storage);
+ DVI.setExpression(Expr);
// We only hoist dbg.declare today since it doesn't make sense to hoist
// dbg.value since it does not have the same function wide guarantees that
// dbg.declare does.
@@ -2896,7 +2956,44 @@ void coro::salvageDebugInfo(
else if (isa<Argument>(Storage))
InsertPt = F->getEntryBlock().begin();
if (InsertPt)
- DVI->moveBefore(*(*InsertPt)->getParent(), *InsertPt);
+ DVI.moveBefore(*(*InsertPt)->getParent(), *InsertPt);
+ }
+}
+
+void coro::salvageDebugInfo(
+ SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap, DPValue &DPV,
+ bool OptimizeFrame, bool UseEntryValue) {
+
+ Function *F = DPV.getFunction();
+ // Follow the pointer arithmetic all the way to the incoming
+ // function argument and convert into a DIExpression.
+ bool SkipOutermostLoad = DPV.getType() == DPValue::LocationType::Declare;
+ Value *OriginalStorage = DPV.getVariableLocationOp(0);
+
+ auto SalvagedInfo = ::salvageDebugInfoImpl(
+ ArgToAllocaMap, OptimizeFrame, UseEntryValue, F, OriginalStorage,
+ DPV.getExpression(), SkipOutermostLoad);
+ if (!SalvagedInfo)
+ return;
+
+ Value *Storage = &SalvagedInfo->first;
+ DIExpression *Expr = &SalvagedInfo->second;
+
+ DPV.replaceVariableLocationOp(OriginalStorage, Storage);
+ DPV.setExpression(Expr);
+ // We only hoist dbg.declare today since it doesn't make sense to hoist
+ // dbg.value since it does not have the same function wide guarantees that
+ // dbg.declare does.
+ if (DPV.getType() == DPValue::LocationType::Declare) {
+ std::optional<BasicBlock::iterator> InsertPt;
+ if (auto *I = dyn_cast<Instruction>(Storage))
+ InsertPt = I->getInsertionPointAfterDef();
+ else if (isa<Argument>(Storage))
+ InsertPt = F->getEntryBlock().begin();
+ if (InsertPt) {
+ DPV.removeFromParent();
+ (*InsertPt)->getParent()->insertDPValueBefore(&DPV, *InsertPt);
+ }
}
}
@@ -3087,10 +3184,15 @@ void coro::buildCoroutineFrame(
for (auto &Iter : FrameData.Spills) {
auto *V = Iter.first;
SmallVector<DbgValueInst *, 16> DVIs;
- findDbgValues(DVIs, V);
+ SmallVector<DPValue *, 16> DPVs;
+ findDbgValues(DVIs, V, &DPVs);
for (DbgValueInst *DVI : DVIs)
if (Checker.isDefinitionAcrossSuspend(*V, DVI))
FrameData.Spills[V].push_back(DVI);
+ // Add the instructions which carry debug info that is in the frame.
+ for (DPValue *DPV : DPVs)
+ if (Checker.isDefinitionAcrossSuspend(*V, DPV->Marker->MarkedInstr))
+ FrameData.Spills[V].push_back(DPV->Marker->MarkedInstr);
}
LLVM_DEBUG(dumpSpills("Spills", FrameData.Spills));
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index 0856c4925cc5..fb16a4090689 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -32,7 +32,10 @@ void replaceCoroFree(CoroIdInst *CoroId, bool Elide);
/// OptimizeFrame is false.
void salvageDebugInfo(
SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
- DbgVariableIntrinsic *DVI, bool OptimizeFrame, bool IsEntryPoint);
+ DbgVariableIntrinsic &DVI, bool OptimizeFrame, bool IsEntryPoint);
+void salvageDebugInfo(
+ SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap, DPValue &DPV,
+ bool OptimizeFrame, bool UseEntryValue);
// Keeps data and helper functions for lowering coroutine intrinsics.
struct LowererBase {
@@ -240,10 +243,13 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
return nullptr;
}
- Instruction *getInsertPtAfterFramePtr() const {
- if (auto *I = dyn_cast<Instruction>(FramePtr))
- return I->getNextNode();
- return &cast<Argument>(FramePtr)->getParent()->getEntryBlock().front();
+ BasicBlock::iterator getInsertPtAfterFramePtr() const {
+ if (auto *I = dyn_cast<Instruction>(FramePtr)) {
+ BasicBlock::iterator It = std::next(I->getIterator());
+ It.setHeadBit(true); // Copy pre-RemoveDIs behaviour.
+ return It;
+ }
+ return cast<Argument>(FramePtr)->getParent()->getEntryBlock().begin();
}
/// Allocate memory according to the rules of the active lowering.
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 244580f503d5..7758b52abc20 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -725,13 +725,17 @@ static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape,
}
/// Returns all DbgVariableIntrinsic in F.
-static SmallVector<DbgVariableIntrinsic *, 8>
+static std::pair<SmallVector<DbgVariableIntrinsic *, 8>, SmallVector<DPValue *>>
collectDbgVariableIntrinsics(Function &F) {
SmallVector<DbgVariableIntrinsic *, 8> Intrinsics;
- for (auto &I : instructions(F))
+ SmallVector<DPValue *> DPValues;
+ for (auto &I : instructions(F)) {
+ for (DPValue &DPV : I.getDbgValueRange())
+ DPValues.push_back(&DPV);
if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
Intrinsics.push_back(DVI);
- return Intrinsics;
+ }
+ return {Intrinsics, DPValues};
}
void CoroCloner::replaceSwiftErrorOps() {
@@ -739,15 +743,17 @@ void CoroCloner::replaceSwiftErrorOps() {
}
void CoroCloner::salvageDebugInfo() {
- SmallVector<DbgVariableIntrinsic *, 8> Worklist =
- collectDbgVariableIntrinsics(*NewF);
+ auto [Worklist, DPValues] = collectDbgVariableIntrinsics(*NewF);
SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
// Only 64-bit ABIs have a register we can refer to with the entry value.
bool UseEntryValue =
llvm::Triple(OrigF.getParent()->getTargetTriple()).isArch64Bit();
for (DbgVariableIntrinsic *DVI : Worklist)
- coro::salvageDebugInfo(ArgToAllocaMap, DVI, Shape.OptimizeFrame,
+ coro::salvageDebugInfo(ArgToAllocaMap, *DVI, Shape.OptimizeFrame,
+ UseEntryValue);
+ for (DPValue *DPV : DPValues)
+ coro::salvageDebugInfo(ArgToAllocaMap, *DPV, Shape.OptimizeFrame,
UseEntryValue);
// Remove all salvaged dbg.declare intrinsics that became
@@ -757,7 +763,7 @@ void CoroCloner::salvageDebugInfo() {
return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
&DomTree);
};
- for (DbgVariableIntrinsic *DVI : Worklist) {
+ auto RemoveOne = [&](auto *DVI) {
if (IsUnreachableBlock(DVI->getParent()))
DVI->eraseFromParent();
else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
@@ -770,7 +776,9 @@ void CoroCloner::salvageDebugInfo() {
if (!Uses)
DVI->eraseFromParent();
}
- }
+ };
+ for_each(Worklist, RemoveOne);
+ for_each(DPValues, RemoveOne);
}
void CoroCloner::replaceEntryBlock() {
@@ -1243,7 +1251,7 @@ static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
Function *DestroyFn, Function *CleanupFn) {
assert(Shape.ABI == coro::ABI::Switch);
- IRBuilder<> Builder(Shape.getInsertPtAfterFramePtr());
+ IRBuilder<> Builder(&*Shape.getInsertPtAfterFramePtr());
auto *ResumeAddr = Builder.CreateStructGEP(
Shape.FrameTy, Shape.FramePtr, coro::Shape::SwitchFieldIndex::Resume,
@@ -2039,10 +2047,13 @@ splitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
// original function. The Cloner has already salvaged debug info in the new
// coroutine funclets.
SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
- for (auto *DDI : collectDbgVariableIntrinsics(F))
- coro::salvageDebugInfo(ArgToAllocaMap, DDI, Shape.OptimizeFrame,
+ auto [DbgInsts, DPValues] = collectDbgVariableIntrinsics(F);
+ for (auto *DDI : DbgInsts)
+ coro::salvageDebugInfo(ArgToAllocaMap, *DDI, Shape.OptimizeFrame,
+ false /*UseEntryValue*/);
+ for (DPValue *DPV : DPValues)
+ coro::salvageDebugInfo(ArgToAllocaMap, *DPV, Shape.OptimizeFrame,
false /*UseEntryValue*/);
-
return Shape;
}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 104c73602223..1539fa9a3269 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -4227,12 +4227,9 @@ InstCombinerImpl::foldCommutativeIntrinsicOverSelects(IntrinsicInst &II) {
assert(II.isCommutative());
Value *A, *B, *C;
- bool LHSIsSelect =
- match(II.getOperand(0), m_Select(m_Value(A), m_Value(B), m_Value(C)));
- bool RHSIsSymmetricalSelect = match(
- II.getOperand(1), m_Select(m_Specific(A), m_Specific(C), m_Specific(B)));
-
- if (LHSIsSelect && RHSIsSymmetricalSelect) {
+ if (match(II.getOperand(0), m_Select(m_Value(A), m_Value(B), m_Value(C))) &&
+ match(II.getOperand(1),
+ m_Select(m_Specific(A), m_Specific(C), m_Specific(B)))) {
replaceOperand(II, 0, B);
replaceOperand(II, 1, C);
return &II;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 8d5866e98a8e..e5566578869d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -1205,6 +1205,38 @@ Instruction *InstCombinerImpl::commonIDivTransforms(BinaryOperator &I) {
}
}
+ // (X * Y) / (X * Z) --> Y / Z (and commuted variants)
+ if (match(Op0, m_Mul(m_Value(X), m_Value(Y)))) {
+ auto OB0HasNSW = cast<OverflowingBinaryOperator>(Op0)->hasNoSignedWrap();
+ auto OB0HasNUW = cast<OverflowingBinaryOperator>(Op0)->hasNoUnsignedWrap();
+
+ auto CreateDivOrNull = [&](Value *A, Value *B) -> Instruction * {
+ auto OB1HasNSW = cast<OverflowingBinaryOperator>(Op1)->hasNoSignedWrap();
+ auto OB1HasNUW =
+ cast<OverflowingBinaryOperator>(Op1)->hasNoUnsignedWrap();
+ const APInt *C1, *C2;
+ if (IsSigned && OB0HasNSW) {
+ if (OB1HasNSW && match(B, m_APInt(C1)) && !C1->isAllOnes())
+ return BinaryOperator::CreateSDiv(A, B);
+ }
+ if (!IsSigned && OB0HasNUW) {
+ if (OB1HasNUW)
+ return BinaryOperator::CreateUDiv(A, B);
+ if (match(A, m_APInt(C1)) && match(B, m_APInt(C2)) && C2->ule(*C1))
+ return BinaryOperator::CreateUDiv(A, B);
+ }
+ return nullptr;
+ };
+
+ if (match(Op1, m_c_Mul(m_Specific(X), m_Value(Z)))) {
+ if (auto *Val = CreateDivOrNull(Y, Z))
+ return Val;
+ }
+ if (match(Op1, m_c_Mul(m_Specific(Y), m_Value(Z)))) {
+ if (auto *Val = CreateDivOrNull(X, Z))
+ return Val;
+ }
+ }
return nullptr;
}
@@ -1375,20 +1407,7 @@ Instruction *InstCombinerImpl::visitUDiv(BinaryOperator &I) {
if (Instruction *NarrowDiv = narrowUDivURem(I, *this))
return NarrowDiv;
- // If the udiv operands are non-overflowing multiplies with a common operand,
- // then eliminate the common factor:
- // (A * B) / (A * X) --> B / X (and commuted variants)
- // TODO: The code would be reduced if we had m_c_NUWMul pattern matching.
- // TODO: If -reassociation handled this generally, we could remove this.
Value *A, *B;
- if (match(Op0, m_NUWMul(m_Value(A), m_Value(B)))) {
- if (match(Op1, m_NUWMul(m_Specific(A), m_Value(X))) ||
- match(Op1, m_NUWMul(m_Value(X), m_Specific(A))))
- return BinaryOperator::CreateUDiv(B, X);
- if (match(Op1, m_NUWMul(m_Specific(B), m_Value(X))) ||
- match(Op1, m_NUWMul(m_Value(X), m_Specific(B))))
- return BinaryOperator::CreateUDiv(A, X);
- }
// Look through a right-shift to find the common factor:
// ((Op1 *nuw A) >> B) / Op1 --> A >> B
diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
index 3e5d979f11cc..1fb9d7fff32f 100644
--- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -523,7 +523,8 @@ void ConstantHoistingPass::collectConstantCandidates(Function &Fn) {
if (!DT->isReachableFromEntry(&BB))
continue;
for (Instruction &Inst : BB)
- collectConstantCandidates(ConstCandMap, &Inst);
+ if (!TTI->preferToKeepConstantsAttached(Inst, Fn))
+ collectConstantCandidates(ConstCandMap, &Inst);
}
}
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index fafbe17583f5..18266ba07898 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1109,6 +1109,16 @@ void State::addInfoFor(BasicBlock &BB) {
CmpI->getOperand(1)));
}
+#ifndef NDEBUG
+static void dumpUnpackedICmp(raw_ostream &OS, ICmpInst::Predicate Pred,
+ Value *LHS, Value *RHS) {
+ OS << "icmp " << Pred << ' ';
+ LHS->printAsOperand(OS, /*PrintType=*/true);
+ OS << ", ";
+ RHS->printAsOperand(OS, /*PrintType=*/false);
+}
+#endif
+
namespace {
/// Helper to keep track of a condition and if it should be treated as negated
/// for reproducer construction.
@@ -1241,10 +1251,9 @@ static void generateReproducer(CmpInst *Cond, Module *M,
if (Entry.Pred == ICmpInst::BAD_ICMP_PREDICATE)
continue;
- LLVM_DEBUG(
- dbgs() << " Materializing assumption icmp " << Entry.Pred << ' ';
- Entry.LHS->printAsOperand(dbgs(), /*PrintType=*/true); dbgs() << ", ";
- Entry.RHS->printAsOperand(dbgs(), /*PrintType=*/false); dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << " Materializing assumption ";
+ dumpUnpackedICmp(dbgs(), Entry.Pred, Entry.LHS, Entry.RHS);
+ dbgs() << "\n");
CloneInstructions({Entry.LHS, Entry.RHS}, CmpInst::isSigned(Entry.Pred));
auto *Cmp = Builder.CreateICmp(Entry.Pred, Entry.LHS, Entry.RHS);
@@ -1260,14 +1269,12 @@ static void generateReproducer(CmpInst *Cond, Module *M,
assert(!verifyFunction(*F, &dbgs()));
}
-static std::optional<bool> checkCondition(CmpInst *Cmp, ConstraintInfo &Info,
- unsigned NumIn, unsigned NumOut,
+static std::optional<bool> checkCondition(CmpInst::Predicate Pred, Value *A,
+ Value *B, Instruction *CheckInst,
+ ConstraintInfo &Info, unsigned NumIn,
+ unsigned NumOut,
Instruction *ContextInst) {
- LLVM_DEBUG(dbgs() << "Checking " << *Cmp << "\n");
-
- CmpInst::Predicate Pred = Cmp->getPredicate();
- Value *A = Cmp->getOperand(0);
- Value *B = Cmp->getOperand(1);
+ LLVM_DEBUG(dbgs() << "Checking " << *CheckInst << "\n");
auto R = Info.getConstraintForSolving(Pred, A, B);
if (R.empty() || !R.isValid(Info)){
@@ -1292,13 +1299,10 @@ static std::optional<bool> checkCondition(CmpInst *Cmp, ConstraintInfo &Info,
return std::nullopt;
LLVM_DEBUG({
- if (*ImpliedCondition) {
- dbgs() << "Condition " << *Cmp;
- } else {
- auto InversePred = Cmp->getInversePredicate();
- dbgs() << "Condition " << CmpInst::getPredicateName(InversePred) << " "
- << *A << ", " << *B;
- }
+ dbgs() << "Condition ";
+ dumpUnpackedICmp(
+ dbgs(), *ImpliedCondition ? Pred : CmpInst::getInversePredicate(Pred),
+ A, B);
dbgs() << " implied by dominating constraints\n";
CSToUse.dump();
});
@@ -1338,8 +1342,9 @@ static bool checkAndReplaceCondition(
return true;
};
- if (auto ImpliedCondition =
- checkCondition(Cmp, Info, NumIn, NumOut, ContextInst))
+ if (auto ImpliedCondition = checkCondition(
+ Cmp->getPredicate(), Cmp->getOperand(0), Cmp->getOperand(1), Cmp,
+ Info, NumIn, NumOut, ContextInst))
return ReplaceCmpWithConstant(Cmp, *ImpliedCondition);
return false;
}
@@ -1380,9 +1385,10 @@ static bool checkAndSecondOpImpliedByFirst(
bool Changed = false;
// Check if the second condition can be simplified now.
- if (auto ImpliedCondition =
- checkCondition(cast<ICmpInst>(And->getOperand(1)), Info, CB.NumIn,
- CB.NumOut, CB.getContextInst())) {
+ ICmpInst *Cmp = cast<ICmpInst>(And->getOperand(1));
+ if (auto ImpliedCondition = checkCondition(
+ Cmp->getPredicate(), Cmp->getOperand(0), Cmp->getOperand(1), Cmp,
+ Info, CB.NumIn, CB.NumOut, CB.getContextInst())) {
And->setOperand(1, ConstantInt::getBool(And->getType(), *ImpliedCondition));
Changed = true;
}
@@ -1408,9 +1414,8 @@ void ConstraintInfo::addFact(CmpInst::Predicate Pred, Value *A, Value *B,
if (!R.isValid(*this) || R.isNe())
return;
- LLVM_DEBUG(dbgs() << "Adding '" << Pred << " ";
- A->printAsOperand(dbgs(), false); dbgs() << ", ";
- B->printAsOperand(dbgs(), false); dbgs() << "'\n");
+ LLVM_DEBUG(dbgs() << "Adding '"; dumpUnpackedICmp(dbgs(), Pred, A, B);
+ dbgs() << "'\n");
bool Added = false;
auto &CSToUse = getCS(R.IsSigned);
if (R.Coefficients.empty())
@@ -1616,10 +1621,8 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT, LoopInfo &LI,
}
auto AddFact = [&](CmpInst::Predicate Pred, Value *A, Value *B) {
- LLVM_DEBUG(dbgs() << "fact to add to the system: "
- << CmpInst::getPredicateName(Pred) << " ";
- A->printAsOperand(dbgs()); dbgs() << ", ";
- B->printAsOperand(dbgs(), false); dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << "fact to add to the system: ";
+ dumpUnpackedICmp(dbgs(), Pred, A, B); dbgs() << "\n");
if (Info.getCS(CmpInst::isSigned(Pred)).size() > MaxRows) {
LLVM_DEBUG(
dbgs()
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index d0afe09ce41d..9117378568b7 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -193,7 +193,7 @@ static Instruction *cloneInstructionInExitBlock(
static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
MemorySSAUpdater &MSSAU);
-static void moveInstructionBefore(Instruction &I, Instruction &Dest,
+static void moveInstructionBefore(Instruction &I, BasicBlock::iterator Dest,
ICFLoopSafetyInfo &SafetyInfo,
MemorySSAUpdater &MSSAU, ScalarEvolution *SE);
@@ -1011,7 +1011,8 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
LLVM_DEBUG(dbgs() << "LICM rehoisting to "
<< HoistPoint->getParent()->getNameOrAsOperand()
<< ": " << *I << "\n");
- moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU, SE);
+ moveInstructionBefore(*I, HoistPoint->getIterator(), *SafetyInfo, MSSAU,
+ SE);
HoistPoint = I;
Changed = true;
}
@@ -1491,16 +1492,17 @@ static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
I.eraseFromParent();
}
-static void moveInstructionBefore(Instruction &I, Instruction &Dest,
+static void moveInstructionBefore(Instruction &I, BasicBlock::iterator Dest,
ICFLoopSafetyInfo &SafetyInfo,
MemorySSAUpdater &MSSAU,
ScalarEvolution *SE) {
SafetyInfo.removeInstruction(&I);
- SafetyInfo.insertInstructionTo(&I, Dest.getParent());
- I.moveBefore(&Dest);
+ SafetyInfo.insertInstructionTo(&I, Dest->getParent());
+ I.moveBefore(*Dest->getParent(), Dest);
if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
MSSAU.getMemorySSA()->getMemoryAccess(&I)))
- MSSAU.moveToPlace(OldMemAcc, Dest.getParent(), MemorySSA::BeforeTerminator);
+ MSSAU.moveToPlace(OldMemAcc, Dest->getParent(),
+ MemorySSA::BeforeTerminator);
if (SE)
SE->forgetBlockAndLoopDispositions(&I);
}
@@ -1747,10 +1749,11 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
if (isa<PHINode>(I))
// Move the new node to the end of the phi list in the destination block.
- moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU, SE);
+ moveInstructionBefore(I, Dest->getFirstNonPHIIt(), *SafetyInfo, MSSAU, SE);
else
// Move the new node to the destination block, before its terminator.
- moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE);
+ moveInstructionBefore(I, Dest->getTerminator()->getIterator(), *SafetyInfo,
+ MSSAU, SE);
I.updateLocationAfterHoist();
diff --git a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
index 13e06c79d0d7..9d5e6693c0e5 100644
--- a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
@@ -266,6 +266,11 @@ bool LoopVersioningLICM::legalLoopMemoryAccesses() {
for (const auto &A : AS) {
Value *Ptr = A.getValue();
// Alias tracker should have pointers of same data type.
+ //
+ // FIXME: check no longer effective since opaque pointers?
+ // If the intent is to check that the memory accesses use the
+ // same data type (such that LICM can promote them), then we
+ // can no longer see this from the pointer value types.
TypeCheck = (TypeCheck && (SomePtr->getType() == Ptr->getType()));
}
// At least one alias tracker should have pointers of same data type.
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 76280ed492b3..504f4430dc2c 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -708,12 +708,13 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
// as U1'' and U1' scopes will not be compatible wrt to the local restrict
// Clone the llvm.experimental.noalias.decl again for the NewHeader.
- Instruction *NewHeaderInsertionPoint = &(*NewHeader->getFirstNonPHI());
+ BasicBlock::iterator NewHeaderInsertionPoint =
+ NewHeader->getFirstNonPHIIt();
for (NoAliasScopeDeclInst *NAD : NoAliasDeclInstructions) {
LLVM_DEBUG(dbgs() << " Cloning llvm.experimental.noalias.scope.decl:"
<< *NAD << "\n");
Instruction *NewNAD = NAD->clone();
- NewNAD->insertBefore(NewHeaderInsertionPoint);
+ NewNAD->insertBefore(*NewHeader, NewHeaderInsertionPoint);
}
// Scopes must now be duplicated, once for OrigHeader and once for
diff --git a/llvm/lib/Transforms/Utils/MemoryOpRemark.cpp b/llvm/lib/Transforms/Utils/MemoryOpRemark.cpp
index 5c3776683d5d..47c6bcbaf26e 100644
--- a/llvm/lib/Transforms/Utils/MemoryOpRemark.cpp
+++ b/llvm/lib/Transforms/Utils/MemoryOpRemark.cpp
@@ -322,8 +322,9 @@ void MemoryOpRemark::visitVariable(const Value *V,
// Try to get an llvm.dbg.declare, which has a DILocalVariable giving us the
// real debug info name and size of the variable.
SmallVector<DbgDeclareInst *, 1> DbgDeclares;
- findDbgDeclares(DbgDeclares, const_cast<Value *>(V));
- for (const DbgVariableIntrinsic *DVI : DbgDeclares) {
+ SmallVector<DPValue *, 1> DPValues;
+ findDbgDeclares(DbgDeclares, const_cast<Value *>(V), &DPValues);
+ auto FindDI = [&](const auto *DVI) {
if (DILocalVariable *DILV = DVI->getVariable()) {
std::optional<uint64_t> DISize = getSizeInBytes(DILV->getSizeInBits());
VariableInfo Var{DILV->getName(), DISize};
@@ -332,7 +333,10 @@ void MemoryOpRemark::visitVariable(const Value *V,
FoundDI = true;
}
}
- }
+ };
+ for_each(DbgDeclares, FindDI);
+ for_each(DPValues, FindDI);
+
if (FoundDI) {
assert(!Result.empty());
return;
generated by cgit v1.2.3 (git 2.25.1) at 2026-05-31 11:58:09 +0000