summaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/SelectionDAG
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/SelectionDAG')
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp167
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp2
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp125
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp22
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp49
5 files changed, 246 insertions, 119 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 231184587d68..fed5e7238433 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -609,6 +609,8 @@ namespace {
SDValue foldABSToABD(SDNode *N, const SDLoc &DL);
SDValue foldSelectToABD(SDValue LHS, SDValue RHS, SDValue True,
SDValue False, ISD::CondCode CC, const SDLoc &DL);
+ SDValue foldSelectToUMin(SDValue LHS, SDValue RHS, SDValue True,
+ SDValue False, ISD::CondCode CC, const SDLoc &DL);
SDValue unfoldMaskedMerge(SDNode *N);
SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
@@ -859,7 +861,7 @@ namespace {
auto LK = TLI.getTypeConversion(*DAG.getContext(), VT);
return (LK.first == TargetLoweringBase::TypeLegal ||
LK.first == TargetLoweringBase::TypePromoteInteger) &&
- TLI.isOperationLegal(ISD::UMIN, LK.second);
+ TLI.isOperationLegalOrCustom(ISD::UMIN, LK.second);
}
public:
@@ -2606,9 +2608,7 @@ SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
return SDValue();
}
- SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
- SelectOp->setFlags(BO->getFlags());
- return SelectOp;
+ return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF, BO->getFlags());
}
static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, const SDLoc &DL,
@@ -4095,6 +4095,26 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
return N0;
}
+ // (sub x, ([v]select (ult x, y), 0, y)) -> (umin x, (sub x, y))
+ // (sub x, ([v]select (uge x, y), y, 0)) -> (umin x, (sub x, y))
+ if (N1.hasOneUse() && hasUMin(VT)) {
+ SDValue Y;
+ if (sd_match(N1, m_Select(m_SetCC(m_Specific(N0), m_Value(Y),
+ m_SpecificCondCode(ISD::SETULT)),
+ m_Zero(), m_Deferred(Y))) ||
+ sd_match(N1, m_Select(m_SetCC(m_Specific(N0), m_Value(Y),
+ m_SpecificCondCode(ISD::SETUGE)),
+ m_Deferred(Y), m_Zero())) ||
+ sd_match(N1, m_VSelect(m_SetCC(m_Specific(N0), m_Value(Y),
+ m_SpecificCondCode(ISD::SETULT)),
+ m_Zero(), m_Deferred(Y))) ||
+ sd_match(N1, m_VSelect(m_SetCC(m_Specific(N0), m_Value(Y),
+ m_SpecificCondCode(ISD::SETUGE)),
+ m_Deferred(Y), m_Zero())))
+ return DAG.getNode(ISD::UMIN, DL, VT, N0,
+ DAG.getNode(ISD::SUB, DL, VT, N0, Y));
+ }
+
if (SDValue NewSel = foldBinOpIntoSelect(N))
return NewSel;
@@ -4444,20 +4464,6 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
sd_match(N1, m_UMaxLike(m_Specific(A), m_Specific(B))))
return DAG.getNegative(DAG.getNode(ISD::ABDU, DL, VT, A, B), DL, VT);
- // (sub x, (select (ult x, y), 0, y)) -> (umin x, (sub x, y))
- // (sub x, (select (uge x, y), y, 0)) -> (umin x, (sub x, y))
- if (hasUMin(VT)) {
- SDValue Y;
- if (sd_match(N1, m_OneUse(m_Select(m_SetCC(m_Specific(N0), m_Value(Y),
- m_SpecificCondCode(ISD::SETULT)),
- m_Zero(), m_Deferred(Y)))) ||
- sd_match(N1, m_OneUse(m_Select(m_SetCC(m_Specific(N0), m_Value(Y),
- m_SpecificCondCode(ISD::SETUGE)),
- m_Deferred(Y), m_Zero()))))
- return DAG.getNode(ISD::UMIN, DL, VT, N0,
- DAG.getNode(ISD::SUB, DL, VT, N0, Y));
- }
-
return SDValue();
}
@@ -7635,7 +7641,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
if (SDValue(GN0, 0).hasOneUse() &&
isConstantSplatVectorMaskForType(N1.getNode(), ScalarVT) &&
- TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+ TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
@@ -9149,7 +9155,7 @@ calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
if (Op.getOpcode() != ISD::LOAD && VectorIndex.has_value())
return std::nullopt;
- unsigned BitWidth = Op.getValueSizeInBits();
+ unsigned BitWidth = Op.getScalarValueSizeInBits();
if (BitWidth % 8 != 0)
return std::nullopt;
unsigned ByteWidth = BitWidth / 8;
@@ -9248,7 +9254,7 @@ calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
if (!L->isSimple() || L->isIndexed())
return std::nullopt;
- unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
+ unsigned NarrowBitWidth = L->getMemoryVT().getScalarSizeInBits();
if (NarrowBitWidth % 8 != 0)
return std::nullopt;
uint64_t NarrowByteWidth = NarrowBitWidth / 8;
@@ -12175,6 +12181,30 @@ SDValue DAGCombiner::foldSelectToABD(SDValue LHS, SDValue RHS, SDValue True,
return SDValue();
}
+// ([v]select (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
+// ([v]select (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
+SDValue DAGCombiner::foldSelectToUMin(SDValue LHS, SDValue RHS, SDValue True,
+ SDValue False, ISD::CondCode CC,
+ const SDLoc &DL) {
+ APInt C;
+ EVT VT = True.getValueType();
+ if (sd_match(RHS, m_ConstInt(C)) && hasUMin(VT)) {
+ if (CC == ISD::SETUGT && LHS == False &&
+ sd_match(True, m_Add(m_Specific(False), m_SpecificInt(~C)))) {
+ SDValue AddC = DAG.getConstant(~C, DL, VT);
+ SDValue Add = DAG.getNode(ISD::ADD, DL, VT, False, AddC);
+ return DAG.getNode(ISD::UMIN, DL, VT, Add, False);
+ }
+ if (CC == ISD::SETULT && LHS == True &&
+ sd_match(False, m_Add(m_Specific(True), m_SpecificInt(-C)))) {
+ SDValue AddC = DAG.getConstant(-C, DL, VT);
+ SDValue Add = DAG.getNode(ISD::ADD, DL, VT, True, AddC);
+ return DAG.getNode(ISD::UMIN, DL, VT, True, Add);
+ }
+ }
+ return SDValue();
+}
+
SDValue DAGCombiner::visitSELECT(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
@@ -12191,11 +12221,8 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
return V;
// select (not Cond), N1, N2 -> select Cond, N2, N1
- if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) {
- SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1);
- SelectOp->setFlags(Flags);
- return SelectOp;
- }
+ if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
+ return DAG.getSelect(DL, VT, F, N2, N1, Flags);
if (SDValue V = foldSelectOfConstants(N))
return V;
@@ -12363,24 +12390,8 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
// (select (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
// (select (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
- APInt C;
- if (sd_match(Cond1, m_ConstInt(C)) && hasUMin(VT)) {
- if (CC == ISD::SETUGT && Cond0 == N2 &&
- sd_match(N1, m_Add(m_Specific(N2), m_SpecificInt(~C)))) {
- // The resulting code relies on an unsigned wrap in ADD.
- // Recreating ADD to drop possible nuw/nsw flags.
- SDValue AddC = DAG.getConstant(~C, DL, VT);
- SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N2, AddC);
- return DAG.getNode(ISD::UMIN, DL, VT, Add, N2);
- }
- if (CC == ISD::SETULT && Cond0 == N1 &&
- sd_match(N2, m_Add(m_Specific(N1), m_SpecificInt(-C)))) {
- // Ditto.
- SDValue AddC = DAG.getConstant(-C, DL, VT);
- SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, AddC);
- return DAG.getNode(ISD::UMIN, DL, VT, N1, Add);
- }
- }
+ if (SDValue UMin = foldSelectToUMin(Cond0, Cond1, N1, N2, CC, DL))
+ return UMin;
}
if (!VT.isVector())
@@ -13417,6 +13428,11 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) {
}
}
}
+
+ // (vselect (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
+ // (vselect (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
+ if (SDValue UMin = foldSelectToUMin(LHS, RHS, N1, N2, CC, DL))
+ return UMin;
}
if (SimplifySelectOps(N, N1, N2))
@@ -13490,11 +13506,9 @@ SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
// Fold to a simpler select_cc
if (SCC.getOpcode() == ISD::SETCC) {
- SDValue SelectOp =
- DAG.getNode(ISD::SELECT_CC, DL, N2.getValueType(), SCC.getOperand(0),
- SCC.getOperand(1), N2, N3, SCC.getOperand(2));
- SelectOp->setFlags(SCC->getFlags());
- return SelectOp;
+ return DAG.getNode(ISD::SELECT_CC, DL, N2.getValueType(),
+ SCC.getOperand(0), SCC.getOperand(1), N2, N3,
+ SCC.getOperand(2), SCC->getFlags());
}
}
@@ -15731,7 +15745,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
// fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x)
if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
if (SDValue(GN0, 0).hasOneUse() && ExtVT == GN0->getMemoryVT() &&
- TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+ TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
@@ -16758,12 +16772,8 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
if (DAG.isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ false,
/*Depth*/ 1))
continue;
- bool HadMaybePoisonOperands = !MaybePoisonOperands.empty();
- bool IsNewMaybePoisonOperand = MaybePoisonOperands.insert(Op).second;
- if (IsNewMaybePoisonOperand)
+ if (MaybePoisonOperands.insert(Op).second)
MaybePoisonOperandNumbers.push_back(OpNo);
- if (!HadMaybePoisonOperands)
- continue;
}
// NOTE: the whole op may be not guaranteed to not be undef or poison because
// it could create undef or poison due to it's poison-generating flags.
@@ -18713,6 +18723,12 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
return FTrunc;
+ // fold (sint_to_fp (trunc nsw x)) -> (sint_to_fp x)
+ if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoSignedWrap() &&
+ TLI.isTypeDesirableForOp(ISD::SINT_TO_FP,
+ N0.getOperand(0).getValueType()))
+ return DAG.getNode(ISD::SINT_TO_FP, DL, VT, N0.getOperand(0));
+
return SDValue();
}
@@ -18750,6 +18766,12 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
return FTrunc;
+ // fold (uint_to_fp (trunc nuw x)) -> (uint_to_fp x)
+ if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoUnsignedWrap() &&
+ TLI.isTypeDesirableForOp(ISD::UINT_TO_FP,
+ N0.getOperand(0).getValueType()))
+ return DAG.getNode(ISD::UINT_TO_FP, DL, VT, N0.getOperand(0));
+
return SDValue();
}
@@ -28194,14 +28216,16 @@ SDValue DAGCombiner::SimplifyVCastOp(SDNode *N, const SDLoc &DL) {
TLI.preferScalarizeSplat(N)) {
EVT SrcVT = N0.getValueType();
EVT SrcEltVT = SrcVT.getVectorElementType();
- SDValue IndexC = DAG.getVectorIdxConstant(Index0, DL);
- SDValue Elt =
- DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcEltVT, Src0, IndexC);
- SDValue ScalarBO = DAG.getNode(Opcode, DL, EltVT, Elt, N->getFlags());
- if (VT.isScalableVector())
- return DAG.getSplatVector(VT, DL, ScalarBO);
- SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), ScalarBO);
- return DAG.getBuildVector(VT, DL, Ops);
+ if (!LegalTypes || TLI.isTypeLegal(SrcEltVT)) {
+ SDValue IndexC = DAG.getVectorIdxConstant(Index0, DL);
+ SDValue Elt =
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcEltVT, Src0, IndexC);
+ SDValue ScalarBO = DAG.getNode(Opcode, DL, EltVT, Elt, N->getFlags());
+ if (VT.isScalableVector())
+ return DAG.getSplatVector(VT, DL, ScalarBO);
+ SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), ScalarBO);
+ return DAG.getBuildVector(VT, DL, Ops);
+ }
}
return SDValue();
@@ -28343,10 +28367,8 @@ SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1,
SCC.getOperand(0), SCC.getOperand(1),
SCC.getOperand(4), Flags);
AddToWorklist(SETCC.getNode());
- SDValue SelectNode = DAG.getSelect(SDLoc(SCC), SCC.getValueType(), SETCC,
- SCC.getOperand(2), SCC.getOperand(3));
- SelectNode->setFlags(Flags);
- return SelectNode;
+ return DAG.getSelect(SDLoc(SCC), SCC.getValueType(), SETCC,
+ SCC.getOperand(2), SCC.getOperand(3), Flags);
}
return SCC;
@@ -28647,9 +28669,9 @@ SDValue DAGCombiner::foldSelectOfBinops(SDNode *N) {
SDValue N10 = N1.getOperand(0);
SDValue N20 = N2.getOperand(0);
SDValue NewSel = DAG.getSelect(DL, N10.getValueType(), N0, N10, N20);
- SDValue NewBinOp = DAG.getNode(BinOpc, DL, OpVTs, NewSel, N1.getOperand(1));
- NewBinOp->setFlags(N1->getFlags());
- NewBinOp->intersectFlagsWith(N2->getFlags());
+ SDNodeFlags Flags = N1->getFlags() & N2->getFlags();
+ SDValue NewBinOp =
+ DAG.getNode(BinOpc, DL, OpVTs, {NewSel, N1.getOperand(1)}, Flags);
return SDValue(NewBinOp.getNode(), N1.getResNo());
}
@@ -28661,10 +28683,9 @@ SDValue DAGCombiner::foldSelectOfBinops(SDNode *N) {
// Second op VT might be different (e.g. shift amount type)
if (N11.getValueType() == N21.getValueType()) {
SDValue NewSel = DAG.getSelect(DL, N11.getValueType(), N0, N11, N21);
+ SDNodeFlags Flags = N1->getFlags() & N2->getFlags();
SDValue NewBinOp =
- DAG.getNode(BinOpc, DL, OpVTs, N1.getOperand(0), NewSel);
- NewBinOp->setFlags(N1->getFlags());
- NewBinOp->intersectFlagsWith(N2->getFlags());
+ DAG.getNode(BinOpc, DL, OpVTs, {N1.getOperand(0), NewSel}, Flags);
return SDValue(NewBinOp.getNode(), N1.getResNo());
}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 03d3e8eab35d..85efb1bd8aed 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -784,7 +784,7 @@ MachineInstr *
InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
VRBaseMapType &VRBaseMap) {
MDNode *Var = SD->getVariable();
- const DIExpression *Expr = (DIExpression *)SD->getExpression();
+ const DIExpression *Expr = SD->getExpression();
DebugLoc DL = SD->getDebugLoc();
const MCInstrDesc &RefII = TII->get(TargetOpcode::DBG_INSTR_REF);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 528136a55f14..7266940c94bf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -140,12 +140,19 @@ private:
RTLIB::Libcall Call_F128,
RTLIB::Libcall Call_PPCF128,
SmallVectorImpl<SDValue> &Results);
- SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
- RTLIB::Libcall Call_I8,
- RTLIB::Libcall Call_I16,
- RTLIB::Libcall Call_I32,
- RTLIB::Libcall Call_I64,
- RTLIB::Libcall Call_I128);
+
+ void
+ ExpandFastFPLibCall(SDNode *Node, bool IsFast,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
+ SmallVectorImpl<SDValue> &Results);
+
+ SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
+ RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
+ RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
void ExpandArgFPLibCall(SDNode *Node,
RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
@@ -2228,6 +2235,37 @@ void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
ExpandFPLibCall(Node, LC, Results);
}
+void SelectionDAGLegalize::ExpandFastFPLibCall(
+ SDNode *Node, bool IsFast,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
+ std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
+ SmallVectorImpl<SDValue> &Results) {
+
+ EVT VT = Node->getSimpleValueType(0);
+
+ RTLIB::Libcall LC;
+
+ // FIXME: Probably should define fast to respect nan/inf and only be
+ // approximate functions.
+
+ if (IsFast) {
+ LC = RTLIB::getFPLibCall(VT, Call_F32.first, Call_F64.first, Call_F80.first,
+ Call_F128.first, Call_PPCF128.first);
+ }
+
+ if (!IsFast || TLI.getLibcallImpl(LC) == RTLIB::Unsupported) {
+ // Fall back if we don't have a fast implementation.
+ LC = RTLIB::getFPLibCall(VT, Call_F32.second, Call_F64.second,
+ Call_F80.second, Call_F128.second,
+ Call_PPCF128.second);
+ }
+
+ ExpandFPLibCall(Node, LC, Results);
+}
+
SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
RTLIB::Libcall Call_I8,
RTLIB::Libcall Call_I16,
@@ -4514,6 +4552,18 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
return true;
}
+/// Return if we can use the FAST_* variant of a math libcall for the node.
+/// FIXME: This is just guessing, we probably should have unique specific sets
+/// flags required per libcall.
+static bool canUseFastMathLibcall(const SDNode *Node) {
+ // FIXME: Probably should define fast to respect nan/inf and only be
+ // approximate functions.
+
+ SDNodeFlags Flags = Node->getFlags();
+ return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
+ Flags.hasNoInfs() && Flags.hasNoSignedZeros();
+}
+
void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
SmallVector<SDValue, 8> Results;
@@ -4634,11 +4684,18 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
RTLIB::FMAXIMUM_NUM_PPCF128, Results);
break;
case ISD::FSQRT:
- case ISD::STRICT_FSQRT:
- ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
- RTLIB::SQRT_F80, RTLIB::SQRT_F128,
- RTLIB::SQRT_PPCF128, Results);
+ case ISD::STRICT_FSQRT: {
+ // FIXME: Probably should define fast to respect nan/inf and only be
+ // approximate functions.
+ ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),
+ {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
+ {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
+ {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
+ {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
+ {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
+ Results);
break;
+ }
case ISD::FCBRT:
ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
RTLIB::CBRT_F80, RTLIB::CBRT_F128,
@@ -4875,11 +4932,15 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
RTLIB::LLRINT_PPCF128, Results);
break;
case ISD::FDIV:
- case ISD::STRICT_FDIV:
- ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
- RTLIB::DIV_F80, RTLIB::DIV_F128,
- RTLIB::DIV_PPCF128, Results);
+ case ISD::STRICT_FDIV: {
+ ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),
+ {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
+ {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
+ {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
+ {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
+ {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
break;
+ }
case ISD::FREM:
case ISD::STRICT_FREM:
ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
@@ -4893,17 +4954,25 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
RTLIB::FMA_PPCF128, Results);
break;
case ISD::FADD:
- case ISD::STRICT_FADD:
- ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
- RTLIB::ADD_F80, RTLIB::ADD_F128,
- RTLIB::ADD_PPCF128, Results);
+ case ISD::STRICT_FADD: {
+ ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),
+ {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
+ {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
+ {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
+ {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
+ {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
break;
+ }
case ISD::FMUL:
- case ISD::STRICT_FMUL:
- ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
- RTLIB::MUL_F80, RTLIB::MUL_F128,
- RTLIB::MUL_PPCF128, Results);
+ case ISD::STRICT_FMUL: {
+ ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),
+ {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
+ {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
+ {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
+ {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
+ {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
break;
+ }
case ISD::FP16_TO_FP:
if (Node->getValueType(0) == MVT::f32) {
Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first);
@@ -5076,11 +5145,15 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
break;
}
case ISD::FSUB:
- case ISD::STRICT_FSUB:
- ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
- RTLIB::SUB_F80, RTLIB::SUB_F128,
- RTLIB::SUB_PPCF128, Results);
+ case ISD::STRICT_FSUB: {
+ ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),
+ {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
+ {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
+ {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
+ {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
+ {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
break;
+ }
case ISD::SREM:
Results.push_back(ExpandIntLibCall(Node, true,
RTLIB::SREM_I8,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 58be4fb7e833..245811587e3b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5544,6 +5544,8 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
case ISD::USUBSAT:
case ISD::MULHU:
case ISD::MULHS:
+ case ISD::ABDU:
+ case ISD::ABDS:
case ISD::SMIN:
case ISD::SMAX:
case ISD::UMIN:
@@ -5569,6 +5571,12 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
case ISD::BUILD_VECTOR:
case ISD::BUILD_PAIR:
case ISD::SPLAT_VECTOR:
+ case ISD::FABS:
+ return false;
+
+ case ISD::ABS:
+ // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
+ // Different to Intrinsic::abs.
return false;
case ISD::ADDC:
@@ -5620,6 +5628,7 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
case ISD::FMUL:
case ISD::FDIV:
case ISD::FREM:
+ case ISD::FCOPYSIGN:
// No poison except from flags (which is handled above)
return false;
@@ -6744,7 +6753,9 @@ SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
return SDValue();
int64_t Offset = C2->getSExtValue();
switch (Opcode) {
- case ISD::ADD: break;
+ case ISD::ADD:
+ case ISD::PTRADD:
+ break;
case ISD::SUB: Offset = -uint64_t(Offset); break;
default: return SDValue();
}
@@ -13866,6 +13877,8 @@ void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
return;
}
+ const SDNode *EntrySDN = getEntryNode().getNode();
+
// We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
// through the replacement of From with To. Otherwise, replacements of a node
// (From) with more complex nodes (To and its operands) may result in lost
@@ -13897,9 +13910,14 @@ void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
return true;
if (!Visited.insert(N).second)
return true;
- if (getEntryNode().getNode() == N)
+ if (EntrySDN == N)
return false;
for (const SDValue &Op : N->op_values()) {
+ if (N == To && Op.getNode() == EntrySDN) {
+ // Special case: New node's operand is the entry node; just need to
+ // copy extra info to new node.
+ break;
+ }
if (!Self(Self, Op.getNode()))
return false;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index ecd1ff87e7fb..01e53123ea7e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -18,6 +18,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/AliasAnalysis.h"
@@ -842,6 +843,23 @@ static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
}
}
+static void failForInvalidBundles(const CallBase &I, StringRef Name,
+ ArrayRef<uint32_t> AllowedBundles) {
+ if (I.hasOperandBundlesOtherThan(AllowedBundles)) {
+ ListSeparator LS;
+ std::string Error;
+ raw_string_ostream OS(Error);
+ for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {
+ OperandBundleUse U = I.getOperandBundleAt(i);
+ if (!is_contained(AllowedBundles, U.getTagID()))
+ OS << LS << U.getTagName();
+ }
+ reportFatalUsageError(
+ Twine("cannot lower ", Name)
+ .concat(Twine(" with arbitrary operand bundles: ", Error)));
+ }
+}
+
RegsForValue::RegsForValue(const SmallVector<Register, 4> &regs, MVT regvt,
EVT valuevt, std::optional<CallingConv::ID> CC)
: ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
@@ -3351,13 +3369,12 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
// Deopt and ptrauth bundles are lowered in helper functions, and we don't
// have to do anything here to lower funclet bundles.
- if (I.hasOperandBundlesOtherThan(
- {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
- LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
- LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
- LLVMContext::OB_clang_arc_attachedcall}))
- reportFatalUsageError(
- "cannot lower invokes with arbitrary operand bundles!");
+ failForInvalidBundles(I, "invokes",
+ {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
+ LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
+ LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
+ LLVMContext::OB_clang_arc_attachedcall,
+ LLVMContext::OB_kcfi});
const Value *Callee(I.getCalledOperand());
const Function *Fn = dyn_cast<Function>(Callee);
@@ -3457,10 +3474,8 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
// have to do anything here to lower funclet bundles.
- if (I.hasOperandBundlesOtherThan(
- {LLVMContext::OB_deopt, LLVMContext::OB_funclet}))
- reportFatalUsageError(
- "cannot lower callbrs with arbitrary operand bundles!");
+ failForInvalidBundles(I, "callbrs",
+ {LLVMContext::OB_deopt, LLVMContext::OB_funclet});
assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
visitInlineAsm(I);
@@ -9568,12 +9583,12 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
// have to do anything here to lower funclet bundles.
// CFGuardTarget bundles are lowered in LowerCallTo.
- if (I.hasOperandBundlesOtherThan(
- {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
- LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
- LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
- LLVMContext::OB_convergencectrl}))
- reportFatalUsageError("cannot lower calls with arbitrary operand bundles!");
+ failForInvalidBundles(
+ I, "calls",
+ {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
+ LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
+ LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
+ LLVMContext::OB_convergencectrl});
SDValue Callee = getValue(I.getCalledOperand());
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-01 02:22:18 +0000