diff options
19 files changed, 248 insertions, 57 deletions
diff --git a/llvm/include/llvm/CodeGen/SDNodeInfo.h b/llvm/include/llvm/CodeGen/SDNodeInfo.h index ba6c343ee183..41154b1d5044 100644 --- a/llvm/include/llvm/CodeGen/SDNodeInfo.h +++ b/llvm/include/llvm/CodeGen/SDNodeInfo.h @@ -26,6 +26,7 @@ enum SDNP { SDNPOptInGlue, SDNPMemOperand, SDNPVariadic, + SDNPMayHaveChain }; enum SDTC : uint8_t { diff --git a/llvm/include/llvm/CodeGen/SDNodeProperties.td b/llvm/include/llvm/CodeGen/SDNodeProperties.td index d32904283a11..640cdd08b9b7 100644 --- a/llvm/include/llvm/CodeGen/SDNodeProperties.td +++ b/llvm/include/llvm/CodeGen/SDNodeProperties.td @@ -29,3 +29,4 @@ def SDNPMayLoad : SDNodeProperty; // May read memory, sets 'mayLoad'. def SDNPSideEffect : SDNodeProperty; // Sets 'HasUnmodelledSideEffects'. def SDNPMemOperand : SDNodeProperty; // Touches memory, has assoc MemOperand def SDNPVariadic : SDNodeProperty; // Node has variable arguments. +def SDNPMayHaveChain: SDNodeProperty; // Optionally has chain operand/result. diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index 5241a51dd8cd..c57f18368b2e 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -151,6 +151,7 @@ public: OPC_RecordChild6, OPC_RecordChild7, OPC_RecordMemRef, + OPC_RecordOptionalChain, OPC_CaptureGlueInput, OPC_MoveChild, OPC_MoveChild0, @@ -493,7 +494,8 @@ private: private: void DoInstructionSelection(); SDNode *MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList, - ArrayRef<SDValue> Ops, unsigned EmitNodeInfo); + ArrayRef<SDValue> Ops, unsigned EmitNodeInfo, + bool OptionalChain); /// Prepares the landing pad to take incoming values or do other EH /// personality specific tasks. Returns true if the block should be diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 116911699ab9..019747136df5 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -717,6 +717,7 @@ public: case ISD::STRICT_FP_TO_FP16: case ISD::STRICT_BF16_TO_FP: case ISD::STRICT_FP_TO_BF16: +#define FP_OPERATION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ case ISD::STRICT_##DAGN: #include "llvm/IR/ConstrainedOps.def" @@ -724,6 +725,23 @@ public: } } + /// Test if this node is a floating-point operation which can exist in two + /// forms, - with chain or without it. + bool isFPOperation() const { + switch (NodeType) { + default: + return false; +#define FP_OPERATION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) case ISD::DAGN: +#include "llvm/IR/ConstrainedOps.def" + return true; + } + } + + /// Test if this node has an input chain. + bool hasChain() const { + return NumOperands > 0 && OperandList[0].getValueType() == MVT::Other; + } + /// Test if this node is an assert operation. bool isAssert() const { switch (NodeType) { diff --git a/llvm/lib/CodeGen/SelectionDAG/SDNodeInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/SDNodeInfo.cpp index e3f6c98a9a90..25f147b3b1ba 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SDNodeInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SDNodeInfo.cpp @@ -47,6 +47,16 @@ void SDNodeInfo::verifyNode(const SelectionDAG &DAG, const SDNode *N) const { bool HasInGlue = Desc.hasProperty(SDNPInGlue); bool HasOptInGlue = Desc.hasProperty(SDNPOptInGlue); bool IsVariadic = Desc.hasProperty(SDNPVariadic); + bool MayHaveChain = Desc.hasProperty(SDNPMayHaveChain); + + if (HasChain && MayHaveChain) + reportNodeError( + DAG, N, "Flags 'HasChain' and 'MayHaveChain' cannot be both specified"); + + if (MayHaveChain && N->getNumOperands() > 0 && + N->getOperand(0).getValueType() == MVT::Other) { + HasChain = true; + } unsigned ActualNumResults = N->getNumValues(); unsigned ExpectedNumResults = Desc.NumResults + HasChain + HasOutGlue; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 6c11c5b815b6..7db90800792d 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -2836,9 +2836,9 @@ HandleMergeInputChains(SmallVectorImpl<SDNode*> &ChainNodesMatched, } /// MorphNode - Handle morphing a node in place for the selector. -SDNode *SelectionDAGISel:: -MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList, - ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) { +SDNode *SelectionDAGISel::MorphNode(SDNode *Node, unsigned TargetOpc, + SDVTList VTList, ArrayRef<SDValue> Ops, + unsigned EmitNodeInfo, bool OptionalChain) { // It is possible we're using MorphNodeTo to replace a node with no // normal results with one that has a normal result (or we could be // adding a chain) and the input could have glue and chains as well. @@ -2880,7 +2880,7 @@ MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList, --ResNumResults; // Move the chain reference if needed. - if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 && + if ((EmitNodeInfo & OPFL_Chain || OptionalChain) && OldChainResultNo != -1 && static_cast<unsigned>(OldChainResultNo) != ResNumResults - 1) ReplaceUses(SDValue(Node, OldChainResultNo), SDValue(Res, ResNumResults - 1)); @@ -3385,6 +3385,12 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // update the chain results when the pattern is complete. SmallVector<SDNode*, 3> ChainNodesMatched; + bool HasNodesWithOptionalChain = false; + + // List of pattern matches nodes that may have input/output chains and + // actually have them. + SmallVector<SDNode *, 8> OptionalChainNodes; + LLVM_DEBUG(dbgs() << "ISEL: Starting pattern match\n"); // Determine where to start the interpreter. Normally we start at opcode #0, @@ -3505,7 +3511,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case OPC_RecordChild2: case OPC_RecordChild3: case OPC_RecordChild4: case OPC_RecordChild5: case OPC_RecordChild6: case OPC_RecordChild7: { - unsigned ChildNo = Opcode-OPC_RecordChild0; + unsigned ChildNo = + Opcode - OPC_RecordChild0 + !OptionalChainNodes.empty(); if (ChildNo >= N.getNumOperands()) break; // Match fails if out of range child #. @@ -3523,6 +3530,17 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, continue; + case OPC_RecordOptionalChain: + HasNodesWithOptionalChain = true; + // If the current node has input chain, record it. + if (N->getNumOperands() != 0) { + SDValue FirstOperand = N->getOperand(0); + if (FirstOperand.getValueType() == MVT::Other) { + OptionalChainNodes.push_back(N.getNode()); + } + } + continue; + case OPC_CaptureGlueInput: // If the current node has an input glue, capture it in InputGlue. if (N->getNumOperands() != 0 && @@ -3531,7 +3549,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, continue; case OPC_MoveChild: { - unsigned ChildNo = MatcherTable[MatcherIndex++]; + unsigned ChildNo = + MatcherTable[MatcherIndex++] + !OptionalChainNodes.empty(); if (ChildNo >= N.getNumOperands()) break; // Match fails if out of range child #. N = N.getOperand(ChildNo); @@ -3543,7 +3562,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case OPC_MoveChild2: case OPC_MoveChild3: case OPC_MoveChild4: case OPC_MoveChild5: case OPC_MoveChild6: case OPC_MoveChild7: { - unsigned ChildNo = Opcode-OPC_MoveChild0; + unsigned ChildNo = Opcode - OPC_MoveChild0 + !OptionalChainNodes.empty(); if (ChildNo >= N.getNumOperands()) break; // Match fails if out of range child #. N = N.getOperand(ChildNo); @@ -3568,6 +3587,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, unsigned SiblingNo = Opcode == OPC_MoveSibling ? MatcherTable[MatcherIndex++] : Opcode - OPC_MoveSibling0; + SiblingNo += !OptionalChainNodes.empty(); if (SiblingNo >= N.getNumOperands()) break; // Match fails if out of range sibling #. N = N.getOperand(SiblingNo); @@ -3998,11 +4018,16 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // Ignore these because the newly token factored chain should not refer to // the old nodes. unsigned NumChains = MatcherTable[MatcherIndex++]; - assert(NumChains != 0 && "Can't TF zero chains"); + assert((NumChains != 0 || HasNodesWithOptionalChain) && + "Can't TF zero chains"); assert(ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"); + if (NumChains == 0 && HasNodesWithOptionalChain && + OptionalChainNodes.empty()) + continue; + // Read all of the chained nodes. for (unsigned i = 0; i != NumChains; ++i) { unsigned RecNo = MatcherTable[MatcherIndex++]; @@ -4020,6 +4045,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, } } + ChainNodesMatched.append(OptionalChainNodes); + // If the inner loop broke out, the match fails. if (ChainNodesMatched.empty()) break; @@ -4164,7 +4191,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, VTs.push_back(VT); } - if (EmitNodeInfo & OPFL_Chain) + if (EmitNodeInfo & OPFL_Chain || !OptionalChainNodes.empty()) VTs.push_back(MVT::Other); if (EmitNodeInfo & OPFL_GlueOutput) VTs.push_back(MVT::Glue); @@ -4186,6 +4213,10 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, unsigned RecNo = MatcherTable[MatcherIndex++]; if (RecNo & 128) RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex); + if (HasNodesWithOptionalChain) { + assert(RecNo > 0); + RecNo--; + } assert(RecNo < RecordedNodes.size() && "Invalid EmitNode"); Ops.push_back(RecordedNodes[RecNo].first); @@ -4209,7 +4240,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, } // If this has chain/glue inputs, add them. - if (EmitNodeInfo & OPFL_Chain) + if (EmitNodeInfo & OPFL_Chain || !OptionalChainNodes.empty()) Ops.push_back(InputChain); if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr) Ops.push_back(InputGlue); @@ -4219,7 +4250,12 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // We need to perform this check before potentially modifying one of the // nodes via MorphNode. bool MayRaiseFPException = - llvm::any_of(ChainNodesMatched, [this](SDNode *N) { + llvm::any_of(ChainNodesMatched, + [this](SDNode *N) { + return mayRaiseFPException(N) && + !N->getFlags().hasNoFPExcept(); + }) || + llvm::any_of(OptionalChainNodes, [this](SDNode *N) { return mayRaiseFPException(N) && !N->getFlags().hasNoFPExcept(); }); @@ -4251,8 +4287,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, "Chain node replaced during MorphNode"); llvm::erase(Chain, N); }); - Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList, - Ops, EmitNodeInfo)); + Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList, Ops, + EmitNodeInfo, + !OptionalChainNodes.empty())); } // Set the NoFPExcept flag when no original matched node could @@ -4264,9 +4301,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // chain and glue. if (EmitNodeInfo & OPFL_GlueOutput) { InputGlue = SDValue(Res, VTs.size()-1); - if (EmitNodeInfo & OPFL_Chain) + if (EmitNodeInfo & OPFL_Chain || !OptionalChainNodes.empty()) InputChain = SDValue(Res, VTs.size()-2); - } else if (EmitNodeInfo & OPFL_Chain) + } else if (EmitNodeInfo & OPFL_Chain || !OptionalChainNodes.empty()) InputChain = SDValue(Res, VTs.size()-1); // If the OPFL_MemRefs glue is set on this node, slap all of the @@ -4430,7 +4467,7 @@ bool SelectionDAGISel::mayRaiseFPException(SDNode *N) const { const SelectionDAGTargetInfo &TSI = CurDAG->getSelectionDAGInfo(); return TSI.mayRaiseFPException(N->getOpcode()); } - return N->isStrictFPOpcode(); + return N->isStrictFPOpcode() || N->isFPOperation(); } bool SelectionDAGISel::isOrEquivalentToAdd(const SDNode *N) const { diff --git a/llvm/test/TableGen/SDNodeInfoEmitter/advanced.td b/llvm/test/TableGen/SDNodeInfoEmitter/advanced.td index d7eeaba9d855..b76f98f38613 100644 --- a/llvm/test/TableGen/SDNodeInfoEmitter/advanced.td +++ b/llvm/test/TableGen/SDNodeInfoEmitter/advanced.td @@ -46,15 +46,22 @@ def my_node_3 : SDNode< SDNPOutGlue, SDNPInGlue, SDNPOptInGlue] >; +def my_node_4 : SDNode< + "MyTargetISD::NODE_4", + SDTypeProfile<1, 1, [SDTCisVT<0, f32>, SDTCisVT<1, f32>]>, + [SDNPMayHaveChain] +>; + // CHECK: namespace llvm::MyTargetISD { // CHECK-EMPTY: // CHECK-NEXT: enum GenNodeType : unsigned { // CHECK-NEXT: NODE_1 = ISD::BUILTIN_OP_END, // CHECK-NEXT: NODE_2, // CHECK-NEXT: NODE_3, +// CHECK-NEXT: NODE_4 // CHECK-NEXT: }; // CHECK-EMPTY: -// CHECK-NEXT: static constexpr unsigned GENERATED_OPCODE_END = NODE_3 + 1; +// CHECK-NEXT: static constexpr unsigned GENERATED_OPCODE_END = NODE_4 + 1; // CHECK-EMPTY: // CHECK-NEXT: } // namespace llvm::MyTargetISD @@ -63,6 +70,7 @@ def my_node_3 : SDNode< // CHECK-NEXT: "MyTargetISD::NODE_1\0" // CHECK-NEXT: "MyTargetISD::NODE_2\0" // CHECK-NEXT: "MyTargetISD::NODE_3\0" +// CHECK-NEXT: "MyTargetISD::NODE_4\0" // CHECK-NEXT: ; // CHECK: static const SDTypeConstraint MyTargetSDTypeConstraints[] = { @@ -81,14 +89,16 @@ def my_node_3 : SDNode< // CHECK-SAME: {SDTCisInt, 2, 0, MVT::INVALID_SIMPLE_VALUE_TYPE}, // CHECK-SAME: {SDTCisPtrTy, 1, 0, MVT::INVALID_SIMPLE_VALUE_TYPE}, // CHECK-SAME: {SDTCisVT, 0, 0, MVT::i1}, +// CHECK-NEXT: /* 15 */ {SDTCisVT, 1, 0, MVT::f32}, {SDTCisVT, 0, 0, MVT::f32}, // CHECK-NEXT: }; // CHECK-EMPTY: // CHECK-NEXT: static const SDNodeDesc MyTargetSDNodeDescs[] = { // CHECK-NEXT: {1, 1, 0|1<<SDNPHasChain, 0, 0, 1, 0, 2}, // NODE_1 // CHECK-NEXT: {3, 1, 0|1<<SDNPVariadic|1<<SDNPMemOperand, 0, 42, 21, 11, 4}, // NODE_2 // CHECK-NEXT: {2, -1, 0|1<<SDNPHasChain|1<<SDNPOutGlue|1<<SDNPInGlue|1<<SDNPOptInGlue, 0|1<<SDNFIsStrictFP, 24, 41, 2, 13}, // NODE_3 +// CHECK-NEXT: {1, 1, 0|1<<SDNPMayHaveChain, 0, 0, 61, 15, 2}, // NODE_4 // CHECK-NEXT: }; // CHECK-EMPTY: // CHECK-NEXT: static const SDNodeInfo MyTargetGenSDNodeInfo( -// CHECK-NEXT: /*NumOpcodes=*/3, MyTargetSDNodeDescs, +// CHECK-NEXT: /*NumOpcodes=*/4, MyTargetSDNodeDescs, // CHECK-NEXT: MyTargetSDNodeNames, MyTargetSDTypeConstraints); diff --git a/llvm/test/TableGen/optional-chain.td b/llvm/test/TableGen/optional-chain.td new file mode 100644 index 000000000000..9905a8f585a2 --- /dev/null +++ b/llvm/test/TableGen/optional-chain.td @@ -0,0 +1,46 @@ +// RUN: llvm-tblgen -gen-dag-isel -I %p/../../include %s 2>&1 | FileCheck %s + +include "llvm/Target/Target.td" + +def TestTargetInstrInfo : InstrInfo; + + +def TestTarget : Target { + let InstructionSet = TestTargetInstrInfo; +} + +def R0 : Register<"r0"> { let Namespace = "MyTarget"; } +def GPR : RegisterClass<"MyTarget", [i32, f32], 32, (add R0)>; + +def a_nearbyint : SDNode< + "MyTargetISD::A_NEARBYINT", + SDTypeProfile<1, 1, [SDTCisFP<0>, SDTCisFP<1>]>, + [SDNPMayHaveChain] +>; + +def I_NEARBYINT : Instruction { + let OutOperandList = (outs GPR:$out); + let InOperandList = (ins GPR:$x); +} + +def : Pat<(a_nearbyint GPR:$x), (I_NEARBYINT GPR:$x)>; +def : Pat<(a_nearbyint (a_nearbyint GPR:$x)), (I_NEARBYINT GPR:$x)>; + +// CHECK-LABEL: OPC_CheckOpcode, TARGET_VAL(MyTargetISD::A_NEARBYINT), +// CHECK-NEXT: OPC_RecordOptionalChain, +// CHECK-NEXT: OPC_Scope, 17 +// CHECK-NEXT: OPC_MoveChild0, +// CHECK-NEXT: OPC_CheckOpcode, TARGET_VAL(MyTargetISD::A_NEARBYINT), +// CHECK-NEXT: OPC_RecordOptionalChain, +// CHECK-NEXT: OPC_CheckFoldableChainNode, +// CHECK-NEXT: OPC_RecordChild0, +// CHECK-NEXT: OPC_MoveParent, +// CHECK-NEXT: OPC_EmitMergeInputChains, 0, +// CHECK-NEXT: OPC_MorphNodeTo1, TARGET_VAL(::I_NEARBYINT), 0, +// CHECK-NEXT: /*MVT::f32*/12, 1/*#Ops*/, 2, + +// CHECK: /*Scope*/ 10, +// CHECK-NEXT: OPC_RecordChild0, +// CHECK-NEXT: OPC_EmitMergeInputChains, 0, +// CHECK-NEXT: OPC_MorphNodeTo1, TARGET_VAL(::I_NEARBYINT), 0, +// CHECK-NEXT: /*MVT::f32*/12, 1/*#Ops*/, 1, diff --git a/llvm/utils/TableGen/Basic/SDNodeProperties.cpp b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp index 46babbbc4194..9a37db6d19b1 100644 --- a/llvm/utils/TableGen/Basic/SDNodeProperties.cpp +++ b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp @@ -28,6 +28,7 @@ unsigned llvm::parseSDPatternOperatorProperties(const Record *R) { .Case("SDNPSideEffect", SDNPSideEffect) .Case("SDNPMemOperand", SDNPMemOperand) .Case("SDNPVariadic", SDNPVariadic) + .Case("SDNPMayHaveChain", SDNPMayHaveChain) .Default(-1u); if (Offset != -1u) Properties |= 1 << Offset; diff --git a/llvm/utils/TableGen/Basic/SDNodeProperties.h b/llvm/utils/TableGen/Basic/SDNodeProperties.h index 97813067341f..516f45e118b3 100644 --- a/llvm/utils/TableGen/Basic/SDNodeProperties.h +++ b/llvm/utils/TableGen/Basic/SDNodeProperties.h @@ -28,6 +28,7 @@ enum SDNP { SDNPSideEffect, SDNPMemOperand, SDNPVariadic, + SDNPMayHaveChain }; unsigned parseSDPatternOperatorProperties(const Record *R); diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp index 8076ce2486f5..9d82a271a465 100644 --- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp +++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp @@ -3681,6 +3681,7 @@ public: bool isBitcast = false; bool isVariadic = false; bool hasChain = false; + bool mayHaveChain = false; InstAnalyzer(const CodeGenDAGPatterns &cdp) : CDP(cdp) {} @@ -3744,6 +3745,8 @@ public: isVariadic = true; if (N.NodeHasProperty(SDNPHasChain, CDP)) hasChain = true; + if (N.NodeHasProperty(SDNPMayHaveChain, CDP)) + mayHaveChain = true; if (const CodeGenIntrinsic *IntInfo = N.getIntrinsicInfo(CDP)) { ModRefInfo MR = IntInfo->ME.getModRef(); @@ -3812,6 +3815,7 @@ static bool InferFromPattern(CodeGenInstruction &InstInfo, InstInfo.isBitcast |= PatInfo.isBitcast; InstInfo.hasChain |= PatInfo.hasChain; InstInfo.hasChain_Inferred = true; + InstInfo.mayHaveChain |= PatInfo.mayHaveChain; } // Don't infer isVariadic. This flag means something different on SDNodes and diff --git a/llvm/utils/TableGen/Common/CodeGenInstruction.h b/llvm/utils/TableGen/Common/CodeGenInstruction.h index ed0bfa7098eb..870afb22d72c 100644 --- a/llvm/utils/TableGen/Common/CodeGenInstruction.h +++ b/llvm/utils/TableGen/Common/CodeGenInstruction.h @@ -268,6 +268,7 @@ public: bool hasChain_Inferred : 1; bool variadicOpsAreDefs : 1; bool isAuthenticated : 1; + bool mayHaveChain : 1; std::string DeprecatedReason; bool HasComplexDeprecationPredicate; diff --git a/llvm/utils/TableGen/Common/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp index 3db0d07eec88..c11304d971a1 100644 --- a/llvm/utils/TableGen/Common/CodeGenTarget.cpp +++ b/llvm/utils/TableGen/Common/CodeGenTarget.cpp @@ -406,6 +406,8 @@ ComplexPattern::ComplexPattern(const Record *R) { Properties |= 1 << SDNPMemOperand; } else if (Prop->getName() == "SDNPVariadic") { Properties |= 1 << SDNPVariadic; + } else if (Prop->getName() == "SDNPMayHaveChain") { + Properties |= 1 << SDNPMayHaveChain; } else { PrintFatalError(R->getLoc(), "Unsupported SD Node property '" + Prop->getName() + diff --git a/llvm/utils/TableGen/Common/DAGISelMatcher.cpp b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp index 4fdb386bf45e..30d83c726593 100644 --- a/llvm/utils/TableGen/Common/DAGISelMatcher.cpp +++ b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp @@ -137,6 +137,11 @@ void CaptureGlueInputMatcher::printImpl(raw_ostream &OS, indent Indent) const { OS << Indent << "CaptureGlueInput\n"; } +void RecordOptionalChainMatcher::printImpl(raw_ostream &OS, + indent Indent) const { + OS << Indent << "CaptureOptionalChain\n"; +} + void MoveChildMatcher::printImpl(raw_ostream &OS, indent Indent) const { OS << Indent << "MoveChild " << ChildNo << '\n'; } diff --git a/llvm/utils/TableGen/Common/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h index f87de757f4f8..e20af40a5e5e 100644 --- a/llvm/utils/TableGen/Common/DAGISelMatcher.h +++ b/llvm/utils/TableGen/Common/DAGISelMatcher.h @@ -53,14 +53,15 @@ class Matcher { public: enum KindTy { // Matcher state manipulation. - Scope, // Push a checking scope. - RecordNode, // Record the current node. - RecordChild, // Record a child of the current node. - RecordMemRef, // Record the memref in the current node. - CaptureGlueInput, // If the current node has an input glue, save it. - MoveChild, // Move current node to specified child. - MoveSibling, // Move current node to specified sibling. - MoveParent, // Move current node to parent. + Scope, // Push a checking scope. + RecordNode, // Record the current node. + RecordChild, // Record a child of the current node. + RecordMemRef, // Record the memref in the current node. + RecordOptionalChain, // If the current node has input chain, record it. + CaptureGlueInput, // If the current node has an input glue, save it. + MoveChild, // Move current node to specified child. + MoveSibling, // Move current node to specified sibling. + MoveParent, // Move current node to parent. // Predicate checking. CheckSame, // Fail if not same as prev match. @@ -322,6 +323,20 @@ private: bool isEqualImpl(const Matcher *M) const override { return true; } }; +/// +class RecordOptionalChainMatcher : public Matcher { +public: + RecordOptionalChainMatcher() : Matcher(RecordOptionalChain) {} + + static bool classof(const Matcher *N) { + return N->getKind() == RecordOptionalChain; + } + +private: + void printImpl(raw_ostream &OS, indent Indent) const override; + bool isEqualImpl(const Matcher *M) const override { return true; } +}; + /// MoveChildMatcher - This tells the interpreter to move into the /// specified child node. class MoveChildMatcher : public Matcher { @@ -1030,7 +1045,7 @@ class EmitNodeMatcherCommon : public Matcher { const CodeGenInstruction &CGI; const SmallVector<MVT::SimpleValueType, 3> VTs; const SmallVector<unsigned, 6> Operands; - bool HasChain, HasInGlue, HasOutGlue, HasMemRefs; + bool HasChain, HasInGlue, HasOutGlue, HasMemRefs, MayHaveChain; /// NumFixedArityOperands - If this is a fixed arity node, this is set to -1. /// If this is a varidic node, this is set to the number of fixed arity @@ -1042,10 +1057,12 @@ public: ArrayRef<MVT::SimpleValueType> vts, ArrayRef<unsigned> operands, bool hasChain, bool hasInGlue, bool hasOutGlue, bool hasmemrefs, - int numfixedarityoperands, bool isMorphNodeTo) + bool mayHaveChain, int numfixedarityoperands, + bool isMorphNodeTo) : Matcher(isMorphNodeTo ? MorphNodeTo : EmitNode), CGI(cgi), VTs(vts), Operands(operands), HasChain(hasChain), HasInGlue(hasInGlue), HasOutGlue(hasOutGlue), HasMemRefs(hasmemrefs), + MayHaveChain(mayHaveChain), NumFixedArityOperands(numfixedarityoperands) {} const CodeGenInstruction &getInstruction() const { return CGI; } @@ -1069,6 +1086,7 @@ public: bool hasInGlue() const { return HasInGlue; } bool hasOutGlue() const { return HasOutGlue; } bool hasMemRefs() const { return HasMemRefs; } + bool mayHaveChain() const { return MayHaveChain; } int getNumFixedArityOperands() const { return NumFixedArityOperands; } static bool classof(const Matcher *N) { @@ -1089,11 +1107,11 @@ public: EmitNodeMatcher(const CodeGenInstruction &cgi, ArrayRef<MVT::SimpleValueType> vts, ArrayRef<unsigned> operands, bool hasChain, bool hasInGlue, - bool hasOutGlue, bool hasmemrefs, int numfixedarityoperands, - unsigned firstresultslot) + bool hasOutGlue, bool hasmemrefs, bool mayHaveChain, + int numfixedarityoperands, unsigned firstresultslot) : EmitNodeMatcherCommon(cgi, vts, operands, hasChain, hasInGlue, - hasOutGlue, hasmemrefs, numfixedarityoperands, - false), + hasOutGlue, hasmemrefs, mayHaveChain, + numfixedarityoperands, false), FirstResultSlot(firstresultslot) {} unsigned getFirstResultSlot() const { return FirstResultSlot; } @@ -1109,11 +1127,11 @@ public: MorphNodeToMatcher(const CodeGenInstruction &cgi, ArrayRef<MVT::SimpleValueType> vts, ArrayRef<unsigned> operands, bool hasChain, bool hasInGlue, - bool hasOutGlue, bool hasmemrefs, + bool hasOutGlue, bool hasmemrefs, bool mayHaveChain, int numfixedarityoperands, const PatternToMatch &pattern) : EmitNodeMatcherCommon(cgi, vts, operands, hasChain, hasInGlue, - hasOutGlue, hasmemrefs, numfixedarityoperands, - true), + hasOutGlue, hasmemrefs, mayHaveChain, + numfixedarityoperands, true), Pattern(pattern) {} const PatternToMatch &getPattern() const { return Pattern; } diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp index ee10500a7ff5..6bffa35f7253 100644 --- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -494,6 +494,10 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N, OS << "OPC_CaptureGlueInput,\n"; return 1; + case Matcher::RecordOptionalChain: + OS << "OPC_RecordOptionalChain,\n"; + return 1; + case Matcher::MoveChild: { const auto *MCM = cast<MoveChildMatcher>(N); @@ -749,6 +753,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N, if (Pattern.hasProperty(SDNPHasChain)) OS << " + chain result"; + if (Pattern.hasProperty(SDNPMayHaveChain)) + OS << " + optional chain result"; } OS << '\n'; return PatternNo < 8 ? 2 : 3; @@ -964,21 +970,23 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N, if (CompressVTs) { OS << EN->getNumVTs(); if (!EN->hasChain() && !EN->hasInGlue() && !EN->hasOutGlue() && - !EN->hasMemRefs() && EN->getNumFixedArityOperands() == -1) { + !EN->hasMemRefs() && EN->getNumFixedArityOperands() == -1 && + !EN->mayHaveChain()) { CompressNodeInfo = true; OS << "None"; } else if (EN->hasChain() && !EN->hasInGlue() && !EN->hasOutGlue() && - !EN->hasMemRefs() && EN->getNumFixedArityOperands() == -1) { + !EN->hasMemRefs() && EN->getNumFixedArityOperands() == -1 && + !EN->mayHaveChain()) { CompressNodeInfo = true; OS << "Chain"; } else if (!IsEmitNode && !EN->hasChain() && EN->hasInGlue() && !EN->hasOutGlue() && !EN->hasMemRefs() && - EN->getNumFixedArityOperands() == -1) { + EN->getNumFixedArityOperands() == -1 && !EN->mayHaveChain()) { CompressNodeInfo = true; OS << "GlueInput"; } else if (!IsEmitNode && !EN->hasChain() && !EN->hasInGlue() && EN->hasOutGlue() && !EN->hasMemRefs() && - EN->getNumFixedArityOperands() == -1) { + EN->getNumFixedArityOperands() == -1 && !EN->mayHaveChain()) { CompressNodeInfo = true; OS << "GlueOutput"; } @@ -1333,6 +1341,8 @@ static StringRef getOpcodeString(Matcher::KindTy Kind) { return "OPC_EmitNodeXForm"; case Matcher::CompleteMatch: return "OPC_CompleteMatch"; + case Matcher::RecordOptionalChain: + return "OPC_RecordOptionalChain"; } llvm_unreachable("Unhandled opcode?"); diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp index d84bfa8d0c92..0bb385f5f411 100644 --- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -82,6 +82,10 @@ class MatcherGen { /// array of all of the recorded input nodes that have chains. SmallVector<unsigned, 2> MatchedChainNodes; + /// This maintains the position in the recorded nodes array for all recorded + /// input nodes that may have chains. + SmallVector<unsigned, 2> MatchedOptionalChainNodes; + /// MatchedComplexPatterns - This maintains a list of all of the /// ComplexPatterns that we need to check. The second element of each pair /// is the recorded operand number of the input node. @@ -370,18 +374,8 @@ void MatcherGen::EmitOperatorMatchCode(const TreePatternNode &N, // If this node has a chain, then the chain is operand #0 is the SDNode, and // the child numbers of the node are all offset by one. unsigned OpNo = 0; - if (N.NodeHasProperty(SDNPHasChain, CGP)) { - // Record the node and remember it in our chained nodes list. - AddMatcher(new RecordMatcher("'" + N.getOperator()->getName().str() + - "' chained node", - NextRecordedOperandNo)); - // Remember all of the input chains our pattern will match. - MatchedChainNodes.push_back(NextRecordedOperandNo++); - - // Don't look at the input chain when matching the tree pattern to the - // SDNode. - OpNo = 1; + const auto checkFoldableChain = [&]() { // If this node is not the root and the subtree underneath it produces a // chain, then the result of matching the node is also produce a chain. // Beyond that, this means that we're also folding (at least) the root node @@ -420,12 +414,36 @@ void MatcherGen::EmitOperatorMatchCode(const TreePatternNode &N, Root.getOperator() == CGP.get_intrinsic_w_chain_sdnode() || Root.getOperator() == CGP.get_intrinsic_wo_chain_sdnode() || PInfo.getNumOperands() > 1 || PInfo.hasProperty(SDNPHasChain) || - PInfo.hasProperty(SDNPInGlue) || PInfo.hasProperty(SDNPOptInGlue); + PInfo.hasProperty(SDNPInGlue) || PInfo.hasProperty(SDNPOptInGlue) || + PInfo.hasProperty(SDNPMayHaveChain); } if (NeedCheck) AddMatcher(new CheckFoldableChainNodeMatcher()); } + }; + + if (N.NodeHasProperty(SDNPHasChain, CGP)) { + // Record the node and remember it in our chained nodes list. + AddMatcher(new RecordMatcher("'" + N.getOperator()->getName().str() + + "' chained node", + NextRecordedOperandNo)); + // Remember all of the input chains our pattern will match. + MatchedChainNodes.push_back(NextRecordedOperandNo++); + + // Don't look at the input chain when matching the tree pattern to the + // SDNode. + OpNo = 1; + + checkFoldableChain(); + } + + if (N.NodeHasProperty(SDNPMayHaveChain, CGP)) { + // Record the node and remember it in our possibly-chained nodes list. + AddMatcher(new RecordOptionalChainMatcher()); + MatchedOptionalChainNodes.push_back(NextRecordedOperandNo++); + + checkFoldableChain(); } // If this node has an output glue and isn't the root, remember it. @@ -692,7 +710,7 @@ void MatcherGen::EmitResultLeafAsOperand(const TreePatternNode &N, const Record *ImpDef = Def->getRecords().getDef("IMPLICIT_DEF"); CodeGenInstruction &II = CGP.getTargetInfo().getInstruction(ImpDef); AddMatcher(new EmitNodeMatcher(II, ResultVT, {}, false, false, false, - false, -1, IDOperandNo)); + false, false, -1, IDOperandNo)); ResultOps.push_back(IDOperandNo); return; } @@ -968,9 +986,12 @@ void MatcherGen::EmitResultInstructionAsOperand( assert((!ResultVTs.empty() || TreeHasOutGlue || NodeHasChain) && "Node has no result"); + bool NodeMayHaveChain = + Pattern.getSrcPattern().TreeHasProperty(SDNPMayHaveChain, CGP); AddMatcher(new EmitNodeMatcher(II, ResultVTs, InstOps, NodeHasChain, TreeHasInGlue, TreeHasOutGlue, NodeHasMemRefs, - NumFixedArityOperands, NextRecordedOperandNo)); + NodeMayHaveChain, NumFixedArityOperands, + NextRecordedOperandNo)); // The non-chain and non-glue results of the newly emitted node get recorded. for (MVT::SimpleValueType ResultVT : ResultVTs) { @@ -1023,7 +1044,7 @@ void MatcherGen::EmitResultCode() { // Patterns that match nodes with (potentially multiple) chain inputs have to // merge them together into a token factor. This informs the generated code // what all the chained nodes are. - if (!MatchedChainNodes.empty()) + if (!MatchedChainNodes.empty() || !MatchedOptionalChainNodes.empty()) AddMatcher(new EmitMergeInputChainsMatcher(MatchedChainNodes)); // Codegen the root of the result pattern, capturing the resulting values. diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp index 268e6bbc4eee..43cfb92b66b4 100644 --- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp @@ -287,7 +287,7 @@ static void ContractNodes(std::unique_ptr<Matcher> &InputMatcherPtr, MatcherPtr->reset(new MorphNodeToMatcher( EN->getInstruction(), VTs, Operands, EN->hasChain(), EN->hasInGlue(), EN->hasOutGlue(), EN->hasMemRefs(), - EN->getNumFixedArityOperands(), Pattern)); + EN->mayHaveChain(), EN->getNumFixedArityOperands(), Pattern)); return; } } diff --git a/llvm/utils/TableGen/SDNodeInfoEmitter.cpp b/llvm/utils/TableGen/SDNodeInfoEmitter.cpp index 64f03dae83e7..e728c85d3e3b 100644 --- a/llvm/utils/TableGen/SDNodeInfoEmitter.cpp +++ b/llvm/utils/TableGen/SDNodeInfoEmitter.cpp @@ -67,7 +67,8 @@ static bool haveCompatibleDescriptions(const SDNodeInfo &N1, // and sometimes differ between nodes sharing the same enum name. constexpr unsigned PropMask = (1 << SDNPHasChain) | (1 << SDNPOutGlue) | (1 << SDNPInGlue) | (1 << SDNPOptInGlue) | - (1 << SDNPMemOperand) | (1 << SDNPVariadic); + (1 << SDNPMemOperand) | (1 << SDNPVariadic) | + (1 << SDNPMayHaveChain); return (N1.getProperties() & PropMask) == (N2.getProperties() & PropMask); } @@ -312,6 +313,8 @@ static void emitDesc(raw_ostream &OS, StringRef EnumName, OS << "|1<<SDNPVariadic"; if (Properties & (1 << SDNPMemOperand)) OS << "|1<<SDNPMemOperand"; + if (Properties & (1 << SDNPMayHaveChain)) + OS << "|1<<SDNPMayHaveChain"; OS << ", 0"; if (N.isStrictFP()) |