summaryrefslogtreecommitdiff
path: root/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp')
-rw-r--r--llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp95
1 files changed, 80 insertions, 15 deletions
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 340f4ac6f57a..c03cde0a6d54 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -189,11 +189,12 @@ private:
bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
SDValue &SOffset, SDValue &Offset, SDValue &Offen,
SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
- SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
+ SDValue &TFE, SDValue &DLC, SDValue &SWZ,
+ SDValue &SCCB) const;
bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
SDValue &SOffset, SDValue &Offset, SDValue &GLC,
SDValue &SLC, SDValue &TFE, SDValue &DLC,
- SDValue &SWZ) const;
+ SDValue &SWZ, SDValue &SCCB) const;
bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
SDValue &SLC) const;
@@ -206,7 +207,8 @@ private:
bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
SDValue &Offset, SDValue &GLC, SDValue &SLC,
- SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
+ SDValue &TFE, SDValue &DLC, SDValue &SWZ,
+ SDValue &SCCB) const;
bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
SDValue &Offset, SDValue &SLC) const;
bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
@@ -322,6 +324,16 @@ static SDValue stripBitcast(SDValue Val) {
// Figure out if this is really an extract of the high 16-bits of a dword.
static bool isExtractHiElt(SDValue In, SDValue &Out) {
In = stripBitcast(In);
+
+ if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
+ if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
+ if (!Idx->isOne())
+ return false;
+ Out = In.getOperand(0);
+ return true;
+ }
+ }
+
if (In.getOpcode() != ISD::TRUNCATE)
return false;
@@ -341,6 +353,13 @@ static bool isExtractHiElt(SDValue In, SDValue &Out) {
// Look through operations that obscure just looking at the low 16-bits of the
// same register.
static SDValue stripExtractLoElt(SDValue In) {
+ if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
+ if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
+ if (Idx->isNullValue() && In.getValueSizeInBits() <= 32)
+ return In.getOperand(0);
+ }
+ }
+
if (In.getOpcode() == ISD::TRUNCATE) {
SDValue Src = In.getOperand(0);
if (Src.getValueType().getSizeInBits() == 32)
@@ -1380,7 +1399,7 @@ bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
SDValue &Idxen, SDValue &Addr64,
SDValue &GLC, SDValue &SLC,
SDValue &TFE, SDValue &DLC,
- SDValue &SWZ) const {
+ SDValue &SWZ, SDValue &SCCB) const {
// Subtarget prefers to use flat instruction
// FIXME: This should be a pattern predicate and not reach here
if (Subtarget->useFlatForGlobal())
@@ -1395,6 +1414,7 @@ bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
DLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1);
+ SCCB = CurDAG->getTargetConstant(0, DL, MVT::i1);
Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
@@ -1474,7 +1494,8 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
SDValue &VAddr, SDValue &SOffset,
SDValue &Offset, SDValue &GLC,
SDValue &SLC, SDValue &TFE,
- SDValue &DLC, SDValue &SWZ) const {
+ SDValue &DLC, SDValue &SWZ,
+ SDValue &SCCB) const {
SDValue Ptr, Offen, Idxen, Addr64;
// addr64 bit was removed for volcanic islands.
@@ -1483,7 +1504,7 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
return false;
if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
- GLC, SLC, TFE, DLC, SWZ))
+ GLC, SLC, TFE, DLC, SWZ, SCCB))
return false;
ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
@@ -1505,9 +1526,9 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
SDValue &Offset,
SDValue &SLC) const {
SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
- SDValue GLC, TFE, DLC, SWZ;
+ SDValue GLC, TFE, DLC, SWZ, SCCB;
- return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ);
+ return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ, SCCB);
}
static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
@@ -1631,13 +1652,13 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
SDValue &SOffset, SDValue &Offset,
SDValue &GLC, SDValue &SLC,
SDValue &TFE, SDValue &DLC,
- SDValue &SWZ) const {
+ SDValue &SWZ, SDValue &SCCB) const {
SDValue Ptr, VAddr, Offen, Idxen, Addr64;
const SIInstrInfo *TII =
static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
- GLC, SLC, TFE, DLC, SWZ))
+ GLC, SLC, TFE, DLC, SWZ, SCCB))
return false;
if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
@@ -1659,16 +1680,16 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
SDValue &Soffset, SDValue &Offset
) const {
- SDValue GLC, SLC, TFE, DLC, SWZ;
+ SDValue GLC, SLC, TFE, DLC, SWZ, SCCB;
- return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
+ return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ, SCCB);
}
bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
SDValue &Soffset, SDValue &Offset,
SDValue &SLC) const {
- SDValue GLC, TFE, DLC, SWZ;
+ SDValue GLC, TFE, DLC, SWZ, SCCB;
- return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
+ return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ, SCCB);
}
// Find a load or store from corresponding pattern root.
@@ -2773,18 +2794,62 @@ bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
if (isExtractHiElt(Hi, Hi))
Mods |= SISrcMods::OP_SEL_1;
+ unsigned VecSize = Src.getValueSizeInBits();
Lo = stripExtractLoElt(Lo);
Hi = stripExtractLoElt(Hi);
+ if (Lo.getValueSizeInBits() > VecSize) {
+ Lo = CurDAG->getTargetExtractSubreg(
+ (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
+ MVT::getIntegerVT(VecSize), Lo);
+ }
+
+ if (Hi.getValueSizeInBits() > VecSize) {
+ Hi = CurDAG->getTargetExtractSubreg(
+ (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
+ MVT::getIntegerVT(VecSize), Hi);
+ }
+
+ assert(Lo.getValueSizeInBits() <= VecSize &&
+ Hi.getValueSizeInBits() <= VecSize);
+
if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
// Really a scalar input. Just select from the low half of the register to
// avoid packing.
- Src = Lo;
+ if (VecSize == 32 || VecSize == Lo.getValueSizeInBits()) {
+ Src = Lo;
+ } else {
+ assert(Lo.getValueSizeInBits() == 32 && VecSize == 64);
+
+ SDLoc SL(In);
+ SDValue Undef = SDValue(
+ CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL,
+ Lo.getValueType()), 0);
+ auto RC = Lo->isDivergent() ? AMDGPU::VReg_64RegClassID
+ : AMDGPU::SReg_64RegClassID;
+ const SDValue Ops[] = {
+ CurDAG->getTargetConstant(RC, SL, MVT::i32),
+ Lo, CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
+ Undef, CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32) };
+
+ Src = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
+ Src.getValueType(), Ops), 0);
+ }
SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
return true;
}
+ if (VecSize == 64 && Lo == Hi && isa<ConstantFPSDNode>(Lo)) {
+ uint64_t Lit = cast<ConstantFPSDNode>(Lo)->getValueAPF()
+ .bitcastToAPInt().getZExtValue();
+ if (AMDGPU::isInlinableLiteral32(Lit, Subtarget->hasInv2PiInlineImm())) {
+ Src = CurDAG->getTargetConstant(Lit, SDLoc(In), MVT::i64);;
+ SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
+ return true;
+ }
+ }
+
Mods = VecMods;
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-10-22 02:29:19 +0000